diff --git a/vp9/common/arm/neon/vp9_loopfilter_neon.asm b/vp9/common/arm/neon/vp9_loopfilter_neon.asm
index 5011315438273f11fda3b4d9373ac3afa2e873e9..02cf2f5e9db196c2469b381a239ea62a015e4394 100644
--- a/vp9/common/arm/neon/vp9_loopfilter_neon.asm
+++ b/vp9/common/arm/neon/vp9_loopfilter_neon.asm
@@ -10,6 +10,8 @@
EXPORT |vp9_loop_filter_horizontal_edge_neon|
EXPORT |vp9_loop_filter_vertical_edge_neon|
+ EXPORT |vp9_mbloop_filter_horizontal_edge_neon|
+ EXPORT |vp9_mbloop_filter_vertical_edge_neon|
ARM
AREA ||.text||, CODE, READONLY, ALIGN=2
@@ -33,50 +35,47 @@
; sp const uint8_t *thresh,
; sp+4 int count
|vp9_loop_filter_horizontal_edge_neon| PROC
- push {r4-r6, lr}
+ push {lr}
- ldr r12, [sp,#20] ; load count
- ldrb r4, [r2] ; load *blimit
- ldrb r5, [r3] ; load *limit
+ ldr r12, [sp,#8] ; load count
cmp r12, #0
beq end_vp9_lf_h_edge
- ldr r3, [sp, #16] ; load thresh
- vdup.u8 d0, r4 ; duplicate blimit
- ldrb r6, [r3] ; load *thresh
- vdup.u8 d1, r5 ; duplicate limit
- vdup.u8 d2, r6 ; duplicate thresh
+ vld1.8 {d0[]}, [r2] ; duplicate *blimit
+ ldr r2, [sp, #4] ; load thresh
+ vld1.8 {d1[]}, [r3] ; duplicate *limit
+ vld1.8 {d2[]}, [r2] ; duplicate *thresh
count_lf_h_loop
sub r2, r0, r1, lsl #2 ; move src pointer down by 4 lines
- add r6, r2, r1
+ add r3, r2, r1
add r1, r1, r1
vld1.u8 {d3}, [r2@64], r1 ; p3
- vld1.u8 {d4}, [r6@64], r1 ; p2
+ vld1.u8 {d4}, [r3@64], r1 ; p2
vld1.u8 {d5}, [r2@64], r1 ; p1
- vld1.u8 {d6}, [r6@64], r1 ; p0
+ vld1.u8 {d6}, [r3@64], r1 ; p0
vld1.u8 {d7}, [r2@64], r1 ; q0
- vld1.u8 {d16}, [r6@64], r1 ; q1
+ vld1.u8 {d16}, [r3@64], r1 ; q1
vld1.u8 {d17}, [r2@64] ; q2
- vld1.u8 {d18}, [r6@64] ; q3
+ vld1.u8 {d18}, [r3@64] ; q3
sub r2, r2, r1, lsl #1
- sub r6, r6, r1, lsl #1
+ sub r3, r3, r1, lsl #1
bl vp9_loop_filter_neon
vst1.u8 {d4}, [r2@64], r1 ; store op1
- vst1.u8 {d5}, [r6@64], r1 ; store op0
+ vst1.u8 {d5}, [r3@64], r1 ; store op0
vst1.u8 {d6}, [r2@64], r1 ; store oq0
- vst1.u8 {d7}, [r6@64], r1 ; store oq1
+ vst1.u8 {d7}, [r3@64], r1 ; store oq1
add r0, r0, #8
subs r12, r12, #1
bne count_lf_h_loop
end_vp9_lf_h_edge
- pop {r4-r6, pc}
+ pop {pc}
ENDP ; |vp9_loop_filter_horizontal_edge_neon|
; Currently vp9 only works on iterations 8 at a time. The vp8 loop filter
@@ -98,31 +97,28 @@ end_vp9_lf_h_edge
; sp const uint8_t *thresh,
; sp+4 int count
|vp9_loop_filter_vertical_edge_neon| PROC
- push {r4-r6, lr}
+ push {lr}
- ldr r12, [sp,#20] ; load count
- ldrb r4, [r2] ; load *blimit
- ldrb r5, [r3] ; load *limit
+ ldr r12, [sp,#8] ; load count
cmp r12, #0
beq end_vp9_lf_v_edge
- ldr r3, [sp, #16] ; load thresh
- vdup.u8 d0, r4 ; duplicate blimit
- ldrb r6, [r3] ; load *thresh
- vdup.u8 d1, r5 ; duplicate limit
- vdup.u8 d2, r6 ; duplicate thresh
+ vld1.8 {d0[]}, [r2] ; duplicate *blimit
+ ldr r2, [sp, #4] ; load thresh
+ vld1.8 {d1[]}, [r3] ; duplicate *limit
+ vld1.8 {d2[]}, [r2] ; duplicate *thresh
count_lf_v_loop
- sub r6, r0, #4 ; move s pointer down by 4 columns
+ sub r2, r0, #4 ; move s pointer down by 4 columns
- vld1.u8 {d3}, [r6], r1 ; load s data
- vld1.u8 {d4}, [r6], r1
- vld1.u8 {d5}, [r6], r1
- vld1.u8 {d6}, [r6], r1
- vld1.u8 {d7}, [r6], r1
- vld1.u8 {d16}, [r6], r1
- vld1.u8 {d17}, [r6], r1
- vld1.u8 {d18}, [r6]
+ vld1.u8 {d3}, [r2], r1 ; load s data
+ vld1.u8 {d4}, [r2], r1
+ vld1.u8 {d5}, [r2], r1
+ vld1.u8 {d6}, [r2], r1
+ vld1.u8 {d7}, [r2], r1
+ vld1.u8 {d16}, [r2], r1
+ vld1.u8 {d17}, [r2], r1
+ vld1.u8 {d18}, [r2]
;transpose to 8x16 matrix
vtrn.32 d3, d7
@@ -159,7 +155,7 @@ count_lf_v_loop
bne count_lf_v_loop
end_vp9_lf_v_edge
- pop {r4-r6, pc}
+ pop {pc}
ENDP ; |vp9_loop_filter_vertical_edge_neon|
; void vp9_loop_filter_neon();
@@ -224,7 +220,7 @@ end_vp9_lf_v_edge
vsub.s8 d28, d7, d6 ; ( qs0 - ps0)
- vcge.u8 d17, d0, d17 ; (a > blimit * 2 + limit) * -1
+ vcge.u8 d17, d0, d17 ; a > blimit
vqsub.s8 d27, d5, d16 ; filter = clamp(ps1-qs1)
vorr d22, d21, d22 ; hevmask
@@ -267,4 +263,331 @@ end_vp9_lf_v_edge
bx lr
ENDP ; |vp9_loop_filter_neon|
+; void vp9_mbloop_filter_horizontal_edge_neon(uint8_t *s, int p,
+; const uint8_t *blimit,
+; const uint8_t *limit,
+; const uint8_t *thresh,
+; int count)
+; r0 uint8_t *s,
+; r1 int p, /* pitch */
+; r2 const uint8_t *blimit,
+; r3 const uint8_t *limit,
+; sp const uint8_t *thresh,
+; sp+4 int count
+|vp9_mbloop_filter_horizontal_edge_neon| PROC
+ push {lr}
+
+ ldr r12, [sp,#8] ; load count
+ cmp r12, #0
+ beq end_vp9_mblf_h_edge
+
+ vld1.8 {d0[]}, [r2] ; duplicate *blimit
+ ldr r2, [sp, #4] ; load thresh
+ vld1.8 {d1[]}, [r3] ; duplicate *limit
+ vld1.8 {d2[]}, [r2] ; duplicate *thresh
+
+count_mblf_h_loop
+ sub r3, r0, r1, lsl #2 ; move src pointer down by 4 lines
+ add r2, r3, r1
+ add r1, r1, r1
+
+ vld1.u8 {d3}, [r3@64], r1 ; p3
+ vld1.u8 {d4}, [r2@64], r1 ; p2
+ vld1.u8 {d5}, [r3@64], r1 ; p1
+ vld1.u8 {d6}, [r2@64], r1 ; p0
+ vld1.u8 {d7}, [r3@64], r1 ; q0
+ vld1.u8 {d16}, [r2@64], r1 ; q1
+ vld1.u8 {d17}, [r3@64] ; q2
+ vld1.u8 {d18}, [r2@64], r1 ; q3
+
+ sub r3, r3, r1, lsl #1
+ sub r2, r2, r1, lsl #2
+
+ bl vp9_mbloop_filter_neon
+
+ vst1.u8 {d2}, [r2@64], r1 ; store op2
+ vst1.u8 {d3}, [r3@64], r1 ; store op1
+ vst1.u8 {d4}, [r2@64], r1 ; store op0
+ vst1.u8 {d5}, [r3@64], r1 ; store oq0
+ vst1.u8 {d6}, [r2@64], r1 ; store oq1
+ vst1.u8 {d7}, [r3@64], r1 ; store oq2
+
+ add r0, r0, #8
+ subs r12, r12, #1
+ bne count_mblf_h_loop
+
+end_vp9_mblf_h_edge
+ pop {pc}
+
+ ENDP ; |vp9_mbloop_filter_horizontal_edge_neon|
+
+; void vp9_mbloop_filter_vertical_edge_neon(uint8_t *s,
+; int pitch,
+; const uint8_t *blimit,
+; const uint8_t *limit,
+; const uint8_t *thresh,
+; int count)
+;
+; r0 uint8_t *s,
+; r1 int pitch,
+; r2 const uint8_t *blimit,
+; r3 const uint8_t *limit,
+; sp const uint8_t *thresh,
+; sp+4 int count
+|vp9_mbloop_filter_vertical_edge_neon| PROC
+ push {lr}
+
+ ldr r12, [sp,#8] ; load count
+ cmp r12, #0
+ beq end_vp9_mblf_v_edge
+
+ vld1.8 {d0[]}, [r2] ; duplicate *blimit
+ ldr r2, [sp, #4] ; load thresh
+ vld1.8 {d1[]}, [r3] ; duplicate *limit
+ vld1.8 {d2[]}, [r2] ; duplicate *thresh
+
+count_mblf_v_loop
+ sub r2, r0, #4 ; move s pointer down by 4 columns
+
+ vld1.u8 {d3}, [r2], r1 ; load s data
+ vld1.u8 {d4}, [r2], r1
+ vld1.u8 {d5}, [r2], r1
+ vld1.u8 {d6}, [r2], r1
+ vld1.u8 {d7}, [r2], r1
+ vld1.u8 {d16}, [r2], r1
+ vld1.u8 {d17}, [r2], r1
+ vld1.u8 {d18}, [r2]
+
+ ;transpose to 8x16 matrix
+ vtrn.32 d3, d7
+ vtrn.32 d4, d16
+ vtrn.32 d5, d17
+ vtrn.32 d6, d18
+
+ vtrn.16 d3, d5
+ vtrn.16 d4, d6
+ vtrn.16 d7, d17
+ vtrn.16 d16, d18
+
+ vtrn.8 d3, d4
+ vtrn.8 d5, d6
+ vtrn.8 d7, d16
+ vtrn.8 d17, d18
+
+ sub r2, r0, #3
+ add r3, r0, #1
+
+ bl vp9_mbloop_filter_neon
+
+ ;store op2, op1, op0, oq0
+ vst4.8 {d2[0], d3[0], d4[0], d5[0]}, [r2], r1
+ vst4.8 {d2[1], d3[1], d4[1], d5[1]}, [r2], r1
+ vst4.8 {d2[2], d3[2], d4[2], d5[2]}, [r2], r1
+ vst4.8 {d2[3], d3[3], d4[3], d5[3]}, [r2], r1
+ vst4.8 {d2[4], d3[4], d4[4], d5[4]}, [r2], r1
+ vst4.8 {d2[5], d3[5], d4[5], d5[5]}, [r2], r1
+ vst4.8 {d2[6], d3[6], d4[6], d5[6]}, [r2], r1
+ vst4.8 {d2[7], d3[7], d4[7], d5[7]}, [r2]
+
+ ;store oq1, oq2
+ vst2.8 {d6[0], d7[0]}, [r3], r1
+ vst2.8 {d6[1], d7[1]}, [r3], r1
+ vst2.8 {d6[2], d7[2]}, [r3], r1
+ vst2.8 {d6[3], d7[3]}, [r3], r1
+ vst2.8 {d6[4], d7[4]}, [r3], r1
+ vst2.8 {d6[5], d7[5]}, [r3], r1
+ vst2.8 {d6[6], d7[6]}, [r3], r1
+ vst2.8 {d6[7], d7[7]}, [r3]
+
+ add r0, r0, r1, lsl #3 ; s += pitch * 8
+ subs r12, r12, #1
+ bne count_mblf_v_loop
+
+end_vp9_mblf_v_edge
+ pop {pc}
+ ENDP ; |vp9_mbloop_filter_vertical_edge_neon|
+
+; void vp9_mbloop_filter_neon();
+; This is a helper function for the loopfilters. The invidual functions do the
+; necessary load, transpose (if necessary) and store. The function does not use
+; registers d8-d15.
+;
+; r0-r3 PRESERVE
+; d0 blimit
+; d1 limit
+; d2 thresh
+; d3 p3
+; d4 p2
+; d5 p1
+; d6 p0
+; d7 q0
+; d16 q1
+; d17 q2
+; d18 q3
+|vp9_mbloop_filter_neon| PROC
+ ; filter_mask
+ vabd.u8 d19, d3, d4 ; abs(p3 - p2)
+ vabd.u8 d20, d4, d5 ; abs(p2 - p1)
+ vabd.u8 d21, d5, d6 ; abs(p1 - p0)
+ vabd.u8 d22, d16, d7 ; abs(q1 - q0)
+ vabd.u8 d23, d17, d16 ; abs(q2 - q1)
+ vabd.u8 d24, d18, d17 ; abs(q3 - q2)
+
+ ; only compare the largest value to limit
+ vmax.u8 d19, d19, d20 ; max(abs(p3 - p2), abs(p2 - p1))
+ vmax.u8 d20, d21, d22 ; max(abs(p1 - p0), abs(q1 - q0))
+ vmax.u8 d23, d23, d24 ; max(abs(q2 - q1), abs(q3 - q2))
+ vmax.u8 d19, d19, d20
+
+ vabd.u8 d24, d6, d7 ; abs(p0 - q0)
+
+ vmax.u8 d19, d19, d23
+
+ vabd.u8 d23, d5, d16 ; a = abs(p1 - q1)
+ vqadd.u8 d24, d24, d24 ; b = abs(p0 - q0) * 2
+
+ ; abs () > limit
+ vcge.u8 d19, d1, d19
+
+ ; flatmask4
+ vabd.u8 d25, d6, d4 ; abs(p0 - p2)
+ vabd.u8 d26, d7, d17 ; abs(q0 - q2)
+ vabd.u8 d27, d3, d6 ; abs(p3 - p0)
+ vabd.u8 d28, d18, d7 ; abs(q3 - q0)
+
+ ; only compare the largest value to thresh
+ vmax.u8 d25, d25, d26 ; max(abs(p0 - p2), abs(q0 - q2))
+ vmax.u8 d26, d27, d28 ; max(abs(p3 - p0), abs(q3 - q0))
+ vmax.u8 d25, d25, d26
+ vmax.u8 d20, d20, d25
+
+ vshr.u8 d23, d23, #1 ; a = a / 2
+ vqadd.u8 d24, d24, d23 ; a = b + a
+
+ vmov.u8 d23, #1
+ vcge.u8 d24, d0, d24 ; a > blimit
+
+ vcge.u8 d20, d23, d20 ; flat
+
+ vand d19, d19, d24 ; mask
+
+ ; hevmask
+ vcgt.u8 d21, d21, d2 ; (abs(p1 - p0) > thresh)*-1
+ vcgt.u8 d22, d22, d2 ; (abs(q1 - q0) > thresh)*-1
+ vorr d21, d21, d22 ; hev
+
+ vmov.u8 d22, #0x80
+
+ ; mbfilter() function
+
+ ; filter() function
+ ; convert to signed
+ veor d23, d7, d22 ; qs0
+ veor d24, d6, d22 ; ps0
+ veor d25, d5, d22 ; ps1
+ veor d26, d16, d22 ; qs1
+
+ vmov.u8 d27, #3
+
+ vsub.s8 d28, d23, d24 ; ( qs0 - ps0)
+
+ vqsub.s8 d29, d25, d26 ; filter = clamp(ps1-qs1)
+
+ vmull.s8 q15, d28, d27 ; 3 * ( qs0 - ps0)
+
+ vand d29, d29, d21 ; filter &= hev
+
+ vaddw.s8 q15, q15, d29 ; filter + 3 * (qs0 - ps0)
+
+ vmov.u8 d29, #4
+
+ ; filter = clamp(filter + 3 * ( qs0 - ps0))
+ vqmovn.s16 d28, q15
+
+ vand d28, d28, d19 ; filter &= mask
+
+ vqadd.s8 d30, d28, d27 ; filter2 = clamp(filter+3)
+ vqadd.s8 d29, d28, d29 ; filter1 = clamp(filter+4)
+ vshr.s8 d30, d30, #3 ; filter2 >>= 3
+ vshr.s8 d29, d29, #3 ; filter1 >>= 3
+
+ vand d20, d20, d19 ; flat & mask
+
+ vqadd.s8 d24, d24, d30 ; op0 = clamp(ps0 + filter2)
+ vqsub.s8 d23, d23, d29 ; oq0 = clamp(qs0 - filter1)
+
+ ; outer tap adjustments: ++filter1 >> 1
+ vrshr.s8 d29, d29, #1
+ vbic d29, d29, d21 ; filter &= ~hev
+
+ vqadd.s8 d25, d25, d29 ; op1 = clamp(ps1 + filter)
+ vqsub.s8 d26, d26, d29 ; oq1 = clamp(qs1 - filter)
+
+ veor d24, d24, d22 ; *f_op0 = u^0x80
+ veor d23, d23, d22 ; *f_oq0 = u^0x80
+ veor d25, d25, d22 ; *f_op1 = u^0x80
+ veor d26, d26, d22 ; *f_oq1 = u^0x80
+
+ ; mbfilter flat && mask branch
+ ; TODO(fgalligan): Can I decrease the cycles shifting to consective d's
+ ; and using vibt on the q's?
+ vmov.u8 d21, #2
+ vaddl.u8 q14, d6, d7 ; op2 = p0 + q0
+ vmlal.u8 q14, d3, d27 ; op2 += p3 * 3
+ vmlal.u8 q14, d4, d21 ; op2 += p2 * 2
+ vaddw.u8 q14, d5 ; op2 += p1
+ vqrshrn.u16 d30, q14, #3 ; r_op2
+
+ vsubw.u8 q14, d3 ; op1 = op2 - p3
+ vsubw.u8 q14, d4 ; op1 -= p2
+ vaddw.u8 q14, d5 ; op1 += p1
+ vaddw.u8 q14, d16 ; op1 += q1
+ vqrshrn.u16 d31, q14, #3 ; r_op1
+
+ vsubw.u8 q14, d3 ; op0 = op1 - p3
+ vsubw.u8 q14, d5 ; op0 -= p1
+ vaddw.u8 q14, d6 ; op0 += p0
+ vaddw.u8 q14, d17 ; op0 += q2
+ vqrshrn.u16 d21, q14, #3 ; r_op0
+
+ vsubw.u8 q14, d3 ; oq0 = op0 - p3
+ vsubw.u8 q14, d6 ; oq0 -= p0
+ vaddw.u8 q14, d7 ; oq0 += q0
+ vaddw.u8 q14, d18 ; oq0 += q3
+ vqrshrn.u16 d22, q14, #3 ; r_oq0
+
+ vsubw.u8 q14, d4 ; oq1 = oq0 - p2
+ vsubw.u8 q14, d7 ; oq1 -= q0
+ vaddw.u8 q14, d16 ; oq1 += q1
+ vaddw.u8 q14, d18 ; oq1 += q3
+ vqrshrn.u16 d0, q14, #3 ; r_oq1
+
+ vsubw.u8 q14, d5 ; oq2 = oq0 - p1
+ vsubw.u8 q14, d16 ; oq2 -= q1
+ vaddw.u8 q14, d17 ; oq2 += q2
+ vaddw.u8 q14, d18 ; oq2 += q3
+ vqrshrn.u16 d1, q14, #3 ; r_oq2
+
+ ; Filter does not set op2 or oq2, so use p2 and q2.
+ vbit d2, d30, d20 ; op2 |= r_op2 & (flat & mask)
+ vbif d2, d4, d20 ; op2 |= op2 & ~(flat & mask)
+
+ vbit d3, d31, d20 ; op1 |= r_op1 & (flat & mask)
+ vbif d3, d25, d20 ; op1 |= f_op1 & ~(flat & mask)
+
+ vbit d4, d21, d20 ; op0 |= r_op0 & (flat & mask)
+ vbif d4, d24, d20 ; op0 |= f_op0 & ~(flat & mask)
+
+ vbit d5, d22, d20 ; oq0 |= r_oq0 & (flat & mask)
+ vbif d5, d23, d20 ; oq0 |= f_oq0 & ~(flat & mask)
+
+ vbit d6, d0, d20 ; oq1 |= r_oq1 & (flat & mask)
+ vbif d6, d26, d20 ; oq1 |= f_oq1 & ~(flat & mask)
+
+ vbit d7, d1, d20 ; oq2 |= r_oq2 & (flat & mask)
+ vbif d7, d17, d20 ; oq2 |= oq2 & ~(flat & mask)
+
+ bx lr
+ ENDP ; |vp9_mbloop_filter_neon|
+
END
diff --git a/vp9/common/vp9_rtcd_defs.sh b/vp9/common/vp9_rtcd_defs.sh
index 7e2cc6de19123cb36f4ebd04baafff9d0a9ec6eb..c6167d279501960344110d279f9154ded2f01d68 100644
--- a/vp9/common/vp9_rtcd_defs.sh
+++ b/vp9/common/vp9_rtcd_defs.sh
@@ -109,7 +109,7 @@ prototype void vp9_mb_lpf_vertical_edge_w "uint8_t *s, int pitch, const uint8_t
specialize vp9_mb_lpf_vertical_edge_w sse2
prototype void vp9_mbloop_filter_vertical_edge "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int count"
-specialize vp9_mbloop_filter_vertical_edge sse2
+specialize vp9_mbloop_filter_vertical_edge sse2 neon
prototype void vp9_loop_filter_vertical_edge "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int count"
specialize vp9_loop_filter_vertical_edge mmx neon
@@ -118,7 +118,7 @@ prototype void vp9_mb_lpf_horizontal_edge_w "uint8_t *s, int pitch, const uint8_
specialize vp9_mb_lpf_horizontal_edge_w sse2
prototype void vp9_mbloop_filter_horizontal_edge "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int count"
-specialize vp9_mbloop_filter_horizontal_edge sse2
+specialize vp9_mbloop_filter_horizontal_edge sse2 neon
prototype void vp9_loop_filter_horizontal_edge "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int count"
specialize vp9_loop_filter_horizontal_edge mmx neon