Commit 19c45ccd authored by Scott LaVarnway's avatar Scott LaVarnway

vpxdsp: [x86] add highbd_d153_predictor functions

C vs SSE2 speed gains:
_4x4 : ~1.95x

C vs SSSE3 speed gains:
_8x8 : ~3.30x
_16x16 : ~5.67x
_32x32 : ~3.87x

BUG=webm:1411

Change-Id: Ib483989b25614aa89b635e8c087d0879a5d71904
parent d203a91a
......@@ -484,9 +484,9 @@ HIGHBD_INTRA_PRED_TEST(
SSE2, TestHighbdIntraPred4, vpx_highbd_dc_predictor_4x4_sse2,
vpx_highbd_dc_left_predictor_4x4_sse2, vpx_highbd_dc_top_predictor_4x4_sse2,
vpx_highbd_dc_128_predictor_4x4_sse2, vpx_highbd_v_predictor_4x4_sse2,
vpx_highbd_h_predictor_4x4_sse2, NULL, NULL, NULL, NULL,
vpx_highbd_d207_predictor_4x4_sse2, vpx_highbd_d63_predictor_4x4_sse2,
vpx_highbd_tm_predictor_4x4_c)
vpx_highbd_h_predictor_4x4_sse2, NULL, NULL, NULL,
vpx_highbd_d153_predictor_4x4_sse2, vpx_highbd_d207_predictor_4x4_sse2,
vpx_highbd_d63_predictor_4x4_sse2, vpx_highbd_tm_predictor_4x4_c)
HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred8,
vpx_highbd_dc_predictor_8x8_sse2,
......@@ -522,15 +522,18 @@ HIGHBD_INTRA_PRED_TEST(SSSE3, TestHighbdIntraPred4, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL)
HIGHBD_INTRA_PRED_TEST(SSSE3, TestHighbdIntraPred8, NULL, NULL, NULL, NULL,
NULL, NULL, vpx_highbd_d45_predictor_8x8_ssse3, NULL,
NULL, NULL, vpx_highbd_d207_predictor_8x8_ssse3,
NULL, vpx_highbd_d153_predictor_8x8_ssse3,
vpx_highbd_d207_predictor_8x8_ssse3,
vpx_highbd_d63_predictor_8x8_ssse3, NULL)
HIGHBD_INTRA_PRED_TEST(SSSE3, TestHighbdIntraPred16, NULL, NULL, NULL, NULL,
NULL, NULL, vpx_highbd_d45_predictor_16x16_ssse3, NULL,
NULL, NULL, vpx_highbd_d207_predictor_16x16_ssse3,
NULL, vpx_highbd_d153_predictor_16x16_ssse3,
vpx_highbd_d207_predictor_16x16_ssse3,
vpx_highbd_d63_predictor_16x16_ssse3, NULL)
HIGHBD_INTRA_PRED_TEST(SSSE3, TestHighbdIntraPred32, NULL, NULL, NULL, NULL,
NULL, NULL, vpx_highbd_d45_predictor_32x32_ssse3, NULL,
NULL, NULL, vpx_highbd_d207_predictor_32x32_ssse3,
NULL, vpx_highbd_d153_predictor_32x32_ssse3,
vpx_highbd_d207_predictor_32x32_ssse3,
vpx_highbd_d63_predictor_32x32_ssse3, NULL)
#endif // HAVE_SSSE3
......
......@@ -485,6 +485,12 @@ INSTANTIATE_TEST_CASE_P(
&vpx_highbd_d63_predictor_16x16_c, 16, 8),
HighbdIntraPredParam(&vpx_highbd_d63_predictor_32x32_c,
&vpx_highbd_d63_predictor_32x32_ssse3, 32, 8),
HighbdIntraPredParam(&vpx_highbd_d153_predictor_8x8_ssse3,
&vpx_highbd_d153_predictor_8x8_c, 8, 8),
HighbdIntraPredParam(&vpx_highbd_d153_predictor_16x16_ssse3,
&vpx_highbd_d153_predictor_16x16_c, 16, 8),
HighbdIntraPredParam(&vpx_highbd_d153_predictor_32x32_ssse3,
&vpx_highbd_d153_predictor_32x32_c, 32, 8),
HighbdIntraPredParam(&vpx_highbd_d207_predictor_8x8_ssse3,
&vpx_highbd_d207_predictor_8x8_c, 8, 8),
HighbdIntraPredParam(&vpx_highbd_d207_predictor_16x16_ssse3,
......@@ -509,6 +515,12 @@ INSTANTIATE_TEST_CASE_P(
&vpx_highbd_d63_predictor_16x16_c, 16, 10),
HighbdIntraPredParam(&vpx_highbd_d63_predictor_32x32_c,
&vpx_highbd_d63_predictor_32x32_ssse3, 32, 10),
HighbdIntraPredParam(&vpx_highbd_d153_predictor_8x8_ssse3,
&vpx_highbd_d153_predictor_8x8_c, 8, 10),
HighbdIntraPredParam(&vpx_highbd_d153_predictor_16x16_ssse3,
&vpx_highbd_d153_predictor_16x16_c, 16, 10),
HighbdIntraPredParam(&vpx_highbd_d153_predictor_32x32_ssse3,
&vpx_highbd_d153_predictor_32x32_c, 32, 10),
HighbdIntraPredParam(&vpx_highbd_d207_predictor_8x8_ssse3,
&vpx_highbd_d207_predictor_8x8_c, 8, 10),
HighbdIntraPredParam(&vpx_highbd_d207_predictor_16x16_ssse3,
......@@ -533,6 +545,12 @@ INSTANTIATE_TEST_CASE_P(
&vpx_highbd_d63_predictor_16x16_c, 16, 12),
HighbdIntraPredParam(&vpx_highbd_d63_predictor_32x32_c,
&vpx_highbd_d63_predictor_32x32_ssse3, 32, 12),
HighbdIntraPredParam(&vpx_highbd_d153_predictor_8x8_ssse3,
&vpx_highbd_d153_predictor_8x8_c, 8, 12),
HighbdIntraPredParam(&vpx_highbd_d153_predictor_16x16_ssse3,
&vpx_highbd_d153_predictor_16x16_c, 16, 12),
HighbdIntraPredParam(&vpx_highbd_d153_predictor_32x32_ssse3,
&vpx_highbd_d153_predictor_32x32_c, 32, 12),
HighbdIntraPredParam(&vpx_highbd_d207_predictor_8x8_ssse3,
&vpx_highbd_d207_predictor_8x8_c, 8, 12),
HighbdIntraPredParam(&vpx_highbd_d207_predictor_16x16_ssse3,
......@@ -555,6 +573,8 @@ INSTANTIATE_TEST_CASE_P(
&vpx_highbd_dc_128_predictor_32x32_c, 32, 8),
HighbdIntraPredParam(&vpx_highbd_d63_predictor_4x4_sse2,
&vpx_highbd_d63_predictor_4x4_c, 4, 8),
HighbdIntraPredParam(&vpx_highbd_d153_predictor_4x4_sse2,
&vpx_highbd_d153_predictor_4x4_c, 4, 8),
HighbdIntraPredParam(&vpx_highbd_d207_predictor_4x4_sse2,
&vpx_highbd_d207_predictor_4x4_c, 4, 8),
HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_4x4_sse2,
......@@ -619,6 +639,8 @@ INSTANTIATE_TEST_CASE_P(
&vpx_highbd_dc_128_predictor_32x32_c, 32, 10),
HighbdIntraPredParam(&vpx_highbd_d63_predictor_4x4_sse2,
&vpx_highbd_d63_predictor_4x4_c, 4, 10),
HighbdIntraPredParam(&vpx_highbd_d153_predictor_4x4_sse2,
&vpx_highbd_d153_predictor_4x4_c, 4, 10),
HighbdIntraPredParam(&vpx_highbd_d207_predictor_4x4_sse2,
&vpx_highbd_d207_predictor_4x4_c, 4, 10),
HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_4x4_sse2,
......@@ -683,6 +705,8 @@ INSTANTIATE_TEST_CASE_P(
&vpx_highbd_dc_128_predictor_32x32_c, 32, 12),
HighbdIntraPredParam(&vpx_highbd_d63_predictor_4x4_sse2,
&vpx_highbd_d63_predictor_4x4_c, 4, 12),
HighbdIntraPredParam(&vpx_highbd_d153_predictor_4x4_sse2,
&vpx_highbd_d153_predictor_4x4_c, 4, 12),
HighbdIntraPredParam(&vpx_highbd_d207_predictor_4x4_sse2,
&vpx_highbd_d207_predictor_4x4_c, 4, 12),
HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_4x4_sse2,
......
......@@ -206,6 +206,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {
specialize qw/vpx_highbd_d135_predictor_4x4 neon/;
add_proto qw/void vpx_highbd_d153_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd";
specialize qw/vpx_highbd_d153_predictor_4x4 sse2/;
add_proto qw/void vpx_highbd_v_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd";
specialize qw/vpx_highbd_v_predictor_4x4 neon sse2/;
......@@ -243,6 +244,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {
specialize qw/vpx_highbd_d135_predictor_8x8 neon/;
add_proto qw/void vpx_highbd_d153_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd";
specialize qw/vpx_highbd_d153_predictor_8x8 ssse3/;
add_proto qw/void vpx_highbd_v_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd";
specialize qw/vpx_highbd_v_predictor_8x8 neon sse2/;
......@@ -280,6 +282,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {
specialize qw/vpx_highbd_d135_predictor_16x16 neon/;
add_proto qw/void vpx_highbd_d153_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd";
specialize qw/vpx_highbd_d153_predictor_16x16 ssse3/;
add_proto qw/void vpx_highbd_v_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd";
specialize qw/vpx_highbd_v_predictor_16x16 neon sse2/;
......@@ -317,6 +320,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {
specialize qw/vpx_highbd_d135_predictor_32x32 neon/;
add_proto qw/void vpx_highbd_d153_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd";
specialize qw/vpx_highbd_d153_predictor_32x32 ssse3/;
add_proto qw/void vpx_highbd_v_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd";
specialize qw/vpx_highbd_v_predictor_32x32 neon sse2/;
......
......@@ -393,6 +393,37 @@ static INLINE __m128i avg3_epu16(const __m128i *x, const __m128i *y,
return _mm_avg_epu16(b, *y);
}
void vpx_highbd_d153_predictor_4x4_sse2(uint16_t *dst, ptrdiff_t stride,
const uint16_t *above,
const uint16_t *left, int bd) {
const int I = left[0];
const int J = left[1];
const int K = left[2];
const int L = left[3];
const __m128i XXXXXABC = _mm_loadu_si128((const __m128i *)(above - 5));
const __m128i LXXXXABC = _mm_insert_epi16(XXXXXABC, L, 0);
const __m128i LKXXXABC = _mm_insert_epi16(LXXXXABC, K, 1);
const __m128i LKJXXABC = _mm_insert_epi16(LKXXXABC, J, 2);
const __m128i LKJIXABC = _mm_insert_epi16(LKJXXABC, I, 3);
const __m128i KJIXABC0 = _mm_srli_si128(LKJIXABC, 2);
const __m128i JIXABC00 = _mm_srli_si128(LKJIXABC, 4);
const __m128i avg3 = avg3_epu16(&LKJIXABC, &KJIXABC0, &JIXABC00);
const __m128i avg2 = _mm_avg_epu16(LKJIXABC, KJIXABC0);
const __m128i row3 = _mm_unpacklo_epi16(avg2, avg3);
const __m128i row2 = _mm_srli_si128(row3, 4);
const __m128i row1 = _mm_srli_si128(row3, 8);
const __m128i row0 = _mm_srli_si128(avg3, 4);
(void)bd;
_mm_storel_epi64((__m128i *)dst, row0);
dst[0] = _mm_extract_epi16(avg2, 3);
dst += stride;
_mm_storel_epi64((__m128i *)dst, row1);
dst += stride;
_mm_storel_epi64((__m128i *)dst, row2);
dst += stride;
_mm_storel_epi64((__m128i *)dst, row3);
}
void vpx_highbd_d207_predictor_4x4_sse2(uint16_t *dst, ptrdiff_t stride,
const uint16_t *above,
const uint16_t *left, int bd) {
......
......@@ -170,6 +170,212 @@ void vpx_highbd_d45_predictor_32x32_ssse3(uint16_t *dst, ptrdiff_t stride,
}
}
void vpx_highbd_d153_predictor_8x8_ssse3(uint16_t *dst, ptrdiff_t stride,
const uint16_t *above,
const uint16_t *left, int bd) {
const __m128i XABCDEFG = _mm_loadu_si128((const __m128i *)(above - 1));
const __m128i ABCDEFG0 = _mm_srli_si128(XABCDEFG, 2);
const __m128i BCDEFG00 = _mm_srli_si128(XABCDEFG, 4);
const __m128i avg3 = avg3_epu16(&BCDEFG00, &ABCDEFG0, &XABCDEFG);
const __m128i IJKLMNOP = _mm_load_si128((const __m128i *)left);
const __m128i XIJKLMNO =
_mm_alignr_epi8(IJKLMNOP, _mm_slli_si128(XABCDEFG, 14), 14);
const __m128i AXIJKLMN =
_mm_alignr_epi8(XIJKLMNO, _mm_slli_si128(XABCDEFG, 12), 14);
const __m128i avg3_left = avg3_epu16(&IJKLMNOP, &XIJKLMNO, &AXIJKLMN);
const __m128i avg2_left = _mm_avg_epu16(IJKLMNOP, XIJKLMNO);
const __m128i avg2_avg3_lo = _mm_unpacklo_epi16(avg2_left, avg3_left);
const __m128i avg2_avg3_hi = _mm_unpackhi_epi16(avg2_left, avg3_left);
const __m128i row0 =
_mm_alignr_epi8(avg3, _mm_slli_si128(avg2_avg3_lo, 12), 12);
const __m128i row1 =
_mm_alignr_epi8(row0, _mm_slli_si128(avg2_avg3_lo, 8), 12);
const __m128i row2 =
_mm_alignr_epi8(row1, _mm_slli_si128(avg2_avg3_lo, 4), 12);
const __m128i row3 = _mm_alignr_epi8(row2, avg2_avg3_lo, 12);
const __m128i row4 =
_mm_alignr_epi8(row3, _mm_slli_si128(avg2_avg3_hi, 12), 12);
const __m128i row5 =
_mm_alignr_epi8(row4, _mm_slli_si128(avg2_avg3_hi, 8), 12);
const __m128i row6 =
_mm_alignr_epi8(row5, _mm_slli_si128(avg2_avg3_hi, 4), 12);
const __m128i row7 = _mm_alignr_epi8(row6, avg2_avg3_hi, 12);
(void)bd;
_mm_store_si128((__m128i *)dst, row0);
dst += stride;
_mm_store_si128((__m128i *)dst, row1);
dst += stride;
_mm_store_si128((__m128i *)dst, row2);
dst += stride;
_mm_store_si128((__m128i *)dst, row3);
dst += stride;
_mm_store_si128((__m128i *)dst, row4);
dst += stride;
_mm_store_si128((__m128i *)dst, row5);
dst += stride;
_mm_store_si128((__m128i *)dst, row6);
dst += stride;
_mm_store_si128((__m128i *)dst, row7);
}
void vpx_highbd_d153_predictor_16x16_ssse3(uint16_t *dst, ptrdiff_t stride,
const uint16_t *above,
const uint16_t *left, int bd) {
const __m128i A0 = _mm_loadu_si128((const __m128i *)(above - 1));
const __m128i A1 = _mm_loadu_si128((const __m128i *)(above + 7));
const __m128i B0 = _mm_alignr_epi8(A1, A0, 2);
const __m128i B1 = _mm_srli_si128(A1, 2);
const __m128i C0 = _mm_alignr_epi8(A1, A0, 4);
const __m128i C1 = _mm_srli_si128(A1, 4);
const __m128i avg3_0 = avg3_epu16(&A0, &B0, &C0);
const __m128i avg3_1 = avg3_epu16(&A1, &B1, &C1);
const __m128i L0 = _mm_load_si128((const __m128i *)left);
const __m128i L1 = _mm_load_si128((const __m128i *)(left + 8));
const __m128i XL0 = _mm_alignr_epi8(L0, _mm_slli_si128(A0, 14), 14);
const __m128i AXL0 = _mm_alignr_epi8(XL0, _mm_slli_si128(A0, 12), 14);
const __m128i XL1 = _mm_alignr_epi8(L1, L0, 14);
const __m128i AXL1 = _mm_alignr_epi8(L1, L0, 12);
const __m128i avg3_left_0 = avg3_epu16(&L0, &XL0, &AXL0);
const __m128i avg2_left_0 = _mm_avg_epu16(L0, XL0);
const __m128i avg3_left_1 = avg3_epu16(&L1, &XL1, &AXL1);
const __m128i avg2_left_1 = _mm_avg_epu16(L1, XL1);
__m128i row_0 = avg3_0;
__m128i row_1 = avg3_1;
__m128i avg2_avg3_left[2][2];
int i, j;
(void)bd;
avg2_avg3_left[0][0] = _mm_unpacklo_epi16(avg2_left_0, avg3_left_0);
avg2_avg3_left[0][1] = _mm_unpackhi_epi16(avg2_left_0, avg3_left_0);
avg2_avg3_left[1][0] = _mm_unpacklo_epi16(avg2_left_1, avg3_left_1);
avg2_avg3_left[1][1] = _mm_unpackhi_epi16(avg2_left_1, avg3_left_1);
for (j = 0; j < 2; ++j) {
for (i = 0; i < 2; ++i) {
const __m128i avg2_avg3 = avg2_avg3_left[j][i];
row_1 = _mm_alignr_epi8(row_1, row_0, 12);
row_0 = _mm_alignr_epi8(row_0, _mm_slli_si128(avg2_avg3, 12), 12);
_mm_store_si128((__m128i *)dst, row_0);
_mm_store_si128((__m128i *)(dst + 8), row_1);
dst += stride;
row_1 = _mm_alignr_epi8(row_1, row_0, 12);
row_0 = _mm_alignr_epi8(row_0, _mm_slli_si128(avg2_avg3, 8), 12);
_mm_store_si128((__m128i *)dst, row_0);
_mm_store_si128((__m128i *)(dst + 8), row_1);
dst += stride;
row_1 = _mm_alignr_epi8(row_1, row_0, 12);
row_0 = _mm_alignr_epi8(row_0, _mm_slli_si128(avg2_avg3, 4), 12);
_mm_store_si128((__m128i *)dst, row_0);
_mm_store_si128((__m128i *)(dst + 8), row_1);
dst += stride;
row_1 = _mm_alignr_epi8(row_1, row_0, 12);
row_0 = _mm_alignr_epi8(row_0, avg2_avg3, 12);
_mm_store_si128((__m128i *)dst, row_0);
_mm_store_si128((__m128i *)(dst + 8), row_1);
dst += stride;
}
}
}
void vpx_highbd_d153_predictor_32x32_ssse3(uint16_t *dst, ptrdiff_t stride,
const uint16_t *above,
const uint16_t *left, int bd) {
const __m128i A0 = _mm_loadu_si128((const __m128i *)(above - 1));
const __m128i A1 = _mm_loadu_si128((const __m128i *)(above + 7));
const __m128i A2 = _mm_loadu_si128((const __m128i *)(above + 15));
const __m128i A3 = _mm_loadu_si128((const __m128i *)(above + 23));
const __m128i B0 = _mm_alignr_epi8(A1, A0, 2);
const __m128i B1 = _mm_alignr_epi8(A2, A1, 2);
const __m128i B2 = _mm_alignr_epi8(A3, A2, 2);
const __m128i B3 = _mm_srli_si128(A3, 2);
const __m128i C0 = _mm_alignr_epi8(A1, A0, 4);
const __m128i C1 = _mm_alignr_epi8(A2, A1, 4);
const __m128i C2 = _mm_alignr_epi8(A3, A2, 4);
const __m128i C3 = _mm_srli_si128(A3, 4);
const __m128i avg3_0 = avg3_epu16(&A0, &B0, &C0);
const __m128i avg3_1 = avg3_epu16(&A1, &B1, &C1);
const __m128i avg3_2 = avg3_epu16(&A2, &B2, &C2);
const __m128i avg3_3 = avg3_epu16(&A3, &B3, &C3);
const __m128i L0 = _mm_load_si128((const __m128i *)left);
const __m128i L1 = _mm_load_si128((const __m128i *)(left + 8));
const __m128i L2 = _mm_load_si128((const __m128i *)(left + 16));
const __m128i L3 = _mm_load_si128((const __m128i *)(left + 24));
const __m128i XL0 = _mm_alignr_epi8(L0, _mm_slli_si128(A0, 14), 14);
const __m128i XL1 = _mm_alignr_epi8(L1, L0, 14);
const __m128i XL2 = _mm_alignr_epi8(L2, L1, 14);
const __m128i XL3 = _mm_alignr_epi8(L3, L2, 14);
const __m128i AXL0 = _mm_alignr_epi8(XL0, _mm_slli_si128(A0, 12), 14);
const __m128i AXL1 = _mm_alignr_epi8(L1, L0, 12);
const __m128i AXL2 = _mm_alignr_epi8(L2, L1, 12);
const __m128i AXL3 = _mm_alignr_epi8(L3, L2, 12);
const __m128i avg3_left_0 = avg3_epu16(&L0, &XL0, &AXL0);
const __m128i avg3_left_1 = avg3_epu16(&L1, &XL1, &AXL1);
const __m128i avg3_left_2 = avg3_epu16(&L2, &XL2, &AXL2);
const __m128i avg3_left_3 = avg3_epu16(&L3, &XL3, &AXL3);
const __m128i avg2_left_0 = _mm_avg_epu16(L0, XL0);
const __m128i avg2_left_1 = _mm_avg_epu16(L1, XL1);
const __m128i avg2_left_2 = _mm_avg_epu16(L2, XL2);
const __m128i avg2_left_3 = _mm_avg_epu16(L3, XL3);
__m128i row_0 = avg3_0;
__m128i row_1 = avg3_1;
__m128i row_2 = avg3_2;
__m128i row_3 = avg3_3;
__m128i avg2_avg3_left[4][2];
int i, j;
(void)bd;
avg2_avg3_left[0][0] = _mm_unpacklo_epi16(avg2_left_0, avg3_left_0);
avg2_avg3_left[0][1] = _mm_unpackhi_epi16(avg2_left_0, avg3_left_0);
avg2_avg3_left[1][0] = _mm_unpacklo_epi16(avg2_left_1, avg3_left_1);
avg2_avg3_left[1][1] = _mm_unpackhi_epi16(avg2_left_1, avg3_left_1);
avg2_avg3_left[2][0] = _mm_unpacklo_epi16(avg2_left_2, avg3_left_2);
avg2_avg3_left[2][1] = _mm_unpackhi_epi16(avg2_left_2, avg3_left_2);
avg2_avg3_left[3][0] = _mm_unpacklo_epi16(avg2_left_3, avg3_left_3);
avg2_avg3_left[3][1] = _mm_unpackhi_epi16(avg2_left_3, avg3_left_3);
for (j = 0; j < 4; ++j) {
for (i = 0; i < 2; ++i) {
const __m128i avg2_avg3 = avg2_avg3_left[j][i];
row_3 = _mm_alignr_epi8(row_3, row_2, 12);
row_2 = _mm_alignr_epi8(row_2, row_1, 12);
row_1 = _mm_alignr_epi8(row_1, row_0, 12);
row_0 = _mm_alignr_epi8(row_0, _mm_slli_si128(avg2_avg3, 12), 12);
_mm_store_si128((__m128i *)dst, row_0);
_mm_store_si128((__m128i *)(dst + 8), row_1);
_mm_store_si128((__m128i *)(dst + 16), row_2);
_mm_store_si128((__m128i *)(dst + 24), row_3);
dst += stride;
row_3 = _mm_alignr_epi8(row_3, row_2, 12);
row_2 = _mm_alignr_epi8(row_2, row_1, 12);
row_1 = _mm_alignr_epi8(row_1, row_0, 12);
row_0 = _mm_alignr_epi8(row_0, _mm_slli_si128(avg2_avg3, 8), 12);
_mm_store_si128((__m128i *)dst, row_0);
_mm_store_si128((__m128i *)(dst + 8), row_1);
_mm_store_si128((__m128i *)(dst + 16), row_2);
_mm_store_si128((__m128i *)(dst + 24), row_3);
dst += stride;
row_3 = _mm_alignr_epi8(row_3, row_2, 12);
row_2 = _mm_alignr_epi8(row_2, row_1, 12);
row_1 = _mm_alignr_epi8(row_1, row_0, 12);
row_0 = _mm_alignr_epi8(row_0, _mm_slli_si128(avg2_avg3, 4), 12);
_mm_store_si128((__m128i *)dst, row_0);
_mm_store_si128((__m128i *)(dst + 8), row_1);
_mm_store_si128((__m128i *)(dst + 16), row_2);
_mm_store_si128((__m128i *)(dst + 24), row_3);
dst += stride;
row_3 = _mm_alignr_epi8(row_3, row_2, 12);
row_2 = _mm_alignr_epi8(row_2, row_1, 12);
row_1 = _mm_alignr_epi8(row_1, row_0, 12);
row_0 = _mm_alignr_epi8(row_0, avg2_avg3, 12);
_mm_store_si128((__m128i *)dst, row_0);
_mm_store_si128((__m128i *)(dst + 8), row_1);
_mm_store_si128((__m128i *)(dst + 16), row_2);
_mm_store_si128((__m128i *)(dst + 24), row_3);
dst += stride;
}
}
}
static INLINE void d207_store_4x8(uint16_t **dst, const ptrdiff_t stride,
const __m128i *a, const __m128i *b) {
_mm_store_si128((__m128i *)*dst, *a);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment