Commit eb4238ac authored by Johann's avatar Johann
Browse files

Revert "Revert "quantize avx: copy 32x32 implementation""

This reverts commit 8c42237b.

Because ssse3 code is used for the reference, the qcoeff and dqcoeff
reference buffers must be aligned.

Original change's description:
> quantize avx: copy 32x32 implementation
>
> Ensure avx and ssse3 stay in sync by testing them against each other.
>
> Change-Id: I699f3b48785c83260825402d7826231f475f697c

Change-Id: Ieeef11b9406964194028b0d81d84bcb63296ae06
parent c39cd923
......@@ -156,9 +156,11 @@ TEST_P(VP9QuantizeTest, OperationCheck) {
ASSERT_TRUE(qcoeff.Init());
Buffer<tran_low_t> dqcoeff = Buffer<tran_low_t>(max_size_, max_size_, 0, 32);
ASSERT_TRUE(dqcoeff.Init());
Buffer<tran_low_t> ref_qcoeff = Buffer<tran_low_t>(max_size_, max_size_, 0);
Buffer<tran_low_t> ref_qcoeff =
Buffer<tran_low_t>(max_size_, max_size_, 0, 32);
ASSERT_TRUE(ref_qcoeff.Init());
Buffer<tran_low_t> ref_dqcoeff = Buffer<tran_low_t>(max_size_, max_size_, 0);
Buffer<tran_low_t> ref_dqcoeff =
Buffer<tran_low_t>(max_size_, max_size_, 0, 32);
ASSERT_TRUE(ref_dqcoeff.Init());
uint16_t eob, ref_eob;
......@@ -213,9 +215,11 @@ TEST_P(VP9QuantizeTest, EOBCheck) {
ASSERT_TRUE(qcoeff.Init());
Buffer<tran_low_t> dqcoeff = Buffer<tran_low_t>(max_size_, max_size_, 0, 32);
ASSERT_TRUE(dqcoeff.Init());
Buffer<tran_low_t> ref_qcoeff = Buffer<tran_low_t>(max_size_, max_size_, 0);
Buffer<tran_low_t> ref_qcoeff =
Buffer<tran_low_t>(max_size_, max_size_, 0, 32);
ASSERT_TRUE(ref_qcoeff.Init());
Buffer<tran_low_t> ref_dqcoeff = Buffer<tran_low_t>(max_size_, max_size_, 0);
Buffer<tran_low_t> ref_dqcoeff =
Buffer<tran_low_t>(max_size_, max_size_, 0, 32);
ASSERT_TRUE(ref_dqcoeff.Init());
uint16_t eob, ref_eob;
......@@ -391,16 +395,14 @@ INSTANTIATE_TEST_CASE_P(
// TODO(johannkoenig): AVX optimizations do not yet pass the 32x32 test or
// highbitdepth configurations.
#if HAVE_AVX && !CONFIG_VP9_HIGHBITDEPTH
INSTANTIATE_TEST_CASE_P(AVX, VP9QuantizeTest,
::testing::Values(make_tuple(&vpx_quantize_b_avx,
&vpx_quantize_b_c,
VPX_BITS_8, 16)));
#if ARCH_X86_64
INSTANTIATE_TEST_CASE_P(DISABLED_AVX, VP9QuantizeTest,
::testing::Values(make_tuple(&vpx_quantize_b_32x32_avx,
&vpx_quantize_b_32x32_c,
VPX_BITS_8, 32)));
#endif // ARCH_X86_64
INSTANTIATE_TEST_CASE_P(
AVX, VP9QuantizeTest,
::testing::Values(make_tuple(&vpx_quantize_b_avx, &vpx_quantize_b_c,
VPX_BITS_8, 16),
// Even though SSSE3 and AVX do not match the reference
// code, we can keep them in sync with each other.
make_tuple(&vpx_quantize_b_32x32_avx,
&vpx_quantize_b_32x32_ssse3, VPX_BITS_8, 32)));
#endif // HAVE_AVX && !CONFIG_VP9_HIGHBITDEPTH
// TODO(webm:1448): dqcoeff is not handled correctly in HBD builds.
......
......@@ -284,9 +284,6 @@ DSP_SRCS-$(HAVE_NEON) += arm/quantize_neon.c
ifeq ($(CONFIG_VP9_HIGHBITDEPTH),yes)
DSP_SRCS-$(HAVE_SSE2) += x86/highbd_quantize_intrin_sse2.c
endif
ifeq ($(ARCH_X86_64),yes)
DSP_SRCS-$(HAVE_AVX) += x86/quantize_avx_x86_64.asm
endif
# avg
DSP_SRCS-yes += avg.c
......
......@@ -678,7 +678,7 @@ if (vpx_config("CONFIG_VP9_ENCODER") eq "yes") {
specialize qw/vpx_quantize_b neon sse2 ssse3 avx/;
add_proto qw/void vpx_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
specialize qw/vpx_quantize_b_32x32 neon ssse3/, "$avx_x86_64";
specialize qw/vpx_quantize_b_32x32 neon ssse3 avx/;
if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {
add_proto qw/void vpx_highbd_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
......
......@@ -46,7 +46,7 @@ void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
// Setup global values.
zbin = _mm_load_si128((const __m128i *)zbin_ptr);
// x86 has no "greater *or equal* comparison. Subtract 1 from zbin so
// x86 has no "greater *or equal*" comparison. Subtract 1 from zbin so
// it is a strict "greater" comparison.
zbin = _mm_sub_epi16(zbin, _mm_set1_epi16(1));
round = _mm_load_si128((const __m128i *)round_ptr);
......@@ -200,3 +200,216 @@ void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
*eob_ptr = _mm_extract_epi16(eob, 1);
}
}
void vpx_quantize_b_32x32_avx(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block,
const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan_ptr, const int16_t *iscan_ptr) {
const __m128i zero = _mm_setzero_si128();
const __m128i one = _mm_set1_epi16(1);
const __m256i big_zero = _mm256_setzero_si256();
int index;
__m128i zbin, round, quant, dequant, shift;
__m128i coeff0, coeff1;
__m128i qcoeff0, qcoeff1;
__m128i cmp_mask0, cmp_mask1;
__m128i all_zero;
__m128i qtmp0, qtmp1;
__m128i zero_coeff0, zero_coeff1, iscan0, iscan1;
__m128i eob = zero, eob0, eob1;
(void)scan_ptr;
(void)n_coeffs;
(void)skip_block;
assert(!skip_block);
*eob_ptr = 0;
// Setup global values.
// The 32x32 halves zbin and round.
zbin = _mm_load_si128((const __m128i *)zbin_ptr);
// Shift with rounding.
zbin = _mm_add_epi16(zbin, one);
zbin = _mm_srli_epi16(zbin, 1);
// x86 has no "greater *or equal*" comparison. Subtract 1 from zbin so
// it is a strict "greater" comparison.
zbin = _mm_sub_epi16(zbin, one);
round = _mm_load_si128((const __m128i *)round_ptr);
round = _mm_add_epi16(round, one);
round = _mm_srli_epi16(round, 1);
quant = _mm_load_si128((const __m128i *)quant_ptr);
dequant = _mm_load_si128((const __m128i *)dequant_ptr);
shift = _mm_load_si128((const __m128i *)quant_shift_ptr);
shift = _mm_slli_epi16(shift, 1);
// Do DC and first 15 AC.
coeff0 = load_tran_low(coeff_ptr);
coeff1 = load_tran_low(coeff_ptr + 8);
qcoeff0 = _mm_abs_epi16(coeff0);
qcoeff1 = _mm_abs_epi16(coeff1);
cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin);
zbin = _mm_unpackhi_epi64(zbin, zbin); // Switch DC to AC
cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin);
all_zero = _mm_or_si128(cmp_mask0, cmp_mask1);
if (_mm_test_all_zeros(all_zero, all_zero)) {
_mm256_store_si256((__m256i *)(qcoeff_ptr), big_zero);
_mm256_store_si256((__m256i *)(dqcoeff_ptr), big_zero);
#if CONFIG_VP9_HIGHBITDEPTH
_mm256_store_si256((__m256i *)(qcoeff_ptr + 8), big_zero);
_mm256_store_si256((__m256i *)(dqcoeff_ptr + 8), big_zero);
#endif // CONFIG_VP9_HIGHBITDEPTH
round = _mm_unpackhi_epi64(round, round);
quant = _mm_unpackhi_epi64(quant, quant);
shift = _mm_unpackhi_epi64(shift, shift);
dequant = _mm_unpackhi_epi64(dequant, dequant);
} else {
qcoeff0 = _mm_adds_epi16(qcoeff0, round);
round = _mm_unpackhi_epi64(round, round);
qcoeff1 = _mm_adds_epi16(qcoeff1, round);
qtmp0 = _mm_mulhi_epi16(qcoeff0, quant);
quant = _mm_unpackhi_epi64(quant, quant);
qtmp1 = _mm_mulhi_epi16(qcoeff1, quant);
qtmp0 = _mm_add_epi16(qtmp0, qcoeff0);
qtmp1 = _mm_add_epi16(qtmp1, qcoeff1);
qcoeff0 = _mm_mulhi_epi16(qtmp0, shift);
shift = _mm_unpackhi_epi64(shift, shift);
qcoeff1 = _mm_mulhi_epi16(qtmp1, shift);
// Reinsert signs
qcoeff0 = _mm_sign_epi16(qcoeff0, coeff0);
qcoeff1 = _mm_sign_epi16(qcoeff1, coeff1);
// Mask out zbin threshold coeffs
qcoeff0 = _mm_and_si128(qcoeff0, cmp_mask0);
qcoeff1 = _mm_and_si128(qcoeff1, cmp_mask1);
store_tran_low(qcoeff0, qcoeff_ptr);
store_tran_low(qcoeff1, qcoeff_ptr + 8);
// Un-sign to bias rounding like C.
// dequant is almost always negative, so this is probably the backwards way
// to handle the sign. However, it matches the previous assembly.
coeff0 = _mm_abs_epi16(qcoeff0);
coeff1 = _mm_abs_epi16(qcoeff1);
coeff0 = _mm_mullo_epi16(coeff0, dequant);
dequant = _mm_unpackhi_epi64(dequant, dequant);
coeff1 = _mm_mullo_epi16(coeff1, dequant);
// "Divide" by 2.
coeff0 = _mm_srli_epi16(coeff0, 1);
coeff1 = _mm_srli_epi16(coeff1, 1);
coeff0 = _mm_sign_epi16(coeff0, qcoeff0);
coeff1 = _mm_sign_epi16(coeff1, qcoeff1);
store_tran_low(coeff0, dqcoeff_ptr);
store_tran_low(coeff1, dqcoeff_ptr + 8);
// Scan for eob.
zero_coeff0 = _mm_cmpeq_epi16(coeff0, zero);
zero_coeff1 = _mm_cmpeq_epi16(coeff1, zero);
iscan0 = _mm_load_si128((const __m128i *)(iscan_ptr));
iscan1 = _mm_load_si128((const __m128i *)(iscan_ptr + 8));
// Add one to convert from indices to counts
iscan0 = _mm_sub_epi16(iscan0, cmp_mask0);
iscan1 = _mm_sub_epi16(iscan1, cmp_mask1);
eob = _mm_andnot_si128(zero_coeff0, iscan0);
eob1 = _mm_andnot_si128(zero_coeff1, iscan1);
eob = _mm_max_epi16(eob, eob1);
}
// AC only loop.
for (index = 16; index < 32 * 32; index += 16) {
coeff0 = load_tran_low(coeff_ptr + index);
coeff1 = load_tran_low(coeff_ptr + index + 8);
qcoeff0 = _mm_abs_epi16(coeff0);
qcoeff1 = _mm_abs_epi16(coeff1);
cmp_mask0 = _mm_cmpgt_epi16(qcoeff0, zbin);
cmp_mask1 = _mm_cmpgt_epi16(qcoeff1, zbin);
all_zero = _mm_or_si128(cmp_mask0, cmp_mask1);
if (_mm_test_all_zeros(all_zero, all_zero)) {
_mm256_store_si256((__m256i *)(qcoeff_ptr + index), big_zero);
_mm256_store_si256((__m256i *)(dqcoeff_ptr + index), big_zero);
#if CONFIG_VP9_HIGHBITDEPTH
_mm256_store_si256((__m256i *)(qcoeff_ptr + index + 8), big_zero);
_mm256_store_si256((__m256i *)(dqcoeff_ptr + index + 8), big_zero);
#endif // CONFIG_VP9_HIGHBITDEPTH
continue;
}
qcoeff0 = _mm_adds_epi16(qcoeff0, round);
qcoeff1 = _mm_adds_epi16(qcoeff1, round);
qtmp0 = _mm_mulhi_epi16(qcoeff0, quant);
qtmp1 = _mm_mulhi_epi16(qcoeff1, quant);
qtmp0 = _mm_add_epi16(qtmp0, qcoeff0);
qtmp1 = _mm_add_epi16(qtmp1, qcoeff1);
qcoeff0 = _mm_mulhi_epi16(qtmp0, shift);
qcoeff1 = _mm_mulhi_epi16(qtmp1, shift);
qcoeff0 = _mm_sign_epi16(qcoeff0, coeff0);
qcoeff1 = _mm_sign_epi16(qcoeff1, coeff1);
qcoeff0 = _mm_and_si128(qcoeff0, cmp_mask0);
qcoeff1 = _mm_and_si128(qcoeff1, cmp_mask1);
store_tran_low(qcoeff0, qcoeff_ptr + index);
store_tran_low(qcoeff1, qcoeff_ptr + index + 8);
coeff0 = _mm_abs_epi16(qcoeff0);
coeff1 = _mm_abs_epi16(qcoeff1);
coeff0 = _mm_mullo_epi16(coeff0, dequant);
coeff1 = _mm_mullo_epi16(coeff1, dequant);
coeff0 = _mm_srli_epi16(coeff0, 1);
coeff1 = _mm_srli_epi16(coeff1, 1);
coeff0 = _mm_sign_epi16(coeff0, qcoeff0);
coeff1 = _mm_sign_epi16(coeff1, qcoeff1);
store_tran_low(coeff0, dqcoeff_ptr + index);
store_tran_low(coeff1, dqcoeff_ptr + index + 8);
zero_coeff0 = _mm_cmpeq_epi16(coeff0, zero);
zero_coeff1 = _mm_cmpeq_epi16(coeff1, zero);
iscan0 = _mm_load_si128((const __m128i *)(iscan_ptr + index));
iscan1 = _mm_load_si128((const __m128i *)(iscan_ptr + index + 8));
iscan0 = _mm_sub_epi16(iscan0, cmp_mask0);
iscan1 = _mm_sub_epi16(iscan1, cmp_mask1);
eob0 = _mm_andnot_si128(zero_coeff0, iscan0);
eob1 = _mm_andnot_si128(zero_coeff1, iscan1);
eob0 = _mm_max_epi16(eob0, eob1);
eob = _mm_max_epi16(eob, eob0);
}
// Accumulate eob.
{
__m128i eob_shuffled;
eob_shuffled = _mm_shuffle_epi32(eob, 0xe);
eob = _mm_max_epi16(eob, eob_shuffled);
eob_shuffled = _mm_shufflelo_epi16(eob, 0xe);
eob = _mm_max_epi16(eob, eob_shuffled);
eob_shuffled = _mm_shufflelo_epi16(eob, 0x1);
eob = _mm_max_epi16(eob, eob_shuffled);
*eob_ptr = _mm_extract_epi16(eob, 1);
}
}
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment