Commit de3a8f23 authored by Yaowu Xu's avatar Yaowu Xu

vpx_dsp/quantize.c: fix ubsan warnings

BUG=webm:1219

Change-Id: I0c80271c6b78adf40aa7a4cac9e6b431d56958cb
parent e5e998a6
......@@ -53,7 +53,7 @@ void vpx_highbd_quantize_dc(const tran_low_t *coeff_ptr,
const int coeff_sign = (coeff >> 31);
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
const int64_t tmp = abs_coeff + round_ptr[0];
const uint32_t abs_qcoeff = (uint32_t)((tmp * quant) >> 16);
const int abs_qcoeff = (int)((tmp * quant) >> 16);
qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant_ptr;
if (abs_qcoeff)
......@@ -109,7 +109,7 @@ void vpx_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr,
const int coeff_sign = (coeff >> 31);
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
const int64_t tmp = abs_coeff + ROUND_POWER_OF_TWO(round_ptr[0], 1);
const uint32_t abs_qcoeff = (uint32_t)((tmp * quant) >> 15);
const int abs_qcoeff = (int)((tmp * quant) >> 15);
qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant_ptr / 2;
if (abs_qcoeff)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment