From 7f33be9e9625ab2fcaa5d4e1417c57d12bb04820 Mon Sep 17 00:00:00 2001
From: Yaowu Xu <yaowu@google.com>
Date: Thu, 10 Nov 2011 21:14:45 -0800
Subject: [PATCH] fixed the scaling in 8x8 trellis quant
This commit has a few minor fixes to the 8x8 trellis quant, so to
make it work regardless if extend_qrange is enabled or not. It also
borrowed adaptive RDMULT constants from 4x4 trellis that was missed
in the 8x8 trellis quant.
Change-Id: I60d7769071f102c699b5084597e62bca87a1f759
---
vp8/encoder/encodemb.c | 37 ++++++++++++++-----------------------
1 file changed, 14 insertions(+), 23 deletions(-)
diff --git a/vp8/encoder/encodemb.c b/vp8/encoder/encodemb.c
index ead1686b55..da727eb562 100644
--- a/vp8/encoder/encodemb.c
+++ b/vp8/encoder/encodemb.c
@@ -668,6 +668,7 @@ static void check_reset_2nd_coeffs(MACROBLOCKD *x, int type,
*a = *l = (bd->eob != !type);
}
}
+#define SUM_2ND_COEFF_THRESH_8X8 32
static void check_reset_8x8_2nd_coeffs(MACROBLOCKD *x, int type,
ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l)
{
@@ -685,15 +686,7 @@ static void check_reset_8x8_2nd_coeffs(MACROBLOCKD *x, int type,
coef = bd->dqcoeff[8];
sum+= (coef>=0)?coef:-coef;
- /**************************************************************************
- our inverse haar transform effectively is weighted sum of all 4 inputs
- with weight either 1 or -1. It has a last stage scaling of (sum)>>2. And
- dc only idct8x8 is effectively (dc+8)>>4. So if all the sums are between
- -31 and 31, outputs after inverse txfm be all 0s. A sum of absolute value
- smaller than 32 guarantees all 4 different (+1/-1) weighted sums in wht
- fall between -31 and +31.
- **************************************************************************/
- if(sum < 32)
+ if(sum < SUM_2ND_COEFF_THRESH_8X8)
{
bd->qcoeff[0] = 0;
bd->dqcoeff[0] = 0;
@@ -855,6 +848,7 @@ void optimize_b_8x8(MACROBLOCK *mb, int i, int type,
int best;
int band;
int pt;
+ int err_mult = plane_rd_mult[type];
b = &mb->block[i];
d = &mb->e_mbd.block[i];
@@ -874,8 +868,9 @@ void optimize_b_8x8(MACROBLOCK *mb, int i, int type,
eob = d->eob;
/* Now set up a Viterbi trellis to evaluate alternative roundings. */
- /* TODO: These should vary with the block type, since the quantizer does. */
- rdmult = mb->rdmult << 2;
+ rdmult = mb->rdmult * err_mult;
+ if(mb->e_mbd.mode_info_context->mbmi.ref_frame==INTRA_FRAME)
+ rdmult = (rdmult * 9)>>4;
rddiv = mb->rddiv;
best_mask[0] = best_mask[1] = 0;
/* Initialize the sentinel node of the trellis. */
@@ -1055,7 +1050,12 @@ void optimize_b_8x8(MACROBLOCK *mb, int i, int type,
final_eob = i;
rc = vp8_default_zig_zag1d_8x8[i];
qcoeff_ptr[rc] = x;
+#if !CONFIG_EXTEND_QRANGE
dqcoeff_ptr[rc] = x * dequant_ptr[rc!=0];
+#else
+ dqcoeff_ptr[rc] = (x * dequant_ptr[rc!=0]+2)>>2;
+#endif
+
next = tokens[i][best].next;
best = (best_mask[best] >> i) & 1;
}
@@ -1075,9 +1075,6 @@ void optimize_mb_8x8(MACROBLOCK *x, const VP8_ENCODER_RTCD *rtcd)
ENTROPY_CONTEXT *ta;
ENTROPY_CONTEXT *tl;
-#if CONFIG_EXTEND_QRANGE
-return ;
-#endif
vpx_memcpy(&t_above, x->e_mbd.above_context, sizeof(ENTROPY_CONTEXT_PLANES));
vpx_memcpy(&t_left, x->e_mbd.left_context, sizeof(ENTROPY_CONTEXT_PLANES));
@@ -1151,10 +1148,12 @@ return ;
*(tl + vp8_block2left[b]);
}
+
//8x8 always have 2nd roder haar block
check_reset_8x8_2nd_coeffs(&x->e_mbd, PLANE_TYPE_Y2,
ta + vp8_block2above[24], tl + vp8_block2left[24]);
+
}
void vp8_optimize_mby_8x8(MACROBLOCK *x, const VP8_ENCODER_RTCD *rtcd)
@@ -1167,10 +1166,6 @@ void vp8_optimize_mby_8x8(MACROBLOCK *x, const VP8_ENCODER_RTCD *rtcd)
ENTROPY_CONTEXT *ta;
ENTROPY_CONTEXT *tl;
-#if CONFIG_EXTEND_QRANGE
-return ;
-#endif
-
if (!x->e_mbd.above_context)
return;
@@ -1223,10 +1218,10 @@ return ;
}
-
//8x8 always have 2nd roder haar block
check_reset_8x8_2nd_coeffs(&x->e_mbd, PLANE_TYPE_Y2,
ta + vp8_block2above[24], tl + vp8_block2left[24]);
+
}
void vp8_optimize_mbuv_8x8(MACROBLOCK *x, const VP8_ENCODER_RTCD *rtcd)
@@ -1236,10 +1231,6 @@ void vp8_optimize_mbuv_8x8(MACROBLOCK *x, const VP8_ENCODER_RTCD *rtcd)
ENTROPY_CONTEXT *ta;
ENTROPY_CONTEXT *tl;
-#if CONFIG_EXTEND_QRANGE
-return ;
-#endif
-
if (!x->e_mbd.above_context)
return;
--
GitLab