From fff019cea4308fc3c776d7426f315d46a7120ac5 Mon Sep 17 00:00:00 2001
From: Yaowu Xu <yaowu@google.com>
Date: Thu, 20 Sep 2012 12:41:59 -0700
Subject: [PATCH] fixed inconsistency of transform size decision
for MBs using i8x8 prediciton mode when hybridtransform8x8 is not
enabled in configure.
Change-Id: I3e3c8bb1c0261108c1c71213ca9557b0ce558036
---
vp8/common/blockd.h | 32 ++------------------------------
vp8/decoder/decodframe.c | 13 +++++++++++--
vp8/encoder/encodeframe.c | 9 +++++++++
3 files changed, 22 insertions(+), 32 deletions(-)
diff --git a/vp8/common/blockd.h b/vp8/common/blockd.h
index 57dacf841f..96345c2bde 100644
--- a/vp8/common/blockd.h
+++ b/vp8/common/blockd.h
@@ -479,34 +479,6 @@ static void txfm_map(BLOCKD *b, B_PREDICTION_MODE bmode) {
break;
}
}
-
-static TX_TYPE get_tx_type(MACROBLOCKD *xd, BLOCKD *b) {
- TX_TYPE tx_type = DCT_DCT;
-#if CONFIG_HYBRIDTRANSFORM16X16
- if (xd->mode_info_context->mbmi.txfm_size == TX_16X16) {
- if (xd->mode_info_context->mbmi.mode < I8X8_PRED &&
- xd->q_index < ACTIVE_HT16)
- tx_type = b->bmi.as_mode.tx_type;
- return tx_type;
- }
-#endif
-#if CONFIG_HYBRIDTRANSFORM8X8
- if (xd->mode_info_context->mbmi.txfm_size == TX_8X8) {
- if (xd->mode_info_context->mbmi.mode == I8X8_PRED)
- tx_type = b->bmi.as_mode.tx_type;
- return tx_type;
- }
-#endif
-#if CONFIG_HYBRIDTRANSFORM
- if (xd->mode_info_context->mbmi.txfm_size == TX_4X4) {
- if (xd->mode_info_context->mbmi.mode == B_PRED &&
- xd->q_index < ACTIVE_HT)
- tx_type = b->bmi.as_mode.tx_type;
- return tx_type;
- }
-#endif
- return tx_type;
-}
#endif
extern void vp8_build_block_doffsets(MACROBLOCKD *xd);
@@ -516,8 +488,8 @@ static void update_blockd_bmi(MACROBLOCKD *xd) {
int i;
int is_4x4;
is_4x4 = (xd->mode_info_context->mbmi.mode == SPLITMV) ||
- (xd->mode_info_context->mbmi.mode == I8X8_PRED) ||
- (xd->mode_info_context->mbmi.mode == B_PRED);
+ (xd->mode_info_context->mbmi.mode == I8X8_PRED) ||
+ (xd->mode_info_context->mbmi.mode == B_PRED);
if (is_4x4) {
for (i = 0; i < 16; i++) {
diff --git a/vp8/decoder/decodframe.c b/vp8/decoder/decodframe.c
index 80efa58026..e2d0223889 100644
--- a/vp8/decoder/decodframe.c
+++ b/vp8/decoder/decodframe.c
@@ -243,6 +243,9 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
else
#endif
if (pbi->common.txfm_mode == ALLOW_8X8 &&
+#if !CONFIG_HYBRIDTRANSFORM8X8
+ xd->mode_info_context->mbmi.mode != I8X8_PRED &&
+#endif
xd->mode_info_context->mbmi.mode != B_PRED)
xd->mode_info_context->mbmi.txfm_size = TX_8X8;
else
@@ -258,6 +261,9 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
else
#endif
if (pbi->common.txfm_mode == ALLOW_8X8 &&
+#if !CONFIG_HYBRIDTRANSFORM8X8
+ xd->mode_info_context->mbmi.mode != I8X8_PRED &&
+#endif
xd->mode_info_context->mbmi.mode != B_PRED &&
xd->mode_info_context->mbmi.mode != SPLITMV)
xd->mode_info_context->mbmi.txfm_size = TX_8X8;
@@ -616,8 +622,11 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
#if CONFIG_SUPERBLOCKS
if (!xd->mode_info_context->mbmi.encoded_as_sb) {
#endif
- if ((tx_type == TX_8X8 &&
- xd->mode_info_context->mbmi.mode != I8X8_PRED)
+ if ((tx_type == TX_8X8
+#if CONFIG_HYBRIDTRANSFORM8X8
+ && xd->mode_info_context->mbmi.mode != I8X8_PRED
+#endif
+ )
#if CONFIG_TX16X16 || CONFIG_HYBRIDTRANSFORM16X16
|| tx_type == TX_16X16
#endif
diff --git a/vp8/encoder/encodeframe.c b/vp8/encoder/encodeframe.c
index e25b64305d..0d125ec941 100644
--- a/vp8/encoder/encodeframe.c
+++ b/vp8/encoder/encodeframe.c
@@ -1833,6 +1833,9 @@ void vp8cx_encode_intra_macro_block(VP8_COMP *cpi,
else
#endif
if (cpi->common.txfm_mode == ALLOW_8X8
+#if !CONFIG_HYBRIDTRANSFORM8X8
+ && mbmi->mode != I8X8_PRED
+#endif
&& mbmi->mode != B_PRED) {
mbmi->txfm_size = TX_8X8;
cpi->t8x8_count++;
@@ -1929,6 +1932,9 @@ void vp8cx_encode_inter_macroblock (VP8_COMP *cpi, MACROBLOCK *x,
} else
#endif
if (cpi->common.txfm_mode == ALLOW_8X8
+#if !CONFIG_HYBRIDTRANSFORM8X8
+ && mbmi->mode != I8X8_PRED
+#endif
&& mbmi->mode != B_PRED
&& mbmi->mode != SPLITMV) {
mbmi->txfm_size = TX_8X8;
@@ -2109,6 +2115,9 @@ void vp8cx_encode_inter_superblock(VP8_COMP *cpi, MACROBLOCK *x, TOKENEXTRA **t,
/* test code: set transform size based on mode selection */
if (cpi->common.txfm_mode == ALLOW_8X8
+#if !CONFIG_HYBRIDTRANSFORM8X8
+ && x->e_mbd.mode_info_context->mbmi.mode != I8X8_PRED
+#endif
&& x->e_mbd.mode_info_context->mbmi.mode != B_PRED
&& x->e_mbd.mode_info_context->mbmi.mode != SPLITMV) {
x->e_mbd.mode_info_context->mbmi.txfm_size = TX_8X8;
--
GitLab