Commit 3fa0129c authored by hui su's avatar hui su

Extend ext_tx experiment to intra blocks

ext-tx on derflr +2.30% (was +1.84%)

Change-Id: Ic91565cacc38e7a8e1200d054ed7bf99295fe19e
parent f53153db
......@@ -269,11 +269,16 @@ static INLINE TX_TYPE get_tx_type(PLANE_TYPE plane_type,
#if CONFIG_EXT_TX
if (xd->lossless || tx_size >= TX_32X32)
return DCT_DCT;
if (is_inter_block(&mi->mbmi)) {
return ext_tx_to_txtype[mi->mbmi.ext_txfrm];
if (mbmi->sb_type >= BLOCK_8X8) {
if (plane_type == PLANE_TYPE_Y || is_inter_block(mbmi))
return ext_tx_to_txtype[mbmi->ext_txfrm];
}
return intra_mode_to_tx_type_lookup[plane_type == PLANE_TYPE_Y ?
get_y_mode(mi, block_idx) : mbmi->uv_mode];
if (is_inter_block(mbmi))
return DCT_DCT;
else
return intra_mode_to_tx_type_lookup[plane_type == PLANE_TYPE_Y ?
get_y_mode(mi, block_idx) : mbmi->uv_mode];
#else
if (plane_type != PLANE_TYPE_Y || xd->lossless || is_inter_block(mbmi) ||
tx_size >= TX_32X32)
......
......@@ -334,7 +334,8 @@ const vpx_tree_index vp10_ext_tx_tree[TREE_SIZE(EXT_TX_TYPES)] = {
-ALT7, -ALT8
};
static const vpx_prob default_ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1] = {
static const vpx_prob
default_inter_ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1] = {
{ 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
128 },
{ 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
......@@ -342,6 +343,76 @@ static const vpx_prob default_ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1] = {
{ 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
128 },
};
static const vpx_prob
default_intra_ext_tx_prob[EXT_TX_SIZES][INTRA_MODES][EXT_TX_TYPES - 1] = {
{
{ 8, 11, 24, 112, 87, 137, 127, 134,
128, 86, 128, 124, 125, 133, 176, 123, },
{ 10, 9, 39, 106, 73, 155, 163, 228,
35, 62, 129, 127, 133, 114, 213, 234, },
{ 10, 9, 14, 88, 91, 127, 151, 51,
210, 89, 126, 58, 52, 116, 217, 24, },
{ 9, 6, 29, 113, 98, 131, 149, 210,
119, 60, 124, 93, 90, 143, 170, 197, },
{ 8, 8, 38, 101, 111, 166, 167, 141,
130, 105, 128, 75, 75, 118, 197, 117, },
{ 7, 8, 39, 91, 101, 153, 166, 200,
99, 77, 123, 90, 83, 144, 224, 192, },
{ 7, 10, 26, 86, 119, 154, 130, 101,
152, 91, 129, 75, 79, 137, 219, 77, },
{ 10, 13, 20, 86, 102, 162, 112, 76,
171, 86, 134, 122, 106, 124, 196, 44, },
{ 8, 9, 33, 108, 100, 144, 148, 215,
77, 60, 125, 125, 128, 126, 198, 220, },
{ 3, 10, 29, 111, 69, 141, 204, 141,
139, 93, 120, 75, 77, 163, 242, 124, },
},
{
{ 2, 53, 18, 147, 96, 98, 136, 133,
131, 120, 153, 163, 169, 137, 173, 124, },
{ 4, 18, 34, 133, 54, 130, 179, 228,
28, 72, 153, 164, 168, 118, 227, 239, },
{ 4, 18, 13, 125, 72, 110, 176, 36,
221, 104, 148, 75, 72, 117, 225, 19, },
{ 8, 33, 24, 162, 113, 99, 147, 226,
103, 85, 153, 143, 153, 124, 155, 210, },
{ 2, 15, 35, 107, 127, 158, 192, 128,
126, 116, 151, 95, 88, 182, 241, 119, },
{ 3, 15, 36, 112, 100, 146, 194, 189,
90, 98, 152, 99, 100, 165, 235, 175, },
{ 3, 16, 29, 109, 103, 140, 182, 76,
173, 104, 147, 82, 85, 159, 235, 70, },
{ 9, 24, 14, 120, 86, 156, 161, 34,
177, 121, 142, 128, 128, 126, 185, 37, },
{ 5, 24, 29, 152, 98, 99, 174, 228,
82, 76, 147, 149, 128, 132, 191, 225, },
{ 2, 15, 29, 111, 77, 126, 200, 135,
117, 93, 152, 96, 84, 191, 245, 135, },
},
{
{ 2, 69, 13, 173, 111, 69, 137, 159,
159, 146, 151, 193, 203, 131, 180, 123, },
{ 1, 12, 33, 164, 32, 98, 204, 242,
23, 99, 149, 215, 232, 110, 239, 245, },
{ 1, 17, 9, 136, 82, 83, 171, 28,
231, 128, 135, 76, 64, 118, 235, 17, },
{ 4, 41, 17, 195, 131, 58, 161, 237,
141, 97, 153, 189, 191, 117, 182, 202, },
{ 2, 17, 36, 104, 149, 137, 217, 139,
191, 119, 125, 107, 115, 223, 249, 110, },
{ 2, 14, 24, 127, 91, 135, 219, 198,
113, 91, 164, 125, 173, 211, 250, 116, },
{ 3, 19, 24, 120, 102, 130, 209, 81,
187, 95, 143, 102, 50, 190, 244, 56, },
{ 4, 27, 10, 128, 91, 157, 181, 33,
181, 150, 141, 141, 166, 114, 215, 25, },
{ 2, 34, 27, 187, 102, 77, 210, 245,
113, 107, 136, 184, 188, 121, 210, 234, },
{ 1, 15, 22, 141, 59, 94, 208, 133,
154, 95, 152, 112, 105, 191, 242, 111, },
},
};
#endif // CONFIG_EXT_TX
static void init_mode_probs(FRAME_CONTEXT *fc) {
......@@ -357,7 +428,8 @@ static void init_mode_probs(FRAME_CONTEXT *fc) {
vp10_copy(fc->skip_probs, default_skip_probs);
vp10_copy(fc->inter_mode_probs, default_inter_mode_probs);
#if CONFIG_EXT_TX
vp10_copy(fc->ext_tx_prob, default_ext_tx_prob);
vp10_copy(fc->inter_ext_tx_prob, default_inter_ext_tx_prob);
vp10_copy(fc->intra_ext_tx_prob, default_intra_ext_tx_prob);
#endif // CONFIG_EXT_TX
}
......@@ -441,8 +513,13 @@ void vp10_adapt_mode_probs(VP10_COMMON *cm) {
#if CONFIG_EXT_TX
for (i = TX_4X4; i <= TX_16X16; ++i) {
vpx_tree_merge_probs(vp10_ext_tx_tree, pre_fc->ext_tx_prob[i],
counts->ext_tx[i], fc->ext_tx_prob[i]);
vpx_tree_merge_probs(vp10_ext_tx_tree, pre_fc->inter_ext_tx_prob[i],
counts->inter_ext_tx[i], fc->inter_ext_tx_prob[i]);
for (j = 0; j < INTRA_MODES; ++j)
vpx_tree_merge_probs(vp10_ext_tx_tree, pre_fc->intra_ext_tx_prob[i][j],
counts->intra_ext_tx[i][j],
fc->intra_ext_tx_prob[i][j]);
}
#endif // CONFIG_EXT_TX
}
......
......@@ -57,7 +57,8 @@ typedef struct frame_contexts {
vpx_prob skip_probs[SKIP_CONTEXTS];
nmv_context nmvc;
#if CONFIG_EXT_TX
vpx_prob ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1];
vpx_prob inter_ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1];
vpx_prob intra_ext_tx_prob[EXT_TX_SIZES][INTRA_MODES][EXT_TX_TYPES - 1];
#endif // CONFIG_EXT_TX
int initialized;
} FRAME_CONTEXT;
......@@ -80,7 +81,8 @@ typedef struct FRAME_COUNTS {
unsigned int skip[SKIP_CONTEXTS][2];
nmv_context_counts mv;
#if CONFIG_EXT_TX
unsigned int ext_tx[EXT_TX_SIZES][EXT_TX_TYPES];
unsigned int inter_ext_tx[EXT_TX_SIZES][EXT_TX_TYPES];
unsigned int intra_ext_tx[EXT_TX_SIZES][INTRA_MODES][EXT_TX_TYPES];
#endif // CONFIG_EXT_TX
} FRAME_COUNTS;
......
......@@ -2070,11 +2070,18 @@ static size_t read_uncompressed_header(VP10Decoder *pbi,
#if CONFIG_EXT_TX
static void read_ext_tx_probs(FRAME_CONTEXT *fc, vpx_reader *r) {
int i, j;
int i, j, k;
if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) {
for (i = TX_4X4; i <= TX_16X16; ++i)
for (j = 0; j < EXT_TX_TYPES - 1; ++j)
vp10_diff_update_prob(r, &fc->inter_ext_tx_prob[i][j]);
}
if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) {
for (j = TX_4X4; j <= TX_16X16; ++j)
for (i = 0; i < EXT_TX_TYPES - 1; ++i)
vp10_diff_update_prob(r, &fc->ext_tx_prob[j][i]);
for (i = TX_4X4; i <= TX_16X16; ++i)
for (j = 0; j < INTRA_MODES; ++j)
for (k = 0; k < EXT_TX_TYPES - 1; ++k)
vp10_diff_update_prob(r, &fc->intra_ext_tx_prob[i][j][k]);
}
}
#endif // CONFIG_EXT_TX
......
......@@ -244,6 +244,18 @@ static void read_intra_frame_mode_info(VP10_COMMON *const cm,
}
mbmi->uv_mode = read_intra_mode(r, vp10_kf_uv_mode_prob[mbmi->mode]);
#if CONFIG_EXT_TX
if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
mbmi->sb_type >= BLOCK_8X8 && !mbmi->skip &&
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
mbmi->ext_txfrm =
vpx_read_tree(r, vp10_ext_tx_tree,
cm->fc->intra_ext_tx_prob[mbmi->tx_size][mbmi->mode]);
} else {
mbmi->ext_txfrm = NORM;
}
#endif // CONFIG_EXT_TX
}
static int read_mv_component(vpx_reader *r,
......@@ -595,28 +607,37 @@ static void read_inter_frame_mode_info(VP10Decoder *const pbi,
mbmi->skip = read_skip(cm, xd, mbmi->segment_id, r);
inter_block = read_is_inter_block(cm, xd, mbmi->segment_id, r);
mbmi->tx_size = read_tx_size(cm, xd, !mbmi->skip || !inter_block, r);
if (inter_block)
read_inter_block_mode_info(pbi, xd, mi, mi_row, mi_col, r);
else
read_intra_block_mode_info(cm, xd, mi, r);
#if CONFIG_EXT_TX
if (inter_block &&
mbmi->tx_size <= TX_16X16 &&
cm->base_qindex > 0 &&
mbmi->sb_type >= BLOCK_8X8 &&
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP) &&
!mbmi->skip) {
if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
mbmi->sb_type >= BLOCK_8X8 && !mbmi->skip &&
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
FRAME_COUNTS *counts = xd->counts;
mbmi->ext_txfrm = vpx_read_tree(r,
vp10_ext_tx_tree,
cm->fc->ext_tx_prob[mbmi->tx_size]);
if (counts)
++counts->ext_tx[mbmi->tx_size][mbmi->ext_txfrm];
if (inter_block)
mbmi->ext_txfrm =
vpx_read_tree(r, vp10_ext_tx_tree,
cm->fc->inter_ext_tx_prob[mbmi->tx_size]);
else
mbmi->ext_txfrm =
vpx_read_tree(r, vp10_ext_tx_tree,
cm->fc->intra_ext_tx_prob[mbmi->tx_size][mbmi->mode]);
if (counts) {
if (inter_block)
++counts->inter_ext_tx[mbmi->tx_size][mbmi->ext_txfrm];
else
++counts->intra_ext_tx[mbmi->tx_size][mbmi->mode][mbmi->ext_txfrm];
}
} else {
mbmi->ext_txfrm = NORM;
}
#endif // CONFIG_EXT_TX
if (inter_block)
read_inter_block_mode_info(pbi, xd, mi, mi_row, mi_col, r);
else
read_intra_block_mode_info(cm, xd, mi, r);
}
void vp10_read_mode_info(VP10Decoder *const pbi, MACROBLOCKD *xd,
......
......@@ -164,22 +164,39 @@ static void update_switchable_interp_probs(VP10_COMMON *cm, vpx_writer *w,
static void update_ext_tx_probs(VP10_COMMON *cm, vpx_writer *w) {
const int savings_thresh = vp10_cost_one(GROUP_DIFF_UPDATE_PROB) -
vp10_cost_zero(GROUP_DIFF_UPDATE_PROB);
int i;
int i, j;
int savings = 0;
int do_update = 0;
for (i = TX_4X4; i <= TX_16X16; ++i) {
savings += prob_diff_update_savings(
vp10_ext_tx_tree, cm->fc->ext_tx_prob[i],
cm->counts.ext_tx[i], EXT_TX_TYPES);
vp10_ext_tx_tree, cm->fc->inter_ext_tx_prob[i],
cm->counts.inter_ext_tx[i], EXT_TX_TYPES);
}
do_update = savings > savings_thresh;
vpx_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
if (do_update) {
for (i = TX_4X4; i <= TX_16X16; ++i) {
prob_diff_update(vp10_ext_tx_tree, cm->fc->ext_tx_prob[i],
cm->counts.ext_tx[i], EXT_TX_TYPES, w);
prob_diff_update(vp10_ext_tx_tree, cm->fc->inter_ext_tx_prob[i],
cm->counts.inter_ext_tx[i], EXT_TX_TYPES, w);
}
}
savings = 0;
do_update = 0;
for (i = TX_4X4; i <= TX_16X16; ++i)
for (j = 0; j < INTRA_MODES; ++j)
savings += prob_diff_update_savings(
vp10_ext_tx_tree, cm->fc->intra_ext_tx_prob[i][j],
cm->counts.intra_ext_tx[i][j], EXT_TX_TYPES);
do_update = savings > savings_thresh;
vpx_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
if (do_update) {
for (i = TX_4X4; i <= TX_16X16; ++i)
for (j = 0; j < INTRA_MODES; ++j)
prob_diff_update(vp10_ext_tx_tree, cm->fc->intra_ext_tx_prob[i][j],
cm->counts.intra_ext_tx[i][j], EXT_TX_TYPES, w);
}
}
#endif // CONFIG_EXT_TX
......@@ -337,18 +354,6 @@ static void pack_inter_mode_mvs(VP10_COMP *cpi, const MODE_INFO *mi,
write_selected_tx_size(cm, xd, w);
}
#if CONFIG_EXT_TX
if (is_inter &&
mbmi->tx_size <= TX_16X16 &&
cm->base_qindex > 0 &&
bsize >= BLOCK_8X8 &&
!mbmi->skip &&
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
vp10_write_token(w, vp10_ext_tx_tree, cm->fc->ext_tx_prob[mbmi->tx_size],
&ext_tx_encodings[mbmi->ext_txfrm]);
}
#endif // CONFIG_EXT_TX
if (!is_inter) {
if (bsize >= BLOCK_8X8) {
write_intra_mode(w, mode, cm->fc->y_mode_prob[size_group_lookup[bsize]]);
......@@ -412,6 +417,21 @@ static void pack_inter_mode_mvs(VP10_COMP *cpi, const MODE_INFO *mi,
}
}
}
#if CONFIG_EXT_TX
if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
bsize >= BLOCK_8X8 && !mbmi->skip &&
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
if (is_inter)
vp10_write_token(w, vp10_ext_tx_tree,
cm->fc->inter_ext_tx_prob[mbmi->tx_size],
&ext_tx_encodings[mbmi->ext_txfrm]);
else
vp10_write_token(w, vp10_ext_tx_tree,
cm->fc->intra_ext_tx_prob[mbmi->tx_size][mbmi->mode],
&ext_tx_encodings[mbmi->ext_txfrm]);
}
#endif // CONFIG_EXT_TX
}
static void write_mb_modes_kf(const VP10_COMMON *cm, const MACROBLOCKD *xd,
......@@ -448,6 +468,16 @@ static void write_mb_modes_kf(const VP10_COMMON *cm, const MACROBLOCKD *xd,
}
write_intra_mode(w, mbmi->uv_mode, vp10_kf_uv_mode_prob[mbmi->mode]);
#if CONFIG_EXT_TX
if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
bsize >= BLOCK_8X8 && !mbmi->skip &&
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
vp10_write_token(w, vp10_ext_tx_tree,
cm->fc->intra_ext_tx_prob[mbmi->tx_size][mbmi->mode],
&ext_tx_encodings[mbmi->ext_txfrm]);
}
#endif // CONFIG_EXT_TX
}
static void write_modes_b(VP10_COMP *cpi, const TileInfo *const tile,
......
......@@ -2989,13 +2989,13 @@ static void encode_superblock(VP10_COMP *cpi, ThreadData *td,
++td->counts->tx.tx_totals[mbmi->tx_size];
++td->counts->tx.tx_totals[get_uv_tx_size(mbmi, &xd->plane[1])];
#if CONFIG_EXT_TX
if (is_inter_block(mbmi) &&
mbmi->tx_size <= TX_16X16 &&
cm->base_qindex > 0 &&
bsize >= BLOCK_8X8 &&
!mbmi->skip &&
if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
bsize >= BLOCK_8X8 && !mbmi->skip &&
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
++td->counts->ext_tx[mbmi->tx_size][mbmi->ext_txfrm];
if (is_inter_block(mbmi))
++td->counts->inter_ext_tx[mbmi->tx_size][mbmi->ext_txfrm];
else
++td->counts->intra_ext_tx[mbmi->tx_size][mbmi->mode][mbmi->ext_txfrm];
}
#endif // CONFIG_EXT_TX
}
......
......@@ -459,7 +459,8 @@ typedef struct VP10_COMP {
int switchable_interp_costs[SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS];
int partition_cost[PARTITION_CONTEXTS][PARTITION_TYPES];
#if CONFIG_EXT_TX
int ext_tx_costs[EXT_TX_SIZES][EXT_TX_TYPES];
int inter_ext_tx_costs[EXT_TX_SIZES][EXT_TX_TYPES];
int intra_ext_tx_costs[EXT_TX_SIZES][INTRA_MODES][EXT_TX_TYPES];
#endif // CONFIG_EXT_TX
int multi_arf_allowed;
......
......@@ -85,9 +85,13 @@ static void fill_mode_costs(VP10_COMP *cpi) {
vp10_cost_tokens(cpi->switchable_interp_costs[i],
fc->switchable_interp_prob[i], vp10_switchable_interp_tree);
#if CONFIG_EXT_TX
for (i = TX_4X4; i <= TX_16X16; ++i)
vp10_cost_tokens(cpi->ext_tx_costs[i], fc->ext_tx_prob[i],
for (i = TX_4X4; i <= TX_16X16; ++i) {
vp10_cost_tokens(cpi->inter_ext_tx_costs[i], fc->inter_ext_tx_prob[i],
vp10_ext_tx_tree);
for (j = 0; j < INTRA_MODES; ++j)
vp10_cost_tokens(cpi->intra_ext_tx_costs[i][j],
fc->intra_ext_tx_prob[i][j], vp10_ext_tx_tree);
}
#endif // CONFIG_EXT_TX
}
......
......@@ -603,6 +603,11 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x,
#if CONFIG_EXT_TX
if (is_inter_block(mbmi) && bs >= BLOCK_8X8 && !xd->lossless) {
for (tx_type = NORM; tx_type < EXT_TX_TYPES - 1; ++tx_type) {
if (mbmi->ext_txfrm >= ALT11 && mbmi->ext_txfrm < ALT16 &&
best_tx_type == NORM) {
tx_type = ALT16 - 1;
continue;
}
if (tx_type >= GET_EXT_TX_TYPES(mbmi->tx_size))
continue;
......@@ -613,8 +618,13 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x,
if (r == INT_MAX)
continue;
if (mbmi->tx_size <= TX_16X16)
r += cpi->ext_tx_costs[mbmi->tx_size][mbmi->ext_txfrm];
if (mbmi->tx_size <= TX_16X16) {
if (is_inter_block(mbmi))
r += cpi->inter_ext_tx_costs[mbmi->tx_size][mbmi->ext_txfrm];
else
r += cpi->intra_ext_tx_costs[mbmi->tx_size]
[mbmi->mode][mbmi->ext_txfrm];
}
if (s)
this_rd = RDCOST(x->rdmult, x->rddiv, s1, psse);
......@@ -638,10 +648,14 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x,
mbmi->tx_size, cpi->sf.use_fast_coef_costing);
#if CONFIG_EXT_TX
if (is_inter_block(mbmi) && bs >= BLOCK_8X8 &&
mbmi->tx_size <= TX_16X16 &&
!xd->lossless && *rate != INT_MAX)
*rate += cpi->ext_tx_costs[mbmi->tx_size][mbmi->ext_txfrm];
if (bs >= BLOCK_8X8 && mbmi->tx_size <= TX_16X16 &&
!xd->lossless && *rate != INT_MAX) {
if (is_inter_block(mbmi))
*rate += cpi->inter_ext_tx_costs[mbmi->tx_size][mbmi->ext_txfrm];
else
*rate += cpi->intra_ext_tx_costs[mbmi->tx_size]
[mbmi->mode][mbmi->ext_txfrm];
}
#endif // CONFIG_EXT_TX
}
......@@ -693,7 +707,7 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x,
#if CONFIG_EXT_TX
start_tx_type = NORM;
if (is_inter_block(mbmi) && bs >= BLOCK_8X8 && !xd->lossless)
if (bs >= BLOCK_8X8 && !xd->lossless)
end_tx_type = EXT_TX_TYPES - 1;
else
end_tx_type = NORM;
......@@ -726,9 +740,12 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x,
&sse, ref_best_rd, 0, bs, n,
cpi->sf.use_fast_coef_costing);
#if CONFIG_EXT_TX
if (is_inter_block(mbmi) && bs >= BLOCK_8X8 &&
!xd->lossless && r != INT_MAX && n < TX_32X32)
r += cpi->ext_tx_costs[n][mbmi->ext_txfrm];
if (bs >= BLOCK_8X8 && !xd->lossless && r != INT_MAX && n < TX_32X32) {
if (is_inter_block(mbmi))
r += cpi->inter_ext_tx_costs[n][mbmi->ext_txfrm];
else
r += cpi->intra_ext_tx_costs[n][mbmi->mode][mbmi->ext_txfrm];
}
#endif // CONFIG_EXT_TX
if (r == INT_MAX)
......@@ -1136,6 +1153,9 @@ static int64_t rd_pick_intra_sby_mode(VP10_COMP *cpi, MACROBLOCK *x,
int this_rate, this_rate_tokenonly, s;
int64_t this_distortion, this_rd;
TX_SIZE best_tx = TX_4X4;
#if CONFIG_EXT_TX
EXT_TX_TYPE best_tx_type = NORM;
#endif // CONFIG_EXT_TX
int *bmode_costs;
const MODE_INFO *above_mi = xd->above_mi;
const MODE_INFO *left_mi = xd->left_mi;
......@@ -1161,6 +1181,9 @@ static int64_t rd_pick_intra_sby_mode(VP10_COMP *cpi, MACROBLOCK *x,
mode_selected = mode;
best_rd = this_rd;
best_tx = mic->mbmi.tx_size;
#if CONFIG_EXT_TX
best_tx_type = mic->mbmi.ext_txfrm;
#endif // CONFIG_EXT_TX
*rate = this_rate;
*rate_tokenonly = this_rate_tokenonly;
*distortion = this_distortion;
......@@ -1170,6 +1193,9 @@ static int64_t rd_pick_intra_sby_mode(VP10_COMP *cpi, MACROBLOCK *x,
mic->mbmi.mode = mode_selected;
mic->mbmi.tx_size = best_tx;
#if CONFIG_EXT_TX
mic->mbmi.ext_txfrm = best_tx_type;
#endif // CONFIG_EXT_TX
return best_rd;
}
......@@ -3295,9 +3321,6 @@ void vp10_rd_pick_inter_mode_sb(VP10_COMP *cpi,
if (ref_frame == INTRA_FRAME) {
TX_SIZE uv_tx;
struct macroblockd_plane *const pd = &xd->plane[1];
#if CONFIG_EXT_TX
mbmi->ext_txfrm = NORM;
#endif // CONFIG_EXT_TX
memset(x->skip_txfm, 0, sizeof(x->skip_txfm));
super_block_yrd(cpi, x, &rate_y, &distortion_y, &skippable,
NULL, bsize, best_rd);
......@@ -3322,9 +3345,6 @@ void vp10_rd_pick_inter_mode_sb(VP10_COMP *cpi,
rate2 += intra_cost_penalty;
distortion2 = distortion_y + distortion_uv;
} else {
#if CONFIG_EXT_TX
mbmi->ext_txfrm = NORM;
#endif
this_rd = handle_inter_mode(cpi, x, bsize,
&rate2, &distortion2, &skippable,
&rate_y, &rate_uv,
......@@ -3755,9 +3775,6 @@ void vp10_rd_pick_inter_mode_sub8x8(VP10_COMP *cpi,
int64_t filter_cache[SWITCHABLE_FILTER_CONTEXTS];
int internal_active_edge =
vp10_active_edge_sb(cpi, mi_row, mi_col) && vp10_internal_image_edge(cpi);
#if CONFIG_EXT_TX
mbmi->ext_txfrm = NORM;
#endif
memset(x->zcoeff_blk[TX_4X4], 0, 4);
vp10_zero(best_mbmode);
......@@ -3916,9 +3933,6 @@ void vp10_rd_pick_inter_mode_sub8x8(VP10_COMP *cpi,
if (ref_frame == INTRA_FRAME) {
int rate;
#if CONFIG_EXT_TX
mbmi->ext_txfrm = NORM;
#endif
if (rd_pick_intra_sub_8x8_y_mode(cpi, x, &rate, &rate_y,
&distortion_y, best_rd) >= best_rd)
continue;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment