Commit 6d43a53a authored by Jingning Han's avatar Jingning Han Committed by Gerrit Code Review

Merge "Incorporate flexible tx type and tx partition in RD scheme" into nextgenv2

parents 2c328618 4b594d3d
......@@ -223,6 +223,7 @@ typedef struct macroblockd {
// dimension in the unit of 8x8 block of the current block
int16_t n8_w, n8_h;
TX_SIZE max_tx_size;
#endif
#if CONFIG_VP9_HIGHBITDEPTH
......
......@@ -869,6 +869,11 @@ static MB_MODE_INFO *set_offsets(VP10_COMMON *const cm, MACROBLOCKD *const xd,
set_skip_context(xd, mi_row, mi_col);
#if CONFIG_VAR_TX
xd->max_tx_size = max_txsize_lookup[bsize];
#endif
// Distance of Mb to the various image edges. These are specified to 8th pel
// as they are always compared to values that are in 1/8th pel units
set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);
......
......@@ -199,6 +199,7 @@ static void set_offsets(VP10_COMP *cpi, const TileInfo *const tile,
#if CONFIG_VAR_TX
xd->above_txfm_context = cm->above_txfm_context + mi_col;
xd->left_txfm_context = xd->left_txfm_context_buffer + (mi_row & 0x07);
xd->max_tx_size = max_txsize_lookup[bsize];
#endif
mbmi = &xd->mi[0]->mbmi;
......
......@@ -1650,8 +1650,6 @@ static void tx_block_rd_b(const VP10_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
*bsse += (int64_t)tmp_sse * 16;
if (p->eobs[block] > 0) {
// TODO(jingning): integrate multiple transform type experiment
TX_TYPE tx_type = DCT_DCT;
switch (tx_size) {
case TX_32X32:
vp10_inv_txfm_add_32x32(dqcoeff, rec_buffer, 32, p->eobs[block],
......@@ -1878,10 +1876,6 @@ static void inter_block_yrd(const VP10_COMP *cpi, MACROBLOCK *x,
*sse = 0;
*skippable = 1;
#if CONFIG_EXT_TX
xd->mi[0]->mbmi.tx_type = DCT_DCT;
#endif
if (is_cost_valid) {
const struct macroblockd_plane *const pd = &xd->plane[0];
const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
......@@ -1936,6 +1930,99 @@ static void inter_block_yrd(const VP10_COMP *cpi, MACROBLOCK *x,
}
}
#if CONFIG_EXT_TX
static void select_tx_type_yrd(const VP10_COMP *cpi, MACROBLOCK *x,
int *rate, int64_t *distortion, int *skippable,
int64_t *sse, BLOCK_SIZE bsize,
int64_t ref_best_rd) {
const TX_SIZE max_tx_size = max_txsize_lookup[bsize];
const VP10_COMMON *const cm = &cpi->common;
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
int64_t rd = INT64_MAX;
int64_t best_rd = INT64_MAX;
TX_TYPE tx_type, best_tx_type = DCT_DCT;
int ext_tx_set;
const int is_inter = is_inter_block(mbmi);
vpx_prob skip_prob = vp10_get_skip_prob(cm, xd);
int s0 = vp10_cost_bit(skip_prob, 0);
int s1 = vp10_cost_bit(skip_prob, 1);
*distortion = INT64_MAX;
*rate = INT_MAX;
*skippable = 0;
*sse = INT64_MAX;
for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
int this_rate = 0;
int this_skip = 1;
int64_t this_dist = 0;
int64_t this_sse = 0;
ext_tx_set = get_ext_tx_set(max_tx_size, bsize, is_inter);
if (is_inter) {
if (!ext_tx_used_inter[ext_tx_set][tx_type])
continue;
} else {
if (!ext_tx_used_intra[ext_tx_set][tx_type])
continue;
}
mbmi->tx_type = tx_type;
if (ext_tx_set == 1 &&
mbmi->tx_type >= DST_ADST && mbmi->tx_type < IDTX &&
best_tx_type == DCT_DCT) {
tx_type = IDTX - 1;
break;
}
inter_block_yrd(cpi, x, &this_rate, &this_dist, &this_skip, &this_sse,
bsize, ref_best_rd);
if (get_ext_tx_types(max_tx_size, bsize, is_inter) > 1 &&
!xd->lossless[xd->mi[0]->mbmi.segment_id] &&
this_rate != INT_MAX) {
if (is_inter) {
if (ext_tx_set > 0)
this_rate += cpi->inter_tx_type_costs[ext_tx_set]
[mbmi->tx_size][mbmi->tx_type];
} else {
if (ext_tx_set > 0)
this_rate += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size]
[mbmi->mode][mbmi->tx_type];
}
}
if (this_rate == INT_MAX)
continue;
if (this_skip)
rd = RDCOST(x->rdmult, x->rddiv, s1, this_sse);
else
rd = RDCOST(x->rdmult, x->rddiv, this_rate + s0, this_dist);
if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] && !this_skip)
rd = VPXMIN(rd, RDCOST(x->rdmult, x->rddiv, s1, this_sse));
if (rd <
(is_inter && best_tx_type == DCT_DCT ? ext_tx_th : 1) *
best_rd) {
best_rd = rd;
*distortion = this_dist;
*rate = this_rate;
*skippable = this_skip;
*sse = this_sse;
best_tx_type = mbmi->tx_type;
}
}
mbmi->tx_type = best_tx_type;
inter_block_yrd(cpi, x, rate, distortion, skippable, sse,
bsize, ref_best_rd);
}
#endif
static void tx_block_rd(const VP10_COMP *cpi, MACROBLOCK *x,
int blk_row, int blk_col, int plane, int block,
TX_SIZE tx_size, BLOCK_SIZE plane_bsize,
......@@ -3704,8 +3791,13 @@ static int64_t handle_inter_mode(VP10_COMP *cpi, MACROBLOCK *x,
vp10_subtract_plane(x, bsize, 0);
#if CONFIG_VAR_TX
if (cm->tx_mode == TX_MODE_SELECT || xd->lossless[mbmi->segment_id]) {
#if CONFIG_EXT_TX
select_tx_type_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
bsize, ref_best_rd);
#else
inter_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
bsize, ref_best_rd);
#endif
} else {
super_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
bsize, ref_best_rd);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment