Commit 79fe7246 authored by Jingning Han's avatar Jingning Han

Properly handle non-420 color format in recursive transform scheme

This commit makes the recursive transform block partitioning properly
handle the non-420 color format. It resolves an enc/dec mismatch
issue in that setting when var-tx experiment is turned on.

Change-Id: I48a91de02c11b3153f897d1cca0ae948eec15605
parent f1c4b79d
......@@ -115,7 +115,11 @@ static void read_tx_size_inter(VP10_COMMON *cm, MACROBLOCKD *xd,
read_tx_size_inter(cm, xd, mbmi, tx_size - 1, offsetr, offsetc, r);
}
} else {
int idx, idy;
mbmi->inter_tx_size[tx_idx] = tx_size;
for (idy = 0; idy < (1 << tx_size) / 2; ++idy)
for (idx = 0; idx < (1 << tx_size) / 2; ++idx)
mbmi->inter_tx_size[tx_idx + (idy << 3) + idx] = tx_size;
mbmi->tx_size = mbmi->inter_tx_size[tx_idx];
}
}
......
......@@ -1635,9 +1635,14 @@ static void select_tx_block(const VP10_COMP *cpi, MACROBLOCK *x,
}
if (this_rd < sum_rd) {
int idx, idy;
for (i = 0; i < (1 << tx_size); ++i)
pta[i] = ptl[i] = !(tmp_eob == 0);
mbmi->inter_tx_size[tx_idx] = tx_size;
for (idy = 0; idy < (1 << tx_size) / 2; ++idy)
for (idx = 0; idx < (1 << tx_size) / 2; ++idx)
mbmi->inter_tx_size[tx_idx + (idy << 3) + idx] = tx_size;
mbmi->tx_size = tx_size;
} else {
*rate = sum_rate;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment