Commit 5661cd8f authored by James Zern's avatar James Zern
Browse files

vp9_rdopt: correct size to vpx_sum_squares_2d_i16

the current implementations expect pixel size, not the block type

BUG=webm:1392

Change-Id: Ib91e9f30a1f56e13566b1fb76f089dae9bb50cdc
parent 6d71d33d
......@@ -515,7 +515,8 @@ static int64_t sum_squares_visible(const MACROBLOCKD *xd,
pd->subsampling_y, blk_row);
if (tx_bsize == BLOCK_4X4 ||
(b4x4s_to_right_edge >= tx_4x4_w && b4x4s_to_bottom_edge >= tx_4x4_h)) {
sse = (int64_t)vpx_sum_squares_2d_i16(diff, diff_stride, tx_bsize);
assert(tx_4x4_w == tx_4x4_h);
sse = (int64_t)vpx_sum_squares_2d_i16(diff, diff_stride, tx_4x4_w << 2);
} else {
int r, c;
int max_r = VPXMIN(b4x4s_to_bottom_edge, tx_4x4_h);
......@@ -525,7 +526,8 @@ static int64_t sum_squares_visible(const MACROBLOCKD *xd,
for (r = 0; r < max_r; ++r) {
// Skip visiting the sub blocks that are wholly within the UMV.
for (c = 0; c < max_c; ++c) {
sse += (int64_t)vpx_sum_squares_2d_i16(diff, diff_stride, BLOCK_4X4);
sse += (int64_t)vpx_sum_squares_2d_i16(
diff + r * diff_stride * 4 + c * 4, diff_stride, 4);
}
}
}
......
......@@ -123,6 +123,7 @@ uint64_t vpx_sum_squares_2d_i16_sse2(const int16_t *src, int stride, int size) {
return vpx_sum_squares_2d_i16_4x4_sse2(src, stride);
} else {
// Generic case
assert(size % 8 == 0);
return vpx_sum_squares_2d_i16_nxn_sse2(src, stride, size);
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment