Commit 53844275 authored by Jingning Han's avatar Jingning Han
Browse files

Fix potential ioc issue in vp9_get_prob for 4K above sizes

This commit turns on the existing vp9_get_prob function using
64 bit in the intermediate step. It fixes the ioc issue for 4K
above frame sizes (issue 828).

Change-Id: I9f627f3beca2c522f73b38fd2a3e7eefdff01a7c
Showing with 1 addition and 11 deletions
...@@ -89,8 +89,7 @@ TEST_F(VP9FrameSizeTestsLarge, ValidSizes) { ...@@ -89,8 +89,7 @@ TEST_F(VP9FrameSizeTestsLarge, ValidSizes) {
// one for each lag in frames (for 2 pass), and then one for each possible // one for each lag in frames (for 2 pass), and then one for each possible
// reference buffer (8) - we can end up with up to 30 buffers of roughly this // reference buffer (8) - we can end up with up to 30 buffers of roughly this
// size or almost 1 gig of memory. // size or almost 1 gig of memory.
// TODO(jzern): restore this to at least 4096x4096 after issue #828 is fixed. video.SetSize(4096, 4096);
video.SetSize(4096, 2160);
video.set_limit(2); video.set_limit(2);
expected_res_ = VPX_CODEC_OK; expected_res_ = VPX_CODEC_OK;
ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
......
...@@ -47,18 +47,9 @@ static INLINE vp9_prob clip_prob(int p) { ...@@ -47,18 +47,9 @@ static INLINE vp9_prob clip_prob(int p) {
return (p > 255) ? 255u : (p < 1) ? 1u : p; return (p > 255) ? 255u : (p < 1) ? 1u : p;
} }
// int64 is not needed for normal frame level calculations.
// However when outputting entropy stats accumulated over many frames
// or even clips we can overflow int math.
#ifdef ENTROPY_STATS
static INLINE vp9_prob get_prob(int num, int den) { static INLINE vp9_prob get_prob(int num, int den) {
return (den == 0) ? 128u : clip_prob(((int64_t)num * 256 + (den >> 1)) / den); return (den == 0) ? 128u : clip_prob(((int64_t)num * 256 + (den >> 1)) / den);
} }
#else
static INLINE vp9_prob get_prob(int num, int den) {
return (den == 0) ? 128u : clip_prob((num * 256 + (den >> 1)) / den);
}
#endif
static INLINE vp9_prob get_binary_prob(int n0, int n1) { static INLINE vp9_prob get_binary_prob(int n0, int n1) {
return get_prob(n0, n0 + n1); return get_prob(n0, n0 + n1);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment