From 53844275e9e19d0af6f6e5bda98d884cf09aad5d Mon Sep 17 00:00:00 2001
From: Jingning Han <jingning@google.com>
Date: Thu, 24 Jul 2014 15:31:32 -0700
Subject: [PATCH] Fix potential ioc issue in vp9_get_prob for 4K above sizes
This commit turns on the existing vp9_get_prob function using
64 bit in the intermediate step. It fixes the ioc issue for 4K
above frame sizes (issue 828).
Change-Id: I9f627f3beca2c522f73b38fd2a3e7eefdff01a7c
---
test/frame_size_tests.cc | 3 +--
vp9/common/vp9_prob.h | 9 ---------
2 files changed, 1 insertion(+), 11 deletions(-)
diff --git a/test/frame_size_tests.cc b/test/frame_size_tests.cc
index b05d12e9eb..d1ea0967fb 100644
--- a/test/frame_size_tests.cc
+++ b/test/frame_size_tests.cc
@@ -89,8 +89,7 @@ TEST_F(VP9FrameSizeTestsLarge, ValidSizes) {
// one for each lag in frames (for 2 pass), and then one for each possible
// reference buffer (8) - we can end up with up to 30 buffers of roughly this
// size or almost 1 gig of memory.
- // TODO(jzern): restore this to at least 4096x4096 after issue #828 is fixed.
- video.SetSize(4096, 2160);
+ video.SetSize(4096, 4096);
video.set_limit(2);
expected_res_ = VPX_CODEC_OK;
ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
diff --git a/vp9/common/vp9_prob.h b/vp9/common/vp9_prob.h
index f361480350..3920619d88 100644
--- a/vp9/common/vp9_prob.h
+++ b/vp9/common/vp9_prob.h
@@ -47,18 +47,9 @@ static INLINE vp9_prob clip_prob(int p) {
return (p > 255) ? 255u : (p < 1) ? 1u : p;
}
-// int64 is not needed for normal frame level calculations.
-// However when outputting entropy stats accumulated over many frames
-// or even clips we can overflow int math.
-#ifdef ENTROPY_STATS
static INLINE vp9_prob get_prob(int num, int den) {
return (den == 0) ? 128u : clip_prob(((int64_t)num * 256 + (den >> 1)) / den);
}
-#else
-static INLINE vp9_prob get_prob(int num, int den) {
- return (den == 0) ? 128u : clip_prob((num * 256 + (den >> 1)) / den);
-}
-#endif
static INLINE vp9_prob get_binary_prob(int n0, int n1) {
return get_prob(n0, n0 + n1);
--
GitLab