From 215084b31c3e5683c75a558911172d8f34661333 Mon Sep 17 00:00:00 2001
From: James Zern <jzern@google.com>
Date: Tue, 5 Mar 2013 12:02:28 -0800
Subject: [PATCH] vp8: clamp probability values
fixes out of bounds reads on vp8_prob_cost table.
present since:
217591f Added rate-targeted temporal scalability
v0.9.7-p1-71-g217591f
Change-Id: I9194b773098a381f1e3f67bd3307f05df27c24fd
---
vp8/encoder/bitstream.c | 6 ++++++
vp8/encoder/onyx_if.c | 2 ++
2 files changed, 8 insertions(+)
diff --git a/vp8/encoder/bitstream.c b/vp8/encoder/bitstream.c
index e666b6c7e1..8f94171e69 100644
--- a/vp8/encoder/bitstream.c
+++ b/vp8/encoder/bitstream.c
@@ -980,6 +980,12 @@ void vp8_calc_ref_frame_costs(int *ref_frame_cost,
int prob_garf
)
{
+ assert(prob_intra >= 0);
+ assert(prob_intra <= 255);
+ assert(prob_last >= 0);
+ assert(prob_last <= 255);
+ assert(prob_garf >= 0);
+ assert(prob_garf <= 255);
ref_frame_cost[INTRA_FRAME] = vp8_cost_zero(prob_intra);
ref_frame_cost[LAST_FRAME] = vp8_cost_one(prob_intra)
+ vp8_cost_zero(prob_last);
diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c
index 92f9818577..555a9e4bca 100644
--- a/vp8/encoder/onyx_if.c
+++ b/vp8/encoder/onyx_if.c
@@ -2816,6 +2816,8 @@ static void update_rd_ref_frame_probs(VP8_COMP *cpi)
if (cpi->common.refresh_alt_ref_frame)
{
cpi->prob_intra_coded += 40;
+ if (cpi->prob_intra_coded > 255)
+ cpi->prob_intra_coded = 255;
cpi->prob_last_coded = 200;
cpi->prob_gf_coded = 1;
}
--
GitLab