From ff1ae7f713dc079de7e124a023ee3dc6aa9b616d Mon Sep 17 00:00:00 2001
From: Yaowu Xu <yaowu@google.com>
Date: Tue, 24 Sep 2013 09:37:55 -0700
Subject: [PATCH] Prevent using uninitialized value in RD decision
INT64_MAX may be assigned as RDCOST when RDCSOST computation is skipped
for speed, this commit to prevent INT64_MAX from being used as real
RDCOST in transform size decision.
Change-Id: I89a945134191bbdea1f1431ade70424ac079eaac
---
vp9/encoder/vp9_rdopt.c | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index 041316b4f7..354eb611b6 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -3093,8 +3093,12 @@ void vp9_rd_pick_intra_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
vp9_cost_bit(vp9_get_pred_prob_mbskip(cm, xd), 0);
*returndist = dist_y + dist_uv;
if (cpi->sf.tx_size_search_method == USE_FULL_RD)
- for (i = 0; i < TX_MODES; i++)
- ctx->tx_rd_diff[i] = tx_cache[i] - tx_cache[cm->tx_mode];
+ for (i = 0; i < TX_MODES; i++) {
+ if (tx_cache[i] < INT64_MAX && tx_cache[cm->tx_mode] < INT64_MAX)
+ ctx->tx_rd_diff[i] = tx_cache[i] - tx_cache[cm->tx_mode];
+ else
+ ctx->tx_rd_diff[i] = 0;
+ }
}
ctx->mic = *xd->this_mi;
--
GitLab