From ee808e52bd0e371f0caad5ca7f29fbc4d5bf2aee Mon Sep 17 00:00:00 2001
From: "Ronald S. Bultje" <rbultje@google.com>
Date: Fri, 3 May 2013 10:33:07 -0700
Subject: [PATCH] Fix overflow in RD error calculation code.
Change-Id: I61ef1f198c876f9f79787ea7d7385a862cfbae19
---
vp9/encoder/vp9_rdopt.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index f971d91c59..990be6527e 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -603,7 +603,7 @@ static int block_error(int16_t *coeff, int16_t *dqcoeff,
for (i = 0; i < block_size; i++) {
int this_diff = coeff[i] - dqcoeff[i];
- error += this_diff * this_diff;
+ error += (unsigned)this_diff * this_diff;
}
error >>= shift;
--
GitLab