Commit a0993703 authored by Ronald S. Bultje's avatar Ronald S. Bultje
Browse files

Prevent overflow in variance32x32.

Change-Id: I478878c78ef8a770186622d987d318176827ef5f
Showing with 2 additions and 3 deletions
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
#include "variance.h" #include "variance.h"
#include "vp9/common/filter.h" #include "vp9/common/filter.h"
#include "vp9/common/subpelvar.h" #include "vp9/common/subpelvar.h"
#include "vpx/vpx_integer.h"
unsigned int vp9_get_mb_ss_c(const short *src_ptr) { unsigned int vp9_get_mb_ss_c(const short *src_ptr) {
unsigned int i, sum = 0; unsigned int i, sum = 0;
...@@ -37,8 +37,7 @@ unsigned int vp9_variance32x32_c(const unsigned char *src_ptr, ...@@ -37,8 +37,7 @@ unsigned int vp9_variance32x32_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 32, 32, &var, &avg); variance(src_ptr, source_stride, ref_ptr, recon_stride, 32, 32, &var, &avg);
*sse = var; *sse = var;
// TODO(rbultje): in extreme cases these products will rollover. return (var - (((int64_t)avg * avg) >> 10));
return (var - (((unsigned int)avg * avg) >> 10));
} }
#endif #endif
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment