Commit ef665996 authored by Yaowu Xu's avatar Yaowu Xu

Prevent negative variance

Due to rounding used computation, HDB variance computation may produce
slightly negative values. This commit adds clamping to make sure
output variance values for 10 and 12 to be non-negative.

Change-Id: Id679aa55a4c201958c4c7d28cd8733b9246a71c8
parent b3933e2d
......@@ -341,8 +341,10 @@ uint32_t vpx_highbd_10_variance##W##x##H##_c(const uint8_t *a, \
int b_stride, \
uint32_t *sse) { \
int sum; \
int64_t var; \
highbd_10_variance(a, a_stride, b, b_stride, W, H, sse, &sum); \
return *sse - (((int64_t)sum * sum) / (W * H)); \
var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \
return (var >= 0) ? (uint32_t)var : 0; \
} \
\
uint32_t vpx_highbd_12_variance##W##x##H##_c(const uint8_t *a, \
......@@ -351,8 +353,10 @@ uint32_t vpx_highbd_12_variance##W##x##H##_c(const uint8_t *a, \
int b_stride, \
uint32_t *sse) { \
int sum; \
int64_t var; \
highbd_12_variance(a, a_stride, b, b_stride, W, H, sse, &sum); \
return *sse - (((int64_t)sum * sum) / (W * H)); \
var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \
return (var >= 0) ? (uint32_t)var : 0; \
}
#define HIGHBD_GET_VAR(S) \
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment