From 1a01194ab548ca9b5d7dbbdc2350d4c06f54e6d4 Mon Sep 17 00:00:00 2001
From: Jim Bankoski <jimbankoski@google.com>
Date: Thu, 17 Jul 2014 06:33:29 -0700
Subject: [PATCH] fail allocation of buffers if size_t < frame_size
Change-Id: I25c595e8c197ab0a9955d2373f1a74d42fbd1638
---
vpx_scale/generic/yv12config.c | 32 +++++++++++++++++++++-----------
1 file changed, 21 insertions(+), 11 deletions(-)
diff --git a/vpx_scale/generic/yv12config.c b/vpx_scale/generic/yv12config.c
index 3eaf50ed7d..e8003cea44 100644
--- a/vpx_scale/generic/yv12config.c
+++ b/vpx_scale/generic/yv12config.c
@@ -142,34 +142,39 @@ int vp9_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf,
const int aligned_width = (width + 7) & ~7;
const int aligned_height = (height + 7) & ~7;
const int y_stride = ((aligned_width + 2 * border) + 31) & ~31;
- const int yplane_size = (aligned_height + 2 * border) * y_stride;
+ const uint64_t yplane_size = (aligned_height + 2 * border) *
+ (uint64_t)y_stride;
const int uv_width = aligned_width >> ss_x;
const int uv_height = aligned_height >> ss_y;
const int uv_stride = y_stride >> ss_x;
const int uv_border_w = border >> ss_x;
const int uv_border_h = border >> ss_y;
- const int uvplane_size = (uv_height + 2 * uv_border_h) * uv_stride;
+ const uint64_t uvplane_size = (uv_height + 2 * uv_border_h) *
+ (uint64_t)uv_stride;
#if CONFIG_ALPHA
const int alpha_width = aligned_width;
const int alpha_height = aligned_height;
const int alpha_stride = y_stride;
const int alpha_border_w = border;
const int alpha_border_h = border;
- const int alpha_plane_size = (alpha_height + 2 * alpha_border_h) *
- alpha_stride;
- const int frame_size = yplane_size + 2 * uvplane_size +
- alpha_plane_size;
+ const uint64_t alpha_plane_size = (alpha_height + 2 * alpha_border_h) *
+ (uint64_t)alpha_stride;
+ const uint64_t frame_size = yplane_size + 2 * uvplane_size +
+ alpha_plane_size;
#else
- const int frame_size = yplane_size + 2 * uvplane_size;
+ const uint64_t frame_size = yplane_size + 2 * uvplane_size;
#endif
if (cb != NULL) {
const int align_addr_extra_size = 31;
- const size_t external_frame_size = frame_size + align_addr_extra_size;
+ const uint64_t external_frame_size = frame_size + align_addr_extra_size;
assert(fb != NULL);
+ if (external_frame_size != (size_t)external_frame_size)
+ return -1;
+
// Allocation to hold larger frame, or first allocation.
- if (cb(cb_priv, external_frame_size, fb) < 0)
+ if (cb(cb_priv, (size_t)external_frame_size, fb) < 0)
return -1;
if (fb->data == NULL || fb->size < external_frame_size)
@@ -181,10 +186,15 @@ int vp9_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf,
vpx_memset(fb->data, 0, fb->size);
ybf->buffer_alloc = (uint8_t *)yv12_align_addr(fb->data, 32);
- } else if (frame_size > ybf->buffer_alloc_sz) {
+ } else if (frame_size > (size_t)ybf->buffer_alloc_sz) {
// Allocation to hold larger frame, or first allocation.
vpx_free(ybf->buffer_alloc);
- ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, frame_size);
+ ybf->buffer_alloc = NULL;
+
+ if (frame_size != (size_t)frame_size)
+ return -1;
+
+ ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, (size_t)frame_size);
if (!ybf->buffer_alloc)
return -1;
--
GitLab