fail allocation of buffers if size_t < frame_size

Change-Id: I25c595e8c197ab0a9955d2373f1a74d42fbd1638

vpx_scale/generic/yv12config.c

@@ -142,34 +142,39 @@ int vp9_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf,
     const int aligned_width = (width + 7) & ~7;
     const int aligned_height = (height + 7) & ~7;
     const int y_stride = ((aligned_width + 2 * border) + 31) & ~31;
-    const int yplane_size = (aligned_height + 2 * border) * y_stride;
+    const uint64_t yplane_size = (aligned_height + 2 * border) *
+                                 (uint64_t)y_stride;
     const int uv_width = aligned_width >> ss_x;
     const int uv_height = aligned_height >> ss_y;
     const int uv_stride = y_stride >> ss_x;
     const int uv_border_w = border >> ss_x;
     const int uv_border_h = border >> ss_y;
-    const int uvplane_size = (uv_height + 2 * uv_border_h) * uv_stride;
+    const uint64_t uvplane_size = (uv_height + 2 * uv_border_h) *
+                                  (uint64_t)uv_stride;
 #if CONFIG_ALPHA
     const int alpha_width = aligned_width;
     const int alpha_height = aligned_height;
     const int alpha_stride = y_stride;
     const int alpha_border_w = border;
     const int alpha_border_h = border;
-    const int alpha_plane_size = (alpha_height + 2 * alpha_border_h) *
-                                 alpha_stride;
-    const int frame_size = yplane_size + 2 * uvplane_size +
-                           alpha_plane_size;
+    const uint64_t alpha_plane_size = (alpha_height + 2 * alpha_border_h) *
+                                      (uint64_t)alpha_stride;
+    const uint64_t frame_size = yplane_size + 2 * uvplane_size +
+                                alpha_plane_size;
 #else
-    const int frame_size = yplane_size + 2 * uvplane_size;
+    const uint64_t frame_size = yplane_size + 2 * uvplane_size;
 #endif
     if (cb != NULL) {
       const int align_addr_extra_size = 31;
-      const size_t external_frame_size = frame_size + align_addr_extra_size;
+      const uint64_t external_frame_size = frame_size + align_addr_extra_size;
 
       assert(fb != NULL);
 
+      if (external_frame_size != (size_t)external_frame_size)
+        return -1;
+
       // Allocation to hold larger frame, or first allocation.
-      if (cb(cb_priv, external_frame_size, fb) < 0)
+      if (cb(cb_priv, (size_t)external_frame_size, fb) < 0)
         return -1;
 
       if (fb->data == NULL || fb->size < external_frame_size)
@@ -181,10 +186,15 @@ int vp9_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf,
       vpx_memset(fb->data, 0, fb->size);
 
       ybf->buffer_alloc = (uint8_t *)yv12_align_addr(fb->data, 32);
-    } else if (frame_size > ybf->buffer_alloc_sz) {
+    } else if (frame_size > (size_t)ybf->buffer_alloc_sz) {
       // Allocation to hold larger frame, or first allocation.
       vpx_free(ybf->buffer_alloc);
-      ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, frame_size);
+      ybf->buffer_alloc = NULL;
+
+      if (frame_size != (size_t)frame_size)
+        return -1;
+
+      ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, (size_t)frame_size);
       if (!ybf->buffer_alloc)
         return -1;