generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fail allocation of buffers if size_t < frame_size

Browse Source 
Change-Id: I25c595e8c197ab0a9955d2373f1a74d42fbd1638
This commit is contained in:
Jim Bankoski 2014-07-17 06:33:29 -07:00 committed by Gerrit Code Review
parent 93960c869e
commit 1a01194ab5
1 changed files with 21 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
vpx_scale/generic/yv12config.c
View File

@ -142,34 +142,39 @@ int vp9_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf,
const int aligned_width = (width + 7) & ~7;
const int aligned_height = (height + 7) & ~7;
const int y_stride = ((aligned_width + 2 * border) + 31) & ~31;
const int yplane_size = (aligned_height + 2 * border) * y_stride;
const uint64_t yplane_size = (aligned_height + 2 * border) *
(uint64_t)y_stride;
const int uv_width = aligned_width >> ss_x;
const int uv_height = aligned_height >> ss_y;
const int uv_stride = y_stride >> ss_x;
const int uv_border_w = border >> ss_x;
const int uv_border_h = border >> ss_y;
const int uvplane_size = (uv_height + 2 * uv_border_h) * uv_stride;
const uint64_t uvplane_size = (uv_height + 2 * uv_border_h) *
(uint64_t)uv_stride;
#if CONFIG_ALPHA
const int alpha_width = aligned_width;
const int alpha_height = aligned_height;
const int alpha_stride = y_stride;
const int alpha_border_w = border;
const int alpha_border_h = border;
const int alpha_plane_size = (alpha_height + 2 * alpha_border_h) *
alpha_stride;
const int frame_size = yplane_size + 2 * uvplane_size +
alpha_plane_size;
const uint64_t alpha_plane_size = (alpha_height + 2 * alpha_border_h) *
(uint64_t)alpha_stride;
const uint64_t frame_size = yplane_size + 2 * uvplane_size +
alpha_plane_size;
#else
const int frame_size = yplane_size + 2 * uvplane_size;
const uint64_t frame_size = yplane_size + 2 * uvplane_size;
#endif
if (cb != NULL) {
const int align_addr_extra_size = 31;
const size_t external_frame_size = frame_size + align_addr_extra_size;
const uint64_t external_frame_size = frame_size + align_addr_extra_size;
assert(fb != NULL);
if (external_frame_size != (size_t)external_frame_size)
return -1;
// Allocation to hold larger frame, or first allocation.
if (cb(cb_priv, external_frame_size, fb) < 0)
if (cb(cb_priv, (size_t)external_frame_size, fb) < 0)
return -1;
if (fb->data == NULL || fb->size < external_frame_size)
@ -181,10 +186,15 @@ int vp9_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf,
vpx_memset(fb->data, 0, fb->size);
ybf->buffer_alloc = (uint8_t *)yv12_align_addr(fb->data, 32);
} else if (frame_size > ybf->buffer_alloc_sz) {
} else if (frame_size > (size_t)ybf->buffer_alloc_sz) {
// Allocation to hold larger frame, or first allocation.
vpx_free(ybf->buffer_alloc);
ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, frame_size);
ybf->buffer_alloc = NULL;
if (frame_size != (size_t)frame_size)
return -1;
ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, (size_t)frame_size);
if (!ybf->buffer_alloc)
return -1;