generic-library/ffmpeg
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

default_get_buffer() cleanup

Browse Source 
fixes probably exploitable heap overflow
heap overflow found by (Simon Kilvington <s D kilvington A eris D qinetiq D com>)

Originally committed as revision 4714 to svn://svn.ffmpeg.org/ffmpeg/trunk
This commit is contained in:
Michael Niedermayer
2005-12-02 00:12:37 +00:00
parent e852beeee3
commit c7622f9a28
1 changed files with 30 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
libavcodec/utils.c
View File

@@ -292,49 +292,50 @@ int avcodec_default_get_buffer(AVCodecContext *s, AVFrame *pic){
buf->last_pic_num= *picture_number;
}else{
int h_chroma_shift, v_chroma_shift;
int pixel_size;
int pixel_size, size[3];
AVPicture picture;
avcodec_get_chroma_sub_sample(s->pix_fmt, &h_chroma_shift, &v_chroma_shift);
switch(s->pix_fmt){
case PIX_FMT_RGB555:
case PIX_FMT_RGB565:
case PIX_FMT_YUV422:
case PIX_FMT_UYVY422:
pixel_size=2;
break;
case PIX_FMT_RGB24:
case PIX_FMT_BGR24:
pixel_size=3;
break;
case PIX_FMT_RGBA32:
pixel_size=4;
break;
default:
pixel_size=1;
}
avcodec_align_dimensions(s, &w, &h);
if(!(s->flags&CODEC_FLAG_EMU_EDGE)){
w+= EDGE_WIDTH*2;
h+= EDGE_WIDTH*2;
}
avpicture_fill(&picture, NULL, s->pix_fmt, w, h);
pixel_size= picture.linesize[0]*8 / w;
//av_log(NULL, AV_LOG_ERROR, "%d %d %d %d\n", (int)picture.data[1], w, h, s->pix_fmt);
assert(pixel_size>=1);
//FIXME next ensures that linesize= 2^x uvlinesize, thats needed because some MC code assumes it
if(pixel_size == 3*8)
w= ALIGN(w, STRIDE_ALIGN<<h_chroma_shift);
else
w= ALIGN(pixel_size*w, STRIDE_ALIGN<<(h_chroma_shift+3)) / pixel_size;
size[1] = avpicture_fill(&picture, NULL, s->pix_fmt, w, h);
size[0] = picture.linesize[0] * h;
size[1] -= size[0];
if(picture.data[2])
size[1]= size[2]= size[1]/2;
else
size[2]= 0;
buf->last_pic_num= -256*256*256*64;
memset(buf->base, 0, sizeof(buf->base));
memset(buf->data, 0, sizeof(buf->data));
for(i=0; i<3; i++){
for(i=0; i<3 && size[i]; i++){
const int h_shift= i==0 ? 0 : h_chroma_shift;
const int v_shift= i==0 ? 0 : v_chroma_shift;
//FIXME next ensures that linesize= 2^x uvlinesize, thats needed because some MC code assumes it
buf->linesize[i]= ALIGN(pixel_size*w>>h_shift, STRIDE_ALIGN<<(h_chroma_shift-h_shift));
buf->linesize[i]= picture.linesize[i];
buf->base[i]= av_malloc((buf->linesize[i]*h>>v_shift)+16); //FIXME 16
buf->base[i]= av_malloc(size[i]+16); //FIXME 16
if(buf->base[i]==NULL) return -1;
memset(buf->base[i], 128, buf->linesize[i]*h>>v_shift);
memset(buf->base[i], 128, size[i]);
if(s->flags&CODEC_FLAG_EMU_EDGE)
// no edge if EDEG EMU or not planar YUV, we check for PAL8 redundantly to protect against a exploitable bug regression ...
if((s->flags&CODEC_FLAG_EMU_EDGE) || (s->pix_fmt == PIX_FMT_PAL8) || !size[2])
buf->data[i] = buf->base[i];
else
buf->data[i] = buf->base[i] + ALIGN((buf->linesize[i]*EDGE_WIDTH>>v_shift) + (EDGE_WIDTH>>h_shift), STRIDE_ALIGN);