From d9cf5f516974c64e01846ca685301014b38cf224 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 13 Jan 2013 19:52:45 +0100 Subject: [PATCH 1/7] lavc: check for overflow in init_get_bits Fix an undefined behaviour and make the function return a proper error in case of overflow. CC: libav-stable@libav.org --- libavcodec/get_bits.h | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/libavcodec/get_bits.h b/libavcodec/get_bits.h index c56a2c2d10..16cfd5e0fd 100644 --- a/libavcodec/get_bits.h +++ b/libavcodec/get_bits.h @@ -362,20 +362,27 @@ static inline int check_marker(GetBitContext *s, const char *msg) } /** - * Inititalize GetBitContext. - * @param buffer bitstream buffer, must be FF_INPUT_BUFFER_PADDING_SIZE bytes larger than the actual read bits - * because some optimized bitstream readers read 32 or 64 bit at once and could read over the end + * Initialize GetBitContext. + * @param buffer bitstream buffer, must be FF_INPUT_BUFFER_PADDING_SIZE bytes + * larger than the actual read bits because some optimized bitstream + * readers read 32 or 64 bit at once and could read over the end * @param bit_size the size of the buffer in bits + * @return 0 on success, AVERROR_INVALIDDATA if the buffer_size would overflow. */ -static inline void init_get_bits(GetBitContext *s, const uint8_t *buffer, - int bit_size) +static inline int init_get_bits(GetBitContext *s, const uint8_t *buffer, + int bit_size) { - int buffer_size = (bit_size+7)>>3; - if (buffer_size < 0 || bit_size < 0) { + int buffer_size; + int ret = 0; + + if (bit_size > INT_MAX - 7 || bit_size <= 0) { buffer_size = bit_size = 0; buffer = NULL; + ret = AVERROR_INVALIDDATA; } + buffer_size = (bit_size + 7) >> 3; + s->buffer = buffer; s->size_in_bits = bit_size; #if !UNCHECKED_BITSTREAM_READER @@ -383,6 +390,7 @@ static inline void init_get_bits(GetBitContext *s, const uint8_t *buffer, #endif s->buffer_end = buffer + buffer_size; s->index = 0; + return ret; } static inline void align_get_bits(GetBitContext *s) From e28ac6e5e27e64a206e399e958481c1e6f992189 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 13 Jan 2013 23:37:11 +0100 Subject: [PATCH 2/7] lavc: introduce the convenience function init_get_bits8 Accept the buffer size in bytes and check for overflow before passing the value in bits to init_get_bits. --- libavcodec/get_bits.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/libavcodec/get_bits.h b/libavcodec/get_bits.h index 16cfd5e0fd..12770a29a0 100644 --- a/libavcodec/get_bits.h +++ b/libavcodec/get_bits.h @@ -393,6 +393,22 @@ static inline int init_get_bits(GetBitContext *s, const uint8_t *buffer, return ret; } +/** + * Initialize GetBitContext. + * @param buffer bitstream buffer, must be FF_INPUT_BUFFER_PADDING_SIZE bytes + * larger than the actual read bits because some optimized bitstream + * readers read 32 or 64 bit at once and could read over the end + * @param byte_size the size of the buffer in bytes + * @return 0 on success, AVERROR_INVALIDDATA if the buffer_size would overflow. + */ +static inline int init_get_bits8(GetBitContext *s, const uint8_t *buffer, + int byte_size) +{ + if (byte_size > INT_MAX / 8) + return AVERROR_INVALIDDATA; + return init_get_bits(s, buffer, byte_size * 8); +} + static inline void align_get_bits(GetBitContext *s) { int n = -get_bits_count(s) & 7; From 67360ccd51f37a6a4f31363f1c9fc46f43a22cb8 Mon Sep 17 00:00:00 2001 From: Daniel Kang Date: Sun, 13 Jan 2013 12:12:18 -0500 Subject: [PATCH 3/7] yadif: x86: fix build for compilers without aligned stack Manually load registers to avoid using 8 registers on x86_32 with compilers that do not align the stack (e.g. MSVC). Signed-off-by: Diego Biurrun --- libavfilter/x86/yadif.asm | 56 ++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/libavfilter/x86/yadif.asm b/libavfilter/x86/yadif.asm index 5e406a496a..bc4b3ce030 100644 --- a/libavfilter/x86/yadif.asm +++ b/libavfilter/x86/yadif.asm @@ -31,8 +31,8 @@ pw_1: times 8 dw 1 SECTION .text %macro CHECK 2 - movu m2, [curq+mrefsq+%1] - movu m3, [curq+prefsq+%2] + movu m2, [curq+t1+%1] + movu m3, [curq+t0+%2] mova m4, m2 mova m5, m2 pxor m4, m3 @@ -97,8 +97,8 @@ SECTION .text %macro FILTER 3 .loop%1: pxor m7, m7 - LOAD 0, [curq+mrefsq] - LOAD 1, [curq+prefsq] + LOAD 0, [curq+t1] + LOAD 1, [curq+t0] LOAD 2, [%2] LOAD 3, [%3] mova m4, m3 @@ -109,8 +109,8 @@ SECTION .text mova [rsp+32], m1 psubw m2, m4 ABS1 m2, m4 - LOAD 3, [prevq+mrefsq] - LOAD 4, [prevq+prefsq] + LOAD 3, [prevq+t1] + LOAD 4, [prevq+t0] psubw m3, m0 psubw m4, m1 ABS1 m3, m5 @@ -119,8 +119,8 @@ SECTION .text psrlw m2, 1 psrlw m3, 1 pmaxsw m2, m3 - LOAD 3, [nextq+mrefsq] - LOAD 4, [nextq+prefsq] + LOAD 3, [nextq+t1] + LOAD 4, [nextq+t0] psubw m3, m0 psubw m4, m1 ABS1 m3, m5 @@ -136,8 +136,8 @@ SECTION .text psrlw m1, 1 ABS1 m0, m2 - movu m2, [curq+mrefsq-1] - movu m3, [curq+prefsq-1] + movu m2, [curq+t1-1] + movu m3, [curq+t0-1] mova m4, m2 psubusb m2, m3 psubusb m3, m4 @@ -164,12 +164,12 @@ SECTION .text CHECK2 mova m6, [rsp+48] - cmp DWORD modem, 2 + cmp DWORD r8m, 2 jge .end%1 - LOAD 2, [%2+mrefsq*2] - LOAD 4, [%3+mrefsq*2] - LOAD 3, [%2+prefsq*2] - LOAD 5, [%3+prefsq*2] + LOAD 2, [%2+t1*2] + LOAD 4, [%3+t1*2] + LOAD 3, [%2+t0*2] + LOAD 5, [%3+t0*2] paddw m2, m4 paddw m3, m5 psrlw m2, 1 @@ -208,19 +208,31 @@ SECTION .text add prevq, mmsize/2 add curq, mmsize/2 add nextq, mmsize/2 - sub wd, mmsize/2 + sub DWORD r4m, mmsize/2 jg .loop%1 %endmacro %macro YADIF 0 -cglobal yadif_filter_line, 7, 7, 8, 16*5, dst, prev, cur, next, w, prefs, \ - mrefs, parity, mode - test wq, wq +%if ARCH_X86_32 +cglobal yadif_filter_line, 4, 6, 8, 80, dst, prev, cur, next, w, prefs, \ + mrefs, parity, mode +%else +cglobal yadif_filter_line, 4, 7, 8, 80, dst, prev, cur, next, w, prefs, \ + mrefs, parity, mode +%endif + cmp DWORD wm, 0 jle .ret - movsxdifnidn prefsq, prefsd - movsxdifnidn mrefsq, mrefsd +%if ARCH_X86_32 + mov r4, r5mp + mov r5, r6mp + DECLARE_REG_TMP 4,5 +%else + movsxd r5, DWORD r5m + movsxd r6, DWORD r6m + DECLARE_REG_TMP 5,6 +%endif - cmp DWORD paritym, 0 + cmp DWORD paritym, 0 je .parity0 FILTER 1, prevq, curq jmp .ret From 8fd4d1f9b925f3b978756926b40315440c448696 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Sat, 17 Nov 2012 15:45:41 +0100 Subject: [PATCH 4/7] cscd: return meaningful error codes. --- libavcodec/cscd.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/libavcodec/cscd.c b/libavcodec/cscd.c index 2c1b2e7858..b982f5029c 100644 --- a/libavcodec/cscd.c +++ b/libavcodec/cscd.c @@ -143,10 +143,11 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame, int buf_size = avpkt->size; CamStudioContext *c = avctx->priv_data; AVFrame *picture = data; + int ret; if (buf_size < 2) { av_log(avctx, AV_LOG_ERROR, "coded frame too small\n"); - return -1; + return AVERROR_INVALIDDATA; } if (c->pic.data[0]) @@ -154,9 +155,9 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame, c->pic.reference = 1; c->pic.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_READABLE | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE; - if (ff_get_buffer(avctx, &c->pic) < 0) { + if ((ret = ff_get_buffer(avctx, &c->pic)) < 0) { av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n"); - return -1; + return ret; } // decompress data @@ -175,12 +176,12 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame, break; #else av_log(avctx, AV_LOG_ERROR, "compiled without zlib support\n"); - return -1; + return AVERROR(ENOSYS); #endif } default: av_log(avctx, AV_LOG_ERROR, "unknown compression\n"); - return -1; + return AVERROR_INVALIDDATA; } // flip upside down, add difference frame From 9e6764900a0732a4719ca219ceb59a4ae9d9be3b Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Sat, 17 Nov 2012 17:48:38 +0100 Subject: [PATCH 5/7] msrle: return meaningful error codes. --- libavcodec/msrle.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libavcodec/msrle.c b/libavcodec/msrle.c index 7cd8823682..7bca67f42e 100644 --- a/libavcodec/msrle.c +++ b/libavcodec/msrle.c @@ -63,7 +63,7 @@ static av_cold int msrle_decode_init(AVCodecContext *avctx) break; default: av_log(avctx, AV_LOG_ERROR, "unsupported bits per sample\n"); - return -1; + return AVERROR_INVALIDDATA; } s->frame.data[0] = NULL; @@ -79,15 +79,16 @@ static int msrle_decode_frame(AVCodecContext *avctx, int buf_size = avpkt->size; MsrleContext *s = avctx->priv_data; int istride = FFALIGN(avctx->width*avctx->bits_per_coded_sample, 32) / 8; + int ret; s->buf = buf; s->size = buf_size; s->frame.reference = 1; s->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE; - if (avctx->reget_buffer(avctx, &s->frame)) { + if ((ret = avctx->reget_buffer(avctx, &s->frame)) < 0) { av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n"); - return -1; + return ret; } if (avctx->bits_per_coded_sample <= 8) { From 6bf70127d1d49f2a8c46803eda7b893b2b479282 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Sat, 17 Nov 2012 17:52:52 +0100 Subject: [PATCH 6/7] tmv: return meaningful error codes. --- libavcodec/tmv.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libavcodec/tmv.c b/libavcodec/tmv.c index d4e5aa1398..29d8e15fd9 100644 --- a/libavcodec/tmv.c +++ b/libavcodec/tmv.c @@ -47,20 +47,21 @@ static int tmv_decode_frame(AVCodecContext *avctx, void *data, unsigned char_cols = avctx->width >> 3; unsigned char_rows = avctx->height >> 3; unsigned x, y, fg, bg, c; + int ret; if (tmv->pic.data[0]) avctx->release_buffer(avctx, &tmv->pic); - if (ff_get_buffer(avctx, &tmv->pic) < 0) { + if ((ret = ff_get_buffer(avctx, &tmv->pic)) < 0) { av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n"); - return -1; + return ret; } if (avpkt->size < 2*char_rows*char_cols) { av_log(avctx, AV_LOG_ERROR, "Input buffer too small, truncated sample?\n"); *got_frame = 0; - return -1; + return AVERROR_INVALIDDATA; } tmv->pic.pict_type = AV_PICTURE_TYPE_I; From a1c525f7eb0783d31ba7a653865b6cbd3dc880de Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Sat, 17 Nov 2012 17:57:06 +0100 Subject: [PATCH 7/7] pcx: return meaningful error codes. --- libavcodec/pcx.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libavcodec/pcx.c b/libavcodec/pcx.c index 1bd861215e..04770a9ad2 100644 --- a/libavcodec/pcx.c +++ b/libavcodec/pcx.c @@ -94,7 +94,7 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, if (buf[0] != 0x0a || buf[1] > 5) { av_log(avctx, AV_LOG_ERROR, "this is not PCX encoded data\n"); - return -1; + return AVERROR_INVALIDDATA; } compressed = buf[2]; @@ -105,7 +105,7 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, if (xmax < xmin || ymax < ymin) { av_log(avctx, AV_LOG_ERROR, "invalid image dimensions\n"); - return -1; + return AVERROR_INVALIDDATA; } w = xmax - xmin + 1; @@ -118,7 +118,7 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, if (bytes_per_scanline < w * bits_per_pixel * nplanes / 8) { av_log(avctx, AV_LOG_ERROR, "PCX data is corrupted\n"); - return -1; + return AVERROR_INVALIDDATA; } switch ((nplanes<<8) + bits_per_pixel) { @@ -136,7 +136,7 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, break; default: av_log(avctx, AV_LOG_ERROR, "invalid PCX file\n"); - return -1; + return AVERROR_INVALIDDATA; } buf += 128; @@ -144,13 +144,13 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, if (p->data[0]) avctx->release_buffer(avctx, p); - if (av_image_check_size(w, h, 0, avctx)) - return -1; + if ((ret = av_image_check_size(w, h, 0, avctx)) < 0) + return ret; if (w != avctx->width || h != avctx->height) avcodec_set_dimensions(avctx, w, h); - if (ff_get_buffer(avctx, p) < 0) { + if ((ret = ff_get_buffer(avctx, p)) < 0) { av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n"); - return -1; + return ret; } p->pict_type = AV_PICTURE_TYPE_I;