From aedde1a48de4bebdd26d27ca997b2cecc0016a45 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Wed, 3 Jul 2013 14:01:32 +0200 Subject: [PATCH 1/4] indeo: Cosmetic formatting Trim some overly long lines. (cherry picked from commit 6dfacd7ab126aea1392949d1aa10fdc3d3eeb911) Signed-off-by: Luca Barbato Conflicts: libavcodec/ivi_common.c --- libavcodec/ivi_common.c | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/libavcodec/ivi_common.c b/libavcodec/ivi_common.c index 777438cb83..a94285ab36 100644 --- a/libavcodec/ivi_common.c +++ b/libavcodec/ivi_common.c @@ -51,9 +51,10 @@ static uint16_t inv_bits(uint16_t val, int nbits) uint16_t res; if (nbits <= 8) { - res = av_reverse[val] >> (8-nbits); + res = av_reverse[val] >> (8 - nbits); } else - res = ((av_reverse[val & 0xFF] << 8) + (av_reverse[val >> 8])) >> (16-nbits); + res = ((av_reverse[val & 0xFF] << 8) + + (av_reverse[val >> 8])) >> (16 - nbits); return res; } @@ -103,10 +104,12 @@ void ff_ivi_init_static_vlc(void) for (i = 0; i < 8; i++) { ff_ivi_mb_vlc_tabs[i].table = table_data + i * 2 * 8192; ff_ivi_mb_vlc_tabs[i].table_allocated = 8192; - ff_ivi_create_huff_from_desc(&ff_ivi_mb_huff_desc[i], &ff_ivi_mb_vlc_tabs[i], 1); + ff_ivi_create_huff_from_desc(&ff_ivi_mb_huff_desc[i], + &ff_ivi_mb_vlc_tabs[i], 1); ff_ivi_blk_vlc_tabs[i].table = table_data + (i * 2 + 1) * 8192; ff_ivi_blk_vlc_tabs[i].table_allocated = 8192; - ff_ivi_create_huff_from_desc(&ff_ivi_blk_huff_desc[i], &ff_ivi_blk_vlc_tabs[i], 1); + ff_ivi_create_huff_from_desc(&ff_ivi_blk_huff_desc[i], + &ff_ivi_blk_vlc_tabs[i], 1); } initialized_vlcs = 1; } @@ -114,7 +117,7 @@ void ff_ivi_init_static_vlc(void) int ff_ivi_dec_huff_desc(GetBitContext *gb, int desc_coded, int which_tab, IVIHuffTab *huff_tab, AVCodecContext *avctx) { - int i, result; + int i, result; IVIHuffDesc new_huff; if (!desc_coded) { @@ -175,8 +178,9 @@ void ff_ivi_huff_desc_copy(IVIHuffDesc *dst, const IVIHuffDesc *src) int av_cold ff_ivi_init_planes(IVIPlaneDesc *planes, const IVIPicConfig *cfg) { - int p, b; - uint32_t b_width, b_height, align_fac, width_aligned, height_aligned, buf_size; + int p, b; + uint32_t b_width, b_height, align_fac, width_aligned, + height_aligned, buf_size; IVIBandDesc *band; ff_ivi_free_buffers(planes); @@ -199,8 +203,10 @@ int av_cold ff_ivi_init_planes(IVIPlaneDesc *planes, const IVIPicConfig *cfg) /* select band dimensions: if there is only one band then it * has the full size, if there are several bands each of them * has only half size */ - b_width = planes[p].num_bands == 1 ? planes[p].width : (planes[p].width + 1) >> 1; - b_height = planes[p].num_bands == 1 ? planes[p].height : (planes[p].height + 1) >> 1; + b_width = planes[p].num_bands == 1 ? planes[p].width + : (planes[p].width + 1) >> 1; + b_height = planes[p].num_bands == 1 ? planes[p].height + : (planes[p].height + 1) >> 1; /* luma band buffers will be aligned on 16x16 (max macroblock size) */ /* chroma band buffers will be aligned on 8x8 (max macroblock size) */ @@ -228,8 +234,8 @@ int av_cold ff_ivi_init_planes(IVIPlaneDesc *planes, const IVIPicConfig *cfg) if (!band->bufs[2]) return AVERROR(ENOMEM); } - - planes[p].bands[0].blk_vlc.cust_desc.num_rows = 0; /* reset custom vlc */ + /* reset custom vlc */ + planes[p].bands[0].blk_vlc.cust_desc.num_rows = 0; } } @@ -781,7 +787,8 @@ static int decode_band(IVI45DecContext *ctx, int plane_num, } } - /* restore the selected rvmap table by applying its corrections in reverse order */ + /* restore the selected rvmap table by applying its corrections in + * reverse order */ for (i = band->num_corr-1; i >= 0; i--) { idx1 = band->corr[i*2]; idx2 = band->corr[i*2+1]; @@ -794,7 +801,8 @@ static int decode_band(IVI45DecContext *ctx, int plane_num, uint16_t chksum = ivi_calc_band_checksum(band); if (chksum != band->checksum) { av_log(avctx, AV_LOG_ERROR, - "Band checksum mismatch! Plane %d, band %d, received: %x, calculated: %x\n", + "Band checksum mismatch! Plane %d, band %d, " + "received: %x, calculated: %x\n", band->plane, band->band_num, band->checksum, chksum); } } @@ -861,7 +869,8 @@ int ff_ivi_decode_frame(AVCodecContext *avctx, void *data, int *data_size, /* If the bidirectional mode is enabled, next I and the following P frame will */ /* be sent together. Unfortunately the approach below seems to be the only way */ /* to handle the B-frames mode. That's exactly the same Intel decoders do. */ - if (avctx->codec_id == CODEC_ID_INDEO4 && ctx->frame_type == 0/*FRAMETYPE_INTRA*/) { + if (avctx->codec_id == CODEC_ID_INDEO4 && + ctx->frame_type == 0/*FRAMETYPE_INTRA*/) { while (get_bits(&ctx->gb, 8)); // skip version string skip_bits_long(&ctx->gb, 64); // skip padding, TODO: implement correct 8-bytes alignment if (get_bits_left(&ctx->gb) > 18 && show_bits(&ctx->gb, 18) == 0x3FFF8) From efe710f8a009c99a5c4e4dff160c870cb7d95e76 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Wed, 3 Jul 2013 14:55:50 +0200 Subject: [PATCH 2/4] indeo: reject negative array indexes Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 6a10142faa1cca8ba2bfe51b970754f62d60f320) Signed-off-by: Luca Barbato --- libavcodec/ivi_common.c | 42 ++++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/libavcodec/ivi_common.c b/libavcodec/ivi_common.c index a94285ab36..4aab7abfd7 100644 --- a/libavcodec/ivi_common.c +++ b/libavcodec/ivi_common.c @@ -42,6 +42,20 @@ VLC ff_ivi_blk_vlc_tabs[8]; typedef void (*ivi_mc_func) (int16_t *buf, const int16_t *ref_buf, uint32_t pitch, int mc_type); +static int ivi_mc(ivi_mc_func mc, int16_t *buf, const int16_t *ref_buf, + int offs, int mv_x, int mv_y, uint32_t pitch, + int mc_type) +{ + int ref_offs = offs + mv_y * pitch + mv_x; + + if (offs < 0 || ref_offs < 0 || !ref_buf) + return AVERROR_INVALIDDATA; + + mc(buf + offs, ref_buf + ref_offs, pitch, mc_type); + + return 0; +} + /** * Reverse "nbits" bits of the value "val" and return the result * in the least significant bits. @@ -397,7 +411,7 @@ static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band, /* de-zigzag and dequantize */ scan_pos += run; - if (scan_pos >= num_coeffs) + if (scan_pos >= num_coeffs || scan_pos < 0) break; pos = band->scan[scan_pos]; @@ -409,7 +423,7 @@ static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band, col_flags[pos & col_mask] |= !!val; } - if (scan_pos >= num_coeffs && sym != rvmap->eob_sym) + if (scan_pos < 0 || scan_pos >= num_coeffs && sym != rvmap->eob_sym) return AVERROR_INVALIDDATA; /* corrupt block data */ /* undoing DC coeff prediction for intra-blocks */ @@ -425,9 +439,8 @@ static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band, /* apply motion compensation */ if (!is_intra) - mc(band->buf + offs, - band->ref_buf + offs + mv_y * band->pitch + mv_x, - band->pitch, mc_type); + return ivi_mc(mc, band->buf, band->ref_buf, offs, mv_x, mv_y, + band->pitch, mc_type); return 0; } @@ -516,10 +529,12 @@ int ff_ivi_decode_blocks(GetBitContext *gb, IVIBandDesc *band, IVITile *tile) if (band->dc_transform) band->dc_transform(&prev_dc, band->buf + buf_offs, band->pitch, blk_size); - } else - mc_no_delta_func(band->buf + buf_offs, - band->ref_buf + buf_offs + mv_y * band->pitch + mv_x, - band->pitch, mc_type); + } else { + ret = ivi_mc(mc_no_delta_func, band->buf, band->ref_buf, + buf_offs, mv_x, mv_y, band->pitch, mc_type); + if (ret < 0) + return ret; + } } cbp >>= 1; @@ -544,7 +559,7 @@ static int ivi_process_empty_tile(AVCodecContext *avctx, IVIBandDesc *band, IVITile *tile, int32_t mv_scale) { int x, y, need_mc, mbn, blk, num_blocks, mv_x, mv_y, mc_type; - int offs, mb_offset, row_offset; + int offs, mb_offset, row_offset, ret; IVIMbInfo *mb, *ref_mb; const int16_t *src; int16_t *dst; @@ -622,9 +637,10 @@ static int ivi_process_empty_tile(AVCodecContext *avctx, IVIBandDesc *band, for (blk = 0; blk < num_blocks; blk++) { /* adjust block position in the buffer according with its number */ offs = mb->buf_offs + band->blk_size * ((blk & 1) + !!(blk & 2) * band->pitch); - mc_no_delta_func(band->buf + offs, - band->ref_buf + offs + mv_y * band->pitch + mv_x, - band->pitch, mc_type); + ret = ivi_mc(mc_no_delta_func, band->buf, band->ref_buf, + offs, mv_x, mv_y, band->pitch, mc_type); + if (ret < 0) + return ret; } } } else { From c02b9e6e633896617e5f95211665c5521800498b Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Fri, 12 Jul 2013 14:33:24 +0200 Subject: [PATCH 3/4] indeo: Bound-check before applying transform Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit dc79685195a45c9b8b17d7b93d118e0aefa45462) Signed-off-by: Luca Barbato Conflicts: libavcodec/ivi_common.c --- libavcodec/indeo4.c | 7 +++++++ libavcodec/indeo5.c | 38 +++++++++++++++++++++++--------------- libavcodec/ivi_common.c | 31 ++++++++++++++++++++++++++++--- libavcodec/ivi_common.h | 1 + 4 files changed, 59 insertions(+), 18 deletions(-) diff --git a/libavcodec/indeo4.c b/libavcodec/indeo4.c index a3b3c6b2f3..74b3ef001b 100644 --- a/libavcodec/indeo4.c +++ b/libavcodec/indeo4.c @@ -348,6 +348,13 @@ static int decode_band_hdr(IVI45DecContext *ctx, IVIBandDesc *band, band->inv_transform = transforms[transform_id].inv_trans; band->dc_transform = transforms[transform_id].dc_trans; band->is_2d_trans = transforms[transform_id].is_2d_trans; + if (transform_id < 10) + band->transform_size = 8; + else + band->transform_size = 4; + + if (band->blk_size != band->transform_size) + return AVERROR_INVALIDDATA; scan_indx = get_bits(&ctx->gb, 4); if (scan_indx == 15) { diff --git a/libavcodec/indeo5.c b/libavcodec/indeo5.c index 0626454826..c06d46da3e 100644 --- a/libavcodec/indeo5.c +++ b/libavcodec/indeo5.c @@ -147,39 +147,47 @@ static int decode_gop_header(IVI45DecContext *ctx, AVCodecContext *avctx) /* select transform function and scan pattern according to plane and band number */ switch ((p << 2) + i) { case 0: - band->inv_transform = ff_ivi_inverse_slant_8x8; - band->dc_transform = ff_ivi_dc_slant_2d; - band->scan = ff_zigzag_direct; + band->inv_transform = ff_ivi_inverse_slant_8x8; + band->dc_transform = ff_ivi_dc_slant_2d; + band->scan = ff_zigzag_direct; + band->transform_size = 8; break; case 1: - band->inv_transform = ff_ivi_row_slant8; - band->dc_transform = ff_ivi_dc_row_slant; - band->scan = ff_ivi_vertical_scan_8x8; + band->inv_transform = ff_ivi_row_slant8; + band->dc_transform = ff_ivi_dc_row_slant; + band->scan = ff_ivi_vertical_scan_8x8; + band->transform_size = 8; break; case 2: - band->inv_transform = ff_ivi_col_slant8; - band->dc_transform = ff_ivi_dc_col_slant; - band->scan = ff_ivi_horizontal_scan_8x8; + band->inv_transform = ff_ivi_col_slant8; + band->dc_transform = ff_ivi_dc_col_slant; + band->scan = ff_ivi_horizontal_scan_8x8; + band->transform_size = 8; break; case 3: - band->inv_transform = ff_ivi_put_pixels_8x8; - band->dc_transform = ff_ivi_put_dc_pixel_8x8; - band->scan = ff_ivi_horizontal_scan_8x8; + band->inv_transform = ff_ivi_put_pixels_8x8; + band->dc_transform = ff_ivi_put_dc_pixel_8x8; + band->scan = ff_ivi_horizontal_scan_8x8; + band->transform_size = 8; break; case 4: - band->inv_transform = ff_ivi_inverse_slant_4x4; - band->dc_transform = ff_ivi_dc_slant_2d; - band->scan = ff_ivi_direct_scan_4x4; + band->inv_transform = ff_ivi_inverse_slant_4x4; + band->dc_transform = ff_ivi_dc_slant_2d; + band->scan = ff_ivi_direct_scan_4x4; + band->transform_size = 4; break; } band->is_2d_trans = band->inv_transform == ff_ivi_inverse_slant_8x8 || band->inv_transform == ff_ivi_inverse_slant_4x4; + if (band->transform_size != band->blk_size) + return AVERROR_INVALIDDATA; + /* select dequant matrix according to plane and band number */ if (!p) { quant_mat = (pic_conf.luma_bands > 1) ? i+1 : 0; diff --git a/libavcodec/ivi_common.c b/libavcodec/ivi_common.c index 4aab7abfd7..ea9082cadd 100644 --- a/libavcodec/ivi_common.c +++ b/libavcodec/ivi_common.c @@ -365,6 +365,25 @@ int ff_ivi_dec_tile_data_size(GetBitContext *gb) return len; } +static int ivi_dc_transform(IVIBandDesc *band, int *prev_dc, int buf_offs, + int blk_size) +{ + int buf_size = band->pitch * band->aheight - buf_offs; + int min_size = (blk_size - 1) * band->pitch + blk_size; + + if (!band->dc_transform) + return 0; + + + if (min_size > buf_size) + return AVERROR_INVALIDDATA; + + band->dc_transform(prev_dc, band->buf + buf_offs, + band->pitch, blk_size); + + return 0; +} + static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band, ivi_mc_func mc, int mv_x, int mv_y, int *prev_dc, int is_intra, int mc_type, @@ -380,6 +399,12 @@ static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band, int num_coeffs = blk_size * blk_size; int col_mask = blk_size - 1; int scan_pos = -1; + int min_size = band->pitch * (band->transform_size - 1) + + band->transform_size; + int buf_size = band->pitch * band->aheight - offs; + + if (min_size > buf_size) + return AVERROR_INVALIDDATA; if (!band->scan) return AVERROR_INVALIDDATA; @@ -526,9 +551,9 @@ int ff_ivi_decode_blocks(GetBitContext *gb, IVIBandDesc *band, IVITile *tile) /* for intra blocks apply the dc slant transform */ /* for inter - perform the motion compensation without delta */ if (is_intra) { - if (band->dc_transform) - band->dc_transform(&prev_dc, band->buf + buf_offs, - band->pitch, blk_size); + ret = ivi_dc_transform(band, &prev_dc, buf_offs, blk_size); + if (ret < 0) + return ret; } else { ret = ivi_mc(mc_no_delta_func, band->buf, band->ref_buf, buf_offs, mv_x, mv_y, band->pitch, mc_type); diff --git a/libavcodec/ivi_common.h b/libavcodec/ivi_common.h index 07736f25f3..b47d5d26c6 100644 --- a/libavcodec/ivi_common.h +++ b/libavcodec/ivi_common.h @@ -162,6 +162,7 @@ typedef struct { int num_tiles; ///< number of tiles in this band IVITile *tiles; ///< array of tile descriptors InvTransformPtr *inv_transform; + int transform_size; DCTransformPtr *dc_transform; int is_2d_trans; ///< 1 indicates that the two-dimensional inverse transform is used int32_t checksum; ///< for debug purposes From a0b8f85f29883f538a32593bc3c6f712c972ff70 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 14 Jul 2013 14:06:16 +0200 Subject: [PATCH 4/4] indeo: Bound-check before applying motion compensation Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 25a6666f6c07c6ac8449a63d7fbce0dfd29c54cd) Signed-off-by: Luca Barbato --- libavcodec/ivi_common.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/libavcodec/ivi_common.c b/libavcodec/ivi_common.c index ea9082cadd..5289d6c766 100644 --- a/libavcodec/ivi_common.c +++ b/libavcodec/ivi_common.c @@ -42,16 +42,22 @@ VLC ff_ivi_blk_vlc_tabs[8]; typedef void (*ivi_mc_func) (int16_t *buf, const int16_t *ref_buf, uint32_t pitch, int mc_type); -static int ivi_mc(ivi_mc_func mc, int16_t *buf, const int16_t *ref_buf, - int offs, int mv_x, int mv_y, uint32_t pitch, - int mc_type) +static int ivi_mc(IVIBandDesc *band, ivi_mc_func mc, + int offs, int mv_x, int mv_y, int mc_type) { - int ref_offs = offs + mv_y * pitch + mv_x; + int ref_offs = offs + mv_y * band->pitch + mv_x; + int buf_size = band->pitch * band->aheight; + int min_size = band->pitch * (band->blk_size - 1) + band->blk_size; + int ref_size = (mc_type > 1) * band->pitch + (mc_type & 1); - if (offs < 0 || ref_offs < 0 || !ref_buf) + if (offs < 0 || ref_offs < 0 || !band->ref_buf) + return AVERROR_INVALIDDATA; + if (buf_size - min_size < offs) + return AVERROR_INVALIDDATA; + if (buf_size - min_size - ref_size < ref_offs) return AVERROR_INVALIDDATA; - mc(buf + offs, ref_buf + ref_offs, pitch, mc_type); + mc(band->buf + offs, band->ref_buf + ref_offs, band->pitch, mc_type); return 0; } @@ -464,8 +470,7 @@ static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band, /* apply motion compensation */ if (!is_intra) - return ivi_mc(mc, band->buf, band->ref_buf, offs, mv_x, mv_y, - band->pitch, mc_type); + return ivi_mc(band, mc, offs, mv_x, mv_y, mc_type); return 0; } @@ -555,8 +560,8 @@ int ff_ivi_decode_blocks(GetBitContext *gb, IVIBandDesc *band, IVITile *tile) if (ret < 0) return ret; } else { - ret = ivi_mc(mc_no_delta_func, band->buf, band->ref_buf, - buf_offs, mv_x, mv_y, band->pitch, mc_type); + ret = ivi_mc(band, mc_no_delta_func, buf_offs, + mv_x, mv_y, mc_type); if (ret < 0) return ret; } @@ -662,8 +667,8 @@ static int ivi_process_empty_tile(AVCodecContext *avctx, IVIBandDesc *band, for (blk = 0; blk < num_blocks; blk++) { /* adjust block position in the buffer according with its number */ offs = mb->buf_offs + band->blk_size * ((blk & 1) + !!(blk & 2) * band->pitch); - ret = ivi_mc(mc_no_delta_func, band->buf, band->ref_buf, - offs, mv_x, mv_y, band->pitch, mc_type); + ret = ivi_mc(band, mc_no_delta_func, offs, + mv_x, mv_y, mc_type); if (ret < 0) return ret; }