atrac3: Fix crash in tonal component decoding.

Fixes Ticket780
Bug Found by: cosminamironesei

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit 9af6abdc17)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

Changelog

@@ -1,6 +1,16 @@
 Entries are sorted chronologically from oldest to youngest within each release,
 releases are sorted from youngest to oldest.
 
+version 0.6.5:
+- vorbis: An additional defense in the Vorbis codec. (CVE-2011-3895)
+- vorbisdec: Fix decoding bug with channel handling.
+- matroskadec: Fix a bug where a pointer was cached to an array that might
+  later move due to a realloc(). (CVE-2011-3893)
+- vorbis: Avoid some out-of-bounds reads. (CVE-2011-3893)
+- vp3: fix oob read for negative tokens and memleaks on error, (CVE-2011-3892)
+- vp3: fix streams with non-zero last coefficient.
+
+
 version 0.6.4:
 - 4xm: Add a check in decode_i_frame to prevent buffer overreads
 - wma: initialize prev_block_len_bits, next_block_len_bits, and block_len_bits.

RELEASE

@@ -176,3 +176,19 @@ Sierra VMD decoder CVE-2011-4364, and a safety fix in the svq1 decoder
 Distributors and system integrators are encouraged
 to update and share their patches against this branch.  For a full list
 of changes please see the Changelog file.
+
+
+* 0.6.5
+
+General notes
+-------------
+
+This mostly maintenance-only release that addresses a number a number of
+bugs such as security and compilation issues that have been brought to
+our attention. Among other (rather minor) fixes, this release features
+fixes for the VP3 decoder (CVE-2011-3892), vorbis decoder, and matroska
+demuxer (CVE-2011-3893 and CVE-2011-3895).
+
+Distributors and system integrators are encouraged
+to update and share their patches against this branch.  For a full list
+of changes please see the Changelog file.

VERSION

@@ -1 +1 @@
-0.6.4
+0.6.5

libavcodec/atrac3.c

@@ -393,6 +393,8 @@ static int decodeTonalComponents (GetBitContext *gb, tonal_component *pComponent
 
             for (k=0; k<coded_components; k++) {
                 sfIndx = get_bits(gb,6);
+                if(component_count>=64)
+                    return AVERROR_INVALIDDATA;
                 pComponent[component_count].pos = j * 64 + (get_bits(gb,6));
                 max_coded_values = 1024 - pComponent[component_count].pos;
                 coded_values = coded_values_per_component + 1;

libavcodec/h264.h

@@ -1048,7 +1048,7 @@ static void fill_decode_caches(H264Context *h, int mb_type){
                 AV_ZERO32(h->mv_cache [list][scan8[0] + 4 - 1*8]);
                 h->ref_cache[list][scan8[0] + 4 - 1*8]= topright_type ? LIST_NOT_USED : PART_NOT_AVAILABLE;
             }
-            if(h->ref_cache[list][scan8[0] + 4 - 1*8] < 0){
+            if(h->ref_cache[list][scan8[0] + 2 - 1*8] < 0 || h->ref_cache[list][scan8[0] + 4 - 1*8] < 0){
                 if(USES_LIST(topleft_type, list)){
                     const int b_xy = h->mb2b_xy [topleft_xy] + 3 + h->b_stride + (h->topleft_partition & 2*h->b_stride);
                     const int b8_xy= 4*topleft_xy + 1 + (h->topleft_partition & 2);

libavcodec/h264_ps.c

@@ -297,6 +297,10 @@ int ff_h264_decode_seq_parameter_set(H264Context *h){
 
     if(sps->profile_idc >= 100){ //high profile
         sps->chroma_format_idc= get_ue_golomb_31(&s->gb);
+        if (sps->chroma_format_idc > 3U) {
+            av_log(h->s.avctx, AV_LOG_ERROR, "chroma_format_idc %d is illegal\n", sps->chroma_format_idc);
+            goto fail;
+        }
         if(sps->chroma_format_idc == 3)
             sps->residual_color_transform_flag = get_bits1(&s->gb);
         sps->bit_depth_luma   = get_ue_golomb(&s->gb) + 8;

libavcodec/vorbis.c

@@ -156,7 +156,7 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
     }
 }
 
-static inline void render_line_unrolled(intptr_t x, intptr_t y, int x1,
+static inline void render_line_unrolled(intptr_t x, uint8_t y, int x1,
                                         intptr_t sy, int ady, int adx,
                                         float *buf)
 {
@@ -179,7 +179,7 @@ static inline void render_line_unrolled(intptr_t x, intptr_t y, int x1,
     }
 }
 
-static void render_line(int x0, int y0, int x1, int y1, float *buf)
+static void render_line(int x0, uint8_t y0, int x1, int y1, float *buf)
 {
     int dy  = y1 - y0;
     int adx = x1 - x0;
@@ -191,7 +191,7 @@ static void render_line(int x0, int y0, int x1, int y1, float *buf)
     } else {
         int base  = dy / adx;
         int x     = x0;
-        int y    = y0;
+        uint8_t y = y0;
         int err   = -adx;
         ady -= FFABS(base) * adx;
         while (++x < x1) {
@@ -210,7 +210,8 @@ void ff_vorbis_floor1_render_list(vorbis_floor1_entry * list, int values,
                                   uint_fast16_t *y_list, int *flag,
                                   int multiplier, float *out, int samples)
 {
-    int lx, ly, i;
+    int lx, i;
+    uint8_t ly;
     lx = 0;
     ly = y_list[0] * multiplier;
     for (i = 1; i < values; i++) {

libavcodec/vorbis_dec.c

@@ -656,7 +656,7 @@ static int vorbis_parse_setup_hdr_residues(vorbis_context *vc)
         res_setup->partition_size = get_bits(gb, 24) + 1;
         /* Validations to prevent a buffer overflow later. */
         if (res_setup->begin>res_setup->end ||
-            res_setup->end>vc->blocksize[1] / (res_setup->type == 2 ? 1 : 2) ||
+            res_setup->end > (res_setup->type == 2 ? vc->avccontext->channels : 1) * vc->blocksize[1] / 2 ||
             (res_setup->end-res_setup->begin) / res_setup->partition_size > V_MAX_PARTITIONS) {
             av_log(vc->avccontext, AV_LOG_ERROR, "partition out of bounds: type, begin, end, size, blocksize: %"PRIdFAST16", %"PRIdFAST32", %"PRIdFAST32", %"PRIdFAST32", %"PRIdFAST32"\n", res_setup->type, res_setup->begin, res_setup->end, res_setup->partition_size, vc->blocksize[1] / 2);
             return -1;
@@ -1262,6 +1262,7 @@ static av_always_inline int vorbis_residue_decode_internal(vorbis_context *vc,
                                                            uint_fast8_t *do_not_decode,
                                                            float *vec,
                                                            uint_fast16_t vlen,
+                                                           unsigned ch_left,
                                                            int vr_type)
 {
     GetBitContext *gb = &vc->gb;
@@ -1273,6 +1274,7 @@ static av_always_inline int vorbis_residue_decode_internal(vorbis_context *vc,
     uint_fast8_t ch_used;
     uint_fast8_t i,j,l;
     uint_fast16_t k;
+    unsigned max_output = (ch - 1) * vlen;
 
     if (vr_type == 2) {
         for (j = 1; j < ch; ++j)
@@ -1280,8 +1282,15 @@ static av_always_inline int vorbis_residue_decode_internal(vorbis_context *vc,
         if (do_not_decode[0])
             return 0;
         ch_used = 1;
+        max_output += vr->end / ch;
     } else {
         ch_used = ch;
+        max_output += vr->end;
+    }
+
+    if (max_output > ch_left * vlen) {
+        av_log(vc->avccontext, AV_LOG_ERROR, "Insufficient output buffer\n");
+        return -1;
     }
 
     AV_DEBUG(" residue type 0/1/2 decode begin, ch: %d  cpc %d  \n", ch, c_p_c);
@@ -1403,14 +1412,16 @@ static av_always_inline int vorbis_residue_decode_internal(vorbis_context *vc,
 static inline int vorbis_residue_decode(vorbis_context *vc, vorbis_residue *vr,
                                         uint_fast8_t ch,
                                         uint_fast8_t *do_not_decode,
-                                        float *vec, uint_fast16_t vlen)
+                                        float *vec, uint_fast16_t vlen,
+                                        unsigned ch_left)
+
 {
     if (vr->type == 2)
-        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, 2);
+        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, ch_left, 2);
     else if (vr->type == 1)
-        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, 1);
+        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, ch_left, 1);
     else if (vr->type == 0)
-        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, 0);
+        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, ch_left, 0);
     else {
         av_log(vc->avccontext, AV_LOG_ERROR, " Invalid residue type while residue decode?! \n");
         return -1;
@@ -1473,6 +1484,8 @@ static int vorbis_parse_audio_packet(vorbis_context *vc)
     uint_fast8_t res_num = 0;
     int_fast16_t retlen  = 0;
     float fadd_bias = vc->add_bias;
+    unsigned ch_left = vc->audio_channels;
+    unsigned vlen;
 
     if (get_bits1(gb)) {
         av_log(vc->avccontext, AV_LOG_ERROR, "Not a Vorbis I audio packet.\n");
@@ -1491,11 +1504,12 @@ static int vorbis_parse_audio_packet(vorbis_context *vc)
 
     blockflag = vc->modes[mode_number].blockflag;
     blocksize = vc->blocksize[blockflag];
+    vlen = blocksize / 2;
     if (blockflag)
         skip_bits(gb, 2); // previous_window, next_window
 
-    memset(ch_res_ptr,   0, sizeof(float) * vc->audio_channels * blocksize / 2); //FIXME can this be removed ?
+    memset(ch_res_ptr,   0, sizeof(float) * vc->audio_channels * vlen); //FIXME can this be removed ?
-    memset(ch_floor_ptr, 0, sizeof(float) * vc->audio_channels * blocksize / 2); //FIXME can this be removed ?
+    memset(ch_floor_ptr, 0, sizeof(float) * vc->audio_channels * vlen); //FIXME can this be removed ?
 
 // Decode floor
 
@@ -1515,7 +1529,7 @@ static int vorbis_parse_audio_packet(vorbis_context *vc)
             return -1;
         }
         no_residue[i] = ret;
-        ch_floor_ptr += blocksize / 2;
+        ch_floor_ptr += vlen;
     }
 
 // Nonzero vector propagate
@@ -1532,6 +1546,7 @@ static int vorbis_parse_audio_packet(vorbis_context *vc)
     for (i = 0; i < mapping->submaps; ++i) {
         vorbis_residue *residue;
         uint_fast8_t ch = 0;
+        int ret;
 
         for (j = 0; j < vc->audio_channels; ++j) {
             if ((mapping->submaps == 1) || (i == mapping->mux[j])) {
@@ -1546,9 +1561,18 @@ static int vorbis_parse_audio_packet(vorbis_context *vc)
             }
         }
         residue = &vc->residues[mapping->submap_residue[i]];
-        vorbis_residue_decode(vc, residue, ch, do_not_decode, ch_res_ptr, blocksize/2);
+        if (ch_left < ch) {
+            av_log(vc->avccontext, AV_LOG_ERROR, "Too many channels in vorbis_floor_decode.\n");
+            return -1;
+        }
+        if (ch) {
+            ret = vorbis_residue_decode(vc, residue, ch, do_not_decode, ch_res_ptr, vlen, ch_left);
+            if (ret < 0)
+                return ret;
+        }
 
-        ch_res_ptr += ch * blocksize / 2;
+        ch_res_ptr += ch * vlen;
+        ch_left -= ch;
     }
 
 // Inverse coupling

libavcodec/vp3.c

@@ -884,7 +884,7 @@ static int unpack_vlcs(Vp3DecodeContext *s, GetBitContext *gb,
             /* decode a VLC into a token */
             token = get_vlc2(gb, vlc_table, 11, 3);
             /* use the token to get a zero run, a coefficient, and an eob run */
-            if (token <= 6) {
+            if ((unsigned) token <= 6U) {
                 eob_run = eob_run_base[token];
                 if (eob_run_get_bits[token])
                     eob_run += get_bits(gb, eob_run_get_bits[token]);
@@ -902,7 +902,7 @@ static int unpack_vlcs(Vp3DecodeContext *s, GetBitContext *gb,
                     coeff_i        += eob_run;
                     eob_run = 0;
                 }
-            } else {
+            } else if (token >= 0) {
                 bits_to_get = coeff_get_bits[token];
                 if (bits_to_get)
                     bits_to_get = get_bits(gb, bits_to_get);
@@ -936,6 +936,10 @@ static int unpack_vlcs(Vp3DecodeContext *s, GetBitContext *gb,
                 for (i = coeff_index+1; i <= coeff_index+zero_run; i++)
                     s->num_coded_frags[plane][i]--;
                 coeff_i++;
+            } else {
+                av_log(s->avctx, AV_LOG_ERROR,
+                       "Invalid token %d\n", token);
+                return -1;
             }
     }
 
@@ -985,6 +989,8 @@ static int unpack_dct_coeffs(Vp3DecodeContext *s, GetBitContext *gb)
     /* unpack the Y plane DC coefficients */
     residual_eob_run = unpack_vlcs(s, gb, &s->dc_vlc[dc_y_table], 0,
         0, residual_eob_run);
+    if (residual_eob_run < 0)
+        return residual_eob_run;
 
     /* reverse prediction of the Y-plane DC coefficients */
     reverse_dc_prediction(s, 0, s->fragment_width[0], s->fragment_height[0]);
@@ -992,8 +998,12 @@ static int unpack_dct_coeffs(Vp3DecodeContext *s, GetBitContext *gb)
     /* unpack the C plane DC coefficients */
     residual_eob_run = unpack_vlcs(s, gb, &s->dc_vlc[dc_c_table], 0,
         1, residual_eob_run);
+    if (residual_eob_run < 0)
+        return residual_eob_run;
     residual_eob_run = unpack_vlcs(s, gb, &s->dc_vlc[dc_c_table], 0,
         2, residual_eob_run);
+    if (residual_eob_run < 0)
+        return residual_eob_run;
 
     /* reverse prediction of the C-plane DC coefficients */
     if (!(s->avctx->flags & CODEC_FLAG_GRAY))
@@ -1030,11 +1040,17 @@ static int unpack_dct_coeffs(Vp3DecodeContext *s, GetBitContext *gb)
     for (i = 1; i <= 63; i++) {
             residual_eob_run = unpack_vlcs(s, gb, y_tables[i], i,
                 0, residual_eob_run);
+            if (residual_eob_run < 0)
+                return residual_eob_run;
 
             residual_eob_run = unpack_vlcs(s, gb, c_tables[i], i,
                 1, residual_eob_run);
+            if (residual_eob_run < 0)
+                return residual_eob_run;
             residual_eob_run = unpack_vlcs(s, gb, c_tables[i], i,
                 2, residual_eob_run);
+            if (residual_eob_run < 0)
+                return residual_eob_run;
     }
 
     return 0;
@@ -1300,6 +1316,8 @@ static inline int vp3_dequant(Vp3DecodeContext *s, Vp3Fragment *frag,
             return i;
         }
     } while (i < 64);
+    // return value is expected to be a valid level
+    i--;
 end:
     // the actual DC+prediction is in the fragment structure
     block[0] = frag->dc * s->qmat[0][inter][plane][0];
@@ -1462,10 +1480,7 @@ static void render_slice(Vp3DecodeContext *s, int slice)
                     /* invert DCT and place (or add) in final output */
 
                     if (s->all_fragments[i].coding_method == MODE_INTRA) {
-                        int index;
+                        vp3_dequant(s, s->all_fragments + i, plane, 0, block);
-                        index = vp3_dequant(s, s->all_fragments + i, plane, 0, block);
-                        if (index > 63)
-                            continue;
                         if(s->avctx->idct_algo!=FF_IDCT_VP3)
                             block[0] += 128<<3;
                         s->dsp.idct_put(
@@ -1473,10 +1488,7 @@ static void render_slice(Vp3DecodeContext *s, int slice)
                             stride,
                             block);
                     } else {
-                        int index = vp3_dequant(s, s->all_fragments + i, plane, 1, block);
+                        if (vp3_dequant(s, s->all_fragments + i, plane, 1, block)) {
-                        if (index > 63)
-                            continue;
-                        if (index > 0) {
                         s->dsp.idct_add(
                             output_plane + first_pixel,
                             stride,

libavformat/matroskadec.c

@@ -1074,13 +1074,13 @@ static void matroska_convert_tags(AVFormatContext *s)
 static void matroska_execute_seekhead(MatroskaDemuxContext *matroska)
 {
     EbmlList *seekhead_list = &matroska->seekhead;
-    MatroskaSeekhead *seekhead = seekhead_list->elem;
     uint32_t level_up = matroska->level_up;
     int64_t before_pos = url_ftell(matroska->ctx->pb);
     MatroskaLevel level;
     int i;
 
     for (i=0; i<seekhead_list->nb_elem; i++) {
+        MatroskaSeekhead *seekhead = seekhead_list->elem;
         int64_t offset = seekhead[i].pos + matroska->segment_start;
 
         if (seekhead[i].pos <= before_pos