Merge commit '163196562fe744149ef599d754c30c08a9898381' into release/1.1

* commit '163196562fe744149ef599d754c30c08a9898381':
  oggparseogm: Convert to use bytestream2
  rv34: Check the return value from ff_rv34_decode_init
  matroskadec: Verify realaudio codec parameters
  mace: Make sure that the channel count is set to a valid value
  svq3: Check for any negative return value from ff_h264_check_intra_pred_mode
  vp3: Check the framerate for validity
  cavsdec: Make sure a sequence header has been decoded before decoding pictures
  vocdec: Don't update codec parameters mid-stream
  sierravmd: Do sanity checking of frame sizes
  omadec: Properly check lengths before incrementing the position
  mpc8: Make sure the first stream exists before parsing the seek table

Conflicts:
	libavcodec/mace.c
	libavformat/oggparseogm.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/cavsdec.c

@@ -944,6 +944,11 @@ static int decode_pic(AVSContext *h)
     int ret;
     enum cavs_mb mb_type;
 
+    if (!h->top_qp) {
+        av_log(h->avctx, AV_LOG_ERROR, "No sequence header decoded yet\n");
+        return AVERROR_INVALIDDATA;
+    }
+
     skip_bits(&h->gb, 16);//bbv_dwlay
     if (h->stc == PIC_PB_START_CODE) {
         h->cur.f->pict_type = get_bits(&h->gb, 2) + AV_PICTURE_TYPE_I;

libavcodec/mace.c

@@ -229,8 +229,8 @@ static av_cold int mace_decode_init(AVCodecContext * avctx)
 {
     MACEContext *ctx = avctx->priv_data;
 
-    if (avctx->channels > 2 || avctx->channels <= 0)
+    if (avctx->channels > 2 || avctx->channels < 1)
-        return -1;
+        return AVERROR(EINVAL);
     avctx->sample_fmt = AV_SAMPLE_FMT_S16P;
 
     avcodec_get_frame_defaults(&ctx->frame);

libavcodec/rv30.c

@@ -249,9 +249,11 @@ static void rv30_loop_filter(RV34DecContext *r, int row)
 static av_cold int rv30_decode_init(AVCodecContext *avctx)
 {
     RV34DecContext *r = avctx->priv_data;
+    int ret;
 
     r->rv30 = 1;
-    ff_rv34_decode_init(avctx);
+    if ((ret = ff_rv34_decode_init(avctx)) < 0)
+        return ret;
     if(avctx->extradata_size < 2){
         av_log(avctx, AV_LOG_ERROR, "Extradata is too small.\n");
         return -1;

libavcodec/rv40.c

@@ -548,9 +548,11 @@ static void rv40_loop_filter(RV34DecContext *r, int row)
 static av_cold int rv40_decode_init(AVCodecContext *avctx)
 {
     RV34DecContext *r = avctx->priv_data;
+    int ret;
 
     r->rv30 = 0;
-    ff_rv34_decode_init(avctx);
+    if ((ret = ff_rv34_decode_init(avctx)) < 0)
+        return ret;
     if(!aic_top_vlc.bits)
         rv40_init_tables();
     r->parse_slice_header = rv40_parse_slice_header;

libavcodec/svq3.c

@@ -638,9 +638,9 @@ static int svq3_decode_mb(SVQ3Context *svq3, unsigned int mb_type)
         dir = i_mb_type_info[mb_type - 8].pred_mode;
         dir = (dir >> 1) ^ 3 * (dir & 1) ^ 1;
 
-        if ((h->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, dir, 0)) == -1) {
+        if ((h->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, dir, 0)) < 0) {
-            av_log(h->s.avctx, AV_LOG_ERROR, "check_intra_pred_mode = -1\n");
+            av_log(h->s.avctx, AV_LOG_ERROR, "ff_h264_check_intra_pred_mode < 0\n");
-            return -1;
+            return h->intra16x16_pred_mode;
         }
 
         cbp     = i_mb_type_info[mb_type - 8].cbp;

libavcodec/vp3.c

@@ -2154,6 +2154,10 @@ static int theora_decode_header(AVCodecContext *avctx, GetBitContext *gb)
     fps.num = get_bits_long(gb, 32);
     fps.den = get_bits_long(gb, 32);
     if (fps.num && fps.den) {
+        if (fps.num < 0 || fps.den < 0) {
+            av_log(avctx, AV_LOG_ERROR, "Invalid framerate\n");
+            return AVERROR_INVALIDDATA;
+        }
         av_reduce(&avctx->time_base.num, &avctx->time_base.den,
                   fps.den, fps.num, 1<<30);
     }

libavformat/matroskadec.c

@@ -1697,6 +1697,10 @@ static int matroska_read_header(AVFormatContext *s)
             track->audio.sub_packet_h    = avio_rb16(&b);
             track->audio.frame_size      = avio_rb16(&b);
             track->audio.sub_packet_size = avio_rb16(&b);
+            if (flavor <= 0 || track->audio.coded_framesize <= 0 ||
+                track->audio.sub_packet_h <= 0 || track->audio.frame_size <= 0 ||
+                track->audio.sub_packet_size <= 0)
+                return AVERROR_INVALIDDATA;
             track->audio.buf = av_malloc(track->audio.frame_size * track->audio.sub_packet_h);
             if (codec_id == AV_CODEC_ID_RA_288) {
                 st->codec->block_align = track->audio.coded_framesize;

libavformat/mpc8.c

@@ -139,6 +139,11 @@ static void mpc8_parse_seektable(AVFormatContext *s, int64_t off)
     int i, t, seekd;
     GetBitContext gb;
 
+    if (s->nb_streams == 0) {
+        av_log(s, AV_LOG_ERROR, "No stream added before parsing seek table\n");
+        return;
+    }
+
     avio_seek(s->pb, off, SEEK_SET);
     mpc8_get_chunk_header(s->pb, &tag, &size);
     if(tag != TAG_SEEKTABLE){

libavformat/oggparseogm.c

@@ -38,34 +38,35 @@ ogm_header(AVFormatContext *s, int idx)
     struct ogg *ogg = s->priv_data;
     struct ogg_stream *os = ogg->streams + idx;
     AVStream *st = s->streams[idx];
-    const uint8_t *p = os->buf + os->pstart;
+    GetByteContext p;
     uint64_t time_unit;
     uint64_t spu;
     uint32_t size;
 
-    if(!(*p & 1))
+    bytestream2_init(&p, os->buf + os->pstart, os->psize);
+    if (!(bytestream2_peek_byte(&p) & 1))
         return 0;
 
-    if(*p == 1) {
+    if (bytestream2_peek_byte(&p) == 1) {
-        p++;
+        bytestream2_skip(&p, 1);
 
-        if(*p == 'v'){
+        if (bytestream2_peek_byte(&p) == 'v'){
             int tag;
             st->codec->codec_type = AVMEDIA_TYPE_VIDEO;
-            p += 8;
+            bytestream2_skip(&p, 8);
-            tag = bytestream_get_le32(&p);
+            tag = bytestream2_get_le32(&p);
             st->codec->codec_id = ff_codec_get_id(ff_codec_bmp_tags, tag);
             st->codec->codec_tag = tag;
-        } else if (*p == 't') {
+        } else if (bytestream2_peek_byte(&p) == 't') {
             st->codec->codec_type = AVMEDIA_TYPE_SUBTITLE;
             st->codec->codec_id = AV_CODEC_ID_TEXT;
-            p += 12;
+            bytestream2_skip(&p, 12);
         } else {
-            uint8_t acid[5];
+            uint8_t acid[5] = { 0 };
             int cid;
             st->codec->codec_type = AVMEDIA_TYPE_AUDIO;
-            p += 8;
+            bytestream2_skip(&p, 8);
-            bytestream_get_buffer(&p, acid, 4);
+            bytestream2_get_buffer(&p, acid, 4);
             acid[4] = 0;
             cid = strtol(acid, NULL, 16);
             st->codec->codec_id = ff_codec_get_id(ff_codec_wav_tags, cid);
@@ -74,25 +75,25 @@ ogm_header(AVFormatContext *s, int idx)
                 st->need_parsing = AVSTREAM_PARSE_FULL;
         }
 
-        size        = bytestream_get_le32(&p);
+        size        = bytestream2_get_le32(&p);
         size        = FFMIN(size, os->psize);
-        time_unit   = bytestream_get_le64(&p);
+        time_unit   = bytestream2_get_le64(&p);
-        spu         = bytestream_get_le64(&p);
+        spu         = bytestream2_get_le64(&p);
-        p += 4;                     /* default_len */
+        bytestream2_skip(&p, 4);    /* default_len */
-        p += 8;                     /* buffersize + bits_per_sample */
+        bytestream2_skip(&p, 8);    /* buffersize + bits_per_sample */
 
         if(st->codec->codec_type == AVMEDIA_TYPE_VIDEO){
-            st->codec->width = bytestream_get_le32(&p);
+            st->codec->width = bytestream2_get_le32(&p);
-            st->codec->height = bytestream_get_le32(&p);
+            st->codec->height = bytestream2_get_le32(&p);
             avpriv_set_pts_info(st, 64, time_unit, spu * 10000000);
         } else {
-            st->codec->channels = bytestream_get_le16(&p);
+            st->codec->channels = bytestream2_get_le16(&p);
-            p += 2;                 /* block_align */
+            bytestream2_skip(&p, 2); /* block_align */
-            st->codec->bit_rate = bytestream_get_le32(&p) * 8;
+            st->codec->bit_rate = bytestream2_get_le32(&p) * 8;
             st->codec->sample_rate = time_unit ? spu * 10000000 / time_unit : 0;
             avpriv_set_pts_info(st, 64, 1, st->codec->sample_rate);
             if (size >= 56 && st->codec->codec_id == AV_CODEC_ID_AAC) {
-                p += 4;
+                bytestream2_skip(&p, 4);
                 size -= 4;
             }
             if (size > 52) {
@@ -100,12 +101,13 @@ ogm_header(AVFormatContext *s, int idx)
                 size -= 52;
                 st->codec->extradata_size = size;
                 st->codec->extradata = av_malloc(size + FF_INPUT_BUFFER_PADDING_SIZE);
-                bytestream_get_buffer(&p, st->codec->extradata, size);
+                bytestream2_get_buffer(&p, st->codec->extradata, size);
             }
         }
-    } else if (*p == 3) {
+    } else if (bytestream2_peek_byte(&p) == 3) {
-        if (os->psize > 8)
+        bytestream2_skip(&p, 7);
-            ff_vorbis_comment(s, &st->metadata, p+7, os->psize-8);
+        if (bytestream2_get_bytes_left(&p) > 1)
+            ff_vorbis_comment(s, &st->metadata, p.buffer, bytestream2_get_bytes_left(&p) - 1);
     }
 
     return 1;

libavformat/omadec.c

@@ -169,7 +169,11 @@ static int nprobe(AVFormatContext *s, uint8_t *enc_header, unsigned size,
     taglen = AV_RB32(&enc_header[pos+32]);
     datalen = AV_RB32(&enc_header[pos+36]) >> 4;
 
-    pos += 44 + taglen;
+    pos += 44;
+    if (size - pos < taglen)
+        return -1;
+
+    pos += taglen;
 
     if (pos + (((uint64_t)datalen) << 4) > size)
         return -1;

libavformat/sierravmd.c

@@ -89,7 +89,7 @@ static int vmd_read_header(AVFormatContext *s)
     unsigned char *raw_frame_table;
     int raw_frame_table_size;
     int64_t current_offset;
-    int i, j;
+    int i, j, ret;
     unsigned int total_frames;
     int64_t current_audio_pts = 0;
     unsigned char chunk[BYTES_PER_FRAME_RECORD];
@@ -176,15 +176,13 @@ static int vmd_read_header(AVFormatContext *s)
     raw_frame_table = av_malloc(raw_frame_table_size);
     vmd->frame_table = av_malloc((vmd->frame_count * vmd->frames_per_block + sound_buffers) * sizeof(vmd_frame));
     if (!raw_frame_table || !vmd->frame_table) {
-        av_free(raw_frame_table);
+        ret = AVERROR(ENOMEM);
-        av_free(vmd->frame_table);
+        goto error;
-        return AVERROR(ENOMEM);
     }
     if (avio_read(pb, raw_frame_table, raw_frame_table_size) !=
         raw_frame_table_size) {
-        av_free(raw_frame_table);
+        ret = AVERROR(EIO);
-        av_free(vmd->frame_table);
+        goto error;
-        return AVERROR(EIO);
     }
 
     total_frames = 0;
@@ -200,6 +198,11 @@ static int vmd_read_header(AVFormatContext *s)
             avio_read(pb, chunk, BYTES_PER_FRAME_RECORD);
             type = chunk[0];
             size = AV_RL32(&chunk[2]);
+            if (size > INT_MAX / 2) {
+                av_log(s, AV_LOG_ERROR, "Invalid frame size\n");
+                ret = AVERROR_INVALIDDATA;
+                goto error;
+            }
             if(!size && type != 1)
                 continue;
             switch(type) {
@@ -236,6 +239,11 @@ static int vmd_read_header(AVFormatContext *s)
     vmd->frame_count = total_frames;
 
     return 0;
+
+error:
+    av_free(raw_frame_table);
+    av_free(vmd->frame_table);
+    return ret;
 }
 
 static int vmd_read_packet(AVFormatContext *s,

libavformat/vocdec.c

@@ -91,11 +91,11 @@ ff_voc_get_packet(AVFormatContext *s, AVPacket *pkt, AVStream *st, int max_size)
                 if (sample_rate)
                     dec->sample_rate = sample_rate;
                 avpriv_set_pts_info(st, 64, 1, dec->sample_rate);
+                dec->channels = channels;
+                dec->bits_per_coded_sample = av_get_bits_per_sample(dec->codec_id);
             } else
                 avio_skip(pb, 1);
-            dec->channels = channels;
             tmp_codec = avio_r8(pb);
-            dec->bits_per_coded_sample = av_get_bits_per_sample(dec->codec_id);
             voc->remaining_size -= 2;
             max_size -= 2;
             channels = 1;
@@ -117,10 +117,10 @@ ff_voc_get_packet(AVFormatContext *s, AVPacket *pkt, AVStream *st, int max_size)
             if (!dec->sample_rate) {
                 dec->sample_rate = avio_rl32(pb);
                 avpriv_set_pts_info(st, 64, 1, dec->sample_rate);
-            } else
-                avio_skip(pb, 4);
                 dec->bits_per_coded_sample = avio_r8(pb);
                 dec->channels = avio_r8(pb);
+            } else
+                avio_skip(pb, 6);
             tmp_codec = avio_rl16(pb);
             avio_skip(pb, 4);
             voc->remaining_size -= 12;