Merge remote-tracking branch 'qatar/release/9' into release/1.1

* qatar/release/9:
  update Changelog
  h264: set ref_count to 0 for intra slices.
  h264: on reference overflow, reset the reference count to 0, not 1.
  flvdec: Check the return value of a malloc

Conflicts:
	Changelog
	libavcodec/h264.c
	libavformat/flvdec.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

Changelog

@@ -2,6 +2,11 @@ Entries are sorted chronologically from oldest to youngest within each release,
 releases are sorted from youngest to oldest.
 
 version <next>:
+- h264: fix deadlocks with broken/fuzzed files
+- flvdec: make decoder more robust
+- vorbisdec: fix buffer overflow (CVE-2013-0894)
+- ac3dec: validate channel output mode against channel count
+- doc: minor improvements
 - loco: check that there is data left after decoding a plane.
 - mov: use the format context for logging.
 - lagarith: avoid infinite loop in lag_rac_refill() with corrupted files

libavcodec/h264.c

@@ -3119,7 +3119,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
 
         if (h->ref_count[0]-1 > max[0] || h->ref_count[1]-1 > max[1]){
             av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow %u > %u or %u > %u\n", h->ref_count[0]-1, max[0], h->ref_count[1]-1, max[1]);
-            h->ref_count[0] = h->ref_count[1] = 1;
+            h->ref_count[0] = h->ref_count[1] = 0;
             return AVERROR_INVALIDDATA;
         }
 
@@ -3127,8 +3127,10 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
             h->list_count = 2;
         else
             h->list_count = 1;
-    } else
-        h->ref_count[1]= h->ref_count[0]= h->list_count= 0;
+    } else {
+        h->list_count = 0;
+        h->ref_count[0] = h->ref_count[1] = 0;
+    }
 
     if (!default_ref_list_done)
         ff_h264_fill_default_ref_list(h);

libavformat/flvdec.c

@@ -212,10 +212,14 @@ static int flv_set_video_codec(AVFormatContext *s, AVStream *vstream, int flv_co
                 vcodec->codec_id = AV_CODEC_ID_VP6A;
             if (read) {
                 if (vcodec->extradata_size != 1) {
-                    vcodec->extradata_size = 1;
                     vcodec->extradata = av_malloc(1 + FF_INPUT_BUFFER_PADDING_SIZE);
+                    if (vcodec->extradata)
+                        vcodec->extradata_size = 1;
                 }
-                vcodec->extradata[0] = avio_r8(s->pb);
+                if (vcodec->extradata)
+                    vcodec->extradata[0] = avio_r8(s->pb);
+                else
+                    avio_skip(s->pb, 1);
             }
             return 1; // 1 byte body size adjustment for flv_read_packet()
         case FLV_CODECID_H264: