avcodec/h264_mp4toannexb_bsf: Reorder operations in nal_size check

Fixes Ticket4778

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 2bb54b82b5)

MAINTAINERS

@@ -14,7 +14,6 @@ and related discussions.
 Project Leader
 ==============
 
-Michael Niedermayer
   final design decisions
 
 

RELEASE

@@ -1 +1 @@
-1.2.11
+1.2.12

cmdutils.c

@@ -65,7 +65,7 @@ struct SwsContext *sws_opts;
 AVDictionary *swr_opts;
 AVDictionary *format_opts, *codec_opts, *resample_opts;
 
-const int this_year = 2014;
+const int this_year = 2015;
 
 static FILE *report_file;
 

doc/Doxyfile

@@ -31,7 +31,7 @@ PROJECT_NAME           = FFmpeg
 # This could be handy for archiving the generated documentation or
 # if some version control system is used.
 
-PROJECT_NUMBER         = 1.2.11
+PROJECT_NUMBER         = 1.2.12
 
 # With the PROJECT_LOGO tag one can specify an logo or icon that is included
 # in the documentation. The maximum height of the logo should not exceed 55

ffmpeg.c

@@ -2092,7 +2092,7 @@ static int transcode_init(void)
     AVCodecContext *codec;
     OutputStream *ost;
     InputStream *ist;
-    char error[1024];
+    char error[1024] = {0};
     int want_sdp = 1;
 
     /* init framerate emulation */

libavcodec/dvdsubdec.c

@@ -99,6 +99,12 @@ static int decode_rle(uint8_t *bitmap, int linesize, int w, int h,
     int x, y, len, color;
     uint8_t *d;
 
+    if (start >= buf_size)
+        return -1;
+
+    if (w <= 0 || h <= 0)
+        return -1;
+
     bit_len = (buf_size - start) * 8;
     init_get_bits(&gb, buf + start, bit_len);
 
@@ -340,10 +346,12 @@ static int decode_dvd_subtitles(DVDSubContext *ctx, AVSubtitle *sub_header,
                 sub_header->rects[0] = av_mallocz(sizeof(AVSubtitleRect));
                 sub_header->num_rects = 1;
                 sub_header->rects[0]->pict.data[0] = bitmap;
-                decode_rle(bitmap, w * 2, w, (h + 1) / 2,
-                           buf, offset1, buf_size, is_8bit);
-                decode_rle(bitmap + w, w * 2, w, h / 2,
-                           buf, offset2, buf_size, is_8bit);
+                if (decode_rle(bitmap, w * 2, w, (h + 1) / 2,
+                               buf, offset1, buf_size, is_8bit) < 0)
+                    goto fail;
+                if (decode_rle(bitmap + w, w * 2, w, h / 2,
+                               buf, offset2, buf_size, is_8bit) < 0)
+                    goto fail;
                 sub_header->rects[0]->pict.data[1] = av_mallocz(AVPALETTE_SIZE);
                 if (is_8bit) {
                     if (yuv_palette == 0)

libavcodec/flac_parser.c

@@ -655,7 +655,7 @@ static int flac_parse(AVCodecParserContext *s, AVCodecContext *avctx,
 handle_error:
     *poutbuf      = NULL;
     *poutbuf_size = 0;
-    return read_end - buf;
+    return buf_size ? read_end - buf : 0;
 }
 
 static int flac_parse_init(AVCodecParserContext *c)

libavcodec/h264.c

@@ -3147,6 +3147,9 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
     int last_pic_structure, last_pic_droppable;
     int must_reinit;
     int needs_reinit = 0;
+    int first_slice = h == h0 && !h0->current_slice;
+    int frame_num, picture_structure, droppable;
+    PPS *pps;
 
     h->me.qpel_put = h->h264qpel.put_h264_qpel_pixels_tab;
     h->me.qpel_avg = h->h264qpel.avg_h264_qpel_pixels_tab;
@@ -3200,17 +3203,26 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
                pps_id);
         return -1;
     }
-    h->pps = *h0->pps_buffers[pps_id];
 
-    if (!h0->sps_buffers[h->pps.sps_id]) {
+    pps = h0->pps_buffers[pps_id];
+
+    if (!h0->sps_buffers[pps->sps_id]) {
         av_log(h->avctx, AV_LOG_ERROR,
                "non-existing SPS %u referenced\n",
                h->pps.sps_id);
         return -1;
     }
+    if (first_slice)
+        h->pps = *h0->pps_buffers[pps_id];
 
-    if (h->pps.sps_id != h->current_sps_id ||
-        h0->sps_buffers[h->pps.sps_id]->new) {
+    if (pps->sps_id != h->current_sps_id ||
+        h0->sps_buffers[pps->sps_id]->new) {
+
+        if (!first_slice) {
+            av_log(h->avctx, AV_LOG_ERROR,
+               "SPS changed in the middle of the frame\n");
+            return AVERROR_INVALIDDATA;
+        }
         h0->sps_buffers[h->pps.sps_id]->new = 0;
 
         h->current_sps_id = h->pps.sps_id;
@@ -3242,13 +3254,15 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
                      || 16*h->sps.mb_height * (2 - h->sps.frame_mbs_only_flag) != h->avctx->coded_height
                      || h->avctx->bits_per_raw_sample != h->sps.bit_depth_luma
                      || h->cur_chroma_format_idc != h->sps.chroma_format_idc
-                     || av_cmp_q(h->sps.sar, h->avctx->sample_aspect_ratio)
                      || h->mb_width  != h->sps.mb_width
                      || h->mb_height != h->sps.mb_height * (2 - h->sps.frame_mbs_only_flag)
                     ));
     if (non_j_pixfmt(h0->avctx->pix_fmt) != non_j_pixfmt(get_pixel_format(h0, 0)))
         must_reinit = 1;
 
+    if (first_slice && av_cmp_q(h->sps.sar, h->avctx->sample_aspect_ratio))
+        must_reinit = 1;
+
     h->mb_width  = h->sps.mb_width;
     h->mb_height = h->sps.mb_height * (2 - h->sps.frame_mbs_only_flag);
     h->mb_num    = h->mb_width * h->mb_height;
@@ -3322,37 +3336,40 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
         init_dequant_tables(h);
     }
 
-    h->frame_num = get_bits(&h->gb, h->sps.log2_max_frame_num);
+    frame_num = get_bits(&h->gb, h->sps.log2_max_frame_num);
+    if (!first_slice) {
+        if (h0->frame_num != frame_num) {
+            av_log(h->avctx, AV_LOG_ERROR, "Frame num change from %d to %d\n",
+                   h0->frame_num, frame_num);
+            return AVERROR_INVALIDDATA;
+        }
+    }
 
     h->mb_mbaff        = 0;
     h->mb_aff_frame    = 0;
     last_pic_structure = h0->picture_structure;
     last_pic_droppable = h0->droppable;
-    h->droppable       = h->nal_ref_idc == 0;
+    droppable          = h->nal_ref_idc == 0;
     if (h->sps.frame_mbs_only_flag) {
-        h->picture_structure = PICT_FRAME;
+        picture_structure = PICT_FRAME;
     } else {
         if (!h->sps.direct_8x8_inference_flag && slice_type == AV_PICTURE_TYPE_B) {
             av_log(h->avctx, AV_LOG_ERROR, "This stream was generated by a broken encoder, invalid 8x8 inference\n");
             return -1;
         }
         if (get_bits1(&h->gb)) { // field_pic_flag
-            h->picture_structure = PICT_TOP_FIELD + get_bits1(&h->gb); // bottom_field_flag
+            picture_structure = PICT_TOP_FIELD + get_bits1(&h->gb); // bottom_field_flag
         } else {
-            h->picture_structure = PICT_FRAME;
+            picture_structure = PICT_FRAME;
             h->mb_aff_frame      = h->sps.mb_aff;
         }
     }
-    h->mb_field_decoding_flag = h->picture_structure != PICT_FRAME;
-
-    if (h0->current_slice != 0) {
-        if (last_pic_structure != h->picture_structure ||
-            last_pic_droppable != h->droppable) {
+    if (h0->current_slice) {
+        if (last_pic_structure != picture_structure ||
+            last_pic_droppable != droppable) {
             av_log(h->avctx, AV_LOG_ERROR,
                    "Changing field mode (%d -> %d) between slices is not allowed\n",
                    last_pic_structure, h->picture_structure);
-            h->picture_structure = last_pic_structure;
-            h->droppable         = last_pic_droppable;
             return AVERROR_INVALIDDATA;
         } else if (!h0->cur_pic_ptr) {
             av_log(h->avctx, AV_LOG_ERROR,
@@ -3360,7 +3377,14 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
                    h0->current_slice + 1);
             return AVERROR_INVALIDDATA;
         }
-    } else {
+    }
+
+    h->picture_structure = picture_structure;
+    h->droppable         = droppable;
+    h->frame_num         = frame_num;
+    h->mb_field_decoding_flag = picture_structure != PICT_FRAME;
+
+    if (h0->current_slice == 0) {
         /* Shorten frame num gaps so we don't have to allocate reference
          * frames just to throw them away */
         if (h->frame_num != h->prev_frame_num) {

libavcodec/h264_cabac.c

@@ -1278,7 +1278,7 @@ void ff_h264_init_cabac_states(H264Context *h) {
 }
 
 static int decode_cabac_field_decoding_flag(H264Context *h) {
-    const long mbb_xy = h->mb_xy - 2L*h->mb_stride;
+    const int mbb_xy = h->mb_xy - 2*h->mb_stride;
 
     unsigned long ctx = 0;
 

libavcodec/h264_mp4toannexb_bsf.c

@@ -150,7 +150,7 @@ pps:
         buf += ctx->length_size;
         unit_type = *buf & 0x1f;
 
-        if (buf + nal_size > buf_end || nal_size < 0)
+        if (nal_size > buf_end - buf || nal_size < 0)
             goto fail;
 
         /* prepend only to the first type 5 NAL unit of an IDR picture */

libavcodec/h264_ps.c

@@ -379,7 +379,9 @@ int ff_h264_decode_seq_parameter_set(H264Context *h){
         }
         sps->bit_depth_luma   = get_ue_golomb(&h->gb) + 8;
         sps->bit_depth_chroma = get_ue_golomb(&h->gb) + 8;
-        if (sps->bit_depth_luma > 14U || sps->bit_depth_chroma > 14U || sps->bit_depth_luma != sps->bit_depth_chroma) {
+        if (sps->bit_depth_luma   < 8 || sps->bit_depth_luma   > 14 ||
+            sps->bit_depth_chroma < 8 || sps->bit_depth_chroma > 14 ||
+            sps->bit_depth_luma != sps->bit_depth_chroma) {
             av_log(h->avctx, AV_LOG_ERROR, "illegal bit depth value (%d, %d)\n",
                    sps->bit_depth_luma, sps->bit_depth_chroma);
             goto fail;

libavcodec/mjpegdec.c

@@ -437,9 +437,12 @@ int ff_mjpeg_decode_sof(MJpegDecodeContext *s)
     }
     if (s->ls) {
         s->upscale_h = s->upscale_v = 0;
-        if (s->nb_components > 1)
+        if (s->nb_components == 3) {
             s->avctx->pix_fmt = AV_PIX_FMT_RGB24;
-        else if (s->bits <= 8)
+        } else if (s->nb_components != 1) {
+            av_log(s->avctx, AV_LOG_ERROR, "Unsupported number of components %d\n", s->nb_components);
+            return AVERROR_PATCHWELCOME;
+        } else if (s->bits <= 8)
             s->avctx->pix_fmt = AV_PIX_FMT_GRAY8;
         else
             s->avctx->pix_fmt = AV_PIX_FMT_GRAY16;
@@ -1080,13 +1083,18 @@ static int mjpeg_decode_scan(MJpegDecodeContext *s, int nb_components, int Ah,
 
                     if (s->interlaced && s->bottom_field)
                         block_offset += linesize[c] >> 1;
-                    ptr = data[c] + block_offset;
+                    if (   8*(h * mb_x + x) < s->width
+                        && 8*(v * mb_y + y) < s->height) {
+                        ptr = data[c] + block_offset;
+                    } else
+                        ptr = NULL;
                     if (!s->progressive) {
-                        if (copy_mb)
-                            mjpeg_copy_block(s, ptr, reference_data[c] + block_offset,
-                                             linesize[c], s->avctx->lowres);
+                        if (copy_mb) {
+                            if (ptr)
+                                mjpeg_copy_block(s, ptr, reference_data[c] + block_offset,
+                                                linesize[c], s->avctx->lowres);
 
-                        else {
+                        } else {
                             s->dsp.clear_block(s->block);
                             if (decode_block(s, s->block, i,
                                              s->dc_index[i], s->ac_index[i],
@@ -1095,7 +1103,9 @@ static int mjpeg_decode_scan(MJpegDecodeContext *s, int nb_components, int Ah,
                                        "error y=%d x=%d\n", mb_y, mb_x);
                                 return AVERROR_INVALIDDATA;
                             }
-                            s->dsp.idct_put(ptr, linesize[c], s->block);
+                            if (ptr) {
+                                s->dsp.idct_put(ptr, linesize[c], s->block);
+                            }
                         }
                     } else {
                         int block_idx  = s->block_stride[c] * (v * mb_y + y) +
@@ -1631,6 +1641,10 @@ int ff_mjpeg_find_marker(MJpegDecodeContext *s,
             put_bits(&pb, 8, x);
             if (x == 0xFF) {
                 x = src[b++];
+                if (x & 0x80) {
+                    av_log(s->avctx, AV_LOG_WARNING, "Invalid escape sequence\n");
+                    x &= 0x7f;
+                }
                 put_bits(&pb, 7, x);
                 bit_count--;
             }

libavcodec/mpegaudiodec.c

@@ -1667,9 +1667,11 @@ static int decode_frame(AVCodecContext * avctx, void *data, int *got_frame_ptr,
     uint32_t header;
     int ret;
 
+    int skipped = 0;
     while(buf_size && !*buf){
         buf++;
         buf_size--;
+        skipped++;
     }
 
     if (buf_size < HEADER_SIZE)
@@ -1724,7 +1726,7 @@ static int decode_frame(AVCodecContext * avctx, void *data, int *got_frame_ptr,
             return ret;
     }
     s->frame_size = 0;
-    return buf_size;
+    return buf_size + skipped;
 }
 
 static void mp_flush(MPADecodeContext *ctx)

libavcodec/mpegvideo_enc.c

@@ -412,18 +412,18 @@ av_cold int ff_MPV_encode_init(AVCodecContext *avctx)
         switch(avctx->codec_id) {
         case AV_CODEC_ID_MPEG1VIDEO:
         case AV_CODEC_ID_MPEG2VIDEO:
-            avctx->rc_buffer_size = FFMAX(avctx->rc_max_rate, 15000000) * 112L / 15000000 * 16384;
+            avctx->rc_buffer_size = FFMAX(avctx->rc_max_rate, 15000000) * 112LL / 15000000 * 16384;
             break;
         case AV_CODEC_ID_MPEG4:
         case AV_CODEC_ID_MSMPEG4V1:
         case AV_CODEC_ID_MSMPEG4V2:
         case AV_CODEC_ID_MSMPEG4V3:
             if       (avctx->rc_max_rate >= 15000000) {
-                avctx->rc_buffer_size = 320 + (avctx->rc_max_rate - 15000000L) * (760-320) / (38400000 - 15000000);
+                avctx->rc_buffer_size = 320 + (avctx->rc_max_rate - 15000000LL) * (760-320) / (38400000 - 15000000);
             } else if(avctx->rc_max_rate >=  2000000) {
-                avctx->rc_buffer_size =  80 + (avctx->rc_max_rate -  2000000L) * (320- 80) / (15000000 -  2000000);
+                avctx->rc_buffer_size =  80 + (avctx->rc_max_rate -  2000000LL) * (320- 80) / (15000000 -  2000000);
             } else if(avctx->rc_max_rate >=   384000) {
-                avctx->rc_buffer_size =  40 + (avctx->rc_max_rate -   384000L) * ( 80- 40) / ( 2000000 -   384000);
+                avctx->rc_buffer_size =  40 + (avctx->rc_max_rate -   384000LL) * ( 80- 40) / ( 2000000 -   384000);
             } else
                 avctx->rc_buffer_size = 40;
             avctx->rc_buffer_size *= 16384;

libavcodec/x86/h264_weight.asm

@@ -135,8 +135,11 @@ WEIGHT_FUNC_HALF_MM 8, 8
     add  off_regd, 1
     or   off_regd, 1
     add        r4, 1
+    cmp        r6d, 128
+     je .nonnormal
     cmp        r5, 128
      jne .normal
+.nonnormal
     sar        r5, 1
     sar        r6, 1
     sar  off_regd, 1

libavfilter/x86/vf_hqdn3d.asm

@@ -27,8 +27,8 @@ SECTION .text
 %if lut_bits != 8
     sar    %1q, 8-lut_bits
 %endif
-    movsx  %1d, word [%3q+%1q*2]
-    add    %1d, %2d
+    movsx  %1q, word [%3q+%1q*2]
+    add    %1q, %2q
 %endmacro
 
 %macro LOAD 3 ; dstreg, x, bitdepth

libavformat/avidec.c

@@ -1309,7 +1309,8 @@ static int avi_read_idx1(AVFormatContext *s, int size)
         ast = st->priv_data;
 
         if (first_packet && first_packet_pos) {
-            data_offset = first_packet_pos - pos;
+            if (avi->movi_list + 4 != pos || pos + 500 > first_packet_pos)
+                data_offset  = first_packet_pos - pos;
             first_packet = 0;
         }
         pos += data_offset;

libavformat/cdxl.c

@@ -127,6 +127,8 @@ static int cdxl_read_packet(AVFormatContext *s, AVPacket *pkt)
     height       = AV_RB16(&cdxl->header[16]);
     palette_size = AV_RB16(&cdxl->header[20]);
     audio_size   = AV_RB16(&cdxl->header[22]);
+    if (FFALIGN(width, 16) * (uint64_t)height * cdxl->header[19] > INT_MAX)
+        return AVERROR_INVALIDDATA;
     image_size   = FFALIGN(width, 16) * height * cdxl->header[19] / 8;
     video_size   = palette_size + image_size;
 

libavformat/flvdec.c

@@ -333,7 +333,7 @@ static int amf_parse_object(AVFormatContext *s, AVStream *astream, AVStream *vst
     FLVContext *flv = s->priv_data;
     AVIOContext *ioc;
     AMFDataType amf_type;
-    char str_val[256];
+    char str_val[1024];
     double num_val;
 
     num_val = 0;

libavformat/matroskadec.c

@@ -991,7 +991,7 @@ static void ebml_free(EbmlSyntax *syntax, void *data)
                 char *ptr = list->elem;
                 for (j=0; j<list->nb_elem; j++, ptr+=syntax[i].list_elem_size)
                     ebml_free(syntax[i].def.n, ptr);
-                av_free(list->elem);
+                av_freep(&list->elem);
             } else
                 ebml_free(syntax[i].def.n, data_off);
         default:  break;
@@ -1789,8 +1789,8 @@ static int matroska_read_header(AVFormatContext *s)
                 av_reduce(&st->avg_frame_rate.num, &st->avg_frame_rate.den,
                           1000000000, track->default_duration, 30000);
 #if FF_API_R_FRAME_RATE
-                if (   st->avg_frame_rate.num < st->avg_frame_rate.den * 1000L
-                    && st->avg_frame_rate.num > st->avg_frame_rate.den * 5L)
+                if (   st->avg_frame_rate.num < st->avg_frame_rate.den * 1000LL
+                    && st->avg_frame_rate.num > st->avg_frame_rate.den * 5LL)
                     st->r_frame_rate = st->avg_frame_rate;
 #endif
             }
@@ -1891,7 +1891,7 @@ static int matroska_deliver_packet(MatroskaDemuxContext *matroska,
 {
     if (matroska->num_packets > 0) {
         memcpy(pkt, matroska->packets[0], sizeof(AVPacket));
-        av_free(matroska->packets[0]);
+        av_freep(&matroska->packets[0]);
         if (matroska->num_packets > 1) {
             void *newpackets;
             memmove(&matroska->packets[0], &matroska->packets[1],
@@ -1921,7 +1921,7 @@ static void matroska_clear_queue(MatroskaDemuxContext *matroska)
         int n;
         for (n = 0; n < matroska->num_packets; n++) {
             av_free_packet(matroska->packets[n]);
-            av_free(matroska->packets[n]);
+            av_freep(&matroska->packets[n]);
         }
         av_freep(&matroska->packets);
         matroska->num_packets = 0;
@@ -2514,7 +2514,7 @@ static int matroska_read_close(AVFormatContext *s)
 
     for (n=0; n < matroska->tracks.nb_elem; n++)
         if (tracks[n].type == MATROSKA_TRACK_TYPE_AUDIO)
-            av_free(tracks[n].audio.buf);
+            av_freep(&tracks[n].audio.buf);
     ebml_free(matroska_cluster, &matroska->current_cluster);
     ebml_free(matroska_segment, matroska);
 

libavformat/mov.c

@@ -281,7 +281,11 @@ static int mov_read_covr(MOVContext *c, AVIOContext *pb, int type, int len)
 static int mov_metadata_raw(MOVContext *c, AVIOContext *pb,
                             unsigned len, const char *key)
 {
-    char *value = av_malloc(len + 1);
+    char *value;
+    // Check for overflow.
+    if (len >= INT_MAX)
+        return AVERROR(EINVAL);
+    value = av_malloc(len + 1);
     if (!value)
         return AVERROR(ENOMEM);
     avio_read(pb, value, len);
@@ -385,7 +389,7 @@ static int mov_read_udta_string(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 
     if (!key)
         return 0;
-    if (atom.size < 0)
+    if (atom.size < 0 || str_size >= INT_MAX/2)
         return AVERROR_INVALIDDATA;
 
     str_size = FFMIN3(sizeof(str)-1, str_size, atom.size);
@@ -1162,6 +1166,10 @@ static int mov_read_stco(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     if (entries >= UINT_MAX/sizeof(int64_t))
         return AVERROR_INVALIDDATA;
 
+    if (sc->chunk_offsets)
+        av_log(c->fc, AV_LOG_WARNING, "Duplicate STCO atom\n");
+    av_free(sc->chunk_offsets);
+    sc->chunk_count = 0;
     sc->chunk_offsets = av_malloc(entries * sizeof(int64_t));
     if (!sc->chunk_offsets)
         return AVERROR(ENOMEM);
@@ -1607,6 +1615,10 @@ static int mov_read_stsc(MOVContext *c, AVIOContext *pb, MOVAtom atom)
         return 0;
     if (entries >= UINT_MAX / sizeof(*sc->stsc_data))
         return AVERROR_INVALIDDATA;
+    if (sc->stsc_data)
+        av_log(c->fc, AV_LOG_WARNING, "Duplicate STSC atom\n");
+    av_free(sc->stsc_data);
+    sc->stsc_count = 0;
     sc->stsc_data = av_malloc(entries * sizeof(*sc->stsc_data));
     if (!sc->stsc_data)
         return AVERROR(ENOMEM);
@@ -2815,7 +2827,7 @@ static int mov_read_default(MOVContext *c, AVIOContext *pb, MOVAtom atom)
                 }
             }
             total_size += 8;
-            if (a.size == 1) { /* 64 bit extended size */
+            if (a.size == 1 && total_size + 8 <= atom.size) { /* 64 bit extended size */
                 a.size = avio_rb64(pb) - 8;
                 total_size += 8;
             }

libavformat/movenc.c

@@ -1894,7 +1894,8 @@ static int mov_write_mvhd_tag(AVIOContext *pb, MOVMuxContext *mov)
     }
 
     version = max_track_len < UINT32_MAX ? 0 : 1;
-    (version == 1) ? avio_wb32(pb, 120) : avio_wb32(pb, 108); /* size */
+    avio_wb32(pb, version == 1 ? 120 : 108); /* size */
+
     ffio_wfourcc(pb, "mvhd");
     avio_w8(pb, version);
     avio_wb24(pb, 0); /* flags */

libavformat/mpc8.c

@@ -57,7 +57,7 @@ typedef struct {
 
 static inline int64_t bs_get_v(const uint8_t **bs)
 {
-    int64_t v = 0;
+    uint64_t v = 0;
     int br = 0;
     int c;
 
@@ -91,7 +91,7 @@ static int mpc8_probe(AVProbeData *p)
         size = bs_get_v(&bs);
         if (size < 2)
             return 0;
-        if (bs + size - 2 >= bs_end)
+        if (size >= bs_end - bs + 2)
             return AVPROBE_SCORE_MAX / 4 - 1; //seems to be valid MPC but no header yet
         if (header_found) {
             if (size < 11 || size > 28)
@@ -108,7 +108,7 @@ static int mpc8_probe(AVProbeData *p)
 
 static inline int64_t gb_get_v(GetBitContext *gb)
 {
-    int64_t v = 0;
+    uint64_t v = 0;
     int bits = 0;
     while(get_bits1(gb) && bits < 64-7){
         v <<= 7;
@@ -216,6 +216,10 @@ static int mpc8_read_header(AVFormatContext *s)
     while(!url_feof(pb)){
         pos = avio_tell(pb);
         mpc8_get_chunk_header(pb, &tag, &size);
+        if (size < 0) {
+            av_log(s, AV_LOG_ERROR, "Invalid chunk length\n");
+            return AVERROR_INVALIDDATA;
+        }
         if(tag == TAG_STREAMHDR)
             break;
         mpc8_handle_chunk(s, tag, pos, size);

libavformat/oggenc.c

@@ -256,7 +256,7 @@ static int ogg_buffer_data(AVFormatContext *s, AVStream *st,
         if (i == total_segments)
             page->granule = granule;
 
-        if (!header) {
+        {
             AVStream *st = s->streams[page->stream_index];
 
             int64_t start = av_rescale_q(page->start_granule, st->time_base,
@@ -264,10 +264,13 @@ static int ogg_buffer_data(AVFormatContext *s, AVStream *st,
             int64_t next  = av_rescale_q(page->granule, st->time_base,
                                          AV_TIME_BASE_Q);
 
-            if (page->segments_count == 255 ||
-                (ogg->pref_size     > 0 && page->size   >= ogg->pref_size) ||
-                (ogg->pref_duration > 0 && next - start >= ogg->pref_duration)) {
+            if (page->segments_count == 255) {
                 ogg_buffer_page(s, oggstream);
+            } else if (!header) {
+                if ((ogg->pref_size     > 0 && page->size   >= ogg->pref_size) ||
+                    (ogg->pref_duration > 0 && next - start >= ogg->pref_duration)) {
+                    ogg_buffer_page(s, oggstream);
+                }
             }
         }
     }

libavformat/rmdec.c

@@ -389,7 +389,11 @@ ff_rm_read_mdpr_codecdata (AVFormatContext *s, AVIOContext *pb,
 skip:
     /* skip codec info */
     size = avio_tell(pb) - codec_pos;
-    avio_skip(pb, codec_data_size - size);
+    if (codec_data_size >= size) {
+        avio_skip(pb, codec_data_size - size);
+    } else {
+        av_log(s, AV_LOG_WARNING, "codec_data_size %u < size %d\n", codec_data_size, size);
+    }
 
     return 0;
 }

libavformat/rtpdec_h263_rfc2190.c

@@ -82,6 +82,7 @@ static int h263_handle_packet(AVFormatContext *ctx, PayloadContext *data,
         avio_close_dyn_buf(data->buf, &p);
         av_free(p);
         data->buf = NULL;
+        data->endbyte_bits = 0;
     }
 
     if (len < 4) {

libavformat/segment.c

@@ -717,12 +717,6 @@ fail:
     if (pkt->stream_index == seg->reference_stream_index)
         seg->frame_count++;
 
-    if (ret < 0) {
-        if (seg->list)
-            avio_close(seg->list_pb);
-        avformat_free_context(oc);
-    }
-
     return ret;
 }
 

libavformat/smacker.c

@@ -309,7 +309,7 @@ static int smacker_read_packet(AVFormatContext *s, AVPacket *pkt)
                 uint8_t *tmpbuf;
 
                 size = avio_rl32(s->pb) - 4;
-                if(size + 4L > frame_size)
+                if(size + 4LL > frame_size)
                     return AVERROR_INVALIDDATA;
                 frame_size -= size;
                 frame_size -= 4;

libavformat/swfdec.c

@@ -384,10 +384,8 @@ static int swf_read_packet(AVFormatContext *s, AVPacket *pkt)
             }
             if (st->codec->pix_fmt != AV_PIX_FMT_NONE && st->codec->pix_fmt != pix_fmt) {
                 av_log(s, AV_LOG_ERROR, "pixel format change unsupported\n");
-                res = AVERROR_PATCHWELCOME;
-                goto bitmap_end;
-            }
-            st->codec->pix_fmt = pix_fmt;
+            }else
+                st->codec->pix_fmt = pix_fmt;
 
             if (linesize * height > pkt->size) {
                 res = AVERROR_INVALIDDATA;

libavformat/thp.c

@@ -180,6 +180,8 @@ static int thp_read_packet(AVFormatContext *s,
         pkt->stream_index = thp->video_stream_index;
     } else {
         ret = av_get_packet(pb, pkt, thp->audiosize);
+        if (ret < 0)
+            return ret;
         if (ret != thp->audiosize) {
             av_free_packet(pkt);
             return AVERROR(EIO);

libavformat/tta.c

@@ -96,8 +96,10 @@ static int tta_read_header(AVFormatContext *s)
 
     for (i = 0; i < c->totalframes; i++) {
         uint32_t size = avio_rl32(s->pb);
-        av_add_index_entry(st, framepos, i * c->frame_size, size, 0,
-                           AVINDEX_KEYFRAME);
+        int r;
+        if ((r = av_add_index_entry(st, framepos, i * c->frame_size, size, 0,
+                                    AVINDEX_KEYFRAME)) < 0)
+            return r;
         framepos += size;
     }
     avio_skip(s->pb, 4); // seektable crc
@@ -135,6 +137,11 @@ static int tta_read_packet(AVFormatContext *s, AVPacket *pkt)
     if (c->currentframe >= c->totalframes)
         return AVERROR_EOF;
 
+    if (st->nb_index_entries < c->totalframes) {
+        av_log(s, AV_LOG_ERROR, "Index entry disappeared\n");
+        return AVERROR_INVALIDDATA;
+    }
+
     size = st->index_entries[c->currentframe].size;
 
     ret = av_get_packet(s->pb, pkt, size);

libavformat/utils.c

@@ -2675,8 +2675,8 @@ static int get_std_framerate(int i){
  * And there are "variable" fps files this needs to detect as well.
  */
 static int tb_unreliable(AVCodecContext *c){
-    if(   c->time_base.den >= 101L*c->time_base.num
-       || c->time_base.den <    5L*c->time_base.num
+    if(   c->time_base.den >= 101LL*c->time_base.num
+       || c->time_base.den <    5LL*c->time_base.num
 /*       || c->codec_tag == AV_RL32("DIVX")
        || c->codec_tag == AV_RL32("XVID")*/
        || c->codec_tag == AV_RL32("mp4v")

libavutil/opt.c

@@ -74,8 +74,8 @@ static int read_number(const AVOption *o, void *dst, double *num, int *den, int6
 {
     switch (o->type) {
     case AV_OPT_TYPE_FLAGS:     *intnum = *(unsigned int*)dst;return 0;
-    case AV_OPT_TYPE_PIXEL_FMT:
-    case AV_OPT_TYPE_SAMPLE_FMT:
+    case AV_OPT_TYPE_PIXEL_FMT: *intnum = *(enum AVPixelFormat *)dst;return 0;
+    case AV_OPT_TYPE_SAMPLE_FMT:*intnum = *(enum AVSampleFormat*)dst;return 0;
     case AV_OPT_TYPE_INT:       *intnum = *(int         *)dst;return 0;
     case AV_OPT_TYPE_INT64:     *intnum = *(int64_t     *)dst;return 0;
     case AV_OPT_TYPE_FLOAT:     *num    = *(float       *)dst;return 0;
@@ -97,9 +97,9 @@ static int write_number(void *obj, const AVOption *o, void *dst, double num, int
     }
 
     switch (o->type) {
+    case AV_OPT_TYPE_PIXEL_FMT: *(enum AVPixelFormat *)dst = llrint(num/den) * intnum; break;
+    case AV_OPT_TYPE_SAMPLE_FMT:*(enum AVSampleFormat*)dst = llrint(num/den) * intnum; break;
     case AV_OPT_TYPE_FLAGS:
-    case AV_OPT_TYPE_PIXEL_FMT:
-    case AV_OPT_TYPE_SAMPLE_FMT:
     case AV_OPT_TYPE_INT:   *(int       *)dst= llrint(num/den)*intnum; break;
     case AV_OPT_TYPE_INT64: *(int64_t   *)dst= llrint(num/den)*intnum; break;
     case AV_OPT_TYPE_FLOAT: *(float     *)dst= num*intnum/den;         break;
@@ -149,6 +149,8 @@ static int set_string_binary(void *obj, const AVOption *o, const char *val, uint
     len /= 2;
 
     ptr = bin = av_malloc(len);
+    if (!ptr)
+        return AVERROR(ENOMEM);
     while (*val) {
         int a = hexchar2int(*val++);
         int b = hexchar2int(*val++);

libswresample/dither.c

@@ -84,13 +84,13 @@ int swri_dither_init(SwrContext *s, enum AVSampleFormat out_fmt, enum AVSampleFo
     in_fmt  = av_get_packed_sample_fmt( in_fmt);
 
     if(in_fmt == AV_SAMPLE_FMT_FLT || in_fmt == AV_SAMPLE_FMT_DBL){
-        if(out_fmt == AV_SAMPLE_FMT_S32) scale = 1.0/(1L<<31);
-        if(out_fmt == AV_SAMPLE_FMT_S16) scale = 1.0/(1L<<15);
-        if(out_fmt == AV_SAMPLE_FMT_U8 ) scale = 1.0/(1L<< 7);
+        if(out_fmt == AV_SAMPLE_FMT_S32) scale = 1.0/(1LL<<31);
+        if(out_fmt == AV_SAMPLE_FMT_S16) scale = 1.0/(1LL<<15);
+        if(out_fmt == AV_SAMPLE_FMT_U8 ) scale = 1.0/(1LL<< 7);
     }
-    if(in_fmt == AV_SAMPLE_FMT_S32 && out_fmt == AV_SAMPLE_FMT_S16) scale = 1L<<16;
-    if(in_fmt == AV_SAMPLE_FMT_S32 && out_fmt == AV_SAMPLE_FMT_U8 ) scale = 1L<<24;
-    if(in_fmt == AV_SAMPLE_FMT_S16 && out_fmt == AV_SAMPLE_FMT_U8 ) scale = 1L<<8;
+    if(in_fmt == AV_SAMPLE_FMT_S32 && out_fmt == AV_SAMPLE_FMT_S16) scale = 1<<16;
+    if(in_fmt == AV_SAMPLE_FMT_S32 && out_fmt == AV_SAMPLE_FMT_U8 ) scale = 1<<24;
+    if(in_fmt == AV_SAMPLE_FMT_S16 && out_fmt == AV_SAMPLE_FMT_U8 ) scale = 1<<8;
 
     scale *= s->dither.scale;
 

libswscale/utils.c

@@ -571,14 +571,15 @@ static int initFilter(int16_t **outFilter, int32_t **filterPos,
         }
 
         if ((*filterPos)[i] + filterSize > srcW) {
-            int shift = (*filterPos)[i] + filterSize - srcW;
+            int shift = (*filterPos)[i] + FFMIN(filterSize - srcW, 0);
+
             // move filter coefficients right to compensate for filterPos
             for (j = filterSize - 2; j >= 0; j--) {
                 int right = FFMIN(j + shift, filterSize - 1);
                 filter[i * filterSize + right] += filter[i * filterSize + j];
                 filter[i * filterSize + j]      = 0;
             }
-            (*filterPos)[i]= srcW - filterSize;
+            (*filterPos)[i]-= shift;
         }
     }
 

libswscale/yuv2rgb.c

@@ -782,9 +782,13 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
 
     av_free(c->yuvTable);
 
+#define ALLOC_YUV_TABLE(x)          \
+        c->yuvTable = av_malloc(x); \
+        if (!c->yuvTable)           \
+            return AVERROR(ENOMEM);
     switch (bpp) {
     case 1:
-        c->yuvTable = av_malloc(1024);
+        ALLOC_YUV_TABLE(1024);
         y_table     = c->yuvTable;
         yb = -(384 << 16) - oy;
         for (i = 0; i < 1024 - 110; i++) {
@@ -799,7 +803,7 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
         rbase       = isRgb ? 3 : 0;
         gbase       = 1;
         bbase       = isRgb ? 0 : 3;
-        c->yuvTable = av_malloc(1024 * 3);
+        ALLOC_YUV_TABLE(1024 * 3);
         y_table     = c->yuvTable;
         yb = -(384 << 16) - oy;
         for (i = 0; i < 1024 - 110; i++) {
@@ -818,7 +822,7 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
         rbase       = isRgb ? 5 : 0;
         gbase       = isRgb ? 2 : 3;
         bbase       = isRgb ? 0 : 6;
-        c->yuvTable = av_malloc(1024 * 3);
+        ALLOC_YUV_TABLE(1024 * 3);
         y_table     = c->yuvTable;
         yb = -(384 << 16) - oy;
         for (i = 0; i < 1024 - 38; i++) {
@@ -837,7 +841,7 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
         rbase       = isRgb ? 8 : 0;
         gbase       = 4;
         bbase       = isRgb ? 0 : 8;
-        c->yuvTable = av_malloc(1024 * 3 * 2);
+        ALLOC_YUV_TABLE(1024 * 3 * 2);
         y_table16   = c->yuvTable;
         yb = -(384 << 16) - oy;
         for (i = 0; i < 1024; i++) {
@@ -860,7 +864,7 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
         rbase       = isRgb ? bpp - 5 : 0;
         gbase       = 5;
         bbase       = isRgb ? 0 : (bpp - 5);
-        c->yuvTable = av_malloc(1024 * 3 * 2);
+        ALLOC_YUV_TABLE(1024 * 3 * 2);
         y_table16   = c->yuvTable;
         yb = -(384 << 16) - oy;
         for (i = 0; i < 1024; i++) {
@@ -880,7 +884,7 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
         break;
     case 24:
     case 48:
-        c->yuvTable = av_malloc(1024);
+        ALLOC_YUV_TABLE(1024);
         y_table     = c->yuvTable;
         yb = -(384 << 16) - oy;
         for (i = 0; i < 1024; i++) {
@@ -901,7 +905,7 @@ av_cold int ff_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4],
         needAlpha = CONFIG_SWSCALE_ALPHA && isALPHA(c->srcFormat);
         if (!needAlpha)
             abase = (base + 24) & 31;
-        c->yuvTable = av_malloc(1024 * 3 * 4);
+        ALLOC_YUV_TABLE(1024 * 3 * 4);
         y_table32   = c->yuvTable;
         yb = -(384 << 16) - oy;
         for (i = 0; i < 1024; i++) {