generic-library/ffmpeg
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

avcodec/motion_est: Fix mv_penalty table size

Browse Source 
Fixes out of array read

Found-by: Tyson Smith <twsmith@mozilla.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 5b4da8a38a5ed211df9504c85ce401c30af86b97)

Conflicts:

	libavcodec/motion_est.h
This commit is contained in:
Michael Niedermayer 2016-01-05 14:41:04 +01:00
parent 5b14f436c6
commit 2c58d73413
6 changed files with 18 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
libavcodec/ituh263enc.c
View File

@ -44,7 +44,7 @@
/** /**
* Table of number of bits a motion vector component needs. * Table of number of bits a motion vector component needs.
*/ */
static uint8_t mv_penalty[MAX_FCODE+1][MAX_MV*2+1]; static uint8_t mv_penalty[MAX_FCODE+1][MAX_DMV*2+1];
/** /**
* Minimal fcode that a motion vector component would need. * Minimal fcode that a motion vector component would need.
@ -677,7 +677,7 @@ static av_cold void init_mv_penalty_and_fcode(MpegEncContext *s)
int mv; int mv;
for(f_code=1; f_code<=MAX_FCODE; f_code++){ for(f_code=1; f_code<=MAX_FCODE; f_code++){
for(mv=-MAX_MV; mv<=MAX_MV; mv++){ for(mv=-MAX_DMV; mv<=MAX_DMV; mv++){
int len; int len;
if(mv==0) len= ff_mvtab[0][1]; if(mv==0) len= ff_mvtab[0][1];
@ -698,7 +698,7 @@ static av_cold void init_mv_penalty_and_fcode(MpegEncContext *s)
} }
} }
mv_penalty[f_code][mv+MAX_MV]= len; mv_penalty[f_code][mv+MAX_DMV]= len;
} }
} }

16
libavcodec/motion_est.c
View File

@ -908,7 +908,7 @@ void ff_estimate_p_frame_motion(MpegEncContext * s,
c->penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp); c->penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp);
c->sub_penalty_factor= get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_sub_cmp); c->sub_penalty_factor= get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_sub_cmp);
c->mb_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->mb_cmp); c->mb_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->mb_cmp);
c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV; c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV;
get_limits(s, 16*mb_x, 16*mb_y); get_limits(s, 16*mb_x, 16*mb_y);
c->skip=0; c->skip=0;
@ -1084,7 +1084,7 @@ int ff_pre_estimate_p_frame_motion(MpegEncContext * s,
av_assert0(s->quarter_sample==0 || s->quarter_sample==1); av_assert0(s->quarter_sample==0 || s->quarter_sample==1);
c->pre_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_pre_cmp); c->pre_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_pre_cmp);
c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV; c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV;
get_limits(s, 16*mb_x, 16*mb_y); get_limits(s, 16*mb_x, 16*mb_y);
c->skip=0; c->skip=0;
@ -1133,7 +1133,7 @@ static int estimate_motion_b(MpegEncContext *s, int mb_x, int mb_y,
const int shift= 1+s->quarter_sample; const int shift= 1+s->quarter_sample;
const int mot_stride = s->mb_stride; const int mot_stride = s->mb_stride;
const int mot_xy = mb_y*mot_stride + mb_x; const int mot_xy = mb_y*mot_stride + mb_x;
uint8_t * const mv_penalty= c->mv_penalty[f_code] + MAX_MV; uint8_t * const mv_penalty= c->mv_penalty[f_code] + MAX_DMV;
int mv_scale; int mv_scale;
c->penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp); c->penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp);
@ -1207,8 +1207,8 @@ static inline int check_bidir_mv(MpegEncContext * s,
//FIXME better f_code prediction (max mv & distance) //FIXME better f_code prediction (max mv & distance)
//FIXME pointers //FIXME pointers
MotionEstContext * const c= &s->me; MotionEstContext * const c= &s->me;
uint8_t * const mv_penalty_f= c->mv_penalty[s->f_code] + MAX_MV; // f_code of the prev frame uint8_t * const mv_penalty_f= c->mv_penalty[s->f_code] + MAX_DMV; // f_code of the prev frame
uint8_t * const mv_penalty_b= c->mv_penalty[s->b_code] + MAX_MV; // f_code of the prev frame uint8_t * const mv_penalty_b= c->mv_penalty[s->b_code] + MAX_DMV; // f_code of the prev frame
int stride= c->stride; int stride= c->stride;
uint8_t *dest_y = c->scratchpad; uint8_t *dest_y = c->scratchpad;
uint8_t *ptr; uint8_t *ptr;
@ -1421,7 +1421,7 @@ static inline int direct_search(MpegEncContext * s, int mb_x, int mb_y)
int mx, my, xmin, xmax, ymin, ymax; int mx, my, xmin, xmax, ymin, ymax;
int16_t (*mv_table)[2]= s->b_direct_mv_table; int16_t (*mv_table)[2]= s->b_direct_mv_table;
c->current_mv_penalty= c->mv_penalty[1] + MAX_MV; c->current_mv_penalty= c->mv_penalty[1] + MAX_DMV;
ymin= xmin=(-32)>>shift; ymin= xmin=(-32)>>shift;
ymax= xmax= 31>>shift; ymax= xmax= 31>>shift;
@ -1557,11 +1557,11 @@ void ff_estimate_b_frame_motion(MpegEncContext * s,
if (s->avctx->flags & CODEC_FLAG_INTERLACED_ME) { if (s->avctx->flags & CODEC_FLAG_INTERLACED_ME) {
//FIXME mb type penalty //FIXME mb type penalty
c->skip=0; c->skip=0;
c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV; c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_DMV;
fimin= interlaced_search(s, 0, fimin= interlaced_search(s, 0,
s->b_field_mv_table[0], s->b_field_select_table[0], s->b_field_mv_table[0], s->b_field_select_table[0],
s->b_forw_mv_table[xy][0], s->b_forw_mv_table[xy][1], 0); s->b_forw_mv_table[xy][0], s->b_forw_mv_table[xy][1], 0);
c->current_mv_penalty= c->mv_penalty[s->b_code] + MAX_MV; c->current_mv_penalty= c->mv_penalty[s->b_code] + MAX_DMV;
bimin= interlaced_search(s, 2, bimin= interlaced_search(s, 2,
s->b_field_mv_table[1], s->b_field_select_table[1], s->b_field_mv_table[1], s->b_field_select_table[1],
s->b_back_mv_table[xy][0], s->b_back_mv_table[xy][1], 0); s->b_back_mv_table[xy][0], s->b_back_mv_table[xy][1], 0);

3
libavcodec/motion_est.h
View File

@ -30,6 +30,7 @@
struct MpegEncContext; struct MpegEncContext;
#define MAX_MV 4096 #define MAX_MV 4096
#define MAX_DMV (2*MAX_MV)
/** /**
* Motion estimation context. * Motion estimation context.
@ -80,7 +81,7 @@ typedef struct MotionEstContext {
op_pixels_func(*hpel_avg)[4]; op_pixels_func(*hpel_avg)[4];
qpel_mc_func(*qpel_put)[16]; qpel_mc_func(*qpel_put)[16];
qpel_mc_func(*qpel_avg)[16]; qpel_mc_func(*qpel_avg)[16];
uint8_t (*mv_penalty)[MAX_MV * 2 + 1]; ///< bit amount needed to encode a MV uint8_t (*mv_penalty)[MAX_DMV * 2 + 1]; ///< bit amount needed to encode a MV
uint8_t *current_mv_penalty; uint8_t *current_mv_penalty;
int (*sub_motion_search)(struct MpegEncContext *s, int (*sub_motion_search)(struct MpegEncContext *s,
int *mx_ptr, int *my_ptr, int dmin, int *mx_ptr, int *my_ptr, int dmin,

6
libavcodec/mpeg12enc.c
View File

@ -52,7 +52,7 @@ static const uint8_t svcd_scan_offset_placeholder[] = {
0x81, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x81, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
}; };
static uint8_t mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1]; static uint8_t mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1];
static uint8_t fcode_tab[MAX_MV * 2 + 1]; static uint8_t fcode_tab[MAX_MV * 2 + 1];
static uint8_t uni_mpeg1_ac_vlc_len[64 * 64 * 2]; static uint8_t uni_mpeg1_ac_vlc_len[64 * 64 * 2];
@ -1051,7 +1051,7 @@ av_cold void ff_mpeg1_encode_init(MpegEncContext *s)
} }
for (f_code = 1; f_code <= MAX_FCODE; f_code++) for (f_code = 1; f_code <= MAX_FCODE; f_code++)
for (mv = -MAX_MV; mv <= MAX_MV; mv++) { for (mv = -MAX_DMV; mv <= MAX_DMV; mv++) {
int len; int len;
if (mv == 0) { if (mv == 0) {
@ -1074,7 +1074,7 @@ av_cold void ff_mpeg1_encode_init(MpegEncContext *s)
2 + bit_size; 2 + bit_size;
} }
mv_penalty[f_code][mv + MAX_MV] = len; mv_penalty[f_code][mv + MAX_DMV] = len;
} }

2
libavcodec/mpegvideo_enc.c
View File

@ -72,7 +72,7 @@ static int sse_mb(MpegEncContext *s);
static void denoise_dct_c(MpegEncContext *s, int16_t *block); static void denoise_dct_c(MpegEncContext *s, int16_t *block);
static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow); static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow);
static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1]; static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1];
static uint8_t default_fcode_tab[MAX_MV * 2 + 1]; static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
const AVOption ff_mpv_generic_options[] = { const AVOption ff_mpv_generic_options[] = {

2
libavcodec/snowenc.c
View File

@ -284,7 +284,7 @@ static int encode_q_branch(SnowContext *s, int level, int x, int y){
c->penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp); c->penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp);
c->sub_penalty_factor= get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_sub_cmp); c->sub_penalty_factor= get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_sub_cmp);
c->mb_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->mb_cmp); c->mb_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->mb_cmp);
c->current_mv_penalty= c->mv_penalty[s->m.f_code=1] + MAX_MV; c->current_mv_penalty= c->mv_penalty[s->m.f_code=1] + MAX_DMV;
c->xmin = - x*block_w - 16+3; c->xmin = - x*block_w - 16+3;
c->ymin = - y*block_w - 16+3; c->ymin = - y*block_w - 16+3;