generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fix uninitialized read in encode breakout

Browse Source 
Change I3430820 performed an uninitialized read when
encode_breakout == 0, since the sum and sse wouldn't be set:

   if(x->encode_breakout)
       VARIANCE_INVOKE(..., get16x16var)(..., &sum, &sse);
   if (cpi->active_map_enabled && x->active_ptr[0] == 0) {
       ...
   } else if (sse < x->encode_breakout)

Change-Id: I915eb76d1227b4b6d1137a0dedf2c143860098a2
This commit is contained in:
John Koleszar
2010-12-09 15:59:22 -05:00
parent c63fc881e1
commit cb9698951c
1 changed files with 24 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
vp8/encoder/rdopt.c
View File

@@ -1642,8 +1642,6 @@ int vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
//int intermodecost[MAX_MODES];
MB_PREDICTION_MODE uv_intra_mode;
int sse;
int sum;
int uvintra_eob = 0;
int tteob = 0;
int force_no_skip = 0;
@@ -2238,36 +2236,44 @@ int vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
vp8_set_mbmode_and_mvs(x, this_mode, &mode_mv[this_mode]);
vp8_build_inter_predictors_mby(&x->e_mbd);
if(x->encode_breakout)
VARIANCE_INVOKE(&cpi->rtcd.variance, get16x16var)(x->src.y_buffer, x->src.y_stride, x->e_mbd.predictor, 16, (unsigned int *)(&sse), &sum);
if (cpi->active_map_enabled && x->active_ptr[0] == 0) {
x->skip = 1;
}
else if (sse < x->encode_breakout)
else if (x->encode_breakout)
{
// Check u and v to make sure skip is ok
int sse2 = 0;
int sum, sse;
// add dc check
if (abs(sum) < (cpi->common.Y2dequant[0][0] << 2))
VARIANCE_INVOKE(&cpi->rtcd.variance, get16x16var)
(x->src.y_buffer, x->src.y_stride,
x->e_mbd.predictor, 16, (unsigned int *)(&sse), &sum);
if (sse < x->encode_breakout)
{
sse2 = VP8_UVSSE(x, IF_RTCD(&cpi->rtcd.variance));
// Check u and v to make sure skip is ok
int sse2 = 0;
if (sse2 * 2 < x->encode_breakout)
// add dc check
if (abs(sum) < (cpi->common.Y2dequant[0][0] << 2))
{
x->skip = 1;
distortion2 = sse;
rate2 = 500;
disable_skip = 1;
this_rd = RDFUNC(x->rdmult, x->rddiv, rate2,
distortion2, cpi->target_bits_per_mb);
sse2 = VP8_UVSSE(x, IF_RTCD(&cpi->rtcd.variance));
break;
if (sse2 * 2 < x->encode_breakout)
{
x->skip = 1;
distortion2 = sse;
rate2 = 500;
disable_skip = 1;
this_rd = RDFUNC(x->rdmult, x->rddiv, rate2,
distortion2, cpi->target_bits_per_mb);
break;
}
}
}
}
//intermodecost[mode_index] = vp8_cost_mv_ref(this_mode, mdcounts); // Experimental debug code
// Add in the Mv/mode cost