generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix cost array overrun bug in search_bilateral_level()

Browse Source 
Change-Id: I9b7b727cb45328040a82477a2ce8eb3d15feec77
This commit is contained in:
Jack Haughton 2015-06-09 14:21:55 +01:00
parent 5a69abc66b
commit fa4b1ca714
1 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
vp9/encoder/vp9_picklpf.c
View File

@ -91,7 +91,7 @@ static int search_bilateral_level(const YV12_BUFFER_CONFIG *sd,
VP9_COMMON *const cm = &cpi->common;
int i, bilateral_best, err;
double best_cost;
double cost[BILATERAL_LEVELS_KF];
double cost;
const int bilateral_level_bits = vp9_bilateral_level_bits(&cpi->common);
const int bilateral_levels = 1 << bilateral_level_bits;
#ifdef USE_RD_LOOP_POSTFILTER_SEARCH
@ -109,11 +109,11 @@ static int search_bilateral_level(const YV12_BUFFER_CONFIG *sd,
err = try_bilateral_frame(sd, cpi, 0, partial_frame);
#ifdef USE_RD_LOOP_POSTFILTER_SEARCH
bits = cm->lf.last_bilateral_level == 0 ? 0 : bilateral_level_bits;
cost[0] = RDCOST_DBL(x->rdmult, x->rddiv, (bits << 2), err);
cost = RDCOST_DBL(x->rdmult, x->rddiv, (bits << 2), err);
#else
cost[0] = (double)err;
cost = (double)err;
#endif
best_cost = cost[0];
best_cost = cost;
for (i = 1; i <= bilateral_levels; ++i) {
err = try_bilateral_frame(sd, cpi, i, partial_frame);
#ifdef USE_RD_LOOP_POSTFILTER_SEARCH
@ -121,13 +121,13 @@ static int search_bilateral_level(const YV12_BUFFER_CONFIG *sd,
// when RDCOST is used. However below we just scale both in the correct
// ratios appropriately but not exactly by these values.
bits = cm->lf.last_bilateral_level == i ? 0 : bilateral_level_bits;
cost[i] = RDCOST_DBL(x->rdmult, x->rddiv, (bits << 2), err);
cost = RDCOST_DBL(x->rdmult, x->rddiv, (bits << 2), err);
#else
cost[i] = (double)err;
cost = (double)err;
#endif
if (cost[i] < best_cost) {
if (cost < best_cost) {
bilateral_best = i;
best_cost = cost[i];
best_cost = cost;
}
}
if (best_cost_ret) *best_cost_ret = best_cost;