generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

change to avoid out-of-range computation

Browse Source 
Change-Id: Id5e31833a0ef40de9f64c2f5674af7083233bf14
This commit is contained in:
Yaowu Xu 2013-10-10 11:00:57 -07:00
parent 850a919640
commit b47cef056e
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
vp9/encoder/vp9_rdopt.c
View File

@ -3642,10 +3642,17 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
// values, which actually are bigger than this_rd itself. This can
// cause negative best_filter_rd[] values, which is obviously silly.
// Therefore, if filter_cache < ref, we do an adjusted calculation.
if (cpi->rd_filter_cache[i] >= ref)
if (cpi->rd_filter_cache[i] >= ref) {
adj_rd = this_rd + cpi->rd_filter_cache[i] - ref;
else // FIXME(rbultje) do this for comppred also
adj_rd = this_rd - (ref - cpi->rd_filter_cache[i]) * this_rd / ref;
} else {
// FIXME(rbultje) do this for comppsred also
//
// To prevent out-of-range computation in
// adj_rd = cpi->rd_filter_cache[i] * this_rd / ref
// cpi->rd_filter_cache[i] / ref is converted to a 256 based ratio.
int tmp = cpi->rd_filter_cache[i] * 256 / ref;
adj_rd = (this_rd * tmp) >> 8;
}
best_filter_rd[i] = MIN(best_filter_rd[i], adj_rd);
}
}