generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix integer overflow issue in rtc coding flow intra mode search

Browse Source 
The overflow issue affects a variable that is only used in inter
mode. This commit fixes the ioc warning triggered in the intra
mode. It does not affect the compression performance.

Change-Id: I593d1b5650599de07f3e68176dd1442c6cb7bdbc
This commit is contained in:
Jingning Han 2015-06-16 12:00:50 -07:00
parent e820ca6973
commit bc7074508a
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
vp9/encoder/vp9_pickmode.c
View File

@ -658,7 +658,8 @@ static void block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate, int64_t *dist,
block = 0;
*rate = 0;
*dist = 0;
*sse = (*sse << 6) >> shift;
if (*sse < INT64_MAX)
*sse = (*sse << 6) >> shift;
for (r = 0; r < max_blocks_high; r += block_step) {
for (c = 0; c < num_4x4_w; c += block_step) {
if (c < max_blocks_wide) {
@ -912,6 +913,8 @@ static void estimate_block_intra(int plane, int block, BLOCK_SIZE plane_bsize,
// TODO(jingning): This needs further refactoring.
block_yrd(cpi, x, &rate, &dist, &is_skippable, &this_sse, 0,
bsize_tx, MIN(tx_size, TX_16X16));
// this_sse is a dummy variable here. Its value should remain INT64_MAX.
assert(this_sse == INT64_MAX);
x->skip_txfm[0] = is_skippable;
rate += vp9_cost_bit(vp9_get_skip_prob(&cpi->common, xd), is_skippable);