generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge "Fix integer overflow issue in rtc coding flow intra mode search"

Browse Source
This commit is contained in:
James Zern 2015-06-17 05:29:30 +00:00 committed by Gerrit Code Review
commit 0d51a97ae9
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
vp9/encoder/vp9_pickmode.c
View File

@ -658,7 +658,8 @@ static void block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate, int64_t *dist,
block = 0;
*rate = 0;
*dist = 0;
*sse = (*sse << 6) >> shift;
if (*sse < INT64_MAX)
*sse = (*sse << 6) >> shift;
for (r = 0; r < max_blocks_high; r += block_step) {
for (c = 0; c < num_4x4_w; c += block_step) {
if (c < max_blocks_wide) {
@ -912,6 +913,8 @@ static void estimate_block_intra(int plane, int block, BLOCK_SIZE plane_bsize,
// TODO(jingning): This needs further refactoring.
block_yrd(cpi, x, &rate, &dist, &is_skippable, &this_sse, 0,
bsize_tx, MIN(tx_size, TX_16X16));
// this_sse is a dummy variable here. Its value should remain INT64_MAX.
assert(this_sse == INT64_MAX);
x->skip_txfm[0] = is_skippable;
rate += vp9_cost_bit(vp9_get_skip_prob(&cpi->common, xd), is_skippable);