generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added checks to prevent rate/distortion overflow

Browse Source 
At speed 2, due to the threshold scheme used, it is possible the rate
and distortion assigned with INT_MAX value. The patch added checking
to prevent the INT_MAX value is used in further calculation of RD
scores. The patch also changed the assertion in rd_use_partition() to
be mirror similar assertion in rd_pick_partition().

Change-Id: Idb52c543cc1e10abdf6e6a5d6e9cb535a42214dc
This commit is contained in:
Yaowu Xu
2013-07-19 17:38:55 -07:00
parent f00a237a43
commit ea284d6281
1 changed files with 13 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
vp9/encoder/vp9_encodeframe.c
View File

@@ -1248,9 +1248,6 @@ static void rd_use_partition(VP9_COMP *cpi, MODE_INFO *m, TOKENEXTRA **tp,
case PARTITION_NONE: case PARTITION_NONE:
pick_sb_modes(cpi, mi_row, mi_col, &last_part_rate, &last_part_dist, pick_sb_modes(cpi, mi_row, mi_col, &last_part_rate, &last_part_dist,
bsize, get_block_context(x, bsize), INT64_MAX); bsize, get_block_context(x, bsize), INT64_MAX);
set_partition_seg_context(cm, xd, mi_row, mi_col);
pl = partition_plane_context(xd, bsize);
last_part_rate += x->partition_cost[pl][PARTITION_NONE];
break; break;
case PARTITION_HORZ: case PARTITION_HORZ:
*(get_sb_index(xd, subsize)) = 0; *(get_sb_index(xd, subsize)) = 0;
@@ -1267,9 +1264,6 @@ static void rd_use_partition(VP9_COMP *cpi, MODE_INFO *m, TOKENEXTRA **tp,
last_part_rate += rt; last_part_rate += rt;
last_part_dist += dt; last_part_dist += dt;
} }
set_partition_seg_context(cm, xd, mi_row, mi_col);
pl = partition_plane_context(xd, bsize);
last_part_rate += x->partition_cost[pl][PARTITION_HORZ];
break; break;
case PARTITION_VERT: case PARTITION_VERT:
*(get_sb_index(xd, subsize)) = 0; *(get_sb_index(xd, subsize)) = 0;
@@ -1286,9 +1280,6 @@ static void rd_use_partition(VP9_COMP *cpi, MODE_INFO *m, TOKENEXTRA **tp,
last_part_rate += rt; last_part_rate += rt;
last_part_dist += dt; last_part_dist += dt;
} }
set_partition_seg_context(cm, xd, mi_row, mi_col);
pl = partition_plane_context(xd, bsize);
last_part_rate += x->partition_cost[pl][PARTITION_VERT];
break; break;
case PARTITION_SPLIT: case PARTITION_SPLIT:
// Split partition. // Split partition.
@@ -1308,16 +1299,23 @@ static void rd_use_partition(VP9_COMP *cpi, MODE_INFO *m, TOKENEXTRA **tp,
rd_use_partition(cpi, m + jj * bss * mis + ii * bss, tp, mi_row + y_idx, rd_use_partition(cpi, m + jj * bss * mis + ii * bss, tp, mi_row + y_idx,
mi_col + x_idx, subsize, &rt, &dt, i != 3); mi_col + x_idx, subsize, &rt, &dt, i != 3);
if (rt == INT_MAX || dt == INT_MAX) {
last_part_rate = INT_MAX;
last_part_dist = INT_MAX;
break;
}
last_part_rate += rt; last_part_rate += rt;
last_part_dist += dt; last_part_dist += dt;
} }
set_partition_seg_context(cm, xd, mi_row, mi_col);
pl = partition_plane_context(xd, bsize);
last_part_rate += x->partition_cost[pl][PARTITION_SPLIT];
break; break;
default: default:
assert(0); assert(0);
} }
set_partition_seg_context(cm, xd, mi_row, mi_col);
pl = partition_plane_context(xd, bsize);
if (last_part_rate < INT_MAX)
last_part_rate += x->partition_cost[pl][partition];
if (cpi->sf.adjust_partitioning_from_last_frame if (cpi->sf.adjust_partitioning_from_last_frame
&& partition != PARTITION_SPLIT && bsize > BLOCK_SIZE_SB8X8 && partition != PARTITION_SPLIT && bsize > BLOCK_SIZE_SB8X8
&& (mi_row + ms < cm->mi_rows || mi_row + (ms >> 1) == cm->mi_rows) && (mi_row + ms < cm->mi_rows || mi_row + (ms >> 1) == cm->mi_rows)
@@ -1392,10 +1390,12 @@ static void rd_use_partition(VP9_COMP *cpi, MODE_INFO *m, TOKENEXTRA **tp,
// We must have chosen a partitioning and encoding or we'll fail later on. // We must have chosen a partitioning and encoding or we'll fail later on.
// No other opportunities for success. // No other opportunities for success.
if ( bsize == BLOCK_SIZE_SB64X64)
assert(chosen_rate < INT_MAX && chosen_dist < INT_MAX); assert(chosen_rate < INT_MAX && chosen_dist < INT_MAX);
if (do_recon) if (do_recon)
encode_sb(cpi, tp, mi_row, mi_col, bsize == BLOCK_SIZE_SB64X64, bsize); encode_sb(cpi, tp, mi_row, mi_col, bsize == BLOCK_SIZE_SB64X64, bsize);
*rate = chosen_rate; *rate = chosen_rate;
*dist = chosen_dist; *dist = chosen_dist;
} }