generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge "Rationalize type to avoid integer out of range"

Browse Source
This commit is contained in:
Yaowu Xu 2016-06-24 13:58:36 +00:00 committed by Gerrit Code Review
commit 26daa30da4
2 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
vp9/encoder/vp9_mcomp.c
View File

@ -8,6 +8,7 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#include <assert.h>
#include <limits.h> #include <limits.h>
#include <math.h> #include <math.h>
#include <stdio.h> #include <stdio.h>
@ -69,6 +70,8 @@ int vp9_init_search_range(int size) {
static INLINE int mv_cost(const MV *mv, static INLINE int mv_cost(const MV *mv,
const int *joint_cost, int *const comp_cost[2]) { const int *joint_cost, int *const comp_cost[2]) {
assert(mv->row >= -MV_MAX && mv->row < MV_MAX);
assert(mv->col >= -MV_MAX && mv->col < MV_MAX);
return joint_cost[vp9_get_mv_joint(mv)] + return joint_cost[vp9_get_mv_joint(mv)] +
comp_cost[0][mv->row] + comp_cost[1][mv->col]; comp_cost[0][mv->row] + comp_cost[1][mv->col];
} }
@ -445,7 +448,7 @@ uint32_t vp9_skip_sub_pixel_tree(
if ((abs(bestmv->col - ref_mv->col) > (MAX_FULL_PEL_VAL << 3)) || if ((abs(bestmv->col - ref_mv->col) > (MAX_FULL_PEL_VAL << 3)) ||
(abs(bestmv->row - ref_mv->row) > (MAX_FULL_PEL_VAL << 3))) (abs(bestmv->row - ref_mv->row) > (MAX_FULL_PEL_VAL << 3)))
return INT_MAX; return UINT32_MAX;
return besterr; return besterr;
} }
@ -599,7 +602,7 @@ uint32_t vp9_find_best_sub_pixel_tree_pruned_more(const MACROBLOCK *x,
if ((abs(bestmv->col - ref_mv->col) > (MAX_FULL_PEL_VAL << 3)) || if ((abs(bestmv->col - ref_mv->col) > (MAX_FULL_PEL_VAL << 3)) ||
(abs(bestmv->row - ref_mv->row) > (MAX_FULL_PEL_VAL << 3))) (abs(bestmv->row - ref_mv->row) > (MAX_FULL_PEL_VAL << 3)))
return INT_MAX; return UINT32_MAX;
return besterr; return besterr;
} }

12
vp9/encoder/vp9_rdopt.c
View File

@ -1594,7 +1594,7 @@ static void joint_motion_search(VP9_COMP *cpi, MACROBLOCK *x,
// Do joint motion search in compound mode to get more accurate mv. // Do joint motion search in compound mode to get more accurate mv.
struct buf_2d backup_yv12[2][MAX_MB_PLANE]; struct buf_2d backup_yv12[2][MAX_MB_PLANE];
int last_besterr[2] = {INT_MAX, INT_MAX}; uint32_t last_besterr[2] = {UINT32_MAX, UINT32_MAX};
const YV12_BUFFER_CONFIG *const scaled_ref_frame[2] = { const YV12_BUFFER_CONFIG *const scaled_ref_frame[2] = {
vp9_get_scaled_ref_frame(cpi, mi->ref_frame[0]), vp9_get_scaled_ref_frame(cpi, mi->ref_frame[0]),
vp9_get_scaled_ref_frame(cpi, mi->ref_frame[1]) vp9_get_scaled_ref_frame(cpi, mi->ref_frame[1])
@ -1640,7 +1640,7 @@ static void joint_motion_search(VP9_COMP *cpi, MACROBLOCK *x,
// and break out of the search loop if it couldn't find a better mv. // and break out of the search loop if it couldn't find a better mv.
for (ite = 0; ite < 4; ite++) { for (ite = 0; ite < 4; ite++) {
struct buf_2d ref_yv12[2]; struct buf_2d ref_yv12[2];
int bestsme = INT_MAX; uint32_t bestsme = UINT32_MAX;
int sadpb = x->sadperbit16; int sadpb = x->sadperbit16;
MV tmp_mv; MV tmp_mv;
int search_range = 3; int search_range = 3;
@ -1705,7 +1705,7 @@ static void joint_motion_search(VP9_COMP *cpi, MACROBLOCK *x,
search_range, search_range,
&cpi->fn_ptr[bsize], &cpi->fn_ptr[bsize],
&ref_mv[id].as_mv, second_pred); &ref_mv[id].as_mv, second_pred);
if (bestsme < INT_MAX) if (bestsme < UINT32_MAX)
bestsme = vp9_get_mvpred_av_var(x, &tmp_mv, &ref_mv[id].as_mv, bestsme = vp9_get_mvpred_av_var(x, &tmp_mv, &ref_mv[id].as_mv,
second_pred, &cpi->fn_ptr[bsize], 1); second_pred, &cpi->fn_ptr[bsize], 1);
@ -1714,7 +1714,7 @@ static void joint_motion_search(VP9_COMP *cpi, MACROBLOCK *x,
x->mv_row_min = tmp_row_min; x->mv_row_min = tmp_row_min;
x->mv_row_max = tmp_row_max; x->mv_row_max = tmp_row_max;
if (bestsme < INT_MAX) { if (bestsme < UINT32_MAX) {
uint32_t dis; /* TODO: use dis in distortion calculation later. */ uint32_t dis; /* TODO: use dis in distortion calculation later. */
uint32_t sse; uint32_t sse;
bestsme = cpi->find_fractional_mv_step( bestsme = cpi->find_fractional_mv_step(
@ -1860,7 +1860,7 @@ static int64_t rd_pick_best_sub8x8_mode(VP9_COMP *cpi, MACROBLOCK *x,
seg_mvs[i][mi->ref_frame[0]].as_int == INVALID_MV) { seg_mvs[i][mi->ref_frame[0]].as_int == INVALID_MV) {
MV *const new_mv = &mode_mv[NEWMV][0].as_mv; MV *const new_mv = &mode_mv[NEWMV][0].as_mv;
int step_param = 0; int step_param = 0;
int bestsme = INT_MAX; uint32_t bestsme = UINT32_MAX;
int sadpb = x->sadperbit4; int sadpb = x->sadperbit4;
MV mvp_full; MV mvp_full;
int max_mv; int max_mv;
@ -1915,7 +1915,7 @@ static int64_t rd_pick_best_sub8x8_mode(VP9_COMP *cpi, MACROBLOCK *x,
&bsi->ref_mv[0]->as_mv, new_mv, &bsi->ref_mv[0]->as_mv, new_mv,
INT_MAX, 1); INT_MAX, 1);
if (bestsme < INT_MAX) { if (bestsme < UINT32_MAX) {
uint32_t distortion; uint32_t distortion;
cpi->find_fractional_mv_step( cpi->find_fractional_mv_step(
x, x,