From b80020d4db1761cfb312de7236cb27363e7c32cd Mon Sep 17 00:00:00 2001 From: Deb Mukherjee Date: Fri, 28 Feb 2014 14:29:22 -0800 Subject: [PATCH] Refactoring motion search libs The core motion estimation fucntions all return sad now consistently. The only exception is vp9_full_pixel_diamond(), however the core diamond and refining search routines called from vp9_full_pixel_diamond() also return SAD. If variance of pred error + mv cost is desired it must be calculated explicitly outside these functions. For very fast encoding, hopefully this will eliminate some redundant computations. Also suggests reimplementing FAST_HEX with the vp9_pattern_search framework. It is not exactly the same as the existing FAST_HEX, but performance is slightly better and speed is very similar. Enables removing a lot of duplicate code. Change-Id: I152736393438c25bdf7e96b37cbb8ce330f4f94a --- vp9/encoder/vp9_firstpass.c | 4 + vp9/encoder/vp9_mcomp.c | 345 +++++++----------------------------- vp9/encoder/vp9_mcomp.h | 65 +++---- vp9/encoder/vp9_pickmode.c | 11 +- vp9/encoder/vp9_rdopt.c | 29 ++- 5 files changed, 133 insertions(+), 321 deletions(-) diff --git a/vp9/encoder/vp9_firstpass.c b/vp9/encoder/vp9_firstpass.c index 8e454e694..b507c6e7c 100644 --- a/vp9/encoder/vp9_firstpass.c +++ b/vp9/encoder/vp9_firstpass.c @@ -415,6 +415,8 @@ static void first_pass_motion_search(VP9_COMP *cpi, MACROBLOCK *x, x->sadperbit16, &num00, &v_fn_ptr, x->nmvjointcost, x->mvcost, ref_mv); + if (tmp_err < INT_MAX) + tmp_err = vp9_get_mvpred_var(x, &tmp_mv, ref_mv, &v_fn_ptr, 1); if (tmp_err < INT_MAX - new_mv_mode_penalty) tmp_err += new_mv_mode_penalty; @@ -439,6 +441,8 @@ static void first_pass_motion_search(VP9_COMP *cpi, MACROBLOCK *x, &num00, &v_fn_ptr, x->nmvjointcost, x->mvcost, ref_mv); + if (tmp_err < INT_MAX) + tmp_err = vp9_get_mvpred_var(x, &tmp_mv, ref_mv, &v_fn_ptr, 1); if (tmp_err < INT_MAX - new_mv_mode_penalty) tmp_err += new_mv_mode_penalty; diff --git a/vp9/encoder/vp9_mcomp.c b/vp9/encoder/vp9_mcomp.c index 7d6fd3b99..48d1e07f0 100644 --- a/vp9/encoder/vp9_mcomp.c +++ b/vp9/encoder/vp9_mcomp.c @@ -728,16 +728,52 @@ static int vp9_pattern_search(const MACROBLOCK *x, best_mv->col; this_mv.row = best_mv->row * 8; this_mv.col = best_mv->col * 8; - if (bestsad == INT_MAX) - return INT_MAX; - - return vfp->vf(what, what_stride, this_offset, in_what_stride, - (unsigned int *)&bestsad) + - use_mvcost ? mv_err_cost(&this_mv, center_mv, - x->nmvjointcost, x->mvcost, x->errorperbit) - : 0; + return bestsad; } +int vp9_get_mvpred_var(const MACROBLOCK *x, + MV *best_mv, + const MV *center_mv, + const vp9_variance_fn_ptr_t *vfp, + int use_mvcost) { + unsigned int bestsad; + MV this_mv; + const MACROBLOCKD *const xd = &x->e_mbd; + const uint8_t *what = x->plane[0].src.buf; + const int what_stride = x->plane[0].src.stride; + const int in_what_stride = xd->plane[0].pre[0].stride; + const uint8_t *base_offset = xd->plane[0].pre[0].buf; + const uint8_t *this_offset = base_offset + (best_mv->row * in_what_stride) + + best_mv->col; + this_mv.row = best_mv->row * 8; + this_mv.col = best_mv->col * 8; + return vfp->vf(what, what_stride, this_offset, in_what_stride, &bestsad) + + (use_mvcost ? mv_err_cost(&this_mv, center_mv, x->nmvjointcost, + x->mvcost, x->errorperbit) : 0); +} + +int vp9_get_mvpred_av_var(const MACROBLOCK *x, + MV *best_mv, + const MV *center_mv, + const uint8_t *second_pred, + const vp9_variance_fn_ptr_t *vfp, + int use_mvcost) { + unsigned int bestsad; + MV this_mv; + const MACROBLOCKD *const xd = &x->e_mbd; + const uint8_t *what = x->plane[0].src.buf; + const int what_stride = x->plane[0].src.stride; + const int in_what_stride = xd->plane[0].pre[0].stride; + const uint8_t *base_offset = xd->plane[0].pre[0].buf; + const uint8_t *this_offset = base_offset + (best_mv->row * in_what_stride) + + best_mv->col; + this_mv.row = best_mv->row * 8; + this_mv.col = best_mv->col * 8; + return vfp->svaf(this_offset, in_what_stride, 0, 0, what, what_stride, + &bestsad, second_pred) + + (use_mvcost ? mv_err_cost(&this_mv, center_mv, x->nmvjointcost, + x->mvcost, x->errorperbit) : 0); +} int vp9_hex_search(const MACROBLOCK *x, MV *ref_mv, @@ -855,182 +891,18 @@ int vp9_square_search(const MACROBLOCK *x, square_num_candidates, square_candidates); }; -// Number of candidates in first hex search -#define FIRST_HEX_CANDIDATES 6 -// Index of previous hex search's best match -#define PRE_BEST_CANDIDATE 6 -// Number of candidates in following hex search -#define NEXT_HEX_CANDIDATES 3 -// Number of candidates in refining search -#define REFINE_CANDIDATES 4 - int vp9_fast_hex_search(const MACROBLOCK *x, MV *ref_mv, int search_param, int sad_per_bit, + int do_init_search, // must be zero for fast_hex const vp9_variance_fn_ptr_t *vfp, int use_mvcost, const MV *center_mv, MV *best_mv) { - const MACROBLOCKD* const xd = &x->e_mbd; - static const MV hex[FIRST_HEX_CANDIDATES] = { - { -1, -2}, {1, -2}, {2, 0}, {1, 2}, { -1, 2}, { -2, 0} - }; - static const MV next_chkpts[PRE_BEST_CANDIDATE][NEXT_HEX_CANDIDATES] = { - {{ -2, 0}, { -1, -2}, {1, -2}}, - {{ -1, -2}, {1, -2}, {2, 0}}, - {{1, -2}, {2, 0}, {1, 2}}, - {{2, 0}, {1, 2}, { -1, 2}}, - {{1, 2}, { -1, 2}, { -2, 0}}, - {{ -1, 2}, { -2, 0}, { -1, -2}} - }; - static const MV neighbors[REFINE_CANDIDATES] = { - {0, -1}, { -1, 0}, {1, 0}, {0, 1} - }; - int i, j; - - const uint8_t *what = x->plane[0].src.buf; - const int what_stride = x->plane[0].src.stride; - const int in_what_stride = xd->plane[0].pre[0].stride; - int br, bc; - MV this_mv; - unsigned int bestsad = 0x7fffffff; - unsigned int thissad; - const uint8_t *base_offset; - const uint8_t *this_offset; - int k = -1; - int best_site = -1; - const int max_hex_search = 512; - const int max_dia_search = 32; - - const int *mvjsadcost = x->nmvjointsadcost; - int *mvsadcost[2] = {x->nmvsadcost[0], x->nmvsadcost[1]}; - - const MV fcenter_mv = {center_mv->row >> 3, center_mv->col >> 3}; - - // Adjust ref_mv to make sure it is within MV range - clamp_mv(ref_mv, x->mv_col_min, x->mv_col_max, x->mv_row_min, x->mv_row_max); - br = ref_mv->row; - bc = ref_mv->col; - - // Check the start point - base_offset = xd->plane[0].pre[0].buf; - this_offset = base_offset + (br * in_what_stride) + bc; - this_mv.row = br; - this_mv.col = bc; - bestsad = vfp->sdf(what, what_stride, this_offset, in_what_stride, 0x7fffffff) - + mvsad_err_cost(&this_mv, &fcenter_mv, mvjsadcost, mvsadcost, - sad_per_bit); - - // Initial 6-point hex search - if (check_bounds(x, br, bc, 2)) { - for (i = 0; i < FIRST_HEX_CANDIDATES; i++) { - this_mv.row = br + hex[i].row; - this_mv.col = bc + hex[i].col; - this_offset = base_offset + (this_mv.row * in_what_stride) + this_mv.col; - thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride, - bestsad); - CHECK_BETTER - } - } else { - for (i = 0; i < FIRST_HEX_CANDIDATES; i++) { - this_mv.row = br + hex[i].row; - this_mv.col = bc + hex[i].col; - if (!is_mv_in(x, &this_mv)) - continue; - this_offset = base_offset + (this_mv.row * in_what_stride) + this_mv.col; - thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride, - bestsad); - CHECK_BETTER - } - } - - // Continue hex search if we find a better match in first round - if (best_site != -1) { - br += hex[best_site].row; - bc += hex[best_site].col; - k = best_site; - - // Allow search covering maximum MV range - for (j = 1; j < max_hex_search; j++) { - best_site = -1; - - if (check_bounds(x, br, bc, 2)) { - for (i = 0; i < 3; i++) { - this_mv.row = br + next_chkpts[k][i].row; - this_mv.col = bc + next_chkpts[k][i].col; - this_offset = base_offset + (this_mv.row * in_what_stride) + - this_mv.col; - thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride, - bestsad); - CHECK_BETTER - } - } else { - for (i = 0; i < 3; i++) { - this_mv.row = br + next_chkpts[k][i].row; - this_mv.col = bc + next_chkpts[k][i].col; - if (!is_mv_in(x, &this_mv)) - continue; - this_offset = base_offset + (this_mv.row * in_what_stride) + - this_mv.col; - thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride, - bestsad); - CHECK_BETTER - } - } - - if (best_site == -1) { - break; - } else { - br += next_chkpts[k][best_site].row; - bc += next_chkpts[k][best_site].col; - k += 5 + best_site; - if (k >= 12) k -= 12; - else if (k >= 6) k -= 6; - } - } - } - - // Check 4 1-away neighbors - for (j = 0; j < max_dia_search; j++) { - best_site = -1; - - if (check_bounds(x, br, bc, 1)) { - for (i = 0; i < REFINE_CANDIDATES; i++) { - this_mv.row = br + neighbors[i].row; - this_mv.col = bc + neighbors[i].col; - this_offset = base_offset + (this_mv.row * in_what_stride) + - this_mv.col; - thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride, - bestsad); - CHECK_BETTER - } - } else { - for (i = 0; i < REFINE_CANDIDATES; i++) { - this_mv.row = br + neighbors[i].row; - this_mv.col = bc + neighbors[i].col; - if (!is_mv_in(x, &this_mv)) - continue; - this_offset = base_offset + (this_mv.row * in_what_stride) + - this_mv.col; - thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride, - bestsad); - CHECK_BETTER - } - } - - if (best_site == -1) { - break; - } else { - br += neighbors[best_site].row; - bc += neighbors[best_site].col; - } - } - - best_mv->row = br; - best_mv->col = bc; - - return bestsad; + return vp9_hex_search(x, ref_mv, MAX(MAX_MVSEARCH_STEPS - 2, search_param), + sad_per_bit, do_init_search, vfp, use_mvcost, + center_mv, best_mv); } #undef CHECK_BETTER @@ -1045,8 +917,6 @@ int vp9_full_range_search_c(const MACROBLOCK *x, MV *ref_mv, MV *best_mv, const int what_stride = x->plane[0].src.stride; const uint8_t *in_what; const int in_what_stride = xd->plane[0].pre[0].stride; - const uint8_t *best_address; - MV this_mv; unsigned int bestsad = INT_MAX; @@ -1076,7 +946,6 @@ int vp9_full_range_search_c(const MACROBLOCK *x, MV *ref_mv, MV *best_mv, // Work out the start point for the search in_what = xd->plane[0].pre[0].buf + ref_row * in_what_stride + ref_col; - best_address = in_what; // Check the starting position bestsad = fn_ptr->sdf(what, what_stride, in_what, in_what_stride, 0x7fffffff) @@ -1134,20 +1003,9 @@ int vp9_full_range_search_c(const MACROBLOCK *x, MV *ref_mv, MV *best_mv, } } } - best_mv->row += best_tr; best_mv->col += best_tc; - - this_mv.row = best_mv->row * 8; - this_mv.col = best_mv->col * 8; - - if (bestsad == INT_MAX) - return INT_MAX; - - return fn_ptr->vf(what, what_stride, best_address, in_what_stride, - (unsigned int *)(&thissad)) + - mv_err_cost(&this_mv, center_mv, - mvjcost, mvcost, x->errorperbit); + return bestsad; } int vp9_diamond_search_sad_c(const MACROBLOCK *x, @@ -1272,17 +1130,7 @@ int vp9_diamond_search_sad_c(const MACROBLOCK *x, (*num00)++; } } - - this_mv.row = best_mv->row * 8; - this_mv.col = best_mv->col * 8; - - if (bestsad == INT_MAX) - return INT_MAX; - - return fn_ptr->vf(what, what_stride, best_address, in_what_stride, - (unsigned int *)(&thissad)) + - mv_err_cost(&this_mv, center_mv, - mvjcost, mvcost, x->errorperbit); + return bestsad; } int vp9_diamond_search_sadx4(const MACROBLOCK *x, @@ -1448,24 +1296,14 @@ int vp9_diamond_search_sadx4(const MACROBLOCK *x, (*num00)++; } } - - this_mv.row = best_mv->row * 8; - this_mv.col = best_mv->col * 8; - - if (bestsad == INT_MAX) - return INT_MAX; - - return fn_ptr->vf(what, what_stride, best_address, in_what_stride, - (unsigned int *)(&thissad)) + - mv_err_cost(&this_mv, center_mv, - mvjcost, mvcost, x->errorperbit); + return bestsad; } /* do_refine: If last step (1-away) of n-step search doesn't pick the center point as the best match, we will do a final 1-away diamond refining search */ -int vp9_full_pixel_diamond(VP9_COMP *cpi, MACROBLOCK *x, +int vp9_full_pixel_diamond(const VP9_COMP *cpi, MACROBLOCK *x, MV *mvp_full, int step_param, int sadpb, int further_steps, int do_refine, const vp9_variance_fn_ptr_t *fn_ptr, @@ -1476,6 +1314,8 @@ int vp9_full_pixel_diamond(VP9_COMP *cpi, MACROBLOCK *x, step_param, sadpb, &n, fn_ptr, x->nmvjointcost, x->mvcost, ref_mv); + if (bestsme < INT_MAX) + bestsme = vp9_get_mvpred_var(x, &temp_mv, ref_mv, fn_ptr, 1); *dst_mv = temp_mv; // If there won't be more n-step search, check to see if refining search is @@ -1493,6 +1333,8 @@ int vp9_full_pixel_diamond(VP9_COMP *cpi, MACROBLOCK *x, step_param + n, sadpb, &num00, fn_ptr, x->nmvjointcost, x->mvcost, ref_mv); + if (thissme < INT_MAX) + thissme = vp9_get_mvpred_var(x, &temp_mv, ref_mv, fn_ptr, 1); // check to see if refining search is needed. if (num00 > further_steps - n) @@ -1512,12 +1354,13 @@ int vp9_full_pixel_diamond(VP9_COMP *cpi, MACROBLOCK *x, thissme = cpi->refining_search_sad(x, &best_mv, sadpb, search_range, fn_ptr, x->nmvjointcost, x->mvcost, ref_mv); + if (thissme < INT_MAX) + thissme = vp9_get_mvpred_var(x, &best_mv, ref_mv, fn_ptr, 1); if (thissme < bestsme) { bestsme = thissme; *dst_mv = best_mv; } } - return bestsme; } @@ -1562,15 +1405,7 @@ int vp9_full_search_sad_c(const MACROBLOCK *x, const MV *ref_mv, } } } - - if (best_sad < INT_MAX) { - unsigned int unused; - const MV mv = {best_mv->row * 8, best_mv->col * 8}; - return fn_ptr->vf(what, what_stride, best_address, in_what_stride, &unused) - + mv_err_cost(&mv, center_mv, mvjcost, mvcost, x->errorperbit); - } else { - return INT_MAX; - } + return best_sad; } int vp9_full_search_sadx3(const MACROBLOCK *x, const MV *ref_mv, @@ -1665,17 +1500,7 @@ int vp9_full_search_sadx3(const MACROBLOCK *x, const MV *ref_mv, c++; } } - - this_mv.row = best_mv->row * 8; - this_mv.col = best_mv->col * 8; - - if (bestsad < INT_MAX) - return fn_ptr->vf(what, what_stride, bestaddress, in_what_stride, - (unsigned int *)(&thissad)) + - mv_err_cost(&this_mv, center_mv, - mvjcost, mvcost, x->errorperbit); - else - return INT_MAX; + return bestsad; } int vp9_full_search_sadx8(const MACROBLOCK *x, const MV *ref_mv, @@ -1798,17 +1623,7 @@ int vp9_full_search_sadx8(const MACROBLOCK *x, const MV *ref_mv, c++; } } - - this_mv.row = best_mv->row * 8; - this_mv.col = best_mv->col * 8; - - if (bestsad < INT_MAX) - return fn_ptr->vf(what, what_stride, bestaddress, in_what_stride, - (unsigned int *)(&thissad)) + - mv_err_cost(&this_mv, center_mv, - mvjcost, mvcost, x->errorperbit); - else - return INT_MAX; + return bestsad; } int vp9_refining_search_sad_c(const MACROBLOCK *x, @@ -1866,16 +1681,7 @@ int vp9_refining_search_sad_c(const MACROBLOCK *x, best_address = &in_what[ref_mv->row * in_what_stride + ref_mv->col]; } } - - if (bestsad < INT_MAX) { - unsigned int unused; - const MV mv = {ref_mv->row * 8, ref_mv->col * 8}; - return fn_ptr->vf(what, what_stride, best_address, in_what_stride, - &unused) + - mv_err_cost(&mv, center_mv, mvjcost, mvcost, x->errorperbit); - } else { - return INT_MAX; - } + return bestsad; } int vp9_refining_search_sadx4(const MACROBLOCK *x, @@ -1977,17 +1783,7 @@ int vp9_refining_search_sadx4(const MACROBLOCK *x, neighbors[best_site].col; } } - - this_mv.row = ref_mv->row * 8; - this_mv.col = ref_mv->col * 8; - - if (bestsad < INT_MAX) - return fn_ptr->vf(what, what_stride, best_address, in_what_stride, - (unsigned int *)(&thissad)) + - mv_err_cost(&this_mv, center_mv, - mvjcost, mvcost, x->errorperbit); - else - return INT_MAX; + return bestsad; } // This function is called when we do joint motion search in comp_inter_inter @@ -2055,18 +1851,5 @@ int vp9_refining_search_8p_c(const MACROBLOCK *x, best_address = &in_what[ref_mv->row * in_what_stride + ref_mv->col]; } } - - this_mv.row = ref_mv->row * 8; - this_mv.col = ref_mv->col * 8; - - if (bestsad < INT_MAX) { - // FIXME(rbultje, yunqing): add full-pixel averaging variance functions - // so we don't have to use the subpixel with xoff=0,yoff=0 here. - return fn_ptr->svaf(best_address, in_what_stride, 0, 0, what, what_stride, - (unsigned int *)(&thissad), second_pred) + - mv_err_cost(&this_mv, center_mv, - mvjcost, mvcost, x->errorperbit); - } else { - return INT_MAX; - } + return bestsad; } diff --git a/vp9/encoder/vp9_mcomp.h b/vp9/encoder/vp9_mcomp.h index 586a74c9c..39360f1b4 100644 --- a/vp9/encoder/vp9_mcomp.h +++ b/vp9/encoder/vp9_mcomp.h @@ -35,6 +35,19 @@ extern "C" { void vp9_set_mv_search_range(MACROBLOCK *x, const MV *mv); int vp9_mv_bit_cost(const MV *mv, const MV *ref, const int *mvjcost, int *mvcost[2], int weight); + +// Utility to compute variance + MV rate cost for a given MV +int vp9_get_mvpred_var(const MACROBLOCK *x, + MV *best_mv, + const MV *center_mv, + const vp9_variance_fn_ptr_t *vfp, + int use_mvcost); +int vp9_get_mvpred_av_var(const MACROBLOCK *x, + MV *best_mv, + const MV *center_mv, + const uint8_t *second_pred, + const vp9_variance_fn_ptr_t *vfp, + int use_mvcost); void vp9_init_dsmotion_compensation(MACROBLOCK *x, int stride); void vp9_init3smotion_compensation(MACROBLOCK *x, int stride); @@ -42,47 +55,27 @@ struct VP9_COMP; int vp9_init_search_range(struct VP9_COMP *cpi, int size); // Runs sequence of diamond searches in smaller steps for RD -int vp9_full_pixel_diamond(struct VP9_COMP *cpi, MACROBLOCK *x, +int vp9_full_pixel_diamond(const struct VP9_COMP *cpi, MACROBLOCK *x, MV *mvp_full, int step_param, int sadpb, int further_steps, int do_refine, const vp9_variance_fn_ptr_t *fn_ptr, const MV *ref_mv, MV *dst_mv); -int vp9_hex_search(const MACROBLOCK *x, - MV *ref_mv, - int search_param, - int error_per_bit, - int do_init_search, - const vp9_variance_fn_ptr_t *vf, - int use_mvcost, - const MV *center_mv, - MV *best_mv); -int vp9_bigdia_search(const MACROBLOCK *x, - MV *ref_mv, - int search_param, - int error_per_bit, - int do_init_search, - const vp9_variance_fn_ptr_t *vf, - int use_mvcost, - const MV *center_mv, - MV *best_mv); -int vp9_square_search(const MACROBLOCK *x, - MV *ref_mv, - int search_param, - int error_per_bit, - int do_init_search, - const vp9_variance_fn_ptr_t *vf, - int use_mvcost, - const MV *center_mv, - MV *best_mv); -int vp9_fast_hex_search(const MACROBLOCK *x, - MV *ref_mv, - int search_param, - int sad_per_bit, - const vp9_variance_fn_ptr_t *vfp, - int use_mvcost, - const MV *center_mv, - MV *best_mv); +typedef int (integer_mv_pattern_search_fn) ( + const MACROBLOCK *x, + MV *ref_mv, + int search_param, + int error_per_bit, + int do_init_search, + const vp9_variance_fn_ptr_t *vf, + int use_mvcost, + const MV *center_mv, + MV *best_mv); + +integer_mv_pattern_search_fn vp9_hex_search; +integer_mv_pattern_search_fn vp9_bigdia_search; +integer_mv_pattern_search_fn vp9_square_search; +integer_mv_pattern_search_fn vp9_fast_hex_search; typedef int (fractional_mv_step_fp) ( const MACROBLOCK *x, diff --git a/vp9/encoder/vp9_pickmode.c b/vp9/encoder/vp9_pickmode.c index 9ba48a1b5..75122bc6f 100644 --- a/vp9/encoder/vp9_pickmode.c +++ b/vp9/encoder/vp9_pickmode.c @@ -34,7 +34,7 @@ static int full_pixel_motion_search(VP9_COMP *cpi, MACROBLOCK *x, MB_MODE_INFO *mbmi = &xd->mi_8x8[0]->mbmi; struct buf_2d backup_yv12[MAX_MB_PLANE] = {{0}}; int bestsme = INT_MAX; - int further_steps, step_param; + int step_param; int sadpb = x->sadperbit16; MV mvp_full; int ref = mbmi->ref_frame[0]; @@ -67,7 +67,6 @@ static int full_pixel_motion_search(VP9_COMP *cpi, MACROBLOCK *x, // TODO(jingning) exploiting adaptive motion search control in non-RD // mode decision too. step_param = 6; - further_steps = (cpi->sf.max_step_search_steps - 1) - step_param; for (i = LAST_FRAME; i <= LAST_FRAME && cpi->common.show_frame; ++i) { if ((x->pred_mv_sad[ref] >> 3) > x->pred_mv_sad[i]) { @@ -88,22 +87,28 @@ static int full_pixel_motion_search(VP9_COMP *cpi, MACROBLOCK *x, mvp_full.row >>= 3; if (cpi->sf.search_method == FAST_HEX) { - bestsme = vp9_fast_hex_search(x, &mvp_full, step_param, sadpb, + // NOTE: this returns SAD + bestsme = vp9_fast_hex_search(x, &mvp_full, step_param, sadpb, 0, &cpi->fn_ptr[bsize], 1, &ref_mv.as_mv, &tmp_mv->as_mv); } else if (cpi->sf.search_method == HEX) { + // NOTE: this returns SAD bestsme = vp9_hex_search(x, &mvp_full, step_param, sadpb, 1, &cpi->fn_ptr[bsize], 1, &ref_mv.as_mv, &tmp_mv->as_mv); } else if (cpi->sf.search_method == SQUARE) { + // NOTE: this returns SAD bestsme = vp9_square_search(x, &mvp_full, step_param, sadpb, 1, &cpi->fn_ptr[bsize], 1, &ref_mv.as_mv, &tmp_mv->as_mv); } else if (cpi->sf.search_method == BIGDIA) { + // NOTE: this returns SAD bestsme = vp9_bigdia_search(x, &mvp_full, step_param, sadpb, 1, &cpi->fn_ptr[bsize], 1, &ref_mv.as_mv, &tmp_mv->as_mv); } else { + int further_steps = (cpi->sf.max_step_search_steps - 1) - step_param; + // NOTE: this returns variance bestsme = vp9_full_pixel_diamond(cpi, x, &mvp_full, step_param, sadpb, further_steps, 1, &cpi->fn_ptr[bsize], diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index 810f8ba25..a6f54fd08 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c @@ -1848,18 +1848,30 @@ static void rd_check_segment_txsize(VP9_COMP *cpi, MACROBLOCK *x, sadpb, 1, v_fn_ptr, 1, &bsi->ref_mv[0]->as_mv, &new_mv->as_mv); + if (bestsme < INT_MAX) + bestsme = vp9_get_mvpred_var(x, &new_mv->as_mv, + &bsi->ref_mv[0]->as_mv, + v_fn_ptr, 1); } else if (cpi->sf.search_method == SQUARE) { bestsme = vp9_square_search(x, &mvp_full, step_param, sadpb, 1, v_fn_ptr, 1, &bsi->ref_mv[0]->as_mv, &new_mv->as_mv); + if (bestsme < INT_MAX) + bestsme = vp9_get_mvpred_var(x, &new_mv->as_mv, + &bsi->ref_mv[0]->as_mv, + v_fn_ptr, 1); } else if (cpi->sf.search_method == BIGDIA) { bestsme = vp9_bigdia_search(x, &mvp_full, step_param, sadpb, 1, v_fn_ptr, 1, &bsi->ref_mv[0]->as_mv, &new_mv->as_mv); + if (bestsme < INT_MAX) + bestsme = vp9_get_mvpred_var(x, &new_mv->as_mv, + &bsi->ref_mv[0]->as_mv, + v_fn_ptr, 1); } else { bestsme = vp9_full_pixel_diamond(cpi, x, &mvp_full, step_param, sadpb, further_steps, 0, v_fn_ptr, @@ -2470,21 +2482,33 @@ static void single_motion_search(VP9_COMP *cpi, MACROBLOCK *x, further_steps = (cpi->sf.max_step_search_steps - 1) - step_param; if (cpi->sf.search_method == FAST_HEX) { - bestsme = vp9_fast_hex_search(x, &mvp_full, step_param, sadpb, + bestsme = vp9_fast_hex_search(x, &mvp_full, step_param, sadpb, 0, &cpi->fn_ptr[bsize], 1, &ref_mv, &tmp_mv->as_mv); + if (bestsme < INT_MAX) + bestsme = vp9_get_mvpred_var(x, &tmp_mv->as_mv, &ref_mv, + &cpi->fn_ptr[bsize], 1); } else if (cpi->sf.search_method == HEX) { bestsme = vp9_hex_search(x, &mvp_full, step_param, sadpb, 1, &cpi->fn_ptr[bsize], 1, &ref_mv, &tmp_mv->as_mv); + if (bestsme < INT_MAX) + bestsme = vp9_get_mvpred_var(x, &tmp_mv->as_mv, &ref_mv, + &cpi->fn_ptr[bsize], 1); } else if (cpi->sf.search_method == SQUARE) { bestsme = vp9_square_search(x, &mvp_full, step_param, sadpb, 1, &cpi->fn_ptr[bsize], 1, &ref_mv, &tmp_mv->as_mv); + if (bestsme < INT_MAX) + bestsme = vp9_get_mvpred_var(x, &tmp_mv->as_mv, &ref_mv, + &cpi->fn_ptr[bsize], 1); } else if (cpi->sf.search_method == BIGDIA) { bestsme = vp9_bigdia_search(x, &mvp_full, step_param, sadpb, 1, &cpi->fn_ptr[bsize], 1, &ref_mv, &tmp_mv->as_mv); + if (bestsme < INT_MAX) + bestsme = vp9_get_mvpred_var(x, &tmp_mv->as_mv, &ref_mv, + &cpi->fn_ptr[bsize], 1); } else { bestsme = vp9_full_pixel_diamond(cpi, x, &mvp_full, step_param, sadpb, further_steps, 1, @@ -2610,6 +2634,9 @@ static void joint_motion_search(VP9_COMP *cpi, MACROBLOCK *x, x->nmvjointcost, x->mvcost, &ref_mv[id].as_mv, second_pred, pw, ph); + if (bestsme < INT_MAX) + bestsme = vp9_get_mvpred_av_var(x, &tmp_mv.as_mv, &ref_mv[id].as_mv, + second_pred, &cpi->fn_ptr[bsize], 1); x->mv_col_min = tmp_col_min; x->mv_col_max = tmp_col_max;