diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c index c5a68a9e5..c66be39de 100644 --- a/vp10/encoder/encodeframe.c +++ b/vp10/encoder/encodeframe.c @@ -1654,6 +1654,7 @@ static void rd_pick_sb_modes(VP10_COMP *cpi, totalrate_nocoef, #endif // CONFIG_SUPERTX bsize, ctx, best_rd); + assert(*totalrate_nocoef >= 0); } } else { vp10_rd_pick_inter_mode_sub8x8(cpi, tile_data, x, mi_row, mi_col, rd_cost, @@ -1661,6 +1662,7 @@ static void rd_pick_sb_modes(VP10_COMP *cpi, totalrate_nocoef, #endif // CONFIG_SUPERTX bsize, ctx, best_rd); + assert(*totalrate_nocoef >= 0); } } diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c index 6543b339f..830761dbd 100644 --- a/vp10/encoder/rdopt.c +++ b/vp10/encoder/rdopt.c @@ -8333,11 +8333,12 @@ void vp10_rd_pick_inter_mode_sb(VP10_COMP *cpi, rd_cost->rate = rate2; #if CONFIG_SUPERTX - *returnrate_nocoef = rate2 - rate_y - rate_uv; - if (!disable_skip) { - *returnrate_nocoef -= vp10_cost_bit(vp10_get_skip_prob(cm, xd), - skippable || this_skip2); - } + if (x->skip && rate_y == INT_MAX) + *returnrate_nocoef = rate2; + else + *returnrate_nocoef = rate2 - rate_y - rate_uv; + *returnrate_nocoef -= vp10_cost_bit(vp10_get_skip_prob(cm, xd), + disable_skip || skippable || this_skip2); *returnrate_nocoef -= vp10_cost_bit(vp10_get_intra_inter_prob(cm, xd), mbmi->ref_frame[0] != INTRA_FRAME); #if CONFIG_OBMC