From f3dfa0c36a64a52655096f5c1a5e33c99a4eb0bd Mon Sep 17 00:00:00 2001 From: Debargha Mukherjee Date: Thu, 23 Jun 2016 14:49:00 -0700 Subject: [PATCH] Quantization fix for new-quant/var-tx Also use the fp quantizer consistently lowres: -0.07 BDRATE improvement Change-Id: I9174f6ad54a74d38541004b99cb3689d0c09be55 --- vp10/encoder/encodemb.c | 24 ++++++++++++------------ vp10/encoder/rdopt.c | 30 ++++++++++++++---------------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/vp10/encoder/encodemb.c b/vp10/encoder/encodemb.c index 01e72c604..fcacf3a09 100644 --- a/vp10/encoder/encodemb.c +++ b/vp10/encoder/encodemb.c @@ -510,7 +510,7 @@ void vp10_xform_quant_nuq(MACROBLOCK *x, int plane, int block, int blk_row, fwd_txfm_param.tx_type = tx_type; fwd_txfm_param.tx_size = tx_size; - fwd_txfm_param.fwd_txfm_opt = fwd_txfm_opt_list[VP10_XFORM_QUANT_B]; + fwd_txfm_param.fwd_txfm_opt = fwd_txfm_opt_list[VP10_XFORM_QUANT_FP]; fwd_txfm_param.rd_transform = x->use_lp32x32fdct; fwd_txfm_param.lossless = xd->lossless[xd->mi[0]->mbmi.segment_id]; @@ -1020,8 +1020,8 @@ static void encode_block(int plane, int block, int blk_row, int blk_col, if (x->skip_txfm[plane][blk_index] == SKIP_TXFM_NONE) { // full forward transform and quantization #if CONFIG_NEW_QUANT - vp10_xform_quant_nuq(x, plane, block, blk_row, blk_col, plane_bsize, - tx_size, ctx); + vp10_xform_quant_fp_nuq(x, plane, block, blk_row, blk_col, + plane_bsize, tx_size, ctx); #else vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size, VP10_XFORM_QUANT_FP); @@ -1029,8 +1029,8 @@ static void encode_block(int plane, int block, int blk_row, int blk_col, } else if (x->skip_txfm[plane][blk_index] == SKIP_TXFM_AC_ONLY) { // fast path forward transform and quantization #if CONFIG_NEW_QUANT - vp10_xform_quant_dc_nuq(x, plane, block, blk_row, blk_col, - plane_bsize, tx_size, ctx); + vp10_xform_quant_dc_fp_nuq(x, plane, block, blk_row, blk_col, + plane_bsize, tx_size, ctx); #else vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size, VP10_XFORM_QUANT_DC); @@ -1045,8 +1045,8 @@ static void encode_block(int plane, int block, int blk_row, int blk_col, } } else { #if CONFIG_NEW_QUANT - vp10_xform_quant_nuq(x, plane, block, blk_row, blk_col, plane_bsize, - tx_size, ctx); + vp10_xform_quant_fp_nuq(x, plane, block, blk_row, blk_col, plane_bsize, + tx_size, ctx); #else vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size, VP10_XFORM_QUANT_FP); @@ -1163,11 +1163,11 @@ static void encode_block_pass1(int plane, int block, int blk_row, int blk_col, #if CONFIG_NEW_QUANT ctx = 0; - vp10_xform_quant_nuq(x, plane, block, blk_row, blk_col, plane_bsize, - tx_size, ctx); + vp10_xform_quant_fp_nuq(x, plane, block, blk_row, blk_col, plane_bsize, + tx_size, ctx); #else vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, - tx_size, VP10_XFORM_QUANT_B); + tx_size, VP10_XFORM_QUANT_FP); #endif // CONFIG_NEW_QUANT if (p->eobs[block] > 0) { @@ -1340,8 +1340,8 @@ void vp10_encode_block_intra(int plane, int block, int blk_row, int blk_col, if (args->enable_optimize_b) { #if CONFIG_NEW_QUANT - vp10_xform_quant_nuq(x, plane, block, blk_row, blk_col, plane_bsize, - tx_size, ctx); + vp10_xform_quant_fp_nuq(x, plane, block, blk_row, blk_col, plane_bsize, + tx_size, ctx); #else // CONFIG_NEW_QUANT vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size, VP10_XFORM_QUANT_FP); diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c index 708e8fc21..e0c79bce3 100644 --- a/vp10/encoder/rdopt.c +++ b/vp10/encoder/rdopt.c @@ -1251,8 +1251,8 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col, SKIP_TXFM_NONE) { // full forward transform and quantization #if CONFIG_NEW_QUANT - vp10_xform_quant_nuq(x, plane, block, blk_row, blk_col, - plane_bsize, tx_size, ctx); + vp10_xform_quant_fp_nuq(x, plane, block, blk_row, blk_col, + plane_bsize, tx_size, ctx); #else vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size, VP10_XFORM_QUANT_FP); @@ -1267,12 +1267,8 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col, tran_low_t *const coeff = BLOCK_OFFSET(x->plane[plane].coeff, block); tran_low_t *const dqcoeff = BLOCK_OFFSET(xd->plane[plane].dqcoeff, block); #if CONFIG_NEW_QUANT - if (x->quant_fp) - vp10_xform_quant_dc_fp_nuq(x, plane, block, blk_row, blk_col, - plane_bsize, tx_size, ctx); - else - vp10_xform_quant_dc_nuq(x, plane, block, blk_row, blk_col, - plane_bsize, tx_size, ctx); + vp10_xform_quant_dc_fp_nuq(x, plane, block, blk_row, blk_col, + plane_bsize, tx_size, ctx); #else vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size, VP10_XFORM_QUANT_DC); @@ -1303,12 +1299,8 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col, } else { // full forward transform and quantization #if CONFIG_NEW_QUANT - if (x->quant_fp) - vp10_xform_quant_fp_nuq(x, plane, block, blk_row, blk_col, plane_bsize, - tx_size, ctx); - else - vp10_xform_quant_nuq(x, plane, block, blk_row, blk_col, plane_bsize, - tx_size, ctx); + vp10_xform_quant_fp_nuq(x, plane, block, blk_row, blk_col, plane_bsize, + tx_size, ctx); #else vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size, VP10_XFORM_QUANT_FP); @@ -3064,8 +3056,14 @@ void vp10_tx_block_rd_b(const VP10_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, if (xd->mb_to_right_edge < 0) max_blocks_wide += xd->mb_to_right_edge >> (5 + pd->subsampling_x); - vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size, - VP10_XFORM_QUANT_FP); +#if CONFIG_NEW_QUANT + vp10_xform_quant_fp_nuq(x, plane, block, blk_row, blk_col, + plane_bsize, tx_size, coeff_ctx); +#else + vp10_xform_quant(x, plane, block, blk_row, blk_col, + plane_bsize, tx_size, VP10_XFORM_QUANT_FP); +#endif // CONFIG_NEW_QUANT + vp10_optimize_b(x, plane, block, tx_size, coeff_ctx); // TODO(any): Use dist_block to compute distortion