generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Various cosmetics on the new_quant experiment

Browse Source 
Also extends quant profiles to include quality range.

Change-Id: Ia96e45b6425e1d42ca61fc401f63d4fd7214e448
This commit is contained in:
Debargha Mukherjee
2016-06-29 13:18:52 -07:00
parent d5ddb56721
commit a35597fc7f
7 changed files with 135 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
vp10/common/blockd.h
View File

@@ -31,14 +31,6 @@
extern "C" { extern "C" {
#endif #endif
#if CONFIG_NEW_QUANT
#define QUANT_PROFILES 3
static INLINE int get_dq_profile_from_ctx(int q_ctx) {
return VPXMIN(q_ctx, QUANT_PROFILES - 1);
}
#endif // CONFIG_NEW_QUANT
#define MAX_MB_PLANE 3 #define MAX_MB_PLANE 3
typedef enum { typedef enum {

158
vp10/common/quant_common.c
View File

@@ -15,94 +15,112 @@
#include "vp10/common/blockd.h" #include "vp10/common/blockd.h"
#if CONFIG_NEW_QUANT #if CONFIG_NEW_QUANT
// Bin widths expressed as a fraction over 128 of the quant stepsize, // Bin widths expressed as a fraction over 128 of the quant stepsize,
// for the quantization bins 0-4. // for the quantization bins 0-4.
// So a value x indicates the bin is actually factor x/128 of the // So a value x indicates the bin is actually factor x/128 of the
// nominal quantization step. For the zero bin, the width is only // nominal quantization step. For the zero bin, the width is only
// for one side of zero, so the actual width is twice that. // for one side of zero, so the actual width is twice that.
// There are four sets of values for 4 different quantizer ranges.
// //
// Functions with nuq correspond to "non uniform quantization" // Functions with nuq correspond to "non uniform quantization"
// TODO(debargha): Optimize these tables // TODO(sarahparker, debargha): Optimize these tables
static const uint8_t nuq_knots_lossless[COEF_BANDS][NUQ_KNOTS] = {
{64, 128, 128}, // dc, band 0 typedef struct {
{64, 128, 128}, // band 1 uint8_t knots[NUQ_KNOTS]; // offsets
{64, 128, 128}, // band 2 uint8_t doff; // dequantization
{64, 128, 128}, // band 3 } qprofile_type;
{64, 128, 128}, // band 4
{64, 128, 128}, // band 5 static const qprofile_type nuq_lossless[COEF_BANDS] = {
{{64, 128, 128}, 0}, // dc, band 0
{{64, 128, 128}, 0}, // band 1
{{64, 128, 128}, 0}, // band 2
{{64, 128, 128}, 0}, // band 3
{{64, 128, 128}, 0}, // band 4
{{64, 128, 128}, 0}, // band 5
}; };
static const uint8_t nuq_knots[QUANT_PROFILES][COEF_BANDS][NUQ_KNOTS] = { static const qprofile_type nuq[QUANT_PROFILES][QUANT_RANGES][COEF_BANDS] = {
{ {
{91, 133, 139}, // dc, band 0 {
{78, 122, 134}, // band 1 {{91, 133, 139}, 11}, // dc, band 0
{83, 127, 139}, // band 2 {{78, 122, 134}, 12}, // band 1
{84, 117, 128}, // band 3 {{83, 127, 139}, 22}, // band 2
{88, 117, 129}, // band 4 {{84, 117, 128}, 18}, // band 3
{93, 122, 134}, // band 5 {{88, 117, 129}, 20}, // band 4
{{93, 122, 134}, 21} // band 5
}, {
{{91, 133, 139}, 11}, // dc, band 0
{{78, 122, 134}, 12}, // band 1
{{83, 127, 139}, 22}, // band 2
{{84, 117, 128}, 18}, // band 3
{{88, 117, 129}, 20}, // band 4
{{93, 122, 134}, 21} // band 5
}
}, },
#if QUANT_PROFILES > 1 #if QUANT_PROFILES > 1
{ {
{86, 122, 134}, // dc, band 0 {
{78, 122, 134}, // band 1 {{86, 122, 134}, 6}, // dc, band 0
{78, 122, 134}, // band 2 {{78, 122, 134}, 15}, // band 1
{84, 122, 134}, // band 3 {{78, 122, 134}, 17}, // band 2
{88, 122, 134}, // band 4 {{84, 122, 134}, 22}, // band 3
{88, 122, 134}, // band 5 {{88, 122, 134}, 23}, // band 4
{{88, 122, 134}, 23} // band 5
}, {
{{86, 122, 134}, 6}, // dc, band 0
{{78, 122, 134}, 15}, // band 1
{{78, 122, 134}, 17}, // band 2
{{84, 122, 134}, 22}, // band 3
{{88, 122, 134}, 23}, // band 4
{{88, 122, 134}, 23} // band 5
}
}, },
#if QUANT_PROFILES > 2 #if QUANT_PROFILES > 2
{ {
{86, 122, 134}, // dc, band 0 {
{78, 122, 135}, // band 1 {{86, 122, 134}, 6}, // dc, band 0
{78, 122, 134}, // band 2 {{78, 122, 135}, 14}, // band 1
{84, 122, 133}, // band 3 {{78, 122, 134}, 16}, // band 2
{88, 122, 134}, // band 4 {{84, 122, 133}, 22}, // band 3
{88, 122, 134}, // band 5 {{88, 122, 134}, 23}, // band 4
{{88, 122, 134}, 27}, // band 5
}, {
{{86, 122, 134}, 6}, // dc, band 0
{{78, 122, 135}, 14}, // band 1
{{78, 122, 134}, 16}, // band 2
{{84, 122, 133}, 22}, // band 3
{{88, 122, 134}, 23}, // band 4
{{88, 122, 134}, 27}, // band 5
}
} }
#endif // QUANT_PROFILES > 2 #endif // QUANT_PROFILES > 2
#endif // QUANT_PROFILES > 1 #endif // QUANT_PROFILES > 1
}; };
// dequantization offsets static INLINE int qrange_from_qindex(int qindex) {
static const uint8_t nuq_doff_lossless[COEF_BANDS] = {0, 0, 0, 0, 0, 0}; // return high quality (1) or low quality (0)
return qindex < 140 ? 1 : 0;
#if QUANT_PROFILES == 1
static const uint8_t nuq_doff[QUANT_PROFILES][COEF_BANDS] = {
{11, 12, 22, 18, 20, 21} // dq_off_index = 0
};
#elif QUANT_PROFILES == 2
static const uint8_t nuq_doff[QUANT_PROFILES][COEF_BANDS] = {
{11, 12, 22, 18, 20, 21}, // dq_off_index = 0
{13, 20, 21, 27, 28, 29} // dq_off_index = 1
};
#else // QUANT_PROFILES == 3
static const uint8_t nuq_doff[QUANT_PROFILES][COEF_BANDS] = {
{11, 12, 22, 18, 20, 21}, // dq_off_index = 0
{6, 15, 17, 22, 23, 23}, // dq_off_index = 1
{6, 14, 16, 22, 23, 27} // dq_off_index = 2
};
#endif // QUANT_PROFILES == x
static const uint8_t *get_nuq_knots(int lossless, int band, int dq_off_index) {
if (lossless)
return nuq_knots_lossless[band];
else
return nuq_knots[dq_off_index][band];
} }
static INLINE int16_t quant_to_doff_fixed(int lossless, int band, static const uint8_t *get_nuq_knots(int qindex, int band, int q_profile) {
int dq_off_index) { if (!qindex)
if (lossless) return nuq_lossless[band].knots;
return nuq_doff_lossless[band];
else else
return nuq_doff[dq_off_index][band]; return nuq[q_profile][qrange_from_qindex(qindex)][band].knots;
}
static INLINE int16_t quant_to_doff_fixed(int qindex, int band,
int q_profile) {
if (!qindex)
return nuq_lossless[band].doff;
else
return nuq[q_profile][qrange_from_qindex(qindex)][band].doff;
} }
// get cumulative bins // get cumulative bins
static INLINE void get_cuml_bins_nuq(int q, int lossless, int band, static INLINE void get_cuml_bins_nuq(int q, int qindex, int band,
tran_low_t *cuml_bins, int dq_off_index) { tran_low_t *cuml_bins, int q_profile) {
const uint8_t *knots = get_nuq_knots(lossless, band, dq_off_index); const uint8_t *knots = get_nuq_knots(qindex, band, q_profile);
int16_t cuml_knots[NUQ_KNOTS]; int16_t cuml_knots[NUQ_KNOTS];
int i; int i;
cuml_knots[0] = knots[0]; cuml_knots[0] = knots[0];
@@ -112,36 +130,36 @@ static INLINE void get_cuml_bins_nuq(int q, int lossless, int band,
cuml_bins[i] = ROUND_POWER_OF_TWO(cuml_knots[i] * q, 7); cuml_bins[i] = ROUND_POWER_OF_TWO(cuml_knots[i] * q, 7);
} }
void get_dequant_val_nuq(int q, int lossless, int band, void vp10_get_dequant_val_nuq(int q, int qindex, int band,
tran_low_t *dq, tran_low_t *cuml_bins, tran_low_t *dq, tran_low_t *cuml_bins,
int dq_off_index) { int q_profile) {
const uint8_t *knots = get_nuq_knots(lossless, band, dq_off_index); const uint8_t *knots = get_nuq_knots(qindex, band, q_profile);
tran_low_t cuml_bins_[NUQ_KNOTS], *cuml_bins_ptr; tran_low_t cuml_bins_[NUQ_KNOTS], *cuml_bins_ptr;
tran_low_t doff; tran_low_t doff;
int i; int i;
cuml_bins_ptr = (cuml_bins ? cuml_bins : cuml_bins_); cuml_bins_ptr = (cuml_bins ? cuml_bins : cuml_bins_);
get_cuml_bins_nuq(q, lossless, band, cuml_bins_ptr, dq_off_index); get_cuml_bins_nuq(q, qindex, band, cuml_bins_ptr, q_profile);
dq[0] = 0; dq[0] = 0;
for (i = 1; i < NUQ_KNOTS; ++i) { for (i = 1; i < NUQ_KNOTS; ++i) {
doff = quant_to_doff_fixed(lossless, band, dq_off_index); doff = quant_to_doff_fixed(qindex, band, q_profile);
doff = ROUND_POWER_OF_TWO(doff * knots[i], 7); doff = ROUND_POWER_OF_TWO(doff * knots[i], 7);
dq[i] = cuml_bins_ptr[i - 1] + dq[i] = cuml_bins_ptr[i - 1] +
ROUND_POWER_OF_TWO((knots[i] - doff * 2) * q, 8); ROUND_POWER_OF_TWO((knots[i] - doff * 2) * q, 8);
} }
doff = quant_to_doff_fixed(lossless, band, dq_off_index); doff = quant_to_doff_fixed(qindex, band, q_profile);
dq[NUQ_KNOTS] = dq[NUQ_KNOTS] =
cuml_bins_ptr[NUQ_KNOTS - 1] + ROUND_POWER_OF_TWO((64 - doff) * q, 7); cuml_bins_ptr[NUQ_KNOTS - 1] + ROUND_POWER_OF_TWO((64 - doff) * q, 7);
} }
tran_low_t dequant_abscoeff_nuq(int v, int q, const tran_low_t *dq) { tran_low_t vp10_dequant_abscoeff_nuq(int v, int q, const tran_low_t *dq) {
if (v <= NUQ_KNOTS) if (v <= NUQ_KNOTS)
return dq[v]; return dq[v];
else else
return dq[NUQ_KNOTS] + (v - NUQ_KNOTS) * q; return dq[NUQ_KNOTS] + (v - NUQ_KNOTS) * q;
} }
tran_low_t dequant_coeff_nuq(int v, int q, const tran_low_t *dq) { tran_low_t vp10_dequant_coeff_nuq(int v, int q, const tran_low_t *dq) {
tran_low_t dqmag = dequant_abscoeff_nuq(abs(v), q, dq); tran_low_t dqmag = vp10_dequant_abscoeff_nuq(abs(v), q, dq);
return (v < 0 ? -dqmag : dqmag); return (v < 0 ? -dqmag : dqmag);
} }
#endif // CONFIG_NEW_QUANT #endif // CONFIG_NEW_QUANT

14
vp10/common/quant_common.h
View File

@@ -30,14 +30,22 @@ int vp10_get_qindex(const struct segmentation *seg, int segment_id,
int base_qindex); int base_qindex);
#if CONFIG_NEW_QUANT #if CONFIG_NEW_QUANT
#define QUANT_PROFILES 3
#define QUANT_RANGES 2
#define NUQ_KNOTS 3 #define NUQ_KNOTS 3
typedef tran_low_t dequant_val_type_nuq[NUQ_KNOTS + 1]; typedef tran_low_t dequant_val_type_nuq[NUQ_KNOTS + 1];
typedef tran_low_t cuml_bins_type_nuq[NUQ_KNOTS]; typedef tran_low_t cuml_bins_type_nuq[NUQ_KNOTS];
void get_dequant_val_nuq(int q, int lossless, int band, void vp10_get_dequant_val_nuq(int q, int qindex, int band,
tran_low_t *dq, tran_low_t *cuml_bins, tran_low_t *dq, tran_low_t *cuml_bins,
int dq_off_index); int dq_off_index);
tran_low_t dequant_abscoeff_nuq(int v, int q, const tran_low_t *dq); tran_low_t vp10_dequant_abscoeff_nuq(int v, int q, const tran_low_t *dq);
tran_low_t dequant_coeff_nuq(int v, int q, const tran_low_t *dq); tran_low_t vp10_dequant_coeff_nuq(int v, int q, const tran_low_t *dq);
static INLINE int get_dq_profile_from_ctx(int q_ctx) {
return VPXMIN(q_ctx, QUANT_PROFILES - 1);
}
#endif // CONFIG_NEW_QUANT #endif // CONFIG_NEW_QUANT
#ifdef __cplusplus #ifdef __cplusplus

16
vp10/decoder/decodeframe.c
View File

@@ -2093,11 +2093,11 @@ static void setup_segmentation_dequant(VP10_COMMON *const cm) {
#if CONFIG_NEW_QUANT #if CONFIG_NEW_QUANT
for (dq = 0; dq < QUANT_PROFILES; dq ++) { for (dq = 0; dq < QUANT_PROFILES; dq ++) {
for (b = 0; b < COEF_BANDS; ++b) { for (b = 0; b < COEF_BANDS; ++b) {
get_dequant_val_nuq( vp10_get_dequant_val_nuq(
cm->y_dequant[i][b != 0], qindex == 0, b, cm->y_dequant[i][b != 0], qindex, b,
cm->y_dequant_nuq[i][dq][b], NULL, dq); cm->y_dequant_nuq[i][dq][b], NULL, dq);
get_dequant_val_nuq( vp10_get_dequant_val_nuq(
cm->uv_dequant[i][b != 0], qindex == 0, b, cm->uv_dequant[i][b != 0], qindex, b,
cm->uv_dequant_nuq[i][dq][b], NULL, dq); cm->uv_dequant_nuq[i][dq][b], NULL, dq);
} }
} }
@@ -2116,11 +2116,11 @@ static void setup_segmentation_dequant(VP10_COMMON *const cm) {
#if CONFIG_NEW_QUANT #if CONFIG_NEW_QUANT
for (dq = 0; dq < QUANT_PROFILES; dq ++) { for (dq = 0; dq < QUANT_PROFILES; dq ++) {
for (b = 0; b < COEF_BANDS; ++b) { for (b = 0; b < COEF_BANDS; ++b) {
get_dequant_val_nuq( vp10_get_dequant_val_nuq(
cm->y_dequant[0][b != 0], qindex == 0, b, cm->y_dequant[0][b != 0], qindex, b,
cm->y_dequant_nuq[0][dq][b], NULL, dq); cm->y_dequant_nuq[0][dq][b], NULL, dq);
get_dequant_val_nuq( vp10_get_dequant_val_nuq(
cm->uv_dequant[0][b != 0], qindex == 0, b, cm->uv_dequant[0][b != 0], qindex, b,
cm->uv_dequant_nuq[0][dq][b], NULL, dq); cm->uv_dequant_nuq[0][dq][b], NULL, dq);
} }
} }

4
vp10/decoder/detokenize.c
View File

@@ -205,7 +205,7 @@ static int decode_coefs(const MACROBLOCKD *xd,
} }
} }
#if CONFIG_NEW_QUANT #if CONFIG_NEW_QUANT
v = dequant_abscoeff_nuq(val, dqv, dqv_val); v = vp10_dequant_abscoeff_nuq(val, dqv, dqv_val);
v = dq_shift ? ROUND_POWER_OF_TWO(v, dq_shift) : v; v = dq_shift ? ROUND_POWER_OF_TWO(v, dq_shift) : v;
#else #else
v = (val * dqv) >> dq_shift; v = (val * dqv) >> dq_shift;
@@ -389,7 +389,7 @@ static int decode_coefs_ans(const MACROBLOCKD *const xd,
} break; } break;
} }
#if CONFIG_NEW_QUANT #if CONFIG_NEW_QUANT
v = dequant_abscoeff_nuq(val, dqv, dqv_val); v = vp10_dequant_abscoeff_nuq(val, dqv, dqv_val);
v = dq_shift ? ROUND_POWER_OF_TWO(v, dq_shift) : v; v = dq_shift ? ROUND_POWER_OF_TWO(v, dq_shift) : v;
#else #else
v = (val * dqv) >> dq_shift; v = (val * dqv) >> dq_shift;

8
vp10/encoder/encodemb.c
View File

@@ -207,10 +207,10 @@ int vp10_optimize_b(MACROBLOCK *mb, int plane, int block,
} else { } else {
#if CONFIG_NEW_QUANT #if CONFIG_NEW_QUANT
shortcut = ( shortcut = (
(dequant_abscoeff_nuq( (vp10_dequant_abscoeff_nuq(
abs(x), dequant_ptr[rc != 0], abs(x), dequant_ptr[rc != 0],
dequant_val[band_translate[i]]) > (abs(coeff[rc]) << shift)) && dequant_val[band_translate[i]]) > (abs(coeff[rc]) << shift)) &&
(dequant_abscoeff_nuq( (vp10_dequant_abscoeff_nuq(
abs(x) - 1, dequant_ptr[rc != 0], abs(x) - 1, dequant_ptr[rc != 0],
dequant_val[band_translate[i]]) < (abs(coeff[rc]) << shift))); dequant_val[band_translate[i]]) < (abs(coeff[rc]) << shift)));
#else // CONFIG_NEW_QUANT #else // CONFIG_NEW_QUANT
@@ -280,7 +280,7 @@ int vp10_optimize_b(MACROBLOCK *mb, int plane, int block,
} }
#if CONFIG_NEW_QUANT #if CONFIG_NEW_QUANT
dx = dequant_coeff_nuq( dx = vp10_dequant_coeff_nuq(
x, dequant_ptr[rc != 0], x, dequant_ptr[rc != 0],
dequant_val[band_translate[i]]) - (coeff[rc] << shift); dequant_val[band_translate[i]]) - (coeff[rc] << shift);
#if CONFIG_VP9_HIGHBITDEPTH #if CONFIG_VP9_HIGHBITDEPTH
@@ -378,7 +378,7 @@ int vp10_optimize_b(MACROBLOCK *mb, int plane, int block,
dqcoeff[rc] = tokens[i][best].dqc; dqcoeff[rc] = tokens[i][best].dqc;
#if CONFIG_NEW_QUANT #if CONFIG_NEW_QUANT
dqcoeff[rc] = dequant_abscoeff_nuq(abs(x), dequant_ptr[rc != 0], dqcoeff[rc] = vp10_dequant_abscoeff_nuq(abs(x), dequant_ptr[rc != 0],
dequant_val[band_translate[i]]); dequant_val[band_translate[i]]);
if (shift) dqcoeff[rc] = ROUND_POWER_OF_TWO(dqcoeff[rc], shift); if (shift) dqcoeff[rc] = ROUND_POWER_OF_TWO(dqcoeff[rc], shift);
if (x < 0) dqcoeff[rc] = -dqcoeff[rc]; if (x < 0) dqcoeff[rc] = -dqcoeff[rc];

24
vp10/encoder/quantize.c
View File

@@ -48,7 +48,7 @@ static INLINE int quantize_coeff_nuq(const tran_low_t coeffv,
} }
if (q) { if (q) {
*dqcoeff_ptr = *dqcoeff_ptr =
dequant_abscoeff_nuq(q, dequant, dequant_val); vp10_dequant_abscoeff_nuq(q, dequant, dequant_val);
*qcoeff_ptr = (q ^ coeff_sign) - coeff_sign; *qcoeff_ptr = (q ^ coeff_sign) - coeff_sign;
*dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr; *dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr;
} else { } else {
@@ -85,9 +85,9 @@ static INLINE int quantize_coeff_bigtx_nuq(const tran_low_t coeffv,
} }
if (q) { if (q) {
*dqcoeff_ptr = *dqcoeff_ptr =
ROUND_POWER_OF_TWO(dequant_abscoeff_nuq(q, dequant, dequant_val), ROUND_POWER_OF_TWO(vp10_dequant_abscoeff_nuq(q, dequant, dequant_val),
1 + logsizeby32); 1 + logsizeby32);
// *dqcoeff_ptr = dequant_abscoeff_nuq(q, dequant, dequant_val) >> // *dqcoeff_ptr = vp10_dequant_abscoeff_nuq(q, dequant, dequant_val) >>
// (1 + logsizeby32); // (1 + logsizeby32);
*qcoeff_ptr = (q ^ coeff_sign) - coeff_sign; *qcoeff_ptr = (q ^ coeff_sign) - coeff_sign;
*dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr; *dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr;
@@ -122,7 +122,7 @@ static INLINE int quantize_coeff_fp_nuq(const tran_low_t coeffv,
} }
if (q) { if (q) {
*dqcoeff_ptr = *dqcoeff_ptr =
dequant_abscoeff_nuq(q, dequant, dequant_val); vp10_dequant_abscoeff_nuq(q, dequant, dequant_val);
*qcoeff_ptr = (q ^ coeff_sign) - coeff_sign; *qcoeff_ptr = (q ^ coeff_sign) - coeff_sign;
*dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr; *dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr;
} else { } else {
@@ -159,9 +159,9 @@ static INLINE int quantize_coeff_bigtx_fp_nuq(const tran_low_t coeffv,
} }
if (q) { if (q) {
*dqcoeff_ptr = *dqcoeff_ptr =
ROUND_POWER_OF_TWO(dequant_abscoeff_nuq(q, dequant, dequant_val), ROUND_POWER_OF_TWO(vp10_dequant_abscoeff_nuq(q, dequant, dequant_val),
1 + logsizeby32); 1 + logsizeby32);
// *dqcoeff_ptr = dequant_abscoeff_nuq(q, dequant, dequant_val) >> // *dqcoeff_ptr = vp10_dequant_abscoeff_nuq(q, dequant, dequant_val) >>
// (1 + logsizeby32); // (1 + logsizeby32);
*qcoeff_ptr = (q ^ coeff_sign) - coeff_sign; *qcoeff_ptr = (q ^ coeff_sign) - coeff_sign;
*dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr; *dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr;
@@ -557,7 +557,7 @@ static INLINE int highbd_quantize_coeff_nuq(const tran_low_t coeffv,
} }
if (q) { if (q) {
*dqcoeff_ptr = *dqcoeff_ptr =
dequant_abscoeff_nuq(q, dequant, dequant_val); vp10_dequant_abscoeff_nuq(q, dequant, dequant_val);
*qcoeff_ptr = (q ^ coeff_sign) - coeff_sign; *qcoeff_ptr = (q ^ coeff_sign) - coeff_sign;
*dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr; *dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr;
} else { } else {
@@ -591,7 +591,7 @@ static INLINE int highbd_quantize_coeff_fp_nuq(const tran_low_t coeffv,
} }
if (q) { if (q) {
*dqcoeff_ptr = *dqcoeff_ptr =
dequant_abscoeff_nuq(q, dequant, dequant_val); vp10_dequant_abscoeff_nuq(q, dequant, dequant_val);
*qcoeff_ptr = (q ^ coeff_sign) - coeff_sign; *qcoeff_ptr = (q ^ coeff_sign) - coeff_sign;
*dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr; *dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr;
} else { } else {
@@ -629,7 +629,7 @@ static INLINE int highbd_quantize_coeff_bigtx_fp_nuq(
} }
if (q) { if (q) {
*dqcoeff_ptr = *dqcoeff_ptr =
ROUND_POWER_OF_TWO(dequant_abscoeff_nuq(q, dequant, dequant_val), ROUND_POWER_OF_TWO(vp10_dequant_abscoeff_nuq(q, dequant, dequant_val),
1 + logsizeby32); 1 + logsizeby32);
*qcoeff_ptr = (q ^ coeff_sign) - coeff_sign; *qcoeff_ptr = (q ^ coeff_sign) - coeff_sign;
*dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr; *dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr;
@@ -668,7 +668,7 @@ static INLINE int highbd_quantize_coeff_bigtx_nuq(const tran_low_t coeffv,
} }
if (q) { if (q) {
*dqcoeff_ptr = *dqcoeff_ptr =
ROUND_POWER_OF_TWO(dequant_abscoeff_nuq(q, dequant, dequant_val), ROUND_POWER_OF_TWO(vp10_dequant_abscoeff_nuq(q, dequant, dequant_val),
1 + logsizeby32); 1 + logsizeby32);
*qcoeff_ptr = (q ^ coeff_sign) - coeff_sign; *qcoeff_ptr = (q ^ coeff_sign) - coeff_sign;
*dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr; *dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr;
@@ -1228,10 +1228,10 @@ void vp10_init_quantizer(VP10_COMP *cpi) {
for (i = 0; i < COEF_BANDS; i++) { for (i = 0; i < COEF_BANDS; i++) {
const int quant = cpi->y_dequant[q][i != 0]; const int quant = cpi->y_dequant[q][i != 0];
const int uvquant = cpi->uv_dequant[q][i != 0]; const int uvquant = cpi->uv_dequant[q][i != 0];
get_dequant_val_nuq(quant, q == 0, i, vp10_get_dequant_val_nuq(quant, q, i,
cpi->y_dequant_val_nuq[dq][q][i], cpi->y_dequant_val_nuq[dq][q][i],
quants->y_cuml_bins_nuq[dq][q][i], dq); quants->y_cuml_bins_nuq[dq][q][i], dq);
get_dequant_val_nuq(uvquant, q == 0, i, vp10_get_dequant_val_nuq(uvquant, q, i,
cpi->uv_dequant_val_nuq[dq][q][i], cpi->uv_dequant_val_nuq[dq][q][i],
quants->uv_cuml_bins_nuq[dq][q][i], dq); quants->uv_cuml_bins_nuq[dq][q][i], dq);
} }