generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

quantizer: fix assertion in fast quantizer path

Browse Source 
The fast quantizer assembly code has not been updated to match the new
exact quantizer, which was made the default in commit 6adbe09.
Specifically, they are not aware of the potential for the coefficient
to be scaled, which results in the quantized result exceeding the range
of the DCT. This patch restores the previous behavior of using the
non-shifted coefficients when in the fast quantizer code path, but
unfortunately requires rebuilding the tables when switching between the
two.

Change-Id: I0a33f5b3850335011a06906f49fafed54dda9546
This commit is contained in:
John Koleszar 2010-11-11 12:41:07 -05:00
parent 58083cb34d
commit 0a49747b01
3 changed files with 72 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
vp8/encoder/encodeframe.c
View File

@ -146,7 +146,10 @@ static const int qzbin_factors_y2[129] =
#define EXACT_QUANT
#ifdef EXACT_QUANT
static void vp8cx_invert_quant(short *quant, short *shift, short d)
static void vp8cx_invert_quant(int improved_quant, short *quant,
short *shift, short d)
{
if(improved_quant)
{
unsigned t;
int l;
@ -157,6 +160,12 @@ static void vp8cx_invert_quant(short *quant, short *shift, short d)
*quant = (short)(t - (1<<16));
*shift = l;
}
else
{
*quant = (1 << 16) / d;
*shift = 0;
}
}
void vp8cx_init_quantizer(VP8_COMP *cpi)
{
@ -170,7 +179,7 @@ void vp8cx_init_quantizer(VP8_COMP *cpi)
{
// dc values
quant_val = vp8_dc_quant(Q, cpi->common.y1dc_delta_q);
vp8cx_invert_quant(cpi->Y1quant[Q] + 0,
vp8cx_invert_quant(cpi->sf.improved_quant, cpi->Y1quant[Q] + 0,
cpi->Y1quant_shift[Q] + 0, quant_val);
cpi->Y1zbin[Q][0] = ((qzbin_factors[Q] * quant_val) + 64) >> 7;
cpi->Y1round[Q][0] = (qrounding_factors[Q] * quant_val) >> 7;
@ -178,7 +187,7 @@ void vp8cx_init_quantizer(VP8_COMP *cpi)
cpi->zrun_zbin_boost_y1[Q][0] = (quant_val * zbin_boost[0]) >> 7;
quant_val = vp8_dc2quant(Q, cpi->common.y2dc_delta_q);
vp8cx_invert_quant(cpi->Y2quant[Q] + 0,
vp8cx_invert_quant(cpi->sf.improved_quant, cpi->Y2quant[Q] + 0,
cpi->Y2quant_shift[Q] + 0, quant_val);
cpi->Y2zbin[Q][0] = ((qzbin_factors_y2[Q] * quant_val) + 64) >> 7;
cpi->Y2round[Q][0] = (qrounding_factors_y2[Q] * quant_val) >> 7;
@ -186,7 +195,7 @@ void vp8cx_init_quantizer(VP8_COMP *cpi)
cpi->zrun_zbin_boost_y2[Q][0] = (quant_val * zbin_boost[0]) >> 7;
quant_val = vp8_dc_uv_quant(Q, cpi->common.uvdc_delta_q);
vp8cx_invert_quant(cpi->UVquant[Q] + 0,
vp8cx_invert_quant(cpi->sf.improved_quant, cpi->UVquant[Q] + 0,
cpi->UVquant_shift[Q] + 0, quant_val);
cpi->UVzbin[Q][0] = ((qzbin_factors[Q] * quant_val) + 64) >> 7;;
cpi->UVround[Q][0] = (qrounding_factors[Q] * quant_val) >> 7;
@ -199,7 +208,7 @@ void vp8cx_init_quantizer(VP8_COMP *cpi)
int rc = vp8_default_zig_zag1d[i];
quant_val = vp8_ac_yquant(Q);
vp8cx_invert_quant(cpi->Y1quant[Q] + rc,
vp8cx_invert_quant(cpi->sf.improved_quant, cpi->Y1quant[Q] + rc,
cpi->Y1quant_shift[Q] + rc, quant_val);
cpi->Y1zbin[Q][rc] = ((qzbin_factors[Q] * quant_val) + 64) >> 7;
cpi->Y1round[Q][rc] = (qrounding_factors[Q] * quant_val) >> 7;
@ -207,7 +216,7 @@ void vp8cx_init_quantizer(VP8_COMP *cpi)
cpi->zrun_zbin_boost_y1[Q][i] = (quant_val * zbin_boost[i]) >> 7;
quant_val = vp8_ac2quant(Q, cpi->common.y2ac_delta_q);
vp8cx_invert_quant(cpi->Y2quant[Q] + rc,
vp8cx_invert_quant(cpi->sf.improved_quant, cpi->Y2quant[Q] + rc,
cpi->Y2quant_shift[Q] + rc, quant_val);
cpi->Y2zbin[Q][rc] = ((qzbin_factors_y2[Q] * quant_val) + 64) >> 7;
cpi->Y2round[Q][rc] = (qrounding_factors_y2[Q] * quant_val) >> 7;
@ -215,7 +224,7 @@ void vp8cx_init_quantizer(VP8_COMP *cpi)
cpi->zrun_zbin_boost_y2[Q][i] = (quant_val * zbin_boost[i]) >> 7;
quant_val = vp8_ac_uv_quant(Q, cpi->common.uvac_delta_q);
vp8cx_invert_quant(cpi->UVquant[Q] + rc,
vp8cx_invert_quant(cpi->sf.improved_quant, cpi->UVquant[Q] + rc,
cpi->UVquant_shift[Q] + rc, quant_val);
cpi->UVzbin[Q][rc] = ((qzbin_factors[Q] * quant_val) + 64) >> 7;
cpi->UVround[Q][rc] = (qrounding_factors[Q] * quant_val) >> 7;

3
vp8/encoder/onyx_if.c
View File

@ -563,6 +563,7 @@ void vp8_set_speed_features(VP8_COMP *cpi)
int Speed = cpi->Speed;
int i;
VP8_COMMON *cm = &cpi->common;
int last_improved_quant = sf->improved_quant;
// Initialise default mode frequency sampling variables
for (i = 0; i < MAX_MODES; i ++)
@ -1262,6 +1263,8 @@ void vp8_set_speed_features(VP8_COMP *cpi)
{
cpi->mb.quantize_b = QUANTIZE_INVOKE(&cpi->rtcd.quantize, fastquantb);
}
if (cpi->sf.improved_quant != last_improved_quant)
vp8cx_init_quantizer(cpi);
#if CONFIG_RUNTIME_CPU_DETECT
cpi->mb.e_mbd.rtcd = &cpi->common.rtcd;

75
vp8/encoder/quantize.c
View File

@ -17,7 +17,8 @@
#include "predictdc.h"
#define EXACT_QUANT
#ifdef EXACT_QUANT
#ifdef EXACT_FASTQUANT
void vp8_fast_quantize_b_c(BLOCK *b, BLOCKD *d)
{
int i, rc, eob;
@ -64,6 +65,45 @@ void vp8_fast_quantize_b_c(BLOCK *b, BLOCKD *d)
d->eob = eob + 1;
}
#else
void vp8_fast_quantize_b_c(BLOCK *b, BLOCKD *d)
{
int i, rc, eob;
int zbin;
int x, y, z, sz;
short *coeff_ptr = b->coeff;
short *round_ptr = b->round;
short *quant_ptr = b->quant;
short *qcoeff_ptr = d->qcoeff;
short *dqcoeff_ptr = d->dqcoeff;
short *dequant_ptr = d->dequant;
eob = -1;
for (i = 0; i < 16; i++)
{
rc = vp8_default_zig_zag1d[i];
z = coeff_ptr[rc];
sz = (z >> 31); // sign of z
x = (z ^ sz) - sz; // x = abs(z)
y = ((x + round_ptr[rc]) * quant_ptr[rc]) >> 16; // quantize (x)
x = (y ^ sz) - sz; // get the sign back
qcoeff_ptr[rc] = x; // write to destination
dqcoeff_ptr[rc] = x * dequant_ptr[rc]; // dequantized value
if (y)
{
eob = i; // last nonzero coeffs
}
}
d->eob = eob + 1;
}
#endif
#ifdef EXACT_QUANT
void vp8_regular_quantize_b(BLOCK *b, BLOCKD *d)
{
int i, rc, eob;
@ -178,39 +218,6 @@ void vp8_strict_quantize_b(BLOCK *b, BLOCKD *d)
}
#else
void vp8_fast_quantize_b_c(BLOCK *b, BLOCKD *d)
{
int i, rc, eob;
int zbin;
int x, y, z, sz;
short *coeff_ptr = b->coeff;
short *round_ptr = b->round;
short *quant_ptr = b->quant;
short *qcoeff_ptr = d->qcoeff;
short *dqcoeff_ptr = d->dqcoeff;
short *dequant_ptr = d->dequant;
eob = -1;
for (i = 0; i < 16; i++)
{
rc = vp8_default_zig_zag1d[i];
z = coeff_ptr[rc];
sz = (z >> 31); // sign of z
x = (z ^ sz) - sz; // x = abs(z)
y = ((x + round_ptr[rc]) * quant_ptr[rc]) >> 16; // quantize (x)
x = (y ^ sz) - sz; // get the sign back
qcoeff_ptr[rc] = x; // write to destination
dqcoeff_ptr[rc] = x * dequant_ptr[rc]; // dequantized value
if (y)
{
eob = i; // last nonzero coeffs
}
}
d->eob = eob + 1;
}
void vp8_regular_quantize_b(BLOCK *b, BLOCKD *d)
{