generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Avoid an unsigned overflow in invert_quant

Browse Source 
Change-Id: I16a570b2af66b6580d1cd6f8345a25f079009bf4
This commit is contained in:
Alex Converse
2015-11-19 16:46:46 -08:00
parent b2ccb9c189
commit d3fe3b2abb
3 changed files with 9 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
vp10/encoder/quantize.c
View File

@@ -219,12 +219,12 @@ void vp10_regular_quantize_b_4x4(MACROBLOCK *x, int plane, int block,
static void invert_quant(int16_t *quant, int16_t *shift, int d) {
unsigned t;
int l;
int l, m;
t = d;
for (l = 0; t > 1; l++)
t >>= 1;
t = 1 + (1 << (16 + l)) / d;
*quant = (int16_t)(t - (1 << 16));
m = 1 + (1 << (16 + l)) / d;
*quant = (int16_t)(m - (1 << 16));
*shift = 1 << (16 - l);
}

6
vp8/encoder/vp8_quantize.c
View File

@@ -227,12 +227,12 @@ static void invert_quant(int improved_quant, short *quant,
if(improved_quant)
{
unsigned t;
int l;
int l, m;
t = d;
for(l = 0; t > 1; l++)
t>>=1;
t = 1 + (1<<(16+l))/d;
*quant = (short)(t - (1<<16));
m = 1 + (1<<(16+l))/d;
*quant = (short)(m - (1<<16));
*shift = l;
/* use multiplication and constant shift by 16 */
*shift = 1 << (16 - *shift);

6
vp9/encoder/vp9_quantize.c
View File

@@ -219,12 +219,12 @@ void vp9_regular_quantize_b_4x4(MACROBLOCK *x, int plane, int block,
static void invert_quant(int16_t *quant, int16_t *shift, int d) {
unsigned t;
int l;
int l, m;
t = d;
for (l = 0; t > 1; l++)
t >>= 1;
t = 1 + (1 << (16 + l)) / d;
*quant = (int16_t)(t - (1 << 16));
m = 1 + (1 << (16 + l)) / d;
*quant = (int16_t)(m - (1 << 16));
*shift = 1 << (16 - l);
}