generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge "highbd_quantize_fp_32x32: normalize abs_qcoeff type"

Browse Source
This commit is contained in:
James Zern
2017-06-27 23:30:16 +00:00
committed by Gerrit Code Review
parent 0bb31a46a4 9e1d2de67c
commit 515fed8f38
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
vp9/encoder/vp9_quantize.c
View File

@@ -138,7 +138,7 @@ void vp9_highbd_quantize_fp_32x32_c(
if (!skip_block) {
for (i = 0; i < n_coeffs; i++) {
uint32_t abs_qcoeff = 0;
int abs_qcoeff = 0;
const int rc = scan[i];
const int coeff = coeff_ptr[rc];
const int coeff_sign = (coeff >> 31);
@@ -147,7 +147,7 @@ void vp9_highbd_quantize_fp_32x32_c(
if (abs_coeff >= (dequant_ptr[rc != 0] >> 2)) {
const int64_t tmp =
abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1);
abs_qcoeff = (uint32_t)((tmp * quant_ptr[rc != 0]) >> 15);
abs_qcoeff = (int)((tmp * quant_ptr[rc != 0]) >> 15);
qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;
}