generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix an ubsan warning: vp9_quantizer.c

Browse Source 
BUG=https://bugs.chromium.org/p/webm/issues/detail?id=1219

Change-Id: Ie1bfda4d51c1567f551515f0ed113eec1bc1d98c
This commit is contained in:
Yaowu Xu
2016-06-15 17:59:26 -07:00
parent bbe5ddfdfb
commit 25110f2837
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
vp9/encoder/vp9_quantize.c
View File

@@ -94,7 +94,7 @@ void vp9_highbd_quantize_fp_c(const tran_low_t *coeff_ptr,
const int coeff_sign = (coeff >> 31); const int coeff_sign = (coeff >> 31);
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
const int64_t tmp = abs_coeff + round_ptr[rc != 0]; const int64_t tmp = abs_coeff + round_ptr[rc != 0];
const uint32_t abs_qcoeff = (uint32_t)((tmp * quant_ptr[rc != 0]) >> 16); const int abs_qcoeff = (int)((tmp * quant_ptr[rc != 0]) >> 16);
qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0]; dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
if (abs_qcoeff) if (abs_qcoeff)