generic-library/vpx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge changes I1f1edeaa,I89313cac

Browse Source 
* changes:
  quantize: silence unsigned overflow warning
  quantize test: quiet overflow warning
This commit is contained in:
Johann Koenig 2017-08-15 17:37:59 +00:00 committed by Gerrit Code Review
commit c59d1a4dc7
2 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
test/vp9_quantize_test.cc
View File

@ -210,9 +210,9 @@ TEST_P(VP9QuantizeTest, EOBCheck) {
// Two random entries
coeff.Set(0);
coeff.TopLeftPixel()[rnd(count)] =
rnd.RandRange(max_value_ * 2) - max_value_;
static_cast<int>(rnd.RandRange(max_value_ * 2)) - max_value_;
coeff.TopLeftPixel()[rnd(count)] =
rnd.RandRange(max_value_ * 2) - max_value_;
static_cast<int>(rnd.RandRange(max_value_ * 2)) - max_value_;
GenerateHelperArrays(&rnd, zbin_ptr_, round_ptr_, quant_ptr_,
quant_shift_ptr_, dequant_ptr_);

6
vpx_dsp/quantize.c
View File

@ -203,7 +203,9 @@ void vpx_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;
const uint32_t abs_qcoeff =
(uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 16);
qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
// Restoring the sign triggers unsigned overflow warnings with negative
// values because the result of the xor operation is unsigned.
qcoeff_ptr[rc] = (tran_low_t)(abs_qcoeff ^ coeff_sign) - coeff_sign;
dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
if (abs_qcoeff) eob = i;
}
@ -310,7 +312,7 @@ void vpx_highbd_quantize_b_32x32_c(
const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;
const uint32_t abs_qcoeff =
(uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 15);
qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
qcoeff_ptr[rc] = (tran_low_t)(abs_qcoeff ^ coeff_sign) - coeff_sign;
dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;
if (abs_qcoeff) eob = idx_arr[i];
}