diff --git a/test/vp9_quantize_test.cc b/test/vp9_quantize_test.cc index 903958959..c10832590 100644 --- a/test/vp9_quantize_test.cc +++ b/test/vp9_quantize_test.cc @@ -210,9 +210,9 @@ TEST_P(VP9QuantizeTest, EOBCheck) { // Two random entries coeff.Set(0); coeff.TopLeftPixel()[rnd(count)] = - rnd.RandRange(max_value_ * 2) - max_value_; + static_cast(rnd.RandRange(max_value_ * 2)) - max_value_; coeff.TopLeftPixel()[rnd(count)] = - rnd.RandRange(max_value_ * 2) - max_value_; + static_cast(rnd.RandRange(max_value_ * 2)) - max_value_; GenerateHelperArrays(&rnd, zbin_ptr_, round_ptr_, quant_ptr_, quant_shift_ptr_, dequant_ptr_); diff --git a/vpx_dsp/quantize.c b/vpx_dsp/quantize.c index 3c7f9832f..00fa4dc82 100644 --- a/vpx_dsp/quantize.c +++ b/vpx_dsp/quantize.c @@ -203,7 +203,9 @@ void vpx_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; const uint32_t abs_qcoeff = (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 16); - qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + // Restoring the sign triggers unsigned overflow warnings with negative + // values because the result of the xor operation is unsigned. + qcoeff_ptr[rc] = (tran_low_t)(abs_qcoeff ^ coeff_sign) - coeff_sign; dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0]; if (abs_qcoeff) eob = i; } @@ -310,7 +312,7 @@ void vpx_highbd_quantize_b_32x32_c( const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1; const uint32_t abs_qcoeff = (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 15); - qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); + qcoeff_ptr[rc] = (tran_low_t)(abs_qcoeff ^ coeff_sign) - coeff_sign; dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2; if (abs_qcoeff) eob = idx_arr[i]; }