diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index 2afdd69d8..a5079638d 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c @@ -571,16 +571,17 @@ int64_t vp9_high_block_error_c(const tran_low_t *coeff, int64_t *ssz, int bps) { int i; int64_t error = 0, sqcoeff = 0; - int shift = bps - 8; + int shift = 2 * (bps - 8); int rounding = shift > 0 ? 1 << (shift - 1) : 0; for (i = 0; i < block_size; i++) { - const int diff = coeff[i] - dqcoeff[i]; - error += ((diff + rounding) >> shift) * ((diff + rounding) >> shift); - sqcoeff += ((coeff[i] + rounding) >> shift) * - ((coeff[i] + rounding) >> shift); + const int64_t diff = coeff[i] - dqcoeff[i]; + error += diff * diff; + sqcoeff += (int64_t)coeff[i] * (int64_t)coeff[i]; } assert(error >= 0 && sqcoeff >= 0); + error = (error + rounding) >> shift; + sqcoeff = (sqcoeff + rounding) >> shift; *ssz = sqcoeff; return error;