From ee808e52bd0e371f0caad5ca7f29fbc4d5bf2aee Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 3 May 2013 10:33:07 -0700 Subject: [PATCH] Fix overflow in RD error calculation code. Change-Id: I61ef1f198c876f9f79787ea7d7385a862cfbae19 --- vp9/encoder/vp9_rdopt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index f971d91c5..990be6527 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c @@ -603,7 +603,7 @@ static int block_error(int16_t *coeff, int16_t *dqcoeff, for (i = 0; i < block_size; i++) { int this_diff = coeff[i] - dqcoeff[i]; - error += this_diff * this_diff; + error += (unsigned)this_diff * this_diff; } error >>= shift;