From 529540759f285f251a198b6f55bb8cd451ab28e8 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sun, 15 Sep 2013 03:21:03 +0200 Subject: [PATCH] avcodec/simple_idct_template: adjust coeffs for 12bit idct This prevents overflows with 12bit jpegs Fixes Ticket2952 Signed-off-by: Michael Niedermayer --- libavcodec/simple_idct_template.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/libavcodec/simple_idct_template.c b/libavcodec/simple_idct_template.c index ffd75780a8..64a7be02c5 100644 --- a/libavcodec/simple_idct_template.c +++ b/libavcodec/simple_idct_template.c @@ -64,6 +64,7 @@ #elif BIT_DEPTH == 10 || BIT_DEPTH == 12 +#if BIT_DEPTH == 10 #define W1 90901 #define W2 85627 #define W3 77062 @@ -72,13 +73,20 @@ #define W6 35468 #define W7 18081 -#if BIT_DEPTH == 10 #define ROW_SHIFT 15 #define COL_SHIFT 20 #define DC_SHIFT 1 #else -#define ROW_SHIFT 17 -#define COL_SHIFT 18 +#define W1 45451 +#define W2 42813 +#define W3 38531 +#define W4 32767 +#define W5 25746 +#define W6 17734 +#define W7 9041 + +#define ROW_SHIFT 16 +#define COL_SHIFT 17 #define DC_SHIFT -1 #endif