[FFmpeg-cvslog] avcodec/simple_idct_template: adjust coeffs for 12bit idct

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michaelni at gmx.at> | Sun Sep 15 03:21:03 2013 +0200| [529540759f285f251a198b6f55bb8cd451ab28e8] | committer: Michael Niedermayer

avcodec/simple_idct_template: adjust coeffs for 12bit idct

This prevents overflows with 12bit jpegs
Fixes Ticket2952

Signed-off-by: Michael Niedermayer <michaelni at gmx.at>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=529540759f285f251a198b6f55bb8cd451ab28e8
---

 libavcodec/simple_idct_template.c |   14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/libavcodec/simple_idct_template.c b/libavcodec/simple_idct_template.c
index ffd7578..64a7be0 100644
--- a/libavcodec/simple_idct_template.c
+++ b/libavcodec/simple_idct_template.c
@@ -64,6 +64,7 @@
 
 #elif BIT_DEPTH == 10 || BIT_DEPTH == 12
 
+#if BIT_DEPTH == 10
 #define W1 90901
 #define W2 85627
 #define W3 77062
@@ -72,13 +73,20 @@
 #define W6 35468
 #define W7 18081
 
-#if BIT_DEPTH == 10
 #define ROW_SHIFT 15
 #define COL_SHIFT 20
 #define DC_SHIFT 1
 #else
-#define ROW_SHIFT 17
-#define COL_SHIFT 18
+#define W1 45451
+#define W2 42813
+#define W3 38531
+#define W4 32767
+#define W5 25746
+#define W6 17734
+#define W7 9041
+
+#define ROW_SHIFT 16
+#define COL_SHIFT 17
 #define DC_SHIFT -1
 #endif