[FFmpeg-cvslog] commit: Fix overflow in pred16x16_plane x86 simd code. Fixes issue 2547. ( Ronald S. Bultje )

[git at videolan.org]

ffmpeg | branch: master | Ronald S. Bultje <rsbultje at gmail.com> | Sat Jan 15 22:00:44 2011 +0000| [1b3e43e4fdbfe5b6ce10d19ed32a4f20fab76f5a] | committer: Ronald S. Bultje 

Fix overflow in pred16x16_plane x86 simd code. Fixes issue 2547.

Originally committed as revision 26381 to svn://svn.ffmpeg.org/ffmpeg/trunk

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=1b3e43e4fdbfe5b6ce10d19ed32a4f20fab76f5a
---

 libavcodec/x86/h264_intrapred.asm |   43 ++++++++++++++++++-------------------
 1 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/libavcodec/x86/h264_intrapred.asm b/libavcodec/x86/h264_intrapred.asm
index 52e691d..afe6426 100644
--- a/libavcodec/x86/h264_intrapred.asm
+++ b/libavcodec/x86/h264_intrapred.asm
@@ -344,28 +344,6 @@ cglobal pred16x16_plane_%3_%1, 2, 7, %2
 %endif
     paddw        m0, m1           ; sum of H coefficients
 
-%ifidn %3, h264
-    pmullw       m0, [pw_5]
-    paddw        m0, [pw_32]
-    psraw        m0, 6
-%elifidn %3, rv40
-    pmullw       m0, [pw_5]
-    psraw        m0, 6
-%elifidn %3, svq3
-    movd        r3d, m0
-    movsx        r3, r3w
-    test         r3, r3
-    lea          r4, [r3+3]
-    cmovs        r3, r4
-    sar          r3, 2           ; H/4
-    lea          r3, [r3*5]      ; 5*(H/4)
-    test         r3, r3
-    lea          r4, [r3+15]
-    cmovs        r3, r4
-    sar          r3, 4           ; (5*(H/4))/16
-    movd         m0, r3d
-%endif
-
     lea          r4, [r0+r2*8-1]
     lea          r3, [r0+r2*4-1]
     add          r4, r2
@@ -468,8 +446,29 @@ cglobal pred16x16_plane_%3_%1, 2, 7, %2
     movzx        r3, byte [r3+r2*2   ]
     lea          r3, [r3+r4+1]
     shl          r3, 4
+
     movd        r1d, m0
     movsx       r1d, r1w
+%ifnidn %3, svq3
+%ifidn %3, h264
+    lea         r1d, [r1d*5+32]
+%else ; rv40
+    lea         r1d, [r1d*5]
+%endif
+    sar         r1d, 6
+%else ; svq3
+    test        r1d, r1d
+    lea         r4d, [r1d+3]
+    cmovs       r1d, r4d
+    sar         r1d, 2           ; H/4
+    lea         r1d, [r1d*5]     ; 5*(H/4)
+    test        r1d, r1d
+    lea         r4d, [r1d+15]
+    cmovs       r1d, r4d
+    sar         r1d, 4           ; (5*(H/4))/16
+%endif
+    movd         m0, r1d
+
     add         r1d, r5d
     add         r3d, r1d
     shl         r1d, 3