[FFmpeg-cvslog] swscale: prevent overflow during initialization

[Ronald Bultje]

ffmpeg | branch: master | Ronald Bultje <rsbultje at gmail.com> | Tue Oct 18 01:53:04 2011 +0000| [d1d421cbc0d13b08535f7fc08d179572ee352072] | committer: Janne Grunau

swscale: prevent overflow during initialization

Signed-off-by: Janne Grunau <janne-libav at jannau.net>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=d1d421cbc0d13b08535f7fc08d179572ee352072
---

 libswscale/utils.c         |    6 ++++--
 tests/ref/lavfi/crop_scale |    2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/libswscale/utils.c b/libswscale/utils.c
index cfe4be9..8e5daf9 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -271,10 +271,12 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi
                 floatd= d * (1.0/(1<<30));
 
                 if (flags & SWS_BICUBIC) {
+#define SQRT_INT64_MAX 0xb504f333
                     int64_t B= (param[0] != SWS_PARAM_DEFAULT ? param[0] :   0) * (1<<24);
                     int64_t C= (param[1] != SWS_PARAM_DEFAULT ? param[1] : 0.6) * (1<<24);
-                    int64_t dd = ( d*d)>>30;
-                    int64_t ddd= (dd*d)>>30;
+                    int64_t dd  = d > SQRT_INT64_MAX ? ((d  >> 1) * d) >> 29 : (d  * d) >> 30;
+                    int64_t ddd = d > SQRT_INT64_MAX || dd > SQRT_INT64_MAX ?
+                                                       ((dd >> 2) * d) >> 28 : (dd * d) >> 30;
 
                     if      (d < 1LL<<30)
                         coeff = (12*(1<<24)-9*B-6*C)*ddd + (-18*(1<<24)+12*B+6*C)*dd + (6*(1<<24)-2*B)*(1<<30);
diff --git a/tests/ref/lavfi/crop_scale b/tests/ref/lavfi/crop_scale
index 82e5394..ae26c8e 100644
--- a/tests/ref/lavfi/crop_scale
+++ b/tests/ref/lavfi/crop_scale
@@ -1 +1 @@
-crop_scale          0a3d45d58b805b8c47416b9239535f94
+crop_scale          f8cad857d2b7102fc256532ec9849da7