[FFmpeg-cvslog] avcodec/aacsbr_fixed: Fix signed integer overflow in sbr_hf_inverse_filter()

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michael at niedermayer.cc> | Tue Jun 13 17:10:30 2017 +0200| [4cc2a357f5dce9bad36b59fb31ba5cf61cc56272] | committer: Michael Niedermayer

avcodec/aacsbr_fixed: Fix signed integer overflow in sbr_hf_inverse_filter()

Fixes: runtime error: signed integer overflow: 2147483584 + 128 cannot be represented in type 'int'
Fixes: 2164/clusterfuzz-testcase-minimized-4715936172998656

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=4cc2a357f5dce9bad36b59fb31ba5cf61cc56272
---

 libavcodec/aacsbr_fixed.c | 28 ++++++++++++----------------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/libavcodec/aacsbr_fixed.c b/libavcodec/aacsbr_fixed.c
index 2531637194..289bb86a81 100644
--- a/libavcodec/aacsbr_fixed.c
+++ b/libavcodec/aacsbr_fixed.c
@@ -291,10 +291,9 @@ static void sbr_hf_inverse_filter(SBRDSPContext *dsp,
         else if (shift <= -30)
             alpha0[k][0] = 0;
         else {
-            a00.mant *= 2;
-            shift = 2-shift;
-            if (shift == 0)
-                alpha0[k][0] = a00.mant;
+            shift = 1-shift;
+            if (shift <= 0)
+                alpha0[k][0] = a00.mant * (1<<-shift);
             else {
                 round = 1 << (shift-1);
                 alpha0[k][0] = (a00.mant + round) >> shift;
@@ -307,10 +306,9 @@ static void sbr_hf_inverse_filter(SBRDSPContext *dsp,
         else if (shift <= -30)
             alpha0[k][1] = 0;
         else {
-            a01.mant *= 2;
-            shift = 2-shift;
-            if (shift == 0)
-                alpha0[k][1] = a01.mant;
+            shift = 1-shift;
+            if (shift <= 0)
+                alpha0[k][1] = a01.mant * (1<<-shift);
             else {
                 round = 1 << (shift-1);
                 alpha0[k][1] = (a01.mant + round) >> shift;
@@ -322,10 +320,9 @@ static void sbr_hf_inverse_filter(SBRDSPContext *dsp,
         else if (shift <= -30)
             alpha1[k][0] = 0;
         else {
-            a10.mant *= 2;
-            shift = 2-shift;
-            if (shift == 0)
-                alpha1[k][0] = a10.mant;
+            shift = 1-shift;
+            if (shift <= 0)
+                alpha1[k][0] = a10.mant * (1<<-shift);
             else {
                 round = 1 << (shift-1);
                 alpha1[k][0] = (a10.mant + round) >> shift;
@@ -338,10 +335,9 @@ static void sbr_hf_inverse_filter(SBRDSPContext *dsp,
         else if (shift <= -30)
             alpha1[k][1] = 0;
         else {
-            a11.mant *= 2;
-            shift = 2-shift;
-            if (shift == 0)
-                alpha1[k][1] = a11.mant;
+            shift = 1-shift;
+            if (shift <= 0)
+                alpha1[k][1] = a11.mant * (1<<-shift);
             else {
                 round = 1 << (shift-1);
                 alpha1[k][1] = (a11.mant + round) >> shift;