[FFmpeg-cvslog] avcodec/sbrdsp_fixed: Return an error from sbr_hf_apply_noise() if operations are impossible

[Michael Niedermayer]

ffmpeg | branch: release/3.3 | Michael Niedermayer <michael at niedermayer.cc> | Tue Jun 13 13:28:23 2017 +0200| [d2567caea9b12b935a249306697e7bd48e733c3b] | committer: Michael Niedermayer

avcodec/sbrdsp_fixed: Return an error from sbr_hf_apply_noise() if operations are impossible

Fixes: 1775/clusterfuzz-testcase-minimized-5330288148217856

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit d549f026d8b64b879c3ce3b8c7d153c82aa5eb52)
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=d2567caea9b12b935a249306697e7bd48e733c3b
---

 libavcodec/sbrdsp_fixed.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/libavcodec/sbrdsp_fixed.c b/libavcodec/sbrdsp_fixed.c
index 924da83c85..f42708a8a7 100644
--- a/libavcodec/sbrdsp_fixed.c
+++ b/libavcodec/sbrdsp_fixed.c
@@ -242,7 +242,7 @@ static void sbr_hf_g_filt_c(int (*Y)[2], const int (*X_high)[40][2],
     }
 }
 
-static av_always_inline void sbr_hf_apply_noise(int (*Y)[2],
+static av_always_inline int sbr_hf_apply_noise(int (*Y)[2],
                                                 const SoftFloat *s_m,
                                                 const SoftFloat *q_filt,
                                                 int noise,
@@ -260,7 +260,10 @@ static av_always_inline void sbr_hf_apply_noise(int (*Y)[2],
             int shift, round;
 
             shift = 22 - s_m[m].exp;
-            if (shift < 30) {
+            if (shift < 1) {
+                av_log(NULL, AV_LOG_ERROR, "Overflow in sbr_hf_apply_noise, shift=%d\n", shift);
+                return AVERROR(ERANGE);
+            } else if (shift < 30) {
                 round = 1 << (shift-1);
                 y0 += (s_m[m].mant * phi_sign0 + round) >> shift;
                 y1 += (s_m[m].mant * phi_sign1 + round) >> shift;
@@ -270,7 +273,10 @@ static av_always_inline void sbr_hf_apply_noise(int (*Y)[2],
             int64_t accu;
 
             shift = 22 - q_filt[m].exp;
-            if (shift < 30) {
+            if (shift < 1) {
+                av_log(NULL, AV_LOG_ERROR, "Overflow in sbr_hf_apply_noise, shift=%d\n", shift);
+                return AVERROR(ERANGE);
+            } else if (shift < 30) {
                 round = 1 << (shift-1);
 
                 accu = (int64_t)q_filt[m].mant * ff_sbr_noise_table_fixed[noise][0];
@@ -286,6 +292,7 @@ static av_always_inline void sbr_hf_apply_noise(int (*Y)[2],
         Y[m][1] = y1;
         phi_sign1 = -phi_sign1;
     }
+    return 0;
 }
 
 #include "sbrdsp_template.c"