[FFmpeg-cvslog] avcodec/apedec: Fix integer overflow

[Michael Niedermayer]

ffmpeg | branch: release/3.3 | Michael Niedermayer <michael at niedermayer.cc> | Sun Jul 16 14:57:20 2017 +0200| [0272afe70d6205caff05f3548da22a19f1eff9b1] | committer: Michael Niedermayer

avcodec/apedec: Fix integer overflow

Fixes: out of array access
Fixes: PoC.ape and others

Found-by: Bingchang, Liu at VARAS of IIE
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit ba4beaf6149f7241c8bd85fe853318c2f6837ad0)
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=0272afe70d6205caff05f3548da22a19f1eff9b1
---

 libavcodec/apedec.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/libavcodec/apedec.c b/libavcodec/apedec.c
index a6b14b8e24..15eb416ba4 100644
--- a/libavcodec/apedec.c
+++ b/libavcodec/apedec.c
@@ -1412,6 +1412,7 @@ static int ape_decode_frame(AVCodecContext *avctx, void *data,
     int32_t *sample24;
     int i, ch, ret;
     int blockstodecode;
+    uint64_t decoded_buffer_size;
 
     /* this should never be negative, but bad things will happen if it is, so
        check it just to make sure. */
@@ -1467,7 +1468,7 @@ static int ape_decode_frame(AVCodecContext *avctx, void *data,
                 skip_bits_long(&s->gb, offset);
         }
 
-        if (!nblocks || nblocks > INT_MAX) {
+        if (!nblocks || nblocks > INT_MAX / 2 / sizeof(*s->decoded_buffer) - 8) {
             av_log(avctx, AV_LOG_ERROR, "Invalid sample count: %"PRIu32".\n",
                    nblocks);
             return AVERROR_INVALIDDATA;
@@ -1493,8 +1494,9 @@ static int ape_decode_frame(AVCodecContext *avctx, void *data,
         blockstodecode = s->samples;
 
     /* reallocate decoded sample buffer if needed */
-    av_fast_malloc(&s->decoded_buffer, &s->decoded_size,
-                   2 * FFALIGN(blockstodecode, 8) * sizeof(*s->decoded_buffer));
+    decoded_buffer_size = 2LL * FFALIGN(blockstodecode, 8) * sizeof(*s->decoded_buffer);
+    av_assert0(decoded_buffer_size <= INT_MAX);
+    av_fast_malloc(&s->decoded_buffer, &s->decoded_size, decoded_buffer_size);
     if (!s->decoded_buffer)
         return AVERROR(ENOMEM);
     memset(s->decoded_buffer, 0, s->decoded_size);