[FFmpeg-devel] [PATCH] Fix data_size handling for AC3 and dca decoders.

[Reimar Döffinger]

They use now code identical to the AAC decoder.
The AC3 decoder previously did not check the data_size and
the dca decoder checked against and set wrong values for float.
---
 libavcodec/ac3dec.c |    9 +++++++--
 libavcodec/dca.c    |    8 +++++---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/libavcodec/ac3dec.c b/libavcodec/ac3dec.c
index 431f67d..b4aae22 100644
--- a/libavcodec/ac3dec.c
+++ b/libavcodec/ac3dec.c
@@ -1298,6 +1298,7 @@ static int ac3_decode_frame(AVCodecContext * avctx, void *data, int *data_size,
     float *out_samples_flt = (float *)data;
     int16_t *out_samples = (int16_t *)data;
     int blk, ch, err;
+    int data_size_orig, data_size_tmp;
     const uint8_t *channel_map;
     const float *output[AC3_MAX_CHANNELS];
 
@@ -1314,6 +1315,7 @@ static int ac3_decode_frame(AVCodecContext * avctx, void *data, int *data_size,
     init_get_bits(&s->gbc, buf, buf_size * 8);
 
     /* parse the syncinfo */
+    data_size_orig = *data_size;
     *data_size = 0;
     err = parse_frame_header(s);
 
@@ -1397,6 +1399,11 @@ static int ac3_decode_frame(AVCodecContext * avctx, void *data, int *data_size,
     channel_map = ff_ac3_dec_channel_map[s->output_mode & ~AC3_OUTPUT_LFEON][s->lfe_on];
     for (ch = 0; ch < s->out_channels; ch++)
         output[ch] = s->output[channel_map[ch]];
+    data_size_tmp = s->num_blocks * 256 * avctx->channels;
+    data_size_tmp *= avctx->sample_fmt == AV_SAMPLE_FMT_FLT ? sizeof(*out_samples_flt) : sizeof(*out_samples);
+    if (data_size_orig < data_size_tmp)
+        return -1;
+    *data_size = data_size_tmp;
     for (blk = 0; blk < s->num_blocks; blk++) {
         if (!err && decode_audio_block(s, blk)) {
             av_log(avctx, AV_LOG_ERROR, "error decoding the audio block\n");
@@ -1410,8 +1417,6 @@ static int ac3_decode_frame(AVCodecContext * avctx, void *data, int *data_size,
             out_samples += 256 * s->out_channels;
         }
     }
-    *data_size = s->num_blocks * 256 * avctx->channels;
-    *data_size *= avctx->sample_fmt == AV_SAMPLE_FMT_FLT ? sizeof(*out_samples_flt) : sizeof(*out_samples);
     return FFMIN(buf_size, s->frame_size);
 }
 
diff --git a/libavcodec/dca.c b/libavcodec/dca.c
index 1e26eed..03bf7f7 100644
--- a/libavcodec/dca.c
+++ b/libavcodec/dca.c
@@ -1622,6 +1622,7 @@ static int dca_decode_frame(AVCodecContext * avctx,
 {
     const uint8_t *buf = avpkt->data;
     int buf_size = avpkt->size;
+    int data_size_tmp;
 
     int lfe_samples;
     int num_core_channels = 0;
@@ -1813,10 +1814,11 @@ static int dca_decode_frame(AVCodecContext * avctx,
         return -1;
     }
 
-    /* ffdshow custom code */
-    if (*data_size < (s->sample_blocks / 8) * 256 * sizeof(samples[0]) * channels)
+    data_size_tmp = (s->sample_blocks / 8) * 256 * channels;
+    data_size_tmp *= avctx->sample_fmt == AV_SAMPLE_FMT_FLT ? sizeof(*samples_flt) : sizeof(*samples);
+    if (*data_size < data_size_tmp)
         return -1;
-    *data_size = 256 / 8 * s->sample_blocks * sizeof(samples[0]) * channels;
+    *data_size = data_size_tmp;
 
     /* filter to get final output */
     for (i = 0; i < (s->sample_blocks / 8); i++) {
-- 
1.7.4.4