[FFmpeg-devel] [PATCH]Fail on oom in vdpau

[Carl Eugen Hoyos]

Hi!

Attached patch intends to fix a theoretical issue, reported by Gwenole ages 
ago.

Please review, Carl Eugen
-------------- next part --------------
diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 6a6a68c..e582c09 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -4728,10 +4728,20 @@ again:
                                h->avctx->codec->capabilities & CODEC_CAP_HWACCEL_VDPAU) {
                         static const uint8_t start_code[] = {
                             0x00, 0x00, 0x01 };
+                        int ret =
                         ff_vdpau_add_data_chunk(h->cur_pic_ptr->f.data[0], start_code,
                                                 sizeof(start_code));
+                        if (ret < 0) {
+                            buf_index = ret;
+                            goto end;
+                        }
+                        ret =
                         ff_vdpau_add_data_chunk(h->cur_pic_ptr->f.data[0], &buf[buf_index - consumed],
                                                 consumed);
+                        if (ret < 0) {
+                            buf_index = ret;
+                            goto end;
+                        }
                     } else
                         context_count++;
                 }
diff --git a/libavcodec/vdpau.c b/libavcodec/vdpau.c
index 04355b0..2f7ff39 100644
--- a/libavcodec/vdpau.c
+++ b/libavcodec/vdpau.c
@@ -157,21 +157,26 @@ void ff_vdpau_h264_set_reference_frames(H264Context *h)
     }
 }
 
-void ff_vdpau_add_data_chunk(uint8_t *data, const uint8_t *buf, int buf_size)
+int ff_vdpau_add_data_chunk(uint8_t *data, const uint8_t *buf, int buf_size)
 {
+    VdpBitstreamBuffer *new_buffers;
     struct vdpau_render_state *render = (struct vdpau_render_state*)data;
     assert(render);
 
-    render->bitstream_buffers= av_fast_realloc(
+    new_buffers = av_fast_realloc(
         render->bitstream_buffers,
         &render->bitstream_buffers_allocated,
         sizeof(*render->bitstream_buffers)*(render->bitstream_buffers_used + 1)
     );
+    if (!new_buffers)
+        return AVERROR(ENOMEM);
 
+    render->bitstream_buffers = new_buffers;
     render->bitstream_buffers[render->bitstream_buffers_used].struct_version  = VDP_BITSTREAM_BUFFER_VERSION;
     render->bitstream_buffers[render->bitstream_buffers_used].bitstream       = buf;
     render->bitstream_buffers[render->bitstream_buffers_used].bitstream_bytes = buf_size;
     render->bitstream_buffers_used++;
+    return 0;
 }
 
 #if CONFIG_H264_VDPAU_DECODER
diff --git a/libavcodec/vdpau_internal.h b/libavcodec/vdpau_internal.h
index 790b3ef..b2f9f3e 100644
--- a/libavcodec/vdpau_internal.h
+++ b/libavcodec/vdpau_internal.h
@@ -41,7 +41,7 @@ int ff_vdpau_add_buffer(AVCodecContext *avctx,
                         const uint8_t *buf, uint32_t buf_size);
 
 
-void ff_vdpau_add_data_chunk(uint8_t *data, const uint8_t *buf,
+int ff_vdpau_add_data_chunk(uint8_t *data, const uint8_t *buf,
                              int buf_size);
 
 void ff_vdpau_mpeg_picture_complete(MpegEncContext *s, const uint8_t *buf,