[FFmpeg-cvslog] avcodec/vaapi_encode: Fix segfault upon closing uninitialized encoder

[Andreas Rheinhardt]

ffmpeg | branch: release/4.4 | Andreas Rheinhardt <andreas.rheinhardt at outlook.com> | Mon Nov 29 15:29:11 2021 +0100| [de82f92b845e8e5e52ea107702eba11ea277800a] | committer: Andreas Rheinhardt

avcodec/vaapi_encode: Fix segfault upon closing uninitialized encoder

Fixes ticket #9537.
Probably a regression since 2b3206891649f317c20993411efef4bee39ae784.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt at outlook.com>
(cherry picked from commit d1b47f3bfcc625ca1cae210fc198dcbd54381a88)

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=de82f92b845e8e5e52ea107702eba11ea277800a
---

 libavcodec/vaapi_encode.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/libavcodec/vaapi_encode.c b/libavcodec/vaapi_encode.c
index 607858435f..b1fa3307cc 100644
--- a/libavcodec/vaapi_encode.c
+++ b/libavcodec/vaapi_encode.c
@@ -2366,6 +2366,11 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx)
     VAStatus vas;
     int err;
 
+    ctx->va_config  = VA_INVALID_ID;
+    ctx->va_context = VA_INVALID_ID;
+
+    /* If you add something that can fail above this av_frame_alloc(),
+     * modify ff_vaapi_encode_close() accordingly. */
     ctx->frame = av_frame_alloc();
     if (!ctx->frame) {
         return AVERROR(ENOMEM);
@@ -2377,9 +2382,6 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx)
         return AVERROR(EINVAL);
     }
 
-    ctx->va_config  = VA_INVALID_ID;
-    ctx->va_context = VA_INVALID_ID;
-
     ctx->input_frames_ref = av_buffer_ref(avctx->hw_frames_ctx);
     if (!ctx->input_frames_ref) {
         err = AVERROR(ENOMEM);
@@ -2531,6 +2533,11 @@ av_cold int ff_vaapi_encode_close(AVCodecContext *avctx)
     VAAPIEncodeContext *ctx = avctx->priv_data;
     VAAPIEncodePicture *pic, *next;
 
+    /* We check ctx->frame to know whether ff_vaapi_encode_init()
+     * has been called and va_config/va_context initialized. */
+    if (!ctx->frame)
+        return 0;
+
     for (pic = ctx->pic_start; pic; pic = next) {
         next = pic->next;
         vaapi_encode_free(avctx, pic);