[FFmpeg-devel] [PATCH] hevcdec: Fix get_format behaviour when the user declines to pick a format

[Mark Thompson]

Maintain the context pixfmt as NONE in this case, indicating that if the
user requests more decoding then get_format() will need to be called again.
Also remove the pixfmt setting from inside set_sps() and make it explicit,
so that the behaviour is clearer.
---
 libavcodec/hevcdec.c | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/libavcodec/hevcdec.c b/libavcodec/hevcdec.c
index a3b5c8cb71..dca478db6c 100644
--- a/libavcodec/hevcdec.c
+++ b/libavcodec/hevcdec.c
@@ -421,8 +421,7 @@ static enum AVPixelFormat get_format(HEVCContext *s, const HEVCSPS *sps)
     return ff_thread_get_format(s->avctx, pix_fmts);
 }
 
-static int set_sps(HEVCContext *s, const HEVCSPS *sps,
-                   enum AVPixelFormat pix_fmt)
+static int set_sps(HEVCContext *s, const HEVCSPS *sps)
 {
     int ret, i;
 
@@ -439,8 +438,6 @@ static int set_sps(HEVCContext *s, const HEVCSPS *sps,
 
     export_stream_params(s->avctx, &s->ps, sps);
 
-    s->avctx->pix_fmt = pix_fmt;
-
     ff_hevc_pred_init(&s->hpc,     sps->bit_depth);
     ff_hevc_dsp_init (&s->hevcdsp, sps->bit_depth);
     ff_videodsp_init (&s->vdsp,    sps->bit_depth);
@@ -512,7 +509,6 @@ static int hls_slice_header(HEVCContext *s)
     if (s->ps.sps != (HEVCSPS*)s->ps.sps_list[s->ps.pps->sps_id]->data) {
         const HEVCSPS *sps = (HEVCSPS*)s->ps.sps_list[s->ps.pps->sps_id]->data;
         const HEVCSPS *last_sps = s->ps.sps;
-        enum AVPixelFormat pix_fmt;
 
         if (last_sps && IS_IRAP(s) && s->nal_unit_type != HEVC_NAL_CRA_NUT) {
             if (sps->width != last_sps->width || sps->height != last_sps->height ||
@@ -522,17 +518,25 @@ static int hls_slice_header(HEVCContext *s)
         }
         ff_hevc_clear_refs(s);
 
-        ret = set_sps(s, sps, sps->pix_fmt);
+        ret = set_sps(s, sps);
         if (ret < 0)
             return ret;
 
+        s->seq_decode = (s->seq_decode + 1) & 0xff;
+        s->max_ra     = INT_MAX;
+
+        s->avctx->pix_fmt = AV_PIX_FMT_NONE;
+    }
+
+    if (s->avctx->pix_fmt == AV_PIX_FMT_NONE) {
+        const HEVCSPS *sps = (HEVCSPS*)s->ps.sps_list[s->ps.pps->sps_id]->data;
+        enum AVPixelFormat pix_fmt;
+
         pix_fmt = get_format(s, sps);
         if (pix_fmt < 0)
-            return pix_fmt;
-        s->avctx->pix_fmt = pix_fmt;
+            return AVERROR(EINVAL);
 
-        s->seq_decode = (s->seq_decode + 1) & 0xff;
-        s->max_ra     = INT_MAX;
+        s->avctx->pix_fmt = pix_fmt;
     }
 
     sh->dependent_slice_segment_flag = 0;
@@ -3411,9 +3415,11 @@ static int hevc_update_thread_context(AVCodecContext *dst,
     }
 
     if (s->ps.sps != s0->ps.sps)
-        if ((ret = set_sps(s, s0->ps.sps, src->pix_fmt)) < 0)
+        if ((ret = set_sps(s, s0->ps.sps)) < 0)
             return ret;
 
+    dst->pix_fmt = src->pix_fmt;
+
     s->seq_decode = s0->seq_decode;
     s->seq_output = s0->seq_output;
     s->pocTid0    = s0->pocTid0;
-- 
2.19.1