[FFmpeg-devel] [PATCH v2] avcodec/hevc/ps: add a range check for sps_max_sub_layers

[James Almer]

It can't be higher than vps_max_sub_layers.

Do this while keeping the workaround for qsvenc_hevc calling ff_hevc_parse_sps()
without a vps_list, as in some cases it needs to parse an sps to generate a fake
vps derived from it.

Signed-off-by: James Almer <jamrial at gmail.com>
---
 libavcodec/hevc/ps.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/libavcodec/hevc/ps.c b/libavcodec/hevc/ps.c
index 993c60144f..c773d7d221 100644
--- a/libavcodec/hevc/ps.c
+++ b/libavcodec/hevc/ps.c
@@ -1158,6 +1158,7 @@ int ff_hevc_parse_sps(HEVCSPS *sps, GetBitContext *gb, unsigned int *sps_id,
     HEVCWindow *ow;
     int ret = 0;
     int bit_depth_chroma, num_comps, multi_layer_ext;
+    int vps_max_sub_layers;
     int i;
 
     // Coded parameters
@@ -1181,8 +1182,12 @@ int ff_hevc_parse_sps(HEVCSPS *sps, GetBitContext *gb, unsigned int *sps_id,
             return AVERROR(EINVAL);
 
         sps->max_sub_layers = sps->vps->vps_max_sub_layers;
-    }
-    if (sps->max_sub_layers > HEVC_MAX_SUB_LAYERS) {
+        vps_max_sub_layers  = sps->vps->vps_max_sub_layers;
+    } else
+        vps_max_sub_layers = sps->vps ? sps->vps->vps_max_sub_layers
+                                      : FFMIN(sps->max_sub_layers, HEVC_MAX_SUB_LAYERS);
+
+    if (sps->max_sub_layers > vps_max_sub_layers) {
         av_log(avctx, AV_LOG_ERROR, "sps_max_sub_layers out of range: %d\n",
                sps->max_sub_layers);
         return AVERROR_INVALIDDATA;
-- 
2.46.2