[FFmpeg-devel] [PATCH 2/5] avcodec/bsf/hevc_mp4toannexb: Factor creating new extradata out

[Andreas Rheinhardt]

This is in preparation for the next commit.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt at outlook.com>
---
 libavcodec/bsf/hevc_mp4toannexb.c | 64 ++++++++++++++++++++-----------
 1 file changed, 41 insertions(+), 23 deletions(-)

diff --git a/libavcodec/bsf/hevc_mp4toannexb.c b/libavcodec/bsf/hevc_mp4toannexb.c
index c0df2b79a6..a695cba370 100644
--- a/libavcodec/bsf/hevc_mp4toannexb.c
+++ b/libavcodec/bsf/hevc_mp4toannexb.c
@@ -37,38 +37,32 @@ typedef struct HEVCBSFContext {
     int      extradata_parsed;
 } HEVCBSFContext;
 
-static int hevc_extradata_to_annexb(AVBSFContext *ctx)
+static int hevc_extradata_to_annexb_internal(void *logctx, GetByteContext *gb,
+                                             uint8_t **new_extradatap,
+                                             size_t *new_extradata_sizep)
 {
-    GetByteContext gb;
-    int length_size, num_arrays, i, j;
-    int ret = 0;
-
+    int num_arrays = bytestream2_get_byte(gb);
     uint8_t *new_extradata = NULL;
-    size_t   new_extradata_size = 0;
-
-    bytestream2_init(&gb, ctx->par_in->extradata, ctx->par_in->extradata_size);
-
-    bytestream2_skip(&gb, 21);
-    length_size = (bytestream2_get_byte(&gb) & 3) + 1;
-    num_arrays  = bytestream2_get_byte(&gb);
+    size_t new_extradata_size = 0;
+    int ret;
 
-    for (i = 0; i < num_arrays; i++) {
-        int type = bytestream2_get_byte(&gb) & 0x3f;
-        int cnt  = bytestream2_get_be16(&gb);
+    for (int i = 0; i < num_arrays; i++) {
+        int type = bytestream2_get_byte(gb) & 0x3f;
+        int cnt  = bytestream2_get_be16(gb);
 
         if (!(type == HEVC_NAL_VPS || type == HEVC_NAL_SPS || type == HEVC_NAL_PPS ||
               type == HEVC_NAL_SEI_PREFIX || type == HEVC_NAL_SEI_SUFFIX)) {
-            av_log(ctx, AV_LOG_ERROR, "Invalid NAL unit type in extradata: %d\n",
+            av_log(logctx, AV_LOG_ERROR, "Invalid NAL unit type in extradata: %d\n",
                    type);
             ret = AVERROR_INVALIDDATA;
             goto fail;
         }
 
-        for (j = 0; j < cnt; j++) {
-            const int nalu_len = bytestream2_get_be16(&gb);
+        for (int j = 0; j < cnt; j++) {
+            const int nalu_len = bytestream2_get_be16(gb);
 
             if (!nalu_len ||
-                nalu_len > bytestream2_get_bytes_left(&gb) ||
+                nalu_len > bytestream2_get_bytes_left(gb) ||
                 4 + nalu_len > FFMIN(INT_MAX, SIZE_MAX) - AV_INPUT_BUFFER_PADDING_SIZE - new_extradata_size) {
                 ret = AVERROR_INVALIDDATA;
                 goto fail;
@@ -78,11 +72,38 @@ static int hevc_extradata_to_annexb(AVBSFContext *ctx)
                 goto fail;
 
             AV_WB32(new_extradata + new_extradata_size, 1); // add the startcode
-            bytestream2_get_buffer(&gb, new_extradata + new_extradata_size + 4, nalu_len);
+            bytestream2_get_buffer(gb, new_extradata + new_extradata_size + 4, nalu_len);
             new_extradata_size += 4 + nalu_len;
             memset(new_extradata + new_extradata_size, 0, AV_INPUT_BUFFER_PADDING_SIZE);
         }
     }
+    *new_extradatap = new_extradata;
+    *new_extradata_sizep = new_extradata_size;
+
+    return 0;
+fail:
+    av_freep(&new_extradata);
+    return ret;
+}
+
+static int hevc_extradata_to_annexb(AVBSFContext *ctx)
+{
+    GetByteContext gb;
+    int length_size;
+    int ret = 0;
+
+    uint8_t *new_extradata = NULL;
+    size_t   new_extradata_size;
+
+    bytestream2_init(&gb, ctx->par_in->extradata, ctx->par_in->extradata_size);
+
+    bytestream2_skip(&gb, 21);
+    length_size = (bytestream2_get_byte(&gb) & 3) + 1;
+
+    ret = hevc_extradata_to_annexb_internal(ctx, &gb, &new_extradata,
+                                            &new_extradata_size);
+    if (ret < 0)
+        return ret;
 
     av_freep(&ctx->par_out->extradata);
     ctx->par_out->extradata      = new_extradata;
@@ -92,9 +113,6 @@ static int hevc_extradata_to_annexb(AVBSFContext *ctx)
         av_log(ctx, AV_LOG_WARNING, "No parameter sets in the extradata\n");
 
     return length_size;
-fail:
-    av_freep(&new_extradata);
-    return ret;
 }
 
 static int hevc_mp4toannexb_init(AVBSFContext *ctx)
-- 
2.34.1