ffmpeg | branch: release/4.1 | Timo Rothenpieler <timo@rothenpieler.org> | Tue Sep 29 23:19:23 2020 +0200| [7f0db52c534edfb6bf7f8f008b87dc27caac9455] | committer: Timo Rothenpieler avcodec/cuviddec: handle arbitrarily sized extradata
http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=7f0db52c534edfb6bf...
libavcodec/cuviddec.c | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/libavcodec/cuviddec.c b/libavcodec/cuviddec.c index 67e39ac719..fef72385ed 100644 --- a/libavcodec/cuviddec.c +++ b/libavcodec/cuviddec.c @@ -83,7 +83,7 @@ typedef struct CuvidContext CUVIDDECODECAPS caps8, caps10, caps12; CUVIDPARSERPARAMS cuparseinfo; - CUVIDEOFORMATEX cuparse_ext; + CUVIDEOFORMATEX *cuparse_ext; CudaFunctions *cudl; CuvidFunctions *cvdl; @@ -713,6 +713,7 @@ static av_cold int cuvid_decode_end(AVCodecContext *avctx) av_buffer_unref(&ctx->hwdevice); av_freep(&ctx->key_frame); + av_freep(&ctx->cuparse_ext); cuvid_free_functions(&ctx->cvdl); @@ -817,6 +818,8 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx) CUcontext cuda_ctx = NULL; CUcontext dummy; const AVBitStreamFilter *bsf; + uint8_t *extradata; + uint32_t extradata_size; int ret = 0; enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_CUDA, @@ -913,11 +916,8 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx) ctx->cudl = device_hwctx->internal->cuda_dl; memset(&ctx->cuparseinfo, 0, sizeof(ctx->cuparseinfo)); - memset(&ctx->cuparse_ext, 0, sizeof(ctx->cuparse_ext)); memset(&seq_pkt, 0, sizeof(seq_pkt)); - ctx->cuparseinfo.pExtVideoInfo = &ctx->cuparse_ext; - switch (avctx->codec->id) { #if CONFIG_H264_CUVID_DECODER case AV_CODEC_ID_H264: @@ -987,17 +987,25 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx) goto error; } - ctx->cuparse_ext.format.seqhdr_data_length = ctx->bsf->par_out->extradata_size; - memcpy(ctx->cuparse_ext.raw_seqhdr_data, - ctx->bsf->par_out->extradata, - FFMIN(sizeof(ctx->cuparse_ext.raw_seqhdr_data), ctx->bsf->par_out->extradata_size)); + extradata = ctx->bsf->par_out->extradata; + extradata_size = ctx->bsf->par_out->extradata_size; } else if (avctx->extradata_size > 0) { - ctx->cuparse_ext.format.seqhdr_data_length = avctx->extradata_size; - memcpy(ctx->cuparse_ext.raw_seqhdr_data, - avctx->extradata, - FFMIN(sizeof(ctx->cuparse_ext.raw_seqhdr_data), avctx->extradata_size)); + extradata = avctx->extradata; + extradata_size = avctx->extradata_size; } + ctx->cuparse_ext = av_mallocz(sizeof(*ctx->cuparse_ext) + + FFMAX(extradata_size - sizeof(ctx->cuparse_ext->raw_seqhdr_data), 0)); + if (!ctx->cuparse_ext) { + ret = AVERROR(ENOMEM); + goto error; + } + + ctx->cuparse_ext->format.seqhdr_data_length = avctx->extradata_size; + memcpy(ctx->cuparse_ext->raw_seqhdr_data, extradata, extradata_size); + + ctx->cuparseinfo.pExtVideoInfo = ctx->cuparse_ext; + ctx->key_frame = av_mallocz(ctx->nb_surfaces * sizeof(int)); if (!ctx->key_frame) { ret = AVERROR(ENOMEM); @@ -1026,8 +1034,8 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx) if (ret < 0) goto error; - seq_pkt.payload = ctx->cuparse_ext.raw_seqhdr_data; - seq_pkt.payload_size = ctx->cuparse_ext.format.seqhdr_data_length; + seq_pkt.payload = ctx->cuparse_ext->raw_seqhdr_data; + seq_pkt.payload_size = ctx->cuparse_ext->format.seqhdr_data_length; if (seq_pkt.payload && seq_pkt.payload_size) { ret = CHECK_CU(ctx->cvdl->cuvidParseVideoData(ctx->cuparser, &seq_pkt)); @@ -1086,8 +1094,8 @@ static void cuvid_flush(AVCodecContext *avctx) if (ret < 0) goto error; - seq_pkt.payload = ctx->cuparse_ext.raw_seqhdr_data; - seq_pkt.payload_size = ctx->cuparse_ext.format.seqhdr_data_length; + seq_pkt.payload = ctx->cuparse_ext->raw_seqhdr_data; + seq_pkt.payload_size = ctx->cuparse_ext->format.seqhdr_data_length; if (seq_pkt.payload && seq_pkt.payload_size) { ret = CHECK_CU(ctx->cvdl->cuvidParseVideoData(ctx->cuparser, &seq_pkt));