On Mon, 2022-11-21 at 15:58 +0000, Soft Works wrote:
-----Original Message----- From: Xiang, Haihao <haihao.xiang@intel.com> Sent: Monday, November 21, 2022 3:45 AM To: ffmpeg-devel@ffmpeg.org Cc: softworkz@hotmail.com; kierank@obe.tv; haihao.xiang-at- intel.com@ffmpeg.org; andreas.rheinhardt@outlook.com Subject: Re: [FFmpeg-devel] [PATCH v6 3/3] avcodec/qsvdec: Implement SEI parsing for QSV decoders
From: softworkz <softworkz@hotmail.com>
Signed-off-by: softworkz <softworkz@hotmail.com> --- libavcodec/Makefile | 2 +- libavcodec/qsvdec.c | 321 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 322 insertions(+), 1 deletion(-)
diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 90c7f113a3..cbddbb0ace 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP) += mss34dsp.o OBJS-$(CONFIG_PIXBLOCKDSP) += pixblockdsp.o OBJS-$(CONFIG_QPELDSP) += qpeldsp.o OBJS-$(CONFIG_QSV) += qsv.o -OBJS-$(CONFIG_QSVDEC) += qsvdec.o +OBJS-$(CONFIG_QSVDEC) += qsvdec.o h264_sei.o hevc_sei.o OBJS-$(CONFIG_QSVENC) += qsvenc.o OBJS-$(CONFIG_RANGECODER) += rangecoder.o OBJS-$(CONFIG_RDFT) += rdft.o diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c index 73405b5747..467a248224 100644 --- a/libavcodec/qsvdec.c +++ b/libavcodec/qsvdec.c @@ -41,6 +41,7 @@ #include "libavutil/time.h" #include "libavutil/imgutils.h" #include "libavutil/film_grain_params.h" +#include <libavutil/reverse.h>
#include "avcodec.h" #include "codec_internal.h" @@ -49,6 +50,9 @@ #include "hwconfig.h" #include "qsv.h" #include "qsv_internal.h" +#include "h264_sei.h" +#include "hevc_ps.h" +#include "hevc_sei.h"
#if QSV_ONEVPL #include <mfxdispatcher.h> @@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 }; AV_NOPTS_VALUE : pts_tb.num ? \ av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
+#define PAYLOAD_BUFFER_SIZE 65535 + typedef struct QSVAsyncFrame { mfxSyncPoint *sync; QSVFrame *frame; @@ -107,6 +113,9 @@ typedef struct QSVContext {
mfxExtBuffer **ext_buffers; int nb_ext_buffers; + + mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE]; + AVBufferRef *a53_buf_ref; } QSVContext;
static const AVCodecHWConfigInternal *const qsv_hw_configs[] = { @@ -628,6 +637,299 @@ static int qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam } #endif
+static int find_start_offset(mfxU8 data[4]) +{ + if (data[0] == 0 && data[1] == 0 && data[2] == 1) + return 3; + + if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] ==
+ return 4; + + return 0; +} + +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q, AVFrame* out) +{ + H264SEIContext sei = { 0 }; + GetBitContext gb = { 0 }; + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE }; + mfxU64 ts; + int ret; + + while (1) { + int start; + memset(payload.Data, 0, payload.BufSize); + + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload); + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) { + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q- payload_buffer), payload.BufSize); + return 0; + } + if (ret != MFX_ERR_NONE) + return ret; + + if (payload.NumBit == 0 || payload.NumBit >=
+ break; + + start = find_start_offset(payload.Data); + + switch (payload.Type) { + case SEI_TYPE_BUFFERING_PERIOD: + case SEI_TYPE_PIC_TIMING: + continue; + } + + if (init_get_bits(&gb, &payload.Data[start],
8) < 0) + av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader SEI type: %d Numbits %d error: %d\n", payload.Type,
+ else { + ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx); + + if (ret < 0) + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret); + else + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d\n", payload.Type, payload.NumBit); + } + } + + if (out) + return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL, 0); + + return 0; +} + +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q, QSVFrame* out) +{ + HEVCSEI sei = { 0 }; + HEVCParamSets ps = { 0 }; + GetBitContext gb = { 0 }; + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE }; + mfxFrameSurface1 *surface = &out->surface; + mfxU64 ts; + int ret, has_logged = 0; + + while (1) { + int start; + memset(payload.Data, 0, payload.BufSize); + + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload); + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) { + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q- payload_buffer), payload.BufSize); + return 0; + } + if (ret != MFX_ERR_NONE) + return ret; + + if (payload.NumBit == 0 || payload.NumBit >=
On Tue, 2022-10-25 at 04:03 +0000, softworkz wrote: 1) payload.BufSize * 8) payload.NumBit - start * payload.NumBit, ret); payload.BufSize * 8)
+ break; + + if (!has_logged) { + has_logged = 1; + av_log(avctx, AV_LOG_VERBOSE, "-----------------------
---------\n"); + av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI - payload timestamp: %llu - surface timestamp: %llu\n", ts, surface- Data.TimeStamp); + } + + if (ts != surface->Data.TimeStamp) { + av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp (%llu) does not match surface timestamp: (%llu)\n", ts, surface- Data.TimeStamp); + } + + start = find_start_offset(payload.Data); + + av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d Numbits %3d Start: %d\n", payload.Type, payload.NumBit, start); + + switch (payload.Type) { + case SEI_TYPE_BUFFERING_PERIOD: + case SEI_TYPE_PIC_TIMING: + continue; + case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME: + // There seems to be a bug in MSDK + payload.NumBit -= 8; + + break; + case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO: + // There seems to be a bug in MSDK + payload.NumBit = 48; + + break; + case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35: + // There seems to be a bug in MSDK + if (payload.NumBit == 552) + payload.NumBit = 528; + break; + } + + if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start * 8) < 0) + av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret); + else { + ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps, HEVC_NAL_SEI_PREFIX); + + if (ret < 0) + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret); + else + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d\n", payload.Type, payload.NumBit); + } + } + + if (has_logged) { + av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n"); + } + + if (out && out->frame) + return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame, avctx-
framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps- bit_depth_chroma);
I got segfault when trying your patchset,
frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps- bit_depth_chroma); (gdb) bt #0 0x00007ffff67c0497 in parse_sei_hevc (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777 #1 0x00007ffff67c1afe in qsv_decode (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288,
Thread 1 "ffmpeg" received signal SIGSEGV, Segmentation fault. 0x00007ffff67c0497 in parse_sei_hevc (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777 777 return ff_hevc_set_sei_to_frame(avctx, &sei, out- frame=frame@entry=0x5555556df740, got_frame=got_frame@entry=0x7fffffffd6bc, avpkt=avpkt@entry=0x555555635398) at libavcodec/qsvdec.c:1020 BTW the SDK provides support for hevc HDR metadata, we needn't parse SEI payload in qsvdec and may get the corresponding info from the SDK, see https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html
I know. I was the one who had requested this to be added to MSDK :-)
But it's just one small part of SEI information, it's limited to the latest MSDK versions and I'm not sure whether it's working as reliably as this implementation. This would need to be tested.
You should still have access to the repo with the test files for demoing the offset problems (which my patchset is working around).
I worked out a patchset (https://github.com/intel-media-ci/ffmpeg/pull/518), including https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html an d others. I may use your command (with some changes) to convert all HDR clips in your repo to SDR clips except one clip (w/wo my changes, there is a GPU hang issue with this clip). We may only do a small update to support AV1 HDR in the future if applying https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html. Note the command below doesn't work with all test files in your repo with your patchset v6. $ ffmpeg -hwaccel qsv -c:v hevc_qsv -i input.mp4 -f null - Thanks Haihao
But it's also about dynamic HDR data, dovi data, etc. - which MSDK doesn't provide, so this single bit of SEI data doesn't help much.
Best regards, softworkz