July 2025 - ffmpeg-devel - lists.ffmpeg.org

[PATCH 1/5] avcodec/smacker: Move buffer allocation to later
by Michael Niedermayer 06 Jul '25

06 Jul '25

Reduces allocations on random input Fixes: 421650030/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_SMACKAUD_fuzzer-6144441767493632 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer <michael(a)niedermayer.cc> --- libavcodec/smacker.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libavcodec/smacker.c b/libavcodec/smacker.c index db464dfcf7d..bd08bc7be28 100644 --- a/libavcodec/smacker.c +++ b/libavcodec/smacker.c @@ -641,10 +641,6 @@ static int smka_decode_frame(AVCodecContext *avctx, AVFrame *frame, "The buffer does not contain an integer number of samples\n"); return AVERROR_INVALIDDATA; } - if ((ret = ff_get_buffer(avctx, frame, 0)) < 0) - return ret; - samples = (int16_t *)frame->data[0]; - samples8 = frame->data[0]; // Initialize for(i = 0; i < (1 << (bits + stereo)); i++) { @@ -666,6 +662,12 @@ static int smka_decode_frame(AVCodecContext *avctx, AVFrame *frame, } else values[i] = h.entries[0].value; } + + if ((ret = ff_get_buffer(avctx, frame, 0)) < 0) + return ret; + samples = (int16_t *)frame->data[0]; + samples8 = frame->data[0]; + /* this codec relies on wraparound instead of clipping audio */ if(bits) { //decode 16-bit data for(i = stereo; i >= 0; i--) -- 2.49.0

3 9

[PATCH v2] avutil/hwcontext_d3d12va: added resource Flags support to command line
by Dmitrii Ovchinnikov 06 Jul '25

06 Jul '25

--- libavutil/hwcontext_d3d12va.c | 22 +++++++++++++++++++++- libavutil/hwcontext_d3d12va.h | 6 ++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/libavutil/hwcontext_d3d12va.c b/libavutil/hwcontext_d3d12va.c index 6507cf69c1..b2b58ac6f8 100644 --- a/libavutil/hwcontext_d3d12va.c +++ b/libavutil/hwcontext_d3d12va.c @@ -254,7 +254,7 @@ static AVBufferRef *d3d12va_pool_alloc(void *opaque, size_t size) if (!frame) return NULL; - if (FAILED(ID3D12Device_CreateCommittedResource(device_hwctx->device, &props, D3D12_HEAP_FLAG_NONE, &desc, + if (FAILED(ID3D12Device_CreateCommittedResource(device_hwctx->device, &props, hwctx->heap_flags, &desc, D3D12_RESOURCE_STATE_COMMON, NULL, &IID_ID3D12Resource, (void **)&frame->texture))) { av_log(ctx, AV_LOG_ERROR, "Could not create the texture\n"); goto fail; @@ -281,6 +281,7 @@ fail: static int d3d12va_frames_init(AVHWFramesContext *ctx) { AVD3D12VAFramesContext *hwctx = ctx->hwctx; + AVD3D12VADeviceContext *device_hwctx = ctx->device_ctx->hwctx; int i; for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) { @@ -298,6 +299,9 @@ static int d3d12va_frames_init(AVHWFramesContext *ctx) return AVERROR(EINVAL); } + hwctx->flags |= device_hwctx->flags; + hwctx->heap_flags |= device_hwctx->heap_flags; + ffhwframesctx(ctx)->pool_internal = av_buffer_pool_init2(sizeof(AVD3D12VAFrame), ctx, d3d12va_pool_alloc, NULL); @@ -673,6 +677,22 @@ static int d3d12va_device_create(AVHWDeviceContext *hwdev, const char *device, } } + ctx->flags = 0; + ctx->heap_flags = 0; + + // "shader_resouce" already enabled by default. + + if (av_dict_get(opts, "uav_res", NULL, 0)) + ctx->flags |= D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS; + + if (av_dict_get(opts, "render_target_res", NULL, 0)) + ctx->flags |= D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET; + + ctx->heap_flags = D3D12_HEAP_FLAG_NONE; + + if (av_dict_get(opts, "shared_res", NULL, 0)) + ctx->heap_flags |= D3D12_HEAP_FLAG_SHARED; + return 0; } diff --git a/libavutil/hwcontext_d3d12va.h b/libavutil/hwcontext_d3d12va.h index 212a6a6146..2075bd4a48 100644 --- a/libavutil/hwcontext_d3d12va.h +++ b/libavutil/hwcontext_d3d12va.h @@ -75,6 +75,11 @@ typedef struct AVD3D12VADeviceContext { void (*lock)(void *lock_ctx); void (*unlock)(void *lock_ctx); void *lock_ctx; + /** + * Need to pass value to frames context from command line + */ + D3D12_RESOURCE_FLAGS flags; + D3D12_HEAP_FLAGS heap_flags; } AVD3D12VADeviceContext; /** @@ -137,6 +142,7 @@ typedef struct AVD3D12VAFramesContext { * @see https://learn.microsoft.com/en-us/windows/win32/api/d3d12/ne-d3d12-d3d12_re… */ D3D12_RESOURCE_FLAGS flags; + D3D12_HEAP_FLAGS heap_flags; } AVD3D12VAFramesContext; #endif /* AVUTIL_HWCONTEXT_D3D12VA_H */ -- 2.47.1.windows.1

2 1

[PATCH v2 3/3] avcodec/ohenc: Add h264/hevc OpenHarmony encoders
by Zhao Zhili 06 Jul '25

06 Jul '25

From: Zhao Zhili <zhilizhao(a)tencent.com> --- Changelog | 1 + configure | 7 +- libavcodec/Makefile | 2 + libavcodec/allcodecs.c | 2 + libavcodec/ohcodec.c | 8 + libavcodec/ohcodec.h | 1 + libavcodec/ohenc.c | 693 +++++++++++++++++++++++++++++++++++++++++ libavcodec/version.h | 2 +- 8 files changed, 713 insertions(+), 3 deletions(-) create mode 100644 libavcodec/ohenc.c diff --git a/Changelog b/Changelog index f5ec9eafc3..4a8e0ef60d 100644 --- a/Changelog +++ b/Changelog @@ -21,6 +21,7 @@ version <next>: - G.728 decoder - pad_cuda filter - Sanyo LD-ADPCM decoder +- OpenHarmony hardware decoder/encoder version 7.1: diff --git a/configure b/configure index a2ace36cdc..225aaaee99 100755 --- a/configure +++ b/configure @@ -3412,6 +3412,7 @@ h264_nvenc_encoder_deps="nvenc" h264_nvenc_encoder_select="atsc_a53" h264_oh_decoder_deps="ohcodec" h264_oh_decoder_select="h264_mp4toannexb_bsf" +h264_oh_encoder_deps="ohcodec" h264_omx_encoder_deps="omx" h264_qsv_decoder_select="h264_mp4toannexb_bsf qsvdec" h264_qsv_encoder_select="atsc_a53 qsvenc" @@ -3436,6 +3437,7 @@ hevc_nvenc_encoder_deps="nvenc" hevc_nvenc_encoder_select="atsc_a53" hevc_oh_decoder_deps="ohcodec" hevc_oh_decoder_select="hevc_mp4toannexb_bsf" +hevc_oh_encoder_deps="ohcodec" hevc_qsv_decoder_select="hevc_mp4toannexb_bsf qsvdec" hevc_qsv_encoder_select="hevcparse qsvenc" hevc_rkmpp_decoder_deps="rkmpp" @@ -7171,8 +7173,9 @@ enabled mmal && { check_lib mmal interface/mmal/mmal.h mmal_port_co check_lib mmal interface/mmal/mmal.h mmal_port_connect -lmmal_core -lmmal_util -lmmal_vc_client -lbcm_host; } || die "ERROR: mmal not found" && check_func_headers interface/mmal/mmal.h "MMAL_PARAMETER_VIDEO_MAX_NUM_CALLBACKS"; } -enabled ohcodec && { check_lib ohcodec "multimedia/player_framework/native_avcodec_videodecoder.h" \ - OH_VideoDecoder_CreateByName -lnative_media_codecbase -lnative_media_core -lnative_media_vdec || +enabled ohcodec && { check_lib ohcodec "multimedia/player_framework/native_avcodec_videodecoder.h multimedia/player_framework/native_avcodec_videoencoder.h" \ + "OH_VideoDecoder_CreateByName OH_VideoEncoder_CreateByName" \ + -lnative_media_vdec -lnative_media_venc -lnative_media_codecbase -lnative_media_core || die "ERROR: missing native_media libs"; } enabled openal && { check_pkg_config openal "openal >= 1.1" "AL/al.h" alGetError || { for al_extralibs in "${OPENAL_LIBS}" "-lopenal" "-lOpenAL32"; do diff --git a/libavcodec/Makefile b/libavcodec/Makefile index b784bf3f7e..f935cfbe0d 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -429,6 +429,7 @@ OBJS-$(CONFIG_H264_MF_ENCODER) += mfenc.o mf_utils.o OBJS-$(CONFIG_H264_MMAL_DECODER) += mmaldec.o OBJS-$(CONFIG_H264_NVENC_ENCODER) += nvenc_h264.o nvenc.o OBJS-$(CONFIG_H264_OH_DECODER) += ohcodec.o ohdec.o +OBJS-$(CONFIG_H264_OH_ENCODER) += ohcodec.o ohenc.o OBJS-$(CONFIG_H264_OMX_ENCODER) += omx.o OBJS-$(CONFIG_H264_QSV_DECODER) += qsvdec.o OBJS-$(CONFIG_H264_QSV_ENCODER) += qsvenc_h264.o @@ -458,6 +459,7 @@ OBJS-$(CONFIG_HEVC_MEDIACODEC_ENCODER) += mediacodecenc.o OBJS-$(CONFIG_HEVC_MF_ENCODER) += mfenc.o mf_utils.o OBJS-$(CONFIG_HEVC_NVENC_ENCODER) += nvenc_hevc.o nvenc.o OBJS-$(CONFIG_HEVC_OH_DECODER) += ohcodec.o ohdec.o +OBJS-$(CONFIG_HEVC_OH_ENCODER) += ohcodec.o ohenc.o OBJS-$(CONFIG_HEVC_QSV_DECODER) += qsvdec.o OBJS-$(CONFIG_HEVC_QSV_ENCODER) += qsvenc_hevc.o hevc/ps_enc.o OBJS-$(CONFIG_HEVC_RKMPP_DECODER) += rkmppdec.o diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c index 826ccf7329..3aaa351157 100644 --- a/libavcodec/allcodecs.c +++ b/libavcodec/allcodecs.c @@ -861,6 +861,7 @@ extern const FFCodec ff_h264_cuvid_decoder; extern const FFCodec ff_h264_mf_encoder; extern const FFCodec ff_h264_nvenc_encoder; extern const FFCodec ff_h264_oh_decoder; +extern const FFCodec ff_h264_oh_encoder; extern const FFCodec ff_h264_omx_encoder; extern const FFCodec ff_h264_qsv_encoder; extern const FFCodec ff_h264_v4l2m2m_encoder; @@ -876,6 +877,7 @@ extern const FFCodec ff_hevc_mediacodec_encoder; extern const FFCodec ff_hevc_mf_encoder; extern const FFCodec ff_hevc_nvenc_encoder; extern const FFCodec ff_hevc_oh_decoder; +extern const FFCodec ff_hevc_oh_encoder; extern const FFCodec ff_hevc_qsv_encoder; extern const FFCodec ff_hevc_v4l2m2m_encoder; extern const FFCodec ff_hevc_vaapi_encoder; diff --git a/libavcodec/ohcodec.c b/libavcodec/ohcodec.c index 284ff1b2ec..f8a6faaba5 100644 --- a/libavcodec/ohcodec.c +++ b/libavcodec/ohcodec.c @@ -69,3 +69,11 @@ enum AVPixelFormat ff_oh_pix_to_ff_pix(OH_AVPixelFormat oh_pix) return AV_PIX_FMT_NONE; } +int ff_oh_pix_from_ff_pix(enum AVPixelFormat pix) +{ + for (size_t i = 0; i < FF_ARRAY_ELEMS(oh_pix_map); i++) + if (oh_pix_map[i].pix == pix) + return oh_pix_map[i].oh_pix; + + return 0; +} diff --git a/libavcodec/ohcodec.h b/libavcodec/ohcodec.h index 236ddba67e..074747247b 100644 --- a/libavcodec/ohcodec.h +++ b/libavcodec/ohcodec.h @@ -52,5 +52,6 @@ static inline const char *ff_oh_mime(enum AVCodecID codec_id, void *log) } enum AVPixelFormat ff_oh_pix_to_ff_pix(OH_AVPixelFormat oh_pix); +int ff_oh_pix_from_ff_pix(enum AVPixelFormat pix); #endif diff --git a/libavcodec/ohenc.c b/libavcodec/ohenc.c new file mode 100644 index 0000000000..70b17dc029 --- /dev/null +++ b/libavcodec/ohenc.c @@ -0,0 +1,693 @@ +/* + * This file is part of FFmpeg. + * + * Copyright (c) 2025 Zhao Zhili <quinkblack(a)foxmail.com> + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config_components.h" + +#include <stdbool.h> +#include <multimedia/player_framework/native_avcapability.h> +#include <multimedia/player_framework/native_avcodec_videoencoder.h> + +#include "libavutil/fifo.h" +#include "libavutil/imgutils.h" +#include "libavutil/mem.h" +#include "libavutil/opt.h" +#include "libavutil/thread.h" + +#include "avcodec.h" +#include "codec_internal.h" +#include "encode.h" +#include "hwconfig.h" +#include "ohcodec.h" + +typedef struct OHCodecEncContext { + AVClass *avclass; + OH_AVCodec *enc; + + AVMutex input_mutex; + AVCond input_cond; + AVFifo *input_queue; + + AVMutex output_mutex; + AVCond output_cond; + AVFifo *output_queue; + + AVFrame *frame; + uint8_t *extradata; + int extradata_size; + + int encode_status; + bool eof_sent; + + bool got_stream_info; + int stride; + int slice_height; + + uint64_t native_window; + char *name; + int allow_sw; + int bitrate_mode; +} OHCodecEncContext; + +static const enum AVPixelFormat ohcodec_pix_fmts[] = { + AV_PIX_FMT_OHCODEC, + AV_PIX_FMT_NV12, + AV_PIX_FMT_NONE +}; + +static int oh_encode_create(OHCodecEncContext *s, AVCodecContext *avctx) +{ + const char *name = s->name; + + if (!name) { + const char *mime = ff_oh_mime(avctx->codec_id, avctx); + if (!mime) + return AVERROR_BUG; + OH_AVCapability *cap = OH_AVCodec_GetCapabilityByCategory(mime, true, HARDWARE); + if (!cap) { + if (!s->allow_sw) { + av_log(avctx, AV_LOG_ERROR, "Failed to get hardware codec %s\n", mime); + return AVERROR_EXTERNAL; + } + av_log(avctx, AV_LOG_WARNING, + "Failed to get hardware codec %s, try software backend\n", mime); + cap = OH_AVCodec_GetCapabilityByCategory(mime, true, SOFTWARE); + if (!cap) { + av_log(avctx, AV_LOG_ERROR, "Failed to get software codec %s\n", mime); + return AVERROR_EXTERNAL; + } + } + name = OH_AVCapability_GetName(cap); + if (!name) + return AVERROR_EXTERNAL; + } + + s->enc = OH_VideoEncoder_CreateByName(name); + if (!s->enc) { + av_log(avctx, AV_LOG_ERROR, "Create encoder with name %s failed\n", name); + return AVERROR_EXTERNAL; + } + av_log(avctx, AV_LOG_DEBUG, "Create encoder %s success\n", name); + + return 0; +} + +static int oh_encode_set_format(OHCodecEncContext *s, AVCodecContext *avctx) +{ + int ret; + + OH_AVFormat *format = OH_AVFormat_Create(); + if (!format) + return AVERROR(ENOMEM); + + bool b = OH_AVFormat_SetIntValue(format, OH_MD_KEY_WIDTH, avctx->width); + b = b && OH_AVFormat_SetIntValue(format, OH_MD_KEY_HEIGHT, avctx->height); + if (!b) { + av_log(avctx, AV_LOG_ERROR, "Set width/height (%dx%d) failed\n", + avctx->width, avctx->height); + ret = AVERROR_EXTERNAL; + goto out; + } + if (avctx->framerate.num && avctx->framerate.den) + OH_AVFormat_SetDoubleValue(format, OH_MD_KEY_FRAME_RATE, + av_q2d(avctx->framerate)); + int pix = ff_oh_pix_from_ff_pix(avctx->pix_fmt); + if (!pix) { + ret = AVERROR_BUG; + goto out; + } + b = OH_AVFormat_SetIntValue(format, OH_MD_KEY_PIXEL_FORMAT, pix); + if (!b) { + av_log(avctx, AV_LOG_ERROR, "Set pixel format to %d failed\n", pix); + ret = AVERROR_EXTERNAL; + goto out; + } + + if (s->bitrate_mode != -1) { + b = OH_AVFormat_SetIntValue(format, OH_MD_KEY_VIDEO_ENCODE_BITRATE_MODE, s->bitrate_mode); + if (!b) { + av_log(avctx, AV_LOG_ERROR, "Set bitrate mode to %d failed\n", + s->bitrate_mode); + ret = AVERROR_EXTERNAL; + goto out; + } + } + OH_AVFormat_SetLongValue(format, OH_MD_KEY_BITRATE, avctx->bit_rate); + + if (avctx->gop_size > 0) { + if (avctx->framerate.num > 0 && avctx->framerate.den > 0) { + // In milliseconds + int gop = av_rescale_q(avctx->gop_size, + av_make_q(avctx->framerate.den, + avctx->framerate.num), + av_make_q(1, 1000)); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_I_FRAME_INTERVAL, gop); + } else { + av_log(avctx, AV_LOG_WARNING, "Skip setting gop without framerate\n"); + } + } else if (!avctx->gop_size) { + // All frames are key frame + OH_AVFormat_SetIntValue(format, OH_MD_KEY_I_FRAME_INTERVAL, 0); + } else if (avctx->gop_size == -1) { + // Infinite gop + OH_AVFormat_SetIntValue(format, OH_MD_KEY_I_FRAME_INTERVAL, -1); + } + + OH_AVErrCode err = OH_VideoEncoder_Configure(s->enc, format); + if (err != AV_ERR_OK) { + ret = ff_oh_err_to_ff_err(err); + av_log(avctx, AV_LOG_ERROR, "Decoder configure failed, %d, %s\n", + err, av_err2str(ret)); + goto out; + } + + if (avctx->pix_fmt == AV_PIX_FMT_OHCODEC) { + OHNativeWindow *window; + err = OH_VideoEncoder_GetSurface(s->enc, &window); + if (err != AV_ERR_OK) { + ret = ff_oh_err_to_ff_err(err); + av_log(avctx, AV_LOG_ERROR, "Get surface failed, %d, %s\n", + err, av_err2str(ret)); + goto out; + } + s->native_window = (uintptr_t)window; + av_log(avctx, AV_LOG_INFO, "native window %p\n", window); + } + + return 0; +out: + OH_AVFormat_Destroy(format); + return ret; +} + +static void oh_encode_on_err(OH_AVCodec *codec, int32_t err, void *userdata) +{ + AVCodecContext *avctx = userdata; + OHCodecEncContext *s = avctx->priv_data; + + // Careful on the lock order. + // Always lock input first. + ff_mutex_lock(&s->input_mutex); + ff_mutex_lock(&s->output_mutex); + s->encode_status = ff_oh_err_to_ff_err(err); + ff_mutex_unlock(&s->output_mutex); + ff_mutex_unlock(&s->input_mutex); + + ff_cond_signal(&s->output_cond); + ff_cond_signal(&s->input_cond); +} + +static void oh_encode_on_stream_changed(OH_AVCodec *codec, OH_AVFormat *format, + void *userdata) +{ + AVCodecContext *avctx = userdata; + OHCodecEncContext *s = avctx->priv_data; + + if (!OH_AVFormat_GetIntValue(format, OH_MD_KEY_VIDEO_STRIDE, &s->stride)) + s->stride = avctx->width; + if (!OH_AVFormat_GetIntValue(format, OH_MD_KEY_VIDEO_SLICE_HEIGHT, &s->slice_height)) + s->slice_height = avctx->height; + + s->got_stream_info = true; +} + +static void oh_encode_on_need_input(OH_AVCodec *codec, uint32_t index, + OH_AVBuffer *buffer, void *userdata) +{ + AVCodecContext *avctx = userdata; + OHCodecEncContext *s = avctx->priv_data; + OHBufferQueueItem item = { + index, buffer, + }; + + ff_mutex_lock(&s->input_mutex); + int ret = av_fifo_write(s->input_queue, &item, 1); + if (ret >= 0) + ff_cond_signal(&s->input_cond); + ff_mutex_unlock(&s->input_mutex); + + if (ret < 0) + oh_encode_on_err(codec, AV_ERR_NO_MEMORY, userdata); +} + +static void oh_encode_on_output(OH_AVCodec *codec, uint32_t index, + OH_AVBuffer *buffer, void *userdata) +{ + AVCodecContext *avctx = userdata; + OHCodecEncContext *s = avctx->priv_data; + OHBufferQueueItem item = { + index, buffer, + }; + + ff_mutex_lock(&s->output_mutex); + int ret = av_fifo_write(s->output_queue, &item, 1); + if (ret >= 0) + ff_cond_signal(&s->output_cond); + ff_mutex_unlock(&s->output_mutex); + + if (ret < 0) + oh_encode_on_err(codec, AV_ERR_NO_MEMORY, userdata); +} + +static int oh_encode_start(OHCodecEncContext *s, AVCodecContext *avctx) +{ + int ret; + OH_AVErrCode err; + OH_AVCodecCallback cb = { + .onError = oh_encode_on_err, + .onStreamChanged = oh_encode_on_stream_changed, + .onNeedInputBuffer = oh_encode_on_need_input, + .onNewOutputBuffer = oh_encode_on_output, + }; + + err = OH_VideoEncoder_RegisterCallback(s->enc, cb, avctx); + if (err != AV_ERR_OK) { + ret = ff_oh_err_to_ff_err(err); + av_log(avctx, AV_LOG_ERROR, "Register callback failed, %d, %s\n", + err, av_err2str(ret)); + return ret; + } + err = OH_VideoEncoder_Prepare(s->enc); + if (err != AV_ERR_OK) { + ret = ff_oh_err_to_ff_err(err); + av_log(avctx, AV_LOG_ERROR, "Prepare failed, %d, %s\n", + err, av_err2str(ret)); + return ret; + } + err = OH_VideoEncoder_Start(s->enc); + if (err != AV_ERR_OK) { + ret = ff_oh_err_to_ff_err(err); + av_log(avctx, AV_LOG_ERROR, "Start failed, %d, %s\n", + err, av_err2str(ret)); + return ret; + } + + return 0; +} + +static av_cold int oh_encode_init(AVCodecContext *avctx) +{ + OHCodecEncContext *s = avctx->priv_data; + + // Initialize these fields first, so oh_decode_close can destroy them safely + ff_mutex_init(&s->input_mutex, NULL); + ff_cond_init(&s->input_cond, NULL); + ff_mutex_init(&s->output_mutex, NULL); + ff_cond_init(&s->output_cond, NULL); + + int ret = oh_encode_create(s, avctx); + if (ret < 0) + return ret; + ret = oh_encode_set_format(s, avctx); + if (ret < 0) + return ret; + + size_t fifo_size = 16; + s->input_queue = av_fifo_alloc2(fifo_size, sizeof(OHBufferQueueItem), + AV_FIFO_FLAG_AUTO_GROW); + s->output_queue = av_fifo_alloc2(fifo_size, sizeof(OHBufferQueueItem), + AV_FIFO_FLAG_AUTO_GROW); + s->frame = av_frame_alloc(); + if (!s->input_queue || !s->output_queue || !s->frame) + return AVERROR(ENOMEM); + + ret = oh_encode_start(s, avctx); + if (ret < 0) + return ret; + + return 0; +} + +static av_cold int oh_encode_close(AVCodecContext *avctx) +{ + OHCodecEncContext *s = avctx->priv_data; + + if (s->enc) { + OH_VideoEncoder_Stop(s->enc); + OH_AVErrCode err = OH_VideoEncoder_Destroy(s->enc); + if (err == AV_ERR_OK) + av_log(avctx, AV_LOG_DEBUG, "Destroy encoder success\n"); + else + av_log(avctx, AV_LOG_ERROR, "Destroy decoder failed, %d, %s\n", + err, av_err2str(ff_oh_err_to_ff_err(err))); + } + + av_freep(&s->extradata); + av_frame_free(&s->frame); + + ff_mutex_destroy(&s->input_mutex); + ff_cond_destroy(&s->input_cond); + av_fifo_freep2(&s->input_queue); + + ff_mutex_destroy(&s->output_mutex); + ff_cond_destroy(&s->output_cond); + av_fifo_freep2(&s->output_queue); + + return 0; +} + +static int oh_encode_output_packet(AVCodecContext *avctx, AVPacket *pkt, + OHBufferQueueItem *output) +{ + OHCodecEncContext *s = avctx->priv_data; + uint8_t *p; + OH_AVCodecBufferAttr attr; + int ret; + + OH_AVErrCode err = OH_AVBuffer_GetBufferAttr(output->buffer, &attr); + if (err != AV_ERR_OK) { + ret = ff_oh_err_to_ff_err(err); + goto out; + } + if (attr.flags & AVCODEC_BUFFER_FLAGS_EOS) { + av_log(avctx, AV_LOG_DEBUG, "Buffer flag eos\n"); + ret = AVERROR_EOF; + goto out; + } + + p = OH_AVBuffer_GetAddr(output->buffer); + if (!p) { + av_log(avctx, AV_LOG_ERROR, "Failed to get output buffer addr\n"); + ret = AVERROR_EXTERNAL; + goto out; + } + if (attr.flags & AVCODEC_BUFFER_FLAGS_CODEC_DATA) { + av_freep(&s->extradata); + s->extradata = av_malloc(attr.size + AV_INPUT_BUFFER_PADDING_SIZE); + if (!s->extradata) { + ret = AVERROR(ENOMEM); + goto out; + } + memset(s->extradata + attr.size, 0, AV_INPUT_BUFFER_PADDING_SIZE); + memcpy(s->extradata, p + attr.offset, attr.size); + s->extradata_size = attr.size; + ret = 0; + goto out; + } + + int64_t extradata_size = s->extradata_size; + s->extradata_size = 0; + + if (extradata_size && (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER)) { + ret = av_packet_add_side_data(pkt, AV_PKT_DATA_NEW_EXTRADATA, + s->extradata, extradata_size); + if (ret < 0) + goto out; + s->extradata = NULL; + extradata_size = 0; + } + + ret = ff_get_encode_buffer(avctx, pkt, attr.size + extradata_size, 0); + if (ret < 0) + goto out; + + if (extradata_size) + memcpy(pkt->data, s->extradata, extradata_size); + + memcpy(pkt->data + extradata_size, p + attr.offset, attr.size); + pkt->pts = av_rescale_q(attr.pts, AV_TIME_BASE_Q, avctx->time_base); + if (attr.flags & AVCODEC_BUFFER_FLAGS_SYNC_FRAME) + pkt->flags |= AV_PKT_FLAG_KEY; + ret = 0; +out: + OH_VideoEncoder_FreeOutputBuffer(s->enc, output->index); + return ret; +} + +static int oh_encode_send_hw_frame(AVCodecContext *avctx) +{ + OHCodecEncContext *s = avctx->priv_data; + + if (s->eof_sent) + return 0; + + if (s->frame->buf[0]) { + av_frame_unref(s->frame); + return 0; + } + + OH_AVErrCode err = OH_VideoEncoder_NotifyEndOfStream(s->enc); + s->eof_sent = true; + return ff_oh_err_to_ff_err(err); +} + +static int oh_encode_send_sw_frame(AVCodecContext *avctx, OHBufferQueueItem *input) +{ + OHCodecEncContext *s = avctx->priv_data; + AVFrame *frame = s->frame; + OH_AVErrCode err; + int ret; + + if (!s->got_stream_info) { + // This shouldn't happen, add a warning message. + av_log(avctx, AV_LOG_WARNING, + "decoder didn't notify stream info, try get format explicitly\n"); + + OH_AVFormat *format = OH_VideoEncoder_GetOutputDescription(s->enc); + if (!format) { + av_log(avctx, AV_LOG_ERROR, "GetOutputDescription failed\n"); + return AVERROR_EXTERNAL; + } + + oh_encode_on_stream_changed(s->enc, format, avctx); + OH_AVFormat_Destroy(format); + if (!s->got_stream_info) + return AVERROR_EXTERNAL; + } + + if (!frame->buf[0] && !s->eof_sent) { + OH_AVCodecBufferAttr attr = { + .flags = AVCODEC_BUFFER_FLAGS_EOS, + }; + err = OH_AVBuffer_SetBufferAttr(input->buffer, &attr); + if (err != AV_ERR_OK) + return ff_oh_err_to_ff_err(err); + err = OH_VideoEncoder_PushInputBuffer(s->enc, input->index); + if (err != AV_ERR_OK) + return ff_oh_err_to_ff_err(err); + s->eof_sent = true; + return 0; + } + + uint8_t *p = OH_AVBuffer_GetAddr(input->buffer); + int32_t n = OH_AVBuffer_GetCapacity(input->buffer); + if (!p || n <= 0) { + av_log(avctx, AV_LOG_ERROR, + "Failed to get buffer addr (%p) or capacity (%d)\n", + p, n); + return AVERROR_EXTERNAL; + } + + uint8_t *dst[4] = {0}; + int dst_linesizes[4] = {0}; + ret = av_image_fill_linesizes(dst_linesizes, frame->format, s->stride); + if (ret < 0) + return ret; + ret = av_image_fill_pointers(dst, frame->format, s->slice_height, p, + dst_linesizes); + if (ret < 0) + return ret; + + av_image_copy2(dst, dst_linesizes, frame->data, frame->linesize, + frame->format, frame->width, frame->height); + OH_AVCodecBufferAttr attr = { + .size = n, + .offset = 0, + .pts = av_rescale_q(s->frame->pts, avctx->pkt_timebase, + AV_TIME_BASE_Q), + .flags = (s->frame->flags & AV_FRAME_FLAG_KEY) + ? AVCODEC_BUFFER_FLAGS_SYNC_FRAME : 0, + }; + + err = OH_AVBuffer_SetBufferAttr(input->buffer, &attr); + if (err != AV_ERR_OK) { + ret = ff_oh_err_to_ff_err(err); + return ret; + } + err = OH_VideoEncoder_PushInputBuffer(s->enc, input->index); + if (err != AV_ERR_OK) { + ret = ff_oh_err_to_ff_err(err); + av_log(avctx, AV_LOG_ERROR, "Push input buffer failed, %d, %s\n", + err, av_err2str(ret)); + return ret; + } + av_frame_unref(s->frame); + + return 0; +} + +static int oh_encode_receive(AVCodecContext *avctx, AVPacket *pkt) +{ + OHCodecEncContext *s = avctx->priv_data; + + while (1) { + OHBufferQueueItem buffer = {0}; + int ret; + + // Try get output + ff_mutex_lock(&s->output_mutex); + while (!s->encode_status) { + if (av_fifo_read(s->output_queue, &buffer, 1) >= 0) + break; + // Only wait after send EOF + if (s->eof_sent && !s->encode_status) + ff_cond_wait(&s->output_cond, &s->output_mutex); + else + break; + } + + ret = s->encode_status; + ff_mutex_unlock(&s->output_mutex); + + // Got a packet + if (buffer.buffer) + return oh_encode_output_packet(avctx, pkt, &buffer); + if (ret < 0) + return ret; + + if (!s->frame->buf[0]) { + /* fetch new frame or eof */ + ret = ff_encode_get_frame(avctx, s->frame); + if (ret < 0 && ret != AVERROR_EOF) + return ret; + } + + if (s->native_window) { + ret = oh_encode_send_hw_frame(avctx); + if (ret < 0) + return ret; + continue; + } + + // Wait input buffer + ff_mutex_lock(&s->input_mutex); + while (!s->encode_status) { + if (av_fifo_read(s->input_queue, &buffer, 1) >= 0) + break; + ff_cond_wait(&s->input_cond, &s->input_mutex); + } + + ret = s->encode_status; + ff_mutex_unlock(&s->input_mutex); + + if (ret < 0) + return ret; + + ret = oh_encode_send_sw_frame(avctx, &buffer); + if (ret < 0) + return ret; + } + + return AVERROR(EAGAIN); +} + +static void oh_encode_flush(AVCodecContext *avctx) +{ + OHCodecEncContext *s = avctx->priv_data; + + OH_VideoEncoder_Flush(s->enc); + + ff_mutex_lock(&s->input_mutex); + ff_mutex_lock(&s->output_mutex); + av_fifo_reset2(s->input_queue); + av_fifo_reset2(s->output_queue); + s->encode_status = 0; + s->eof_sent = false; + ff_mutex_unlock(&s->output_mutex); + ff_mutex_unlock(&s->input_mutex); + + OH_VideoEncoder_Start(s->enc); +} + +static const AVCodecHWConfigInternal *const oh_hw_configs[] = { + &(const AVCodecHWConfigInternal) { + .public = { + .pix_fmt = AV_PIX_FMT_OHCODEC, + .methods = AV_CODEC_HW_CONFIG_METHOD_AD_HOC, + }, + .hwaccel = NULL, + }, + NULL +}; + +static const FFCodecDefault ohcodec_defaults[] = { + {"g", "-2"}, + {NULL}, +}; + +#define OFFSET(x) offsetof(OHCodecEncContext, x) +#define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM +static const AVOption ohcodec_venc_options[] = { + {"codec_name", "Select codec by name", + OFFSET(name), AV_OPT_TYPE_STRING, .flags = VE}, + {"allow_sw", "Allow software encoding", + OFFSET(allow_sw), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, VE}, + {"native_window", "Export native window created by encoder", + OFFSET(native_window), AV_OPT_TYPE_UINT64, .flags = VE | AV_OPT_FLAG_EXPORT | AV_OPT_FLAG_READONLY}, + { "bitrate_mode", "Bitrate control method", + OFFSET(bitrate_mode), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, VE, .unit = "bitrate_mode" }, + {"cbr", "Constant bitrate mode", + 0, AV_OPT_TYPE_CONST, {.i64 = CBR}, 0, 0, VE, .unit = "bitrate_mode"}, + {"vbr", "Variable bitrate mode", + 0, AV_OPT_TYPE_CONST, {.i64 = VBR}, 0, 0, VE, .unit = "bitrate_mode"}, + {"cq", "Constant quality mode", + 0, AV_OPT_TYPE_CONST, {.i64 = CQ}, 0, 0, VE, .unit = "bitrate_mode"}, + {NULL}, +}; + +#define DECLARE_OHCODEC_CLASS(name) \ +static const AVClass name ## _oh_enc_class = { \ + .class_name = #name "_ohcodec", \ + .item_name = av_default_item_name, \ + .option = ohcodec_venc_options, \ + .version = LIBAVUTIL_VERSION_INT, \ +}; \ + +#define DECLARE_OHCODEC_ENCODER(short_name, long_name, codec_id) \ +DECLARE_OHCODEC_CLASS(short_name) \ +const FFCodec ff_ ## short_name ## _oh_encoder = { \ + .p.name = #short_name "_ohcodec", \ + CODEC_LONG_NAME(long_name " OpenHarmony Codec"), \ + .p.type = AVMEDIA_TYPE_VIDEO, \ + .p.id = codec_id, \ + .p.capabilities = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY | \ + AV_CODEC_CAP_HARDWARE | \ + AV_CODEC_CAP_ENCODER_FLUSH, \ + .priv_data_size = sizeof(OHCodecEncContext), \ + CODEC_PIXFMTS_ARRAY(ohcodec_pix_fmts), \ + .color_ranges = AVCOL_RANGE_MPEG | AVCOL_RANGE_JPEG, \ + .defaults = ohcodec_defaults, \ + .init = oh_encode_init, \ + FF_CODEC_RECEIVE_PACKET_CB(oh_encode_receive), \ + .close = oh_encode_close, \ + .flush = oh_encode_flush, \ + .p.priv_class = &short_name ## _oh_enc_class, \ + .caps_internal = FF_CODEC_CAP_INIT_CLEANUP, \ + .p.wrapper_name = "ohcodec", \ + .hw_configs = oh_hw_configs, \ +}; \ + +#if CONFIG_H264_OH_ENCODER +DECLARE_OHCODEC_ENCODER(h264, "H.264", AV_CODEC_ID_H264) +#endif // CONFIG_H264_OH_ENCODER + +#if CONFIG_HEVC_OH_ENCODER +DECLARE_OHCODEC_ENCODER(hevc, "H.265", AV_CODEC_ID_HEVC) +#endif // CONFIG_HEVC_OH_ENCODER + diff --git a/libavcodec/version.h b/libavcodec/version.h index da54f87887..7acb261bb3 100644 --- a/libavcodec/version.h +++ b/libavcodec/version.h @@ -29,7 +29,7 @@ #include "version_major.h" -#define LIBAVCODEC_VERSION_MINOR 6 +#define LIBAVCODEC_VERSION_MINOR 7 #define LIBAVCODEC_VERSION_MICRO 100 #define LIBAVCODEC_VERSION_INT AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \ -- 2.46.0

1 2

[PATCH v4 0/3] fix leak in avfilter/asrc_sinc and avformat/sapenc
by Lidong Yan 06 Jul '25

06 Jul '25

This patch series fixes memory leaks in `avfilter/asrc_sinc` and `avformat/sapenc`. -----BEGIN PGP PUBLIC KEY BLOCK----- mDMEaEpkmRYJKwYBBAHaRw8BAQdAGwGqH/Dwod+i6kR0/Rhn5GanJ7wK8mM9tWP/ W2qu8Ti0HTUwMjAyNDMzMDA1NkBzbWFpbC5uanUuZWR1LmNuiJkEExYKAEEWIQQC zskBcOehk1y8GoKZR31bPD+6owUCaEpkmQIbAwUJBaOagAULCQgHAgIiAgYVCgkI CwIEFgIDAQIeBwIXgAAKCRCZR31bPD+6o8wHAQCLomsA4XfTd8IdG983gGULUJe/ 0432buy4nX7AsAc87QEA+/QIsWTR6XLJaLa1sLSQCsZkb86U3c17JzG9oivL8gW4 OARoSmSZEgorBgEEAZdVAQUBAQdAfYrEAWd+6bOXkKvHpFmMvKzxAtlhm6ZQKdAq +MlJ7wQDAQgHiHgEGBYKACAWIQQCzskBcOehk1y8GoKZR31bPD+6owUCaEpkmQIb DAAKCRCZR31bPD+6ozWxAQC9OFisWrP/hHXUfj8AnC39r5pf5fEBz7lHvFgWNk2b XwD7Bl6kvIIW7ReqtgXvcl7u78vEo+e9YeTGTlmAogjpeQk= =rP+W -----END PGP PUBLIC KEY BLOCK----- Lidong Yan (3): avfilter/asrc_sinc: fix leak in config_input() avformat/sapenc: fix leak in sap_write_header() avformat/sapenc: reword fail to cleanup in sap_write_header() libavfilter/asrc_sinc.c | 18 +++++++++++------- libavformat/sapenc.c | 34 +++++++++++++++++++--------------- 2 files changed, 30 insertions(+), 22 deletions(-) -- 2.50.0.106.gf0135a9047.dirty

2 6

[PATCH v2 2/3] avcodec/ohdec: Add h264/hevc OpenHarmony decoders
by Zhao Zhili 06 Jul '25

06 Jul '25

From: Zhao Zhili <zhilizhao(a)tencent.com> --- configure | 4 + libavcodec/Makefile | 3 + libavcodec/allcodecs.c | 2 + libavcodec/ohcodec.c | 71 ++++ libavcodec/ohcodec.h | 56 +++ libavcodec/ohdec.c | 771 +++++++++++++++++++++++++++++++++++++++++ libavcodec/version.h | 2 +- 7 files changed, 908 insertions(+), 1 deletion(-) create mode 100644 libavcodec/ohcodec.c create mode 100644 libavcodec/ohcodec.h create mode 100644 libavcodec/ohdec.c diff --git a/configure b/configure index 729d1577a3..a2ace36cdc 100755 --- a/configure +++ b/configure @@ -3410,6 +3410,8 @@ h264_mf_encoder_deps="mediafoundation" h264_mmal_decoder_deps="mmal" h264_nvenc_encoder_deps="nvenc" h264_nvenc_encoder_select="atsc_a53" +h264_oh_decoder_deps="ohcodec" +h264_oh_decoder_select="h264_mp4toannexb_bsf" h264_omx_encoder_deps="omx" h264_qsv_decoder_select="h264_mp4toannexb_bsf qsvdec" h264_qsv_encoder_select="atsc_a53 qsvenc" @@ -3432,6 +3434,8 @@ hevc_mediacodec_encoder_select="extract_extradata_bsf hevc_metadata" hevc_mf_encoder_deps="mediafoundation" hevc_nvenc_encoder_deps="nvenc" hevc_nvenc_encoder_select="atsc_a53" +hevc_oh_decoder_deps="ohcodec" +hevc_oh_decoder_select="hevc_mp4toannexb_bsf" hevc_qsv_decoder_select="hevc_mp4toannexb_bsf qsvdec" hevc_qsv_encoder_select="hevcparse qsvenc" hevc_rkmpp_decoder_deps="rkmpp" diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 215577f7c9..b784bf3f7e 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -428,6 +428,7 @@ OBJS-$(CONFIG_H264_MEDIACODEC_ENCODER) += mediacodecenc.o OBJS-$(CONFIG_H264_MF_ENCODER) += mfenc.o mf_utils.o OBJS-$(CONFIG_H264_MMAL_DECODER) += mmaldec.o OBJS-$(CONFIG_H264_NVENC_ENCODER) += nvenc_h264.o nvenc.o +OBJS-$(CONFIG_H264_OH_DECODER) += ohcodec.o ohdec.o OBJS-$(CONFIG_H264_OMX_ENCODER) += omx.o OBJS-$(CONFIG_H264_QSV_DECODER) += qsvdec.o OBJS-$(CONFIG_H264_QSV_ENCODER) += qsvenc_h264.o @@ -456,6 +457,7 @@ OBJS-$(CONFIG_HEVC_MEDIACODEC_DECODER) += mediacodecdec.o OBJS-$(CONFIG_HEVC_MEDIACODEC_ENCODER) += mediacodecenc.o OBJS-$(CONFIG_HEVC_MF_ENCODER) += mfenc.o mf_utils.o OBJS-$(CONFIG_HEVC_NVENC_ENCODER) += nvenc_hevc.o nvenc.o +OBJS-$(CONFIG_HEVC_OH_DECODER) += ohcodec.o ohdec.o OBJS-$(CONFIG_HEVC_QSV_DECODER) += qsvdec.o OBJS-$(CONFIG_HEVC_QSV_ENCODER) += qsvenc_hevc.o hevc/ps_enc.o OBJS-$(CONFIG_HEVC_RKMPP_DECODER) += rkmppdec.o @@ -1314,6 +1316,7 @@ SKIPHEADERS-$(CONFIG_MEDIACODEC) += mediacodecdec_common.h mediacodec_surf SKIPHEADERS-$(CONFIG_MEDIAFOUNDATION) += mf_utils.h SKIPHEADERS-$(CONFIG_NVDEC) += nvdec.h SKIPHEADERS-$(CONFIG_NVENC) += nvenc.h +SKIPHEADERS-$(CONFIG_OHCODEC) += ohcodec.h SKIPHEADERS-$(CONFIG_QSV) += qsv.h qsv_internal.h SKIPHEADERS-$(CONFIG_QSVENC) += qsvenc.h SKIPHEADERS-$(CONFIG_VAAPI) += vaapi_decode.h vaapi_hevc.h vaapi_encode.h diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c index 7b01453ca2..826ccf7329 100644 --- a/libavcodec/allcodecs.c +++ b/libavcodec/allcodecs.c @@ -860,6 +860,7 @@ extern const FFCodec ff_h264_amf_decoder; extern const FFCodec ff_h264_cuvid_decoder; extern const FFCodec ff_h264_mf_encoder; extern const FFCodec ff_h264_nvenc_encoder; +extern const FFCodec ff_h264_oh_decoder; extern const FFCodec ff_h264_omx_encoder; extern const FFCodec ff_h264_qsv_encoder; extern const FFCodec ff_h264_v4l2m2m_encoder; @@ -874,6 +875,7 @@ extern const FFCodec ff_hevc_mediacodec_decoder; extern const FFCodec ff_hevc_mediacodec_encoder; extern const FFCodec ff_hevc_mf_encoder; extern const FFCodec ff_hevc_nvenc_encoder; +extern const FFCodec ff_hevc_oh_decoder; extern const FFCodec ff_hevc_qsv_encoder; extern const FFCodec ff_hevc_v4l2m2m_encoder; extern const FFCodec ff_hevc_vaapi_encoder; diff --git a/libavcodec/ohcodec.c b/libavcodec/ohcodec.c new file mode 100644 index 0000000000..284ff1b2ec --- /dev/null +++ b/libavcodec/ohcodec.c @@ -0,0 +1,71 @@ +/* + * This file is part of FFmpeg. + * + * Copyright (c) 2025 Zhao Zhili <quinkblack(a)foxmail.com> + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "ohcodec.h" + +#include "libavutil/error.h" + +int ff_oh_err_to_ff_err(OH_AVErrCode err) +{ + switch (err) { + case AV_ERR_OK: + return 0; + case AV_ERR_NO_MEMORY: + return AVERROR(ENOMEM); + case AV_ERR_OPERATE_NOT_PERMIT: + return AVERROR(EPERM); + case AV_ERR_INVALID_VAL: + return AVERROR(EINVAL); + case AV_ERR_IO: + return AVERROR(EIO); + case AV_ERR_TIMEOUT: + return AVERROR(ETIMEDOUT); + case AV_ERR_UNKNOWN: + return AVERROR_UNKNOWN; + case AV_ERR_SERVICE_DIED: + return AVERROR_EXTERNAL; + case AV_ERR_INVALID_STATE: + return AVERROR(EINVAL); + case AV_ERR_UNSUPPORT: + return AVERROR(ENOTSUP); + default: + return AVERROR_EXTERNAL; + } +} + +static const struct { + OH_AVPixelFormat oh_pix; + enum AVPixelFormat pix; +} oh_pix_map[] = { + {AV_PIXEL_FORMAT_NV12, AV_PIX_FMT_NV12}, + {AV_PIXEL_FORMAT_NV21, AV_PIX_FMT_NV21}, + {AV_PIXEL_FORMAT_YUVI420, AV_PIX_FMT_YUV420P}, + {AV_PIXEL_FORMAT_SURFACE_FORMAT, AV_PIX_FMT_OHCODEC}, +}; + +enum AVPixelFormat ff_oh_pix_to_ff_pix(OH_AVPixelFormat oh_pix) +{ + for (size_t i = 0; i < FF_ARRAY_ELEMS(oh_pix_map); i++) + if (oh_pix_map[i].oh_pix == oh_pix) + return oh_pix_map[i].pix; + + return AV_PIX_FMT_NONE; +} + diff --git a/libavcodec/ohcodec.h b/libavcodec/ohcodec.h new file mode 100644 index 0000000000..236ddba67e --- /dev/null +++ b/libavcodec/ohcodec.h @@ -0,0 +1,56 @@ +/* + * This file is part of FFmpeg. + * + * Copyright (c) 2025 Zhao Zhili <quinkblack(a)foxmail.com> + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_OHCODEC_H +#define AVCODEC_OHCODEC_H + +#include <multimedia/player_framework/native_avbuffer.h> +#include <multimedia/player_framework/native_avcodec_base.h> +#include <multimedia/player_framework/native_averrors.h> +#include <multimedia/player_framework/native_avformat.h> + +#include "libavutil/pixfmt.h" + +#include "codec_id.h" + +typedef struct OHBufferQueueItem { + uint32_t index; + OH_AVBuffer *buffer; +} OHBufferQueueItem; + +int ff_oh_err_to_ff_err(OH_AVErrCode err); + +static inline const char *ff_oh_mime(enum AVCodecID codec_id, void *log) +{ + switch (codec_id) { + case AV_CODEC_ID_H264: + return OH_AVCODEC_MIMETYPE_VIDEO_AVC; + case AV_CODEC_ID_HEVC: + return OH_AVCODEC_MIMETYPE_VIDEO_HEVC; + default: + av_log(log, AV_LOG_ERROR, "Unsupported codec %s\n", + avcodec_get_name(codec_id)); + return NULL; + } +} + +enum AVPixelFormat ff_oh_pix_to_ff_pix(OH_AVPixelFormat oh_pix); + +#endif diff --git a/libavcodec/ohdec.c b/libavcodec/ohdec.c new file mode 100644 index 0000000000..cf1177afcd --- /dev/null +++ b/libavcodec/ohdec.c @@ -0,0 +1,771 @@ +/* + * This file is part of FFmpeg. + * + * Copyright (c) 2025 Zhao Zhili <quinkblack(a)foxmail.com> + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config_components.h" + +#include <stdbool.h> +#include <multimedia/player_framework/native_avcapability.h> +#include <multimedia/player_framework/native_avcodec_videodecoder.h> + +#include "libavutil/fifo.h" +#include "libavutil/hwcontext_oh.h" +#include "libavutil/imgutils.h" +#include "libavutil/mem.h" +#include "libavutil/opt.h" +#include "libavutil/thread.h" + +#include "avcodec.h" +#include "codec_internal.h" +#include "decode.h" +#include "hwconfig.h" +#include "ohcodec.h" + +typedef struct OHCodecDecContext { + AVClass *avclass; + OH_AVCodec *dec; + /* A reference count to dec. Each hardware frame has a reference count to + * dec. dec will be destroyed only after oh_decode_close and all hardware + * frames have been released. + */ + AVBufferRef *dec_ref; + + AVMutex input_mutex; + AVCond input_cond; + AVFifo *input_queue; + + AVMutex output_mutex; + AVCond output_cond; + AVFifo *output_queue; + + AVPacket pkt; + + int decode_status; + bool eof_sent; + + bool output_to_window; + bool got_stream_info; + int width; + int height; + int stride; + int slice_height; + OH_AVPixelFormat pix_fmt; + + char *name; + int allow_sw; +} OHCodecDecContext; + +typedef struct OHCodecBuffer { + uint32_t index; + OH_AVBuffer *buffer; + AVBufferRef *dec_ref; +} OHCodecBuffer; + +static void oh_decode_release(void *opaque, uint8_t *data) +{ + OH_AVCodec *dec = (OH_AVCodec *)data; + OH_AVErrCode err = OH_VideoDecoder_Destroy(dec); + if (err == AV_ERR_OK) + av_log(NULL, AV_LOG_DEBUG, "Destroy decoder success\n"); + else + av_log(NULL, AV_LOG_ERROR, "Destroy decoder failed, %d, %s\n", + err, av_err2str(ff_oh_err_to_ff_err(err))); +} + +static int oh_decode_create(OHCodecDecContext *s, AVCodecContext *avctx) +{ + const char *name = s->name; + + if (!name) { + const char *mime = ff_oh_mime(avctx->codec_id, avctx); + if (!mime) + return AVERROR_BUG; + OH_AVCapability *cap = OH_AVCodec_GetCapabilityByCategory(mime, false, HARDWARE); + if (!cap) { + if (!s->allow_sw) { + av_log(avctx, AV_LOG_ERROR, "Failed to get hardware codec %s\n", mime); + return AVERROR_EXTERNAL; + } + av_log(avctx, AV_LOG_WARNING, + "Failed to get hardware codec %s, try software backend\n", mime); + cap = OH_AVCodec_GetCapabilityByCategory(mime, false, SOFTWARE); + if (!cap) { + av_log(avctx, AV_LOG_ERROR, "Failed to get software codec %s\n", mime); + return AVERROR_EXTERNAL; + } + } + name = OH_AVCapability_GetName(cap); + if (!name) + return AVERROR_EXTERNAL; + } + + s->dec = OH_VideoDecoder_CreateByName(name); + if (!s->dec) { + av_log(avctx, AV_LOG_ERROR, "Create decoder with name %s failed\n", name); + return AVERROR_EXTERNAL; + } + av_log(avctx, AV_LOG_DEBUG, "Create decoder %s success\n", name); + + s->dec_ref = av_buffer_create((uint8_t *)s->dec, 0, oh_decode_release, + NULL, 0); + if (!s->dec_ref) + return AVERROR(ENOMEM); + + return 0; +} + +static int oh_decode_set_format(OHCodecDecContext *s, AVCodecContext *avctx) +{ + int ret; + OHNativeWindow *window = NULL; + + if (avctx->hw_device_ctx) { + AVHWDeviceContext *device_ctx = (AVHWDeviceContext*)(avctx->hw_device_ctx->data); + if (device_ctx->type == AV_HWDEVICE_TYPE_OHCODEC) { + AVOHCodecDeviceContext *dev = device_ctx->hwctx; + window = dev->native_window; + s->output_to_window = true; + } else { + av_log(avctx, AV_LOG_WARNING, "Ignore invalid hw device type %s\n", + av_hwdevice_get_type_name(device_ctx->type)); + } + } + + if (avctx->width <= 0 || avctx->height <= 0) { + av_log(avctx, AV_LOG_ERROR, + "Invalid width/height (%dx%d), width and height are mandatory for ohcodec\n", + avctx->width, avctx->height); + return AVERROR(EINVAL); + } + + OH_AVFormat *format = OH_AVFormat_Create(); + if (!format) + return AVERROR(ENOMEM); + + OH_AVFormat_SetIntValue(format, OH_MD_KEY_WIDTH, avctx->width); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_HEIGHT, avctx->height); + if (!s->output_to_window) + OH_AVFormat_SetIntValue(format, OH_MD_KEY_PIXEL_FORMAT, + AV_PIXEL_FORMAT_NV12); + else + OH_AVFormat_SetIntValue(format, OH_MD_KEY_PIXEL_FORMAT, + AV_PIXEL_FORMAT_SURFACE_FORMAT); + OH_AVErrCode err = OH_VideoDecoder_Configure(s->dec, format); + OH_AVFormat_Destroy(format); + if (err != AV_ERR_OK) { + ret = ff_oh_err_to_ff_err(err); + av_log(avctx, AV_LOG_ERROR, "Decoder configure failed, %d, %s\n", + err, av_err2str(ret)); + return ret; + } + + if (s->output_to_window) { + err = OH_VideoDecoder_SetSurface(s->dec, window); + if (err != AV_ERR_OK) { + ret = ff_oh_err_to_ff_err(err); + av_log(avctx, AV_LOG_ERROR, "Set surface failed, %d, %s\n", + err, av_err2str(ret)); + return ret; + } + } + + return 0; +} + +static void oh_decode_on_err(OH_AVCodec *codec, int32_t err, void *userdata) +{ + AVCodecContext *avctx = userdata; + OHCodecDecContext *s = avctx->priv_data; + + // Careful on the lock order. + // Always lock input first. + ff_mutex_lock(&s->input_mutex); + ff_mutex_lock(&s->output_mutex); + s->decode_status = ff_oh_err_to_ff_err(err); + ff_mutex_unlock(&s->output_mutex); + ff_mutex_unlock(&s->input_mutex); + + ff_cond_signal(&s->output_cond); + ff_cond_signal(&s->input_cond); +} + +static void oh_decode_on_stream_changed(OH_AVCodec *codec, OH_AVFormat *format, + void *userdata) +{ + AVCodecContext *avctx = userdata; + OHCodecDecContext *s = avctx->priv_data; + int32_t n; + double d; + + if (!OH_AVFormat_GetIntValue(format, OH_MD_KEY_VIDEO_PIC_WIDTH, &s->width) || + !OH_AVFormat_GetIntValue(format, OH_MD_KEY_VIDEO_PIC_HEIGHT, &s->height) || + !OH_AVFormat_GetIntValue(format, OH_MD_KEY_VIDEO_STRIDE, &s->stride) || + !OH_AVFormat_GetIntValue(format, OH_MD_KEY_VIDEO_SLICE_HEIGHT, + &s->slice_height)) { + av_log(avctx, AV_LOG_ERROR, "Get dimension info from format failed\n"); + goto out; + } + + if (ff_set_dimensions(avctx, s->width, s->height) < 0) + goto out; + + if (s->stride <= 0 || s->slice_height <= 0) { + av_log(avctx, AV_LOG_ERROR, + "Buffer stride (%d) or slice height (%d) is invalid\n", + s->stride, s->slice_height); + goto out; + } + + if (OH_AVFormat_GetIntValue(format, OH_MD_KEY_PIXEL_FORMAT, &n)) { + s->pix_fmt = n; + /* When use output_to_window, the returned format is the memory + * layout of hardware frame, not AV_PIXEL_FORMAT_SURFACE_FORMAT as + * expected. + */ + if (s->output_to_window) + avctx->pix_fmt = AV_PIX_FMT_OHCODEC; + else + avctx->pix_fmt = ff_oh_pix_to_ff_pix(s->pix_fmt); + // Check whether this pixel format is supported + if (avctx->pix_fmt == AV_PIX_FMT_NONE) { + av_log(avctx, AV_LOG_ERROR, "Unsupported OH_AVPixelFormat %d\n", + n); + goto out; + } + } else { + av_log(avctx, AV_LOG_ERROR, "Failed to get pixel format\n"); + goto out; + } + + if (OH_AVFormat_GetIntValue(format, + OH_MD_KEY_MATRIX_COEFFICIENTS, + &n)) + avctx->colorspace = n; + if (OH_AVFormat_GetIntValue(format, + OH_MD_KEY_COLOR_PRIMARIES, + &n)) + avctx->color_primaries = n; + if (OH_AVFormat_GetIntValue(format, + OH_MD_KEY_TRANSFER_CHARACTERISTICS, + &n)) + avctx->color_trc = n; + if (OH_AVFormat_GetIntValue(format, + OH_MD_KEY_RANGE_FLAG, + &n)) + avctx->color_range = n ? AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG; + + if (OH_AVFormat_GetDoubleValue(format, OH_MD_KEY_VIDEO_SAR, &d)) { + AVRational sar = av_d2q(d, 4096 * 4); + ff_set_sar(avctx, sar); + } + + s->got_stream_info = true; + + return; +out: + av_log(avctx, AV_LOG_ERROR, "Invalid format from decoder: %s\n", + OH_AVFormat_DumpInfo(format)); + oh_decode_on_err(codec, AV_ERR_UNKNOWN, userdata); +} + +static void oh_decode_on_need_input(OH_AVCodec *codec, uint32_t index, + OH_AVBuffer *buffer, void *userdata) +{ + AVCodecContext *avctx = userdata; + OHCodecDecContext *s = avctx->priv_data; + OHBufferQueueItem item = { + index, buffer, + }; + + ff_mutex_lock(&s->input_mutex); + int ret = av_fifo_write(s->input_queue, &item, 1); + if (ret >= 0) + ff_cond_signal(&s->input_cond); + ff_mutex_unlock(&s->input_mutex); + + if (ret < 0) + oh_decode_on_err(codec, AV_ERR_NO_MEMORY, userdata); +} + +static void oh_decode_on_output(OH_AVCodec *codec, uint32_t index, + OH_AVBuffer *buffer, void *userdata) +{ + AVCodecContext *avctx = userdata; + OHCodecDecContext *s = avctx->priv_data; + OHBufferQueueItem item = { + index, buffer, + }; + + ff_mutex_lock(&s->output_mutex); + int ret = av_fifo_write(s->output_queue, &item, 1); + if (ret >= 0) + ff_cond_signal(&s->output_cond); + ff_mutex_unlock(&s->output_mutex); + + if (ret < 0) + oh_decode_on_err(codec, AV_ERR_NO_MEMORY, userdata); +} + +static int oh_decode_start(OHCodecDecContext *s, AVCodecContext *avctx) +{ + int ret; + OH_AVErrCode err; + OH_AVCodecCallback cb = { + .onError = oh_decode_on_err, + .onStreamChanged = oh_decode_on_stream_changed, + .onNeedInputBuffer = oh_decode_on_need_input, + .onNewOutputBuffer = oh_decode_on_output, + }; + + err = OH_VideoDecoder_RegisterCallback(s->dec, cb, avctx); + if (err != AV_ERR_OK) { + ret = ff_oh_err_to_ff_err(err); + av_log(avctx, AV_LOG_ERROR, "Register callback failed, %d, %s\n", + err, av_err2str(ret)); + return ret; + } + err = OH_VideoDecoder_Prepare(s->dec); + if (err != AV_ERR_OK) { + ret = ff_oh_err_to_ff_err(err); + av_log(avctx, AV_LOG_ERROR, "Prepare failed, %d, %s\n", + err, av_err2str(ret)); + return ret; + } + err = OH_VideoDecoder_Start(s->dec); + if (err != AV_ERR_OK) { + ret = ff_oh_err_to_ff_err(err); + av_log(avctx, AV_LOG_ERROR, "Start failed, %d, %s\n", + err, av_err2str(ret)); + return ret; + } + + return 0; +} + +static av_cold int oh_decode_init(AVCodecContext *avctx) +{ + OHCodecDecContext *s = avctx->priv_data; + + // Initialize these fields first, so oh_decode_close can destroy them safely + ff_mutex_init(&s->input_mutex, NULL); + ff_cond_init(&s->input_cond, NULL); + ff_mutex_init(&s->output_mutex, NULL); + ff_cond_init(&s->output_cond, NULL); + + int ret = oh_decode_create(s, avctx); + if (ret < 0) + return ret; + ret = oh_decode_set_format(s, avctx); + if (ret < 0) + return ret; + + size_t fifo_size = 16; + s->input_queue = av_fifo_alloc2(fifo_size, sizeof(OHBufferQueueItem), + AV_FIFO_FLAG_AUTO_GROW); + s->output_queue = av_fifo_alloc2(fifo_size, sizeof(OHBufferQueueItem), + AV_FIFO_FLAG_AUTO_GROW); + if (!s->input_queue || !s->output_queue) + return AVERROR(ENOMEM); + + ret = oh_decode_start(s, avctx); + if (ret < 0) + return ret; + + return 0; +} + +static av_cold int oh_decode_close(AVCodecContext *avctx) +{ + OHCodecDecContext *s = avctx->priv_data; + + if (s->dec) { + /* Stop but don't destroy dec directly, to keep hardware frames on + * the fly valid. + */ + OH_AVErrCode err = OH_VideoDecoder_Stop(s->dec); + if (err == AV_ERR_OK) + av_log(avctx, AV_LOG_DEBUG, "Stop decoder success\n"); + else + av_log(avctx, AV_LOG_ERROR, "Stop decoder failed, %d, %s\n", + err, av_err2str(ff_oh_err_to_ff_err(err))); + s->dec = NULL; + av_buffer_unref(&s->dec_ref); + } + + av_packet_unref(&s->pkt); + + ff_mutex_destroy(&s->input_mutex); + ff_cond_destroy(&s->input_cond); + av_fifo_freep2(&s->input_queue); + + ff_mutex_destroy(&s->output_mutex); + ff_cond_destroy(&s->output_cond); + av_fifo_freep2(&s->output_queue); + + return 0; +} + +static void oh_buffer_release(void *opaque, uint8_t *data) +{ + if (!opaque) + return; + + OHCodecBuffer *buffer = opaque; + + if (!buffer->dec_ref) { + av_free(buffer); + return; + } + + if (buffer->buffer) { + OH_AVCodec *dec = (OH_AVCodec *)buffer->dec_ref->data; + OH_AVCodecBufferAttr attr; + OH_AVErrCode err = OH_AVBuffer_GetBufferAttr(buffer->buffer, &attr); + if (err == AV_ERR_OK && !(attr.flags & AVCODEC_BUFFER_FLAGS_DISCARD)) + OH_VideoDecoder_RenderOutputBuffer(dec, buffer->index); + else + OH_VideoDecoder_FreeOutputBuffer(dec, buffer->index); + } + + av_buffer_unref(&buffer->dec_ref); + av_free(buffer); +} + +static int oh_decode_wrap_hw_buffer(AVCodecContext *avctx, AVFrame *frame, + OHBufferQueueItem *output, + const OH_AVCodecBufferAttr *attr) +{ + OHCodecDecContext *s = avctx->priv_data; + + frame->width = s->width; + frame->height = s->height; + int ret = ff_decode_frame_props(avctx, frame); + if (ret < 0) + return ret; + + frame->format = AV_PIX_FMT_OHCODEC; + OHCodecBuffer *buffer = av_mallocz(sizeof(*buffer)); + if (!buffer) + return AVERROR(ENOMEM); + + buffer->dec_ref = av_buffer_ref(s->dec_ref); + if (!buffer->dec_ref) { + oh_buffer_release(buffer, NULL); + return AVERROR(ENOMEM); + } + + buffer->index = output->index; + buffer->buffer = output->buffer; + frame->buf[0] = av_buffer_create((uint8_t *)buffer->buffer, 1, + oh_buffer_release, + buffer, AV_BUFFER_FLAG_READONLY); + if (!frame->buf[0]) { + oh_buffer_release(buffer, NULL); + return AVERROR(ENOMEM); + } + // Point to OH_AVBuffer + frame->data[3] = frame->buf[0]->data; + frame->pts = av_rescale_q(attr->pts, AV_TIME_BASE_Q, avctx->pkt_timebase); + frame->pkt_dts = AV_NOPTS_VALUE; + + return 0; +} + +static int oh_decode_wrap_sw_buffer(AVCodecContext *avctx, AVFrame *frame, + OHBufferQueueItem *output, + const OH_AVCodecBufferAttr *attr) +{ + OHCodecDecContext *s = avctx->priv_data; + + frame->format = avctx->pix_fmt; + frame->width = s->width; + frame->height = s->height; + int ret = ff_get_buffer(avctx, frame, 0); + if (ret < 0) + return ret; + + frame->pts = av_rescale_q(attr->pts, AV_TIME_BASE_Q, avctx->pkt_timebase); + frame->pkt_dts = AV_NOPTS_VALUE; + + uint8_t *p = OH_AVBuffer_GetAddr(output->buffer); + if (!p) { + av_log(avctx, AV_LOG_ERROR, "Failed to get output buffer addr\n"); + return AVERROR_EXTERNAL; + } + + uint8_t *src[4] = {0}; + int src_linesizes[4] = {0}; + + ret = av_image_fill_linesizes(src_linesizes, frame->format, s->stride); + if (ret < 0) + return ret; + ret = av_image_fill_pointers(src, frame->format, s->slice_height, p, + src_linesizes); + if (ret < 0) + return ret; + av_image_copy2(frame->data, frame->linesize, src, src_linesizes, + frame->format, frame->width, frame->height); + + OH_AVErrCode err = OH_VideoDecoder_FreeOutputBuffer(s->dec, output->index); + if (err != AV_ERR_OK) { + ret = ff_oh_err_to_ff_err(err); + av_log(avctx, AV_LOG_ERROR, "FreeOutputBuffer failed, %d, %s\n", err, + av_err2str(ret)); + return ret; + } + + return 0; +} + +static int oh_decode_output_frame(AVCodecContext *avctx, AVFrame *frame, + OHBufferQueueItem *output) +{ + OHCodecDecContext *s = avctx->priv_data; + OH_AVCodecBufferAttr attr; + + OH_AVErrCode err = OH_AVBuffer_GetBufferAttr(output->buffer, &attr); + if (err != AV_ERR_OK) + return ff_oh_err_to_ff_err(err); + + if (attr.flags & AVCODEC_BUFFER_FLAGS_EOS) { + av_log(avctx, AV_LOG_DEBUG, "Buffer flag eos\n"); + OH_VideoDecoder_FreeOutputBuffer(s->dec, output->index); + return AVERROR_EOF; + } + + if (!s->got_stream_info) { + // This shouldn't happen, add a warning message. + av_log(avctx, AV_LOG_WARNING, + "decoder didn't notify stream info, try get format explicitly\n"); + + OH_AVFormat *format = OH_VideoDecoder_GetOutputDescription(s->dec); + if (!format) { + av_log(avctx, AV_LOG_ERROR, "GetOutputDescription failed\n"); + return AVERROR_EXTERNAL; + } + + oh_decode_on_stream_changed(s->dec, format, avctx); + OH_AVFormat_Destroy(format); + if (!s->got_stream_info) + return AVERROR_EXTERNAL; + } + + if (s->output_to_window) + return oh_decode_wrap_hw_buffer(avctx, frame, output, &attr); + return oh_decode_wrap_sw_buffer(avctx, frame, output, &attr); +} + +static int oh_decode_send_pkt(AVCodecContext *avctx, OHBufferQueueItem *input) +{ + OHCodecDecContext *s = avctx->priv_data; + OH_AVErrCode err; + int ret; + + if (!s->pkt.size && !s->eof_sent) { + OH_AVCodecBufferAttr attr = { + .flags = AVCODEC_BUFFER_FLAGS_EOS, + }; + err = OH_AVBuffer_SetBufferAttr(input->buffer, &attr); + if (err != AV_ERR_OK) + return ff_oh_err_to_ff_err(err); + err = OH_VideoDecoder_PushInputBuffer(s->dec, input->index); + if (err != AV_ERR_OK) + return ff_oh_err_to_ff_err(err); + s->eof_sent = true; + return 0; + } + + uint8_t *p = OH_AVBuffer_GetAddr(input->buffer); + int32_t n = OH_AVBuffer_GetCapacity(input->buffer); + if (!p || n <= 0) { + av_log(avctx, AV_LOG_ERROR, + "Failed to get buffer addr (%p) or capacity (%d)\n", + p, n); + return AVERROR_EXTERNAL; + } + n = FFMIN(s->pkt.size, n); + memcpy(p, s->pkt.data, n); + + OH_AVCodecBufferAttr attr = { + .size = n, + .offset = 0, + .pts = av_rescale_q(s->pkt.pts, avctx->pkt_timebase, + AV_TIME_BASE_Q), + .flags = (s->pkt.flags & AV_PKT_FLAG_KEY) + ? AVCODEC_BUFFER_FLAGS_SYNC_FRAME : 0, + }; + + err = OH_AVBuffer_SetBufferAttr(input->buffer, &attr); + if (err != AV_ERR_OK) { + ret = ff_oh_err_to_ff_err(err); + return ret; + } + err = OH_VideoDecoder_PushInputBuffer(s->dec, input->index); + if (err != AV_ERR_OK) { + ret = ff_oh_err_to_ff_err(err); + av_log(avctx, AV_LOG_ERROR, "Push input buffer failed, %d, %s\n", + err, av_err2str(ret)); + return ret; + } + + if (n < s->pkt.size) { + s->pkt.size -= n; + s->pkt.data += n; + } else { + av_packet_unref(&s->pkt); + } + + return 0; +} + +static int oh_decode_receive_frame(AVCodecContext *avctx, AVFrame *frame) +{ + OHCodecDecContext *s = avctx->priv_data; + + while (1) { + OHBufferQueueItem buffer = {0}; + int ret; + + // Try get output + ff_mutex_lock(&s->output_mutex); + while (!s->decode_status) { + if (av_fifo_read(s->output_queue, &buffer, 1) >= 0) + break; + // Only wait after send EOF + if (s->eof_sent && !s->decode_status) + ff_cond_wait(&s->output_cond, &s->output_mutex); + else + break; + } + + ret = s->decode_status; + ff_mutex_unlock(&s->output_mutex); + + // Got a frame + if (buffer.buffer) + return oh_decode_output_frame(avctx, frame, &buffer); + if (ret < 0) + return ret; + + if (!s->pkt.size) { + /* fetch new packet or eof */ + ret = ff_decode_get_packet(avctx, &s->pkt); + if (ret < 0 && ret != AVERROR_EOF) + return ret; + } + + // Wait input buffer + ff_mutex_lock(&s->input_mutex); + while (!s->decode_status) { + if (av_fifo_read(s->input_queue, &buffer, 1) >= 0) + break; + ff_cond_wait(&s->input_cond, &s->input_mutex); + } + + ret = s->decode_status; + ff_mutex_unlock(&s->input_mutex); + + if (ret < 0) + return ret; + + ret = oh_decode_send_pkt(avctx, &buffer); + if (ret < 0) + return ret; + } + + return AVERROR(EAGAIN); +} + +static void oh_decode_flush(AVCodecContext *avctx) +{ + OHCodecDecContext *s = avctx->priv_data; + + OH_VideoDecoder_Flush(s->dec); + + ff_mutex_lock(&s->input_mutex); + ff_mutex_lock(&s->output_mutex); + av_fifo_reset2(s->input_queue); + av_fifo_reset2(s->output_queue); + s->decode_status = 0; + s->eof_sent = false; + ff_mutex_unlock(&s->output_mutex); + ff_mutex_unlock(&s->input_mutex); + + OH_VideoDecoder_Start(s->dec); +} + +static const AVCodecHWConfigInternal *const oh_hw_configs[] = { + &(const AVCodecHWConfigInternal) { + .public = { + .pix_fmt = AV_PIX_FMT_OHCODEC, + .methods = AV_CODEC_HW_CONFIG_METHOD_AD_HOC | + AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX, + .device_type = AV_HWDEVICE_TYPE_OHCODEC, + }, + .hwaccel = NULL, + }, + NULL +}; + +#define OFFSET(x) offsetof(OHCodecDecContext, x) +#define VD (AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM) +static const AVOption ohcodec_vdec_options[] = { + {"codec_name", "Select codec by name", + OFFSET(name), AV_OPT_TYPE_STRING, .flags = VD}, + {"allow_sw", "Allow software decoding", + OFFSET(allow_sw), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, VD}, + {NULL} +}; + +#define DECLARE_OHCODEC_VCLASS(short_name) \ + static const AVClass short_name##_oh_dec_class = { \ + .class_name = #short_name "_ohcodec", \ + .item_name = av_default_item_name, \ + .option = ohcodec_vdec_options, \ + .version = LIBAVUTIL_VERSION_INT, \ + }; + +#define DECLARE_OHCODEC_VDEC(short_name, full_name, codec_id, bsf) \ + DECLARE_OHCODEC_VCLASS(short_name) \ + const FFCodec ff_##short_name##_oh_decoder = { \ + .p.name = #short_name "_ohcodec", \ + CODEC_LONG_NAME(full_name " OpenHarmony Codec"), \ + .p.type = AVMEDIA_TYPE_VIDEO, \ + .p.id = codec_id, \ + .p.priv_class = &short_name##_oh_dec_class, \ + .priv_data_size = sizeof(OHCodecDecContext), \ + .init = oh_decode_init, \ + FF_CODEC_RECEIVE_FRAME_CB(oh_decode_receive_frame), \ + .flush = oh_decode_flush, \ + .close = oh_decode_close, \ + .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING | \ + AV_CODEC_CAP_HARDWARE, \ + .caps_internal = FF_CODEC_CAP_INIT_CLEANUP, \ + .bsfs = bsf, \ + .hw_configs = oh_hw_configs, \ + .p.wrapper_name = "ohcodec", \ + }; + +#if CONFIG_H264_OH_DECODER +DECLARE_OHCODEC_VDEC(h264, "H.264", AV_CODEC_ID_H264, "h264_mp4toannexb") +#endif + +#if CONFIG_HEVC_OH_DECODER +DECLARE_OHCODEC_VDEC(hevc, "H.265", AV_CODEC_ID_HEVC, "hevc_mp4toannexb") +#endif diff --git a/libavcodec/version.h b/libavcodec/version.h index 7aa95fc3f1..da54f87887 100644 --- a/libavcodec/version.h +++ b/libavcodec/version.h @@ -29,7 +29,7 @@ #include "version_major.h" -#define LIBAVCODEC_VERSION_MINOR 5 +#define LIBAVCODEC_VERSION_MINOR 6 #define LIBAVCODEC_VERSION_MICRO 100 #define LIBAVCODEC_VERSION_INT AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \ -- 2.46.0

1 0

[PATCH v2 1/3] avutil/hwcontext: Add ohcodec device and pixel format
by Zhao Zhili 06 Jul '25

06 Jul '25

From: Zhao Zhili <zhilizhao(a)tencent.com> --- configure | 5 ++++ doc/APIchanges | 6 +++++ libavutil/Makefile | 2 ++ libavutil/hwcontext.c | 4 +++ libavutil/hwcontext.h | 2 ++ libavutil/hwcontext_internal.h | 1 + libavutil/hwcontext_oh.c | 47 ++++++++++++++++++++++++++++++++++ libavutil/hwcontext_oh.h | 34 ++++++++++++++++++++++++ libavutil/pixdesc.c | 4 +++ libavutil/pixfmt.h | 2 ++ libavutil/version.h | 4 +-- 11 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 libavutil/hwcontext_oh.c create mode 100644 libavutil/hwcontext_oh.h diff --git a/configure b/configure index 63d11de207..729d1577a3 100755 --- a/configure +++ b/configure @@ -320,6 +320,7 @@ External library support: if openssl, gnutls or libtls is not used [no] --enable-mediacodec enable Android MediaCodec support [no] --enable-mediafoundation enable encoding via MediaFoundation [auto] + --enable-ohcodec enable OpenHarmony Codec support [no] --disable-metal disable Apple Metal framework [autodetect] --enable-libmysofa enable libmysofa, needed for sofalizer filter [no] --enable-openal enable OpenAL 1.1 capture support [no] @@ -1997,6 +1998,7 @@ EXTERNAL_LIBRARY_LIST=" libzvbi lv2 mediacodec + ohcodec openal opengl openssl @@ -7165,6 +7167,9 @@ enabled mmal && { check_lib mmal interface/mmal/mmal.h mmal_port_co check_lib mmal interface/mmal/mmal.h mmal_port_connect -lmmal_core -lmmal_util -lmmal_vc_client -lbcm_host; } || die "ERROR: mmal not found" && check_func_headers interface/mmal/mmal.h "MMAL_PARAMETER_VIDEO_MAX_NUM_CALLBACKS"; } +enabled ohcodec && { check_lib ohcodec "multimedia/player_framework/native_avcodec_videodecoder.h" \ + OH_VideoDecoder_CreateByName -lnative_media_codecbase -lnative_media_core -lnative_media_vdec || + die "ERROR: missing native_media libs"; } enabled openal && { check_pkg_config openal "openal >= 1.1" "AL/al.h" alGetError || { for al_extralibs in "${OPENAL_LIBS}" "-lopenal" "-lOpenAL32"; do check_lib openal 'AL/al.h' alGetError "${al_extralibs}" && break; done } || diff --git a/doc/APIchanges b/doc/APIchanges index 9c03e541fd..92801769b6 100644 --- a/doc/APIchanges +++ b/doc/APIchanges @@ -12,6 +12,12 @@ API changes, most recent first: Add AV3DReferenceDisplaysInfo and AV3DReferenceDisplay structs. Add av_tdrdi_alloc() and av_tdrdi_get_display(). +2025-06-30 - xxxxxxxxxx - lavu 60.4.100 - pixfmt.h + Add AV_PIX_FMT_OHCODEC. + +2025-06-30 - xxxxxxxxxx - lavu 60.4.100 - hwcontext.h + Add AV_HWDEVICE_TYPE_OHCODEC and AVOHCodecDeviceContext. + 2025-05-21 - xxxxxxxxxx - lavu 60.3.100 - avassert.h Add av_unreachable() and av_assume() macros. diff --git a/libavutil/Makefile b/libavutil/Makefile index 94a56bb72f..ee77e51c08 100644 --- a/libavutil/Makefile +++ b/libavutil/Makefile @@ -51,6 +51,7 @@ HEADERS = adler32.h \ hwcontext_qsv.h \ hwcontext_mediacodec.h \ hwcontext_opencl.h \ + hwcontext_oh.h \ hwcontext_vaapi.h \ hwcontext_videotoolbox.h \ hwcontext_vdpau.h \ @@ -210,6 +211,7 @@ OBJS-$(CONFIG_AMF) += hwcontext_amf.o OBJS-$(CONFIG_LIBDRM) += hwcontext_drm.o OBJS-$(CONFIG_MACOS_KPERF) += macos_kperf.o OBJS-$(CONFIG_MEDIACODEC) += hwcontext_mediacodec.o +OBJS-$(CONFIG_OHCODEC) += hwcontext_oh.o OBJS-$(CONFIG_OPENCL) += hwcontext_opencl.o OBJS-$(CONFIG_QSV) += hwcontext_qsv.o OBJS-$(CONFIG_VAAPI) += hwcontext_vaapi.o diff --git a/libavutil/hwcontext.c b/libavutil/hwcontext.c index 3111a44651..83bd7457e8 100644 --- a/libavutil/hwcontext.c +++ b/libavutil/hwcontext.c @@ -68,6 +68,9 @@ static const HWContextType * const hw_table[] = { #endif #if CONFIG_AMF &ff_hwcontext_type_amf, +#endif +#if CONFIG_OHCODEC + &ff_hwcontext_type_oh, #endif NULL, }; @@ -86,6 +89,7 @@ static const char *const hw_type_names[] = { [AV_HWDEVICE_TYPE_MEDIACODEC] = "mediacodec", [AV_HWDEVICE_TYPE_VULKAN] = "vulkan", [AV_HWDEVICE_TYPE_AMF] = "amf", + [AV_HWDEVICE_TYPE_OHCODEC] = "ohcodec", }; typedef struct FFHWDeviceContext { diff --git a/libavutil/hwcontext.h b/libavutil/hwcontext.h index 96042ba197..94fd11da73 100644 --- a/libavutil/hwcontext.h +++ b/libavutil/hwcontext.h @@ -39,6 +39,8 @@ enum AVHWDeviceType { AV_HWDEVICE_TYPE_VULKAN, AV_HWDEVICE_TYPE_D3D12VA, AV_HWDEVICE_TYPE_AMF, + /* OpenHarmony Codec device */ + AV_HWDEVICE_TYPE_OHCODEC, }; /** diff --git a/libavutil/hwcontext_internal.h b/libavutil/hwcontext_internal.h index db23579c9e..dcfdc2016a 100644 --- a/libavutil/hwcontext_internal.h +++ b/libavutil/hwcontext_internal.h @@ -164,5 +164,6 @@ extern const HWContextType ff_hwcontext_type_videotoolbox; extern const HWContextType ff_hwcontext_type_mediacodec; extern const HWContextType ff_hwcontext_type_vulkan; extern const HWContextType ff_hwcontext_type_amf; +extern const HWContextType ff_hwcontext_type_oh; #endif /* AVUTIL_HWCONTEXT_INTERNAL_H */ diff --git a/libavutil/hwcontext_oh.c b/libavutil/hwcontext_oh.c new file mode 100644 index 0000000000..6d1f27cfc5 --- /dev/null +++ b/libavutil/hwcontext_oh.c @@ -0,0 +1,47 @@ +/* + * This file is part of FFmpeg. + * + * Copyright (c) 2025 Zhao Zhili <quinkblack(a)foxmail.com> + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <multimedia/player_framework/native_avcodec_base.h> + +#include "hwcontext.h" +#include "hwcontext_internal.h" +#include "hwcontext_oh.h" + +static int oh_device_create(AVHWDeviceContext *ctx, const char *device, + AVDictionary *opts, int flags) +{ + if (device && device[0]) { + av_log(ctx, AV_LOG_ERROR, "Device selection unsupported.\n"); + return AVERROR_UNKNOWN; + } + + return 0; +} + +const HWContextType ff_hwcontext_type_oh = { + .type = AV_HWDEVICE_TYPE_OHCODEC, + .name = "ohcodec", + .device_hwctx_size = sizeof(AVOHCodecDeviceContext), + .device_create = oh_device_create, + .pix_fmts = (const enum AVPixelFormat[]) { + AV_PIX_FMT_OHCODEC, + AV_PIX_FMT_NONE + }, +}; diff --git a/libavutil/hwcontext_oh.h b/libavutil/hwcontext_oh.h new file mode 100644 index 0000000000..2794f85b7d --- /dev/null +++ b/libavutil/hwcontext_oh.h @@ -0,0 +1,34 @@ +/* + * This file is part of FFmpeg. + * + * Copyright (c) 2025 Zhao Zhili <quinkblack(a)foxmail.com> + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVUTIL_HWCONTEXT_OH_H +#define AVUTIL_HWCONTEXT_OH_H + +/** + * OpenHarmony codec device + */ +typedef struct AVOHCodecDeviceContext { + /** + * Pointer to OHNativeWindow + */ + void *native_window; +} AVOHCodecDeviceContext; + +#endif /* AVUTIL_HWCONTEXT_OH_H */ diff --git a/libavutil/pixdesc.c b/libavutil/pixdesc.c index 53adde5aba..efa4c2db64 100644 --- a/libavutil/pixdesc.c +++ b/libavutil/pixdesc.c @@ -3167,6 +3167,10 @@ static const AVPixFmtDescriptor av_pix_fmt_descriptors[AV_PIX_FMT_NB] = { }, .flags = AV_PIX_FMT_FLAG_BE, }, + [AV_PIX_FMT_OHCODEC] = { + .name = "ohcodec", + .flags = AV_PIX_FMT_FLAG_HWACCEL, + }, }; static const char * const color_range_names[] = { diff --git a/libavutil/pixfmt.h b/libavutil/pixfmt.h index bf1b8ed008..b697306201 100644 --- a/libavutil/pixfmt.h +++ b/libavutil/pixfmt.h @@ -488,6 +488,8 @@ enum AVPixelFormat { AV_PIX_FMT_GBRAP32BE, ///< planar GBRA 4:4:4:4 128bpp, big-endian AV_PIX_FMT_GBRAP32LE, ///< planar GBRA 4:4:4:4 128bpp, little-endian + AV_PIX_FMT_OHCODEC, /// hardware decoding through openharmony + AV_PIX_FMT_NB ///< number of pixel formats, DO NOT USE THIS if you want to link with shared libav* because the number of formats might differ between versions }; diff --git a/libavutil/version.h b/libavutil/version.h index 2c0aa3e35e..29cd31e229 100644 --- a/libavutil/version.h +++ b/libavutil/version.h @@ -79,8 +79,8 @@ */ #define LIBAVUTIL_VERSION_MAJOR 60 -#define LIBAVUTIL_VERSION_MINOR 4 -#define LIBAVUTIL_VERSION_MICRO 101 +#define LIBAVUTIL_VERSION_MINOR 5 +#define LIBAVUTIL_VERSION_MICRO 100 #define LIBAVUTIL_VERSION_INT AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \ LIBAVUTIL_VERSION_MINOR, \ -- 2.46.0

1 0

[PATCH] avfilter/vf_frei0r: fix time not being passed in seconds
by Stefan Breunig 06 Jul '25

06 Jul '25

The frei0r API expects the time in seconds, but was given it in milliseconds. The bug might exist since 41f1d3a (~14 years ago), but plugins depending on the time are unwatchable without this patch. For example: ffmpeg -filter_complex "testsrc2=d=5,frei0r=distort0r" out.mp4 Signed-off-by: Stefan Breunig <stefan-ffmpeg-devel(a)breunig.xyz> --- libavfilter/vf_frei0r.c | 2 +- tests/fate/filter-video.mak | 5 +++++ tests/ref/fate/filter-frei0r-filter | 10 ++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 tests/ref/fate/filter-frei0r-filter diff --git a/libavfilter/vf_frei0r.c b/libavfilter/vf_frei0r.c index 1284e27310..50d81d220f 100644 --- a/libavfilter/vf_frei0r.c +++ b/libavfilter/vf_frei0r.c @@ -379,7 +379,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) in = in2; } - s->update(s->instance, in->pts * av_q2d(inlink->time_base) * 1000, + s->update(s->instance, in->pts * av_q2d(inlink->time_base), (const uint32_t *)in->data[0], (uint32_t *)out->data[0]); diff --git a/tests/fate/filter-video.mak b/tests/fate/filter-video.mak index 34cf7afa83..775febfde4 100644 --- a/tests/fate/filter-video.mak +++ b/tests/fate/filter-video.mak @@ -701,6 +701,11 @@ $(FATE_FILTER_VSYNTH-yes): SRC = $(TARGET_PATH)/tests/vsynth1/%02d.pgm FATE_FFMPEG += $(FATE_FILTER_VSYNTH-yes) +FATE_FILTER_FREI0R-$(call ALLYES, TESTSRC2_FILTER FREI0R_FILTER) = fate-filter-frei0r-filter +fate-filter-frei0r-filter: SRC = testsrc2=r=1:d=5 +fate-filter-frei0r-filter: CMD = framecrc -lavfi "$(SRC),frei0r=enable=gte(n\,3):filter_name=distort0r" +FATE_FFMPEG += $(FATE_FILTER_FREI0R-yes) + # # Metadata tests # diff --git a/tests/ref/fate/filter-frei0r-filter b/tests/ref/fate/filter-frei0r-filter new file mode 100644 index 0000000000..4b1588785d --- /dev/null +++ b/tests/ref/fate/filter-frei0r-filter @@ -0,0 +1,10 @@ +#tb 0: 1/1 +#media_type 0: video +#codec_id 0: rawvideo +#dimensions 0: 320x240 +#sar 0: 1/1 +0, 0, 0, 1, 307200, 0x30630897 +0, 1, 1, 1, 307200, 0xd08784dd +0, 2, 2, 1, 307200, 0xe94387a0 +0, 3, 3, 1, 307200, 0x5df7a70e +0, 4, 4, 1, 307200, 0x9c203210 -- 2.47.2

1 0

[PATCH 00/18] WHIP + TLS + UDP fixes and SChannel DTLS support
by Timo Rothenpieler 06 Jul '25

06 Jul '25

This was originally just me attempting to add DTLS support to SChannel, so it can be used with the WHIP protocol. But on the way there, a lot of random fixes and enhancements accumulated, resulting in this series. The main new features are DTLS support for SChannel, which also happens to enable support for listen-mode, which was previous unsupported for normal TLS as well. To enable that, udp.c had to be enhanced to allow reporting the address a packet was received from. In the process of that I realized that udp.c was clearly not made with bidirectional communication in mind, so that had to be fixed as well ( The fifo buffer was used by both read and write without any checks, meaning the two would interfere with each other). The rest are misc fixes for issues in WHIP and the associated new tls code I found. Timo Rothenpieler (18): avformat/Makefile: don't hardcode openssl for whip muxer avformat/whip: use av_dict_set_int for int avformat/whip: don't leak options dict avformat/whip: remove redundant WHIP: prefix from all logging avformat/whip: fix format string for printing size_t avformat/tls: use non protocol specific error message avformat/tls: don't use http_proxy for udp sockets avformat/tls: move openssl specific init out of generic code avformat/udp: don't override 0 localport avformat/tls: fix udp init avformat/udp: make recv addr of each packet available avformat/udp: separate rx and tx fifo avformat/udp: add function to set remote address directly avformat/tls: remove unused fingerprint option avformat/tls: clean up new whip options avformat/tls: make passing an external socket universal avformat/tls_openssl: use existing context handle avformat/tls_schannel: add DTLS support configure | 6 +- libavformat/Makefile | 2 +- libavformat/network.h | 3 + libavformat/tls.c | 23 +- libavformat/tls.h | 20 +- libavformat/tls_openssl.c | 33 +- libavformat/tls_schannel.c | 853 ++++++++++++++++++++++++++++++++++--- libavformat/udp.c | 100 +++-- libavformat/whip.c | 167 ++++---- 9 files changed, 998 insertions(+), 209 deletions(-) -- 2.49.0

4 39

[PATCH 1/4] tests/fate/audio: G.728 test case
by Peter Ross 06 Jul '25

06 Jul '25

--- tests/fate/audio.mak | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/fate/audio.mak b/tests/fate/audio.mak index 1a2d07304b..c21578947a 100644 --- a/tests/fate/audio.mak +++ b/tests/fate/audio.mak @@ -42,6 +42,12 @@ fate-dsf-dst: CMD = pcm -i $(TARGET_SAMPLES)/dst/dst-64fs44-2ch.dff fate-dsf-dst: CMP = oneoff fate-dsf-dst: REF = $(SAMPLES)/dst/dst-64fs44-2ch.pcm +FATE_SAMPLES_AUDIO-$(call PCM, G728, G728, ARESAMPLE_FILTER) += fate-g728 +fate-g728: CMD = pcm -i $(TARGET_SAMPLES)/g728/CW3.g728 +fate-g728: REF = $(SAMPLES)/g728/OUTA3.BIN +fate-g728: CMP = oneoff +fate-g728: FUZZ = 5 + FATE_SAMPLES_AUDIO-$(call PCM, AVI, IMC, ARESAMPLE_FILTER) += fate-imc fate-imc: CMD = pcm -i $(TARGET_SAMPLES)/imc/imc.avi fate-imc: CMP = oneoff -- 2.47.2 -- Peter (A907 E02F A6E5 0CD2 34CD 20D2 6760 79C5 AC40 DD6B)

1 3

[PATCH] avfilter: add inverse tone mapping
by Sarthak Indurkhya 05 Jul '25

05 Jul '25

Hello FFmpeg developers, This patch introduces a new video filter called inversetonemap for FFmpeg. The filter performs SDR to HDR conversion by mapping SDR BT.709 video to HDR BT.2020 PQ, using local adaptation and inverse tone mapping. The goal is to provide a simple, flexible tool for upconverting SDR content for HDR displays, with local adaptation, tone curve sensitivity, and chroma processing. Please review. Thanks, Sarthak -- >From f563ee1ad93511dfe7dd252578d7801e0cbbe968 Mon Sep 17 00:00:00 2001 From: Sarthak Indurkhya sarthak(a)Sarthaks-MacBook-Pro.local<mailto:sarthak@Sarthaks-MacBook-Pro.local> Date: Sat, 5 Jul 2025 22:33:46 +0530 Subject: [PATCH] avfilter: add inversetonemap filter This filter performs inverse tone mapping from SDR to HDR using local adaptation and PQ mapping. - Added inversetonemap video filter for SDR to HDR conversion. --- Changelog | 1 + libavfilter/Makefile | 1 + libavfilter/allfilters.c | 1 + libavfilter/vf_inversetonemap.c | 98 ++++++++++++++++----------------- 4 files changed, 51 insertions(+), 50 deletions(-) diff --git a/Changelog b/Changelog index 81e2cc813f..0aecf6dbf1 100644 --- a/Changelog +++ b/Changelog @@ -2,6 +2,7 @@ Entries are sorted chronologically from oldest to youngest within each release, releases are sorted from youngest to oldest. version <next>: +- Added inversetonemap video filter for SDR to HDR conversion. - yasm support dropped, users need to use nasm - VVC VAAPI decoder - RealVideo 6.0 decoder diff --git a/libavfilter/Makefile b/libavfilter/Makefile index 97f8f17272..e715a3a5e4 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -191,6 +191,7 @@ OBJS-$(CONFIG_SINE_FILTER) += asrc_sine.o OBJS-$(CONFIG_ANULLSINK_FILTER) += asink_anullsink.o # video filters +OBJS-$(CONFIG_INVERSETONEMAP_FILTER) += vf_inversetonemap.o OBJS-$(CONFIG_ADDROI_FILTER) += vf_addroi.o OBJS-$(CONFIG_ALPHAEXTRACT_FILTER) += vf_extractplanes.o OBJS-$(CONFIG_ALPHAMERGE_FILTER) += vf_alphamerge.o framesync.o diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index 3bc045b28f..2f67300ca1 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -176,6 +176,7 @@ extern const FFFilter ff_asrc_sine; extern const FFFilter ff_asink_anullsink; +extern const FFFilter ff_vf_inversetonemap; extern const FFFilter ff_vf_addroi; extern const FFFilter ff_vf_alphaextract; extern const FFFilter ff_vf_alphamerge; diff --git a/libavfilter/vf_inversetonemap.c b/libavfilter/vf_inversetonemap.c index 28ea1ef29e..d8d8920151 100644 --- a/libavfilter/vf_inversetonemap.c +++ b/libavfilter/vf_inversetonemap.c @@ -1,3 +1,22 @@ +/* + * Copyright (c) 2025 Sarthak Indurkhya + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + /** * @file * @brief SDR to HDR inverse tone mapping filter for FFmpeg @@ -38,7 +57,7 @@ typedef struct FilterContext { float HDR_max; } FilterContext; -//declaring lookup table + static float bt709_gamma_lut[256]; static void init_bt709_gamma_lut(void) { @@ -51,14 +70,13 @@ static void init_bt709_gamma_lut(void) { } } -// Rec.709 to Rec.2020 matrix + static const float bt709_to_bt2020[3][3] = { {0.6274f, 0.3293f, 0.0433f}, {0.0691f, 0.9195f, 0.0114f}, {0.0164f, 0.0880f, 0.8956f} }; -// PQ transfer function #define PQ_M1 0.1593017578125f #define PQ_M2 78.84375f #define PQ_C1 0.8359375f @@ -80,12 +98,10 @@ static void compute_local_adaptation(const float *R_full, float *sigma, int widt float *temp_blur = av_mallocz(npix * sizeof(float)); float *spatial_weights = av_malloc((2 * radius + 1) * sizeof(float)); - // 1D Gaussian kernel for separable blur for (int i = -radius; i <= radius; i++) { spatial_weights[i + radius] = expf(-(i * i) / (2 * sigma_spatial * sigma_spatial)); } - // Scene max luminance float scene_max = 1e-6f; for (int i = 0; i < npix; i++) { if (R_full[i] > scene_max) @@ -93,7 +109,6 @@ static void compute_local_adaptation(const float *R_full, float *sigma, int widt } float hdr_scale = HDR_max / scene_max; - // ---- Horizontal blur pass ---- for (int y = 0; y < height; y++) { for (int x = 0; x < width; x++) { float sum = 0.0f, sum_weights = 0.0f; @@ -113,7 +128,6 @@ static void compute_local_adaptation(const float *R_full, float *sigma, int widt } } - // ---- Vertical blur pass ---- for (int y = 0; y < height; y++) { for (int x = 0; x < width; x++) { float sum = 0.0f, sum_weights = 0.0f; @@ -137,18 +151,15 @@ static void compute_local_adaptation(const float *R_full, float *sigma, int widt av_free(spatial_weights); } static void compute_local_adaptation_fast(const float *R_full, float *sigma, int width, int height, - float sigma_spatial, float sigma_range, float HDR_max) -{ + float sigma_spatial, float sigma_range, float HDR_max){ const int scale = 4; int down_w = width / scale; int down_h = height / scale; int npix_small = down_w * down_h; - // Allocating downsampled and result buffers float *R_small = av_malloc(npix_small * sizeof(float)); float *sigma_small = av_malloc(npix_small * sizeof(float)); - // Simple box downsampling for (int y = 0; y < down_h; y++) { for (int x = 0; x < down_w; x++) { float sum = 0.0f; @@ -167,7 +178,6 @@ static void compute_local_adaptation_fast(const float *R_full, float *sigma, int compute_local_adaptation(R_small, sigma_small, down_w, down_h, sigma_spatial, sigma_range, HDR_max); - // Upsampling sigma_small -> sigma struct SwsContext *sws_ctx = sws_getContext(down_w, down_h, AV_PIX_FMT_GRAYF32, width, height, AV_PIX_FMT_GRAYF32, SWS_BILINEAR, NULL, NULL, NULL); @@ -199,14 +209,12 @@ static void compute_hdr_intensity(const float *R_full, const float *sigma, float static void inverse_tone_map_linear_rgb( const float *R, const float *G, const float *B, float *R_hdr, float *G_hdr, float *B_hdr, - int width, int height, FilterContext *s) -{ + int width, int height, FilterContext *s){ int npix = width * height; float *Y = av_malloc(npix * sizeof(float)); float *Y_sigma = av_malloc(npix * sizeof(float)); float *Y_hdr = av_malloc(npix * sizeof(float)); - // Computing luminance (BT.2020) for (int i = 0; i < npix; i++) Y[i] = 0.2627f * R[i] + 0.6780f * G[i] + 0.0593f * B[i]; @@ -215,7 +223,6 @@ static void inverse_tone_map_linear_rgb( compute_hdr_intensity(Y, Y_sigma, Y_hdr, width, height, s->n, 1.0f); - // Scaling RGB channels for (int i = 0; i < npix; i++) { float scale = Y_hdr[i] / (Y[i] + 1e-6f); R_hdr[i] = R[i] * scale; @@ -249,11 +256,20 @@ static void dither_pq_to_10bit(const float *pq, uint16_t *y_temp, int width, int } static int fil_func(AVFilterLink *inlink, AVFrame *in) { + const float exposure = 3.5f; + const float contrast = 1.02f; + const float black = 0.03f; + const float white = 8.0f; + const float s_curve_pow = 1.00f; + const float white_balance_r = 1.0f; + const float white_balance_g = 1.0f; + const float white_balance_b = 1.1f; + AVFilterContext *ctx = inlink->dst; FilterContext *s = ctx->priv; int width = in->width, height = in->height, npix = width * height; - // Converting to RGB24 + struct SwsContext *sws_ctx = sws_getContext( width, height, in->format, width, height, AV_PIX_FMT_RGB24, @@ -272,9 +288,6 @@ static int fil_func(AVFilterLink *inlink, AVFrame *in) { rgb_frame->data, rgb_frame->linesize); sws_freeContext(sws_ctx); - //Calling gamma lookup table initialization - init_bt709_gamma_lut(); - // Gamma linearization & gamut mapping float *R = av_malloc(npix * sizeof(float)); float *G = av_malloc(npix * sizeof(float)); float *B = av_malloc(npix * sizeof(float)); @@ -290,20 +303,21 @@ static int fil_func(AVFilterLink *inlink, AVFrame *in) { } } - // Inverse Tone Mapping float *R_hdr = av_malloc(npix * sizeof(float)); float *G_hdr = av_malloc(npix * sizeof(float)); float *B_hdr = av_malloc(npix * sizeof(float)); inverse_tone_map_linear_rgb(R, G, B, R_hdr, G_hdr, B_hdr, width, height, s); - float exposure = 3.5f; //4.0 - float contrast = 1.02f; //1.10 - float black = 0.03f, white = 8.0f; //0.04 //12.0 - float s_curve_pow = 1.00f; //0.70 + /* + float exposure = 3.5f; //4.0 + float contrast = 1.02f; //1.10 + float black = 0.03f, white = 8.0f; //0.04 //12.0 + float s_curve_pow = 1.00f; //0.70 + */ for (int i = 0; i < npix; i++) { - float r = R[i] * exposure, g = G[i] * exposure, b = B[i] * exposure; + float r = R_hdr[i] * exposure, g = G_hdr[i] * exposure, b = B_hdr[i] * exposure; float lum = 0.2627f * r + 0.6780f * g + 0.0593f * b; @@ -316,21 +330,14 @@ static int fil_func(AVFilterLink *inlink, AVFrame *in) { g *= scale; b *= scale; - // Contrast r = (r - 0.5f) * contrast + 0.5f; g = (g - 0.5f) * contrast + 0.5f; b = (b - 0.5f) * contrast + 0.5f; - // White balance correction - float white_balance_r = 1.0f; - float white_balance_g = 1.0f; - float white_balance_b = 1.1f; - r *= white_balance_r; g *= white_balance_g; b *= white_balance_b; - // Final clamp r = fminf(fmaxf(r, 0.0f), 1.0f); g = fminf(fmaxf(g, 0.0f), 1.0f); b = fminf(fmaxf(b, 0.0f), 1.0f); @@ -340,18 +347,16 @@ static int fil_func(AVFilterLink *inlink, AVFrame *in) { B_hdr[i] = b; } - //Computing linear luminance (BT.2020) float *Y = av_malloc(npix * sizeof(float)); for (int i = 0; i < npix; i++) Y[i] = 0.2627f * R_hdr[i] + 0.6780f * G_hdr[i] + 0.0593f * B_hdr[i]; - // Applying PQ transfer float *pq = av_malloc(npix * sizeof(float)); for (int i = 0; i < npix; i++) { pq[i] = linear_to_pq(Y[i] * (10000.0f / s->HDR_max)); pq[i] = fminf(fmaxf(pq[i], 0.0f), 1.0f); } - //Allocating output frame (YUV420P10LE) + AVFrame *out = av_frame_alloc(); out->format = AV_PIX_FMT_YUV420P10LE; out->width = width; @@ -359,25 +364,21 @@ static int fil_func(AVFilterLink *inlink, AVFrame *in) { av_frame_get_buffer(out, 32); av_frame_copy_props(out, in); - // Setting HDR metadata + out->color_primaries = AVCOL_PRI_BT2020; - out->color_trc = AVCOL_TRC_SMPTE2084; // PQ + out->color_trc = AVCOL_TRC_SMPTE2084; out->colorspace = AVCOL_SPC_BT2020_NCL; - out->color_range = AVCOL_RANGE_MPEG; // Full range for HDR + out->color_range = AVCOL_RANGE_MPEG; - // Setting mastering display metadata if available if (out->metadata) { - // Mastering display primaries (BT.2020) av_dict_set(&out->metadata, "mastering_display_primaries", "0.708,0.292,0.170,0.797,0.131,0.046,0.3127,0.3290", 0); - // Mastering display luminance (in nits) char luminance_str[64]; snprintf(luminance_str, sizeof(luminance_str), "%.1f,%.1f", - s->HDR_max, 0.001f); // Max luminance, min luminance + s->HDR_max, 0.001f); av_dict_set(&out->metadata, "mastering_display_luminance", luminance_str, 0); - // Content light level char content_light_str[64]; float max_content_light = 0.0f; for (int i = 0; i < npix; i++) { @@ -389,12 +390,10 @@ static int fil_func(AVFilterLink *inlink, AVFrame *in) { av_dict_set(&out->metadata, "content_light_level", content_light_str, 0); } - // Clearing Y, U, V planes av_frame_make_writable(out); for (int y = 0; y < height; y++) memset(out->data[0] + y * out->linesize[0], 0, out->linesize[0]); - // Dithering PQ to 10-bit Y plane uint16_t *y_temp = av_malloc(npix * sizeof(uint16_t)); dither_pq_to_10bit(pq, y_temp, width, height); for (int y = 0; y < height; y++) { @@ -402,7 +401,6 @@ static int fil_func(AVFilterLink *inlink, AVFrame *in) { memcpy(row, y_temp + y * width, width * sizeof(uint16_t)); } - // Computing and encoding U, V chroma planes from original SDR RGB for (int y = 0; y < height / 2; y++) { uint16_t *u_row = (uint16_t *)(out->data[1] + y * out->linesize[1]); uint16_t *v_row = (uint16_t *)(out->data[2] + y * out->linesize[2]); @@ -425,8 +423,8 @@ static int fil_func(AVFilterLink *inlink, AVFrame *in) { float U = (b - Y_sdr) / 1.8814f; float V = (r - Y_sdr) / 1.4746f; float chroma_blend = 1.0f - fminf(Y_sdr, 1.0f); - float chroma_boost = 0.85f * chroma_blend + 0.85f * (1.0f - chroma_blend); //0.8 - // float chroma_boost = 1.08f; + float chroma_boost = 0.85f * chroma_blend + 0.85f * (1.0f - chroma_blend); + U *= chroma_boost; V *= chroma_boost; U = fmaxf(fminf(U, 0.45f), -0.45f); @@ -471,7 +469,7 @@ static const AVFilterPad fil_outputs[] = { }; static int ff_filter_init(AVFilterContext *avctx) { - av_log_set_level(AV_LOG_DEBUG); + init_bt709_gamma_lut(); av_log(avctx, AV_LOG_INFO, "Initializing filter with 1 input and 1 output\n"); return 0; } -- 2.49.0 Get Outlook for Mac <https://aka.ms/GetOutlookForMac>

1 0