[FFmpeg-devel] [PATCH 1/2] lavc: add Vulkan video encoding base code
Lynne
dev at lynne.ee
Sun Sep 15 12:54:10 EEST 2024
On 12/09/2024 15:20, Benjamin Cheng wrote:
> On Wed Sep 11, 2024 at 12:03 AM EDT, Lynne wrote:
>> On 10/09/2024 15:29, Benjamin Cheng wrote:
>>> On Mon Sep 9, 2024 at 6:37 AM EDT, Lynne via ffmpeg-devel wrote:
>>>> This commit adds the common Vulkan video encoding framework.
>>>> It makes full use of the asynchronous features of our new common
>>>> hardware encoding code, and of Vulkan.
>>>> The code is able to handle anything from H264 to AV1 and MJPEG.
>>>> ---
>>>> configure | 2 +
>>>> libavcodec/Makefile | 2 +-
>>>> libavcodec/vulkan_encode.c | 979 +++++++++++++++++++++++++++++++++++++
>>>> libavcodec/vulkan_encode.h | 243 +++++++++
>>>> 4 files changed, 1225 insertions(+), 1 deletion(-)
>>>> create mode 100644 libavcodec/vulkan_encode.c
>>>> create mode 100644 libavcodec/vulkan_encode.h
>>>>
>>>> diff --git a/configure b/configure
>>>> index a8e67d230c..6cfb736a86 100755
>>>> --- a/configure
>>>> +++ b/configure
>>>> @@ -2638,6 +2638,7 @@ CONFIG_EXTRA="
>>>> vp3dsp
>>>> vp56dsp
>>>> vp8dsp
>>>> + vulkan_encode
>>>> wma_freqs
>>>> wmv2dsp
>>>> "
>>>> @@ -3299,6 +3300,7 @@ qsvdec_select="qsv"
>>>> qsvenc_select="qsv"
>>>> qsvvpp_select="qsv"
>>>> vaapi_encode_deps="vaapi"
>>>> +vulkan_encode_deps="vulkan"
>>>> v4l2_m2m_deps="linux_videodev2_h sem_timedwait"
>>>>
>>>> bilateral_cuda_filter_deps="ffnvcodec"
>>>> diff --git a/libavcodec/Makefile b/libavcodec/Makefile
>>>> index 27ef4638ce..ff6a3c4efc 100644
>>>> --- a/libavcodec/Makefile
>>>> +++ b/libavcodec/Makefile
>>>> @@ -1282,7 +1282,7 @@ SKIPHEADERS-$(CONFIG_QSVENC) += qsvenc.h
>>>> SKIPHEADERS-$(CONFIG_VAAPI) += vaapi_decode.h vaapi_hevc.h vaapi_encode.h
>>>> SKIPHEADERS-$(CONFIG_VDPAU) += vdpau.h vdpau_internal.h
>>>> SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX) += videotoolbox.h vt_internal.h
>>>> -SKIPHEADERS-$(CONFIG_VULKAN) += vulkan.h vulkan_video.h vulkan_decode.h
>>>> +SKIPHEADERS-$(CONFIG_VULKAN) += vulkan.h vulkan_video.h vulkan_encode.h vulkan_decode.h
>>>> SKIPHEADERS-$(CONFIG_V4L2_M2M) += v4l2_buffers.h v4l2_context.h v4l2_m2m.h
>>>> SKIPHEADERS-$(CONFIG_ZLIB) += zlib_wrapper.h
>>>>
>>>> diff --git a/libavcodec/vulkan_encode.c b/libavcodec/vulkan_encode.c
>>>> new file mode 100644
>>>> index 0000000000..5e87d4c073
>>>> --- /dev/null
>>>> +++ b/libavcodec/vulkan_encode.c
>>>> @@ -0,0 +1,979 @@
>>>> +/*
>>>> + * This file is part of FFmpeg.
>>>> + *
>>>> + * FFmpeg is free software; you can redistribute it and/or
>>>> + * modify it under the terms of the GNU Lesser General Public
>>>> + * License as published by the Free Software Foundation; either
>>>> + * version 2.1 of the License, or (at your option) any later version.
>>>> + *
>>>> + * FFmpeg is distributed in the hope that it will be useful,
>>>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>>>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
>>>> + * Lesser General Public License for more details.
>>>> + *
>>>> + * You should have received a copy of the GNU Lesser General Public
>>>> + * License along with FFmpeg; if not, write to the Free Software
>>>> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
>>>> + */
>>>> +
>>>> +#include "libavutil/mem.h"
>>>> +#include "libavutil/avassert.h"
>>>> +#include "vulkan_encode.h"
>>>> +#include "config.h"
>>>> +
>>>> +#include "libavutil/vulkan_loader.h"
>>>> +
>>>> +const AVCodecHWConfigInternal *const ff_vulkan_encode_hw_configs[] = {
>>>> + HW_CONFIG_ENCODER_FRAMES(VULKAN, VULKAN),
>>>> + NULL,
>>>> +};
>>>> +
>>>> +av_cold void ff_vulkan_encode_uninit(FFVulkanEncodeContext *ctx)
>>>> +{
>>>> + FFVulkanContext *s = &ctx->s;
>>>> + FFVulkanFunctions *vk = &s->vkfn;
>>>> +
>>>> + /* Wait on and free execution pool */
>>>> + ff_vk_exec_pool_free(s, &ctx->enc_pool);
>>>> +
>>>> + /* Destroy the session params */
>>>> + if (ctx->session_params)
>>>> + vk->DestroyVideoSessionParametersKHR(s->hwctx->act_dev,
>>>> + ctx->session_params,
>>>> + s->hwctx->alloc);
>>>> +
>>>> + ff_hw_base_encode_close(&ctx->base);
>>>> +
>>>> + av_buffer_pool_uninit(&ctx->buf_pool);
>>>> +
>>>> + ff_vk_video_common_uninit(s, &ctx->common);
>>>> +
>>>> + ff_vk_uninit(s);
>>>> +}
>>>> +
>>>> +static int vulkan_encode_init(AVCodecContext *avctx, FFHWBaseEncodePicture *pic)
>>>> +{
>>>> + int err;
>>>> + FFVulkanEncodeContext *ctx = avctx->priv_data;
>>>> + FFVulkanEncodePicture *vp = pic->priv;
>>>> +
>>>> + AVFrame *f = pic->input_image;
>>>> + AVHWFramesContext *hwfc = (AVHWFramesContext *)f->hw_frames_ctx->data;
>>>> + AVVulkanFramesContext *vkfc = hwfc->hwctx;
>>>> + AVVkFrame *vkf = (AVVkFrame *)f->data[0];
>>>> +
>>>> + if (ctx->codec->picture_priv_data_size > 0) {
>>>> + pic->codec_priv = av_mallocz(ctx->codec->picture_priv_data_size);
>>>> + if (!pic->codec_priv)
>>>> + return AVERROR(ENOMEM);
>>>> + }
>>>> +
>>>> + /* Input image view */
>>>> + err = ff_vk_create_view(&ctx->s, &ctx->common,
>>>> + &vp->in.view, &vp->in.aspect,
>>>> + vkf, vkfc->format[0], 0);
>>>> + if (err < 0)
>>>> + return err;
>>>> +
>>>> + /* Reference view */
>>>> + if (!ctx->common.layered_dpb) {
>>>> + AVFrame *rf = pic->recon_image;
>>>> + AVVkFrame *rvkf = (AVVkFrame *)rf->data[0];
>>>> + err = ff_vk_create_view(&ctx->s, &ctx->common,
>>>> + &vp->dpb.view, &vp->dpb.aspect,
>>>> + rvkf, ctx->pic_format, 1);
>>>> + if (err < 0)
>>>> + return err;
>>>> + } else {
>>>> + vp->dpb.view = ctx->common.layered_view;
>>>> + vp->dpb.aspect = ctx->common.layered_aspect;
>>>> + }
>>>> +
>>>> + return 0;
>>>> +}
>>>> +
>>>> +static int vulkan_encode_free(AVCodecContext *avctx, FFHWBaseEncodePicture *pic)
>>>> +{
>>>> + FFVulkanEncodeContext *ctx = avctx->priv_data;
>>>> + FFVulkanFunctions *vk = &ctx->s.vkfn;
>>>> +
>>>> + FFVulkanEncodePicture *vp = pic->priv;
>>>> +
>>>> + if (vp->in.view)
>>>> + vk->DestroyImageView(ctx->s.hwctx->act_dev, vp->in.view,
>>>> + ctx->s.hwctx->alloc);
>>>> +
>>>> + if (!ctx->common.layered_dpb && vp->dpb.view)
>>>> + vk->DestroyImageView(ctx->s.hwctx->act_dev, vp->dpb.view,
>>>> + ctx->s.hwctx->alloc);
>>>> +
>>>> + ctx->slots[vp->dpb_slot.slotIndex] = 0;
>>>> +
>>>> + return 0;
>>>> +}
>>>> +
>>>> +static int init_pic_rc(AVCodecContext *avctx, FFHWBaseEncodePicture *pic,
>>>> + VkVideoEncodeRateControlInfoKHR *rc_info,
>>>> + VkVideoEncodeRateControlLayerInfoKHR *rc_layer /* Goes in ^ */)
>>>> +{
>>>> + FFVulkanEncodeContext *ctx = avctx->priv_data;
>>>> +
>>>> + *rc_info = (VkVideoEncodeRateControlInfoKHR) {
>>>> + .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_RATE_CONTROL_INFO_KHR,
>>>> + .rateControlMode = ctx->opts.rc_mode,
>>>> + };
>>>> +
>>>> + if (ctx->opts.rc_mode > VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR) {
>>>> + *rc_layer = (VkVideoEncodeRateControlLayerInfoKHR) {
>>>> + .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_RATE_CONTROL_LAYER_INFO_KHR,
>>>> + .averageBitrate = avctx->bit_rate,
>>>> + .maxBitrate = avctx->rc_max_rate ? avctx->rc_max_rate : avctx->bit_rate,
>>>> + .frameRateNumerator = avctx->framerate.num,
>>>> + .frameRateDenominator = avctx->framerate.den,
>>>> + };
>>>> + rc_info->layerCount++;
>>>> + rc_info->pLayers = rc_layer;
>>>> + }
>>>> +
>>>> + return ctx->codec->init_pic_rc(avctx, pic, rc_info, rc_layer);
>>>> +}
>>>> +
>>>> +static int vulkan_encode_issue(AVCodecContext *avctx,
>>>> + FFHWBaseEncodePicture *base_pic)
>>>> +{
>>>> + FFVulkanEncodeContext *ctx = avctx->priv_data;
>>>> + FFVulkanFunctions *vk = &ctx->s.vkfn;
>>>> +
>>>> + const size_t size_align = ctx->caps.minBitstreamBufferSizeAlignment;
>>>> +
>>>> + FFVulkanEncodePicture *vp = base_pic->priv;
>>>> + AVFrame *src = (AVFrame *)base_pic->input_image;
>>>> + AVVkFrame *vkf = (AVVkFrame *)src->data[0];
>>>> +
>>>> + int err, max_pkt_size;
>>>> +
>>>> + FFVkBuffer *sd_buf;
>>>> +
>>>> + int slot_index = -1;
>>>> + FFVkExecContext *exec;
>>>> + VkCommandBuffer cmd_buf;
>>>> + VkImageMemoryBarrier2 img_bar[37];
>>>> + int nb_img_bar = 0;
>>>> +
>>>> + /* Coding start/end */
>>>> + VkVideoBeginCodingInfoKHR encode_start;
>>>> + VkVideoEndCodingInfoKHR encode_end = {
>>>> + .sType = VK_STRUCTURE_TYPE_VIDEO_END_CODING_INFO_KHR,
>>>> + };
>>>> +
>>>> + VkVideoEncodeRateControlLayerInfoKHR rc_layer;
>>>> + VkVideoEncodeRateControlInfoKHR rc_info;
>>>> + VkVideoEncodeQualityLevelInfoKHR q_info;
>>>> + VkVideoCodingControlInfoKHR encode_ctrl;
>>>> +
>>>> + VkVideoReferenceSlotInfoKHR ref_slot[37];
>>>> + VkVideoEncodeInfoKHR encode_info;
>>>> +
>>>> + /* Create packet data buffer */
>>>> + max_pkt_size = FFALIGN(3 * ctx->base.surface_width * ctx->base.surface_height + (1 << 16),
>>>> + ctx->caps.minBitstreamBufferSizeAlignment);
>>>> +
>>>> + err = ff_vk_get_pooled_buffer(&ctx->s, &ctx->buf_pool, &vp->pkt_buf,
>>>> + VK_BUFFER_USAGE_VIDEO_ENCODE_DST_BIT_KHR,
>>>> + &ctx->profile_list, max_pkt_size,
>>>> + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
>>>> + VK_MEMORY_PROPERTY_HOST_CACHED_BIT);
>>>> + if (err < 0)
>>>> + return err;
>>>> +
>>>> + sd_buf = (FFVkBuffer *)vp->pkt_buf->data;
>>>> +
>>>> + /* Setup rate control */
>>>> + err = init_pic_rc(avctx, base_pic, &rc_info, &rc_layer);
>>>> + if (err < 0)
>>>> + return err;
>>>> +
>>>> + q_info = (VkVideoEncodeQualityLevelInfoKHR) {
>>>> + .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_QUALITY_LEVEL_INFO_KHR,
>>>> + .pNext = &rc_info,
>>>> + .qualityLevel = ctx->opts.quality,
>>>> + };
>>>> + encode_ctrl = (VkVideoCodingControlInfoKHR) {
>>>> + .sType = VK_STRUCTURE_TYPE_VIDEO_CODING_CONTROL_INFO_KHR,
>>>> + .pNext = &q_info,
>>>> + .flags = VK_VIDEO_CODING_CONTROL_ENCODE_QUALITY_LEVEL_BIT_KHR |
>>>> + VK_VIDEO_CODING_CONTROL_ENCODE_RATE_CONTROL_BIT_KHR |
>>>> + (base_pic->force_idr ? VK_VIDEO_CODING_CONTROL_RESET_BIT_KHR : 0),
>>>> + };
>>>> +
>>>> + /* Current picture's ref slot */
>>>> + vp->dpb_res = (VkVideoPictureResourceInfoKHR) {
>>>> + .sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR,
>>>> + .pNext = NULL,
>>>> + .codedOffset = { 0 },
>>>> + .codedExtent = (VkExtent2D){ ctx->base.surface_width,
>>>> + ctx->base.surface_height },
>>>> + .baseArrayLayer = 0,
>>>> + .imageViewBinding = vp->dpb.view,
>>>> + };
>>>> +
>>>> + for (int i = 0; i < ctx->caps.maxDpbSlots; i++) {
>>>> + if (!ctx->slots[i]) {
>>>> + slot_index = i;
>>>> + ctx->slots[i] = 1;
>>>> + break;
>>>> + }
>>>> + }
>>>> + av_assert0(slot_index >= 0);
>>>> +
>>>> + vp->dpb_slot = (VkVideoReferenceSlotInfoKHR) {
>>>> + .sType = VK_STRUCTURE_TYPE_VIDEO_REFERENCE_SLOT_INFO_KHR,
>>>> + .pNext = NULL, // Set later
>>>> + .slotIndex = slot_index,
>>>> + .pPictureResource = &vp->dpb_res,
>>>> + };
>>>> +
>>>> + encode_info = (VkVideoEncodeInfoKHR) {
>>>> + .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_INFO_KHR,
>>>> + .pNext = NULL, // Set later
>>>> + .flags = 0x0,
>>>> + .srcPictureResource = (VkVideoPictureResourceInfoKHR) {
>>>> + .sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR,
>>>> + .pNext = NULL,
>>>> + .codedOffset = { 0, 0 },
>>>> + .codedExtent = (VkExtent2D){ base_pic->input_image->width,
>>>> + base_pic->input_image->height },
>>>> + .baseArrayLayer = 0,
>>>> + .imageViewBinding = vp->in.view,
>>>> + },
>>>> + .pSetupReferenceSlot = &vp->dpb_slot,
>>>> + .referenceSlotCount = 0,
>>>> + .pReferenceSlots = ref_slot,
>>>> + .dstBuffer = sd_buf->buf,
>>>> + .dstBufferOffset = 0,
>>>> + .dstBufferRange = sd_buf->size,
>>>> + .precedingExternallyEncodedBytes = 0,
>>>> + };
>>>> +
>>>> + for (int i = 0; i < MAX_REFERENCE_LIST_NUM; i++) {
>>>> + for (int j = 0; j < base_pic->nb_refs[i]; j++) {
>>>> + FFHWBaseEncodePicture *ref = base_pic->refs[i][j];
>>>> + FFVulkanEncodePicture *rvp = ref->priv;
>>>> + ref_slot[encode_info.referenceSlotCount++] = rvp->dpb_slot;
>>>> + }
>>>> + }
>>>> +
>>>> + /* Setup picture parameters */
>>>> + err = ctx->codec->init_pic_params(avctx, base_pic,
>>>> + &encode_info);
>>>> + if (err < 0)
>>>> + return err;
>>>> +
>>>> + encode_start = (VkVideoBeginCodingInfoKHR) {
>>>> + .sType = VK_STRUCTURE_TYPE_VIDEO_BEGIN_CODING_INFO_KHR,
>>>> + .pNext = !base_pic->force_idr ? &rc_info : NULL,
>>>> + .videoSession = ctx->common.session,
>>>> + .videoSessionParameters = ctx->session_params,
>>>> + .referenceSlotCount = encode_info.referenceSlotCount,
>>>> + .pReferenceSlots = ref_slot,
>>>> + };
>>>> +
>>>> + /* Calling vkCmdBeginVideoCodingKHR requires to declare all references
>>>> + * being enabled upfront, including the current frame's output ref.
>>>> + * If layered DBPs are used, make sure its not included twice. */
>>>> + if (!ctx->common.layered_dpb || !encode_info.referenceSlotCount) {
>>>> + ref_slot[encode_info.referenceSlotCount] = vp->dpb_slot;
>>>> + ref_slot[encode_info.referenceSlotCount].slotIndex = -1;
>>>> + encode_start.referenceSlotCount++;
>>>> + }
>>>> +
>>>> + /* Write header */
>>>> + if (base_pic->type == FF_HW_PICTURE_TYPE_IDR) {
>>>> + uint8_t *hdr_dst = sd_buf->mapped_mem + encode_info.dstBufferOffset;
>>>> + size_t data_size = encode_info.dstBufferRange;
>>>> + err = ctx->codec->write_sequence_headers(avctx, base_pic, hdr_dst, &data_size);
>>>> + if (err < 0)
>>>> + goto fail;
>>>> + encode_info.dstBufferOffset += data_size;
>>>> + encode_info.dstBufferRange -= data_size;
>>>> + }
>>>> +
>>>> + /* Write extra units */
>>>> + if (ctx->codec->write_extra_headers) {
>>>> + uint8_t *hdr_dst = sd_buf->mapped_mem + encode_info.dstBufferOffset;
>>>> + size_t data_size = encode_info.dstBufferRange;
>>>> + err = ctx->codec->write_extra_headers(avctx, base_pic, hdr_dst, &data_size);
>>>> + if (err < 0)
>>>> + goto fail;
>>>> + encode_info.dstBufferOffset += data_size;
>>>> + encode_info.dstBufferRange -= data_size;
>>>> + }
>>>> +
>>>> + /* Align buffer offset to the required value with filler units */
>>>> + if (ctx->codec->write_filler) {
>>>> + uint8_t *hdr_dst = sd_buf->mapped_mem + encode_info.dstBufferOffset;
>>>> + size_t data_size = encode_info.dstBufferRange;
>>>> +
>>>> + uint32_t offset = encode_info.dstBufferOffset;
>>>> + size_t offset_align = ctx->caps.minBitstreamBufferOffsetAlignment;
>>>> +
>>>> + uint32_t filler_data = FFALIGN(offset, offset_align) - offset;
>>>> +
>>>> + if (filler_data) {
>>>> + while (filler_data < ctx->codec->filler_header_size)
>>>> + filler_data += offset_align;
>>>> +
>>>> + filler_data -= ctx->codec->filler_header_size;
>>>> +
>>>> + err = ctx->codec->write_filler(avctx, filler_data,
>>>> + hdr_dst, &data_size);
>>>> + if (err < 0)
>>>> + goto fail;
>>>> +
>>>> + encode_info.dstBufferOffset += data_size;
>>>> + encode_info.dstBufferRange -= data_size;
>>>> + }
>>>> + }
>>>> +
>>>> + vp->slices_offset = encode_info.dstBufferOffset;
>>>> +
>>>> + /* Align buffer size to the nearest lower alignment requirement. */
>>>> + encode_info.dstBufferRange -= size_align;
>>>> + encode_info.dstBufferRange = FFALIGN(encode_info.dstBufferRange,
>>>> + size_align);
>>>> +
>>>> + /* Start command buffer recording */
>>>> + exec = vp->exec = ff_vk_exec_get(&ctx->enc_pool);
>>>> + ff_vk_exec_start(&ctx->s, exec);
>>>> + cmd_buf = exec->buf;
>>>> +
>>>> + /* Output packet buffer */
>>>> + err = ff_vk_exec_add_dep_buf(&ctx->s, exec, &vp->pkt_buf, 1, 1);
>>>> + if (err < 0)
>>>> + goto fail;
>>>> +
>>>> + /* Source image */
>>>> + err = ff_vk_exec_add_dep_frame(&ctx->s, exec, src,
>>>> + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
>>>> + VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR);
>>>> + if (err < 0)
>>>> + goto fail;
>>>> +
>>>> + /* Source image layout conversion */
>>>> + img_bar[nb_img_bar] = (VkImageMemoryBarrier2) {
>>>> + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
>>>> + .pNext = NULL,
>>>> + .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
>>>> + .srcAccessMask = vkf->access[0],
>>>> + .dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR,
>>>> + .dstAccessMask = VK_ACCESS_2_VIDEO_ENCODE_READ_BIT_KHR,
>>>> + .oldLayout = vkf->layout[0],
>>>> + .newLayout = VK_IMAGE_LAYOUT_VIDEO_ENCODE_SRC_KHR,
>>>> + .srcQueueFamilyIndex = vkf->queue_family[0],
>>>> + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
>>>> + .image = vkf->img[0],
>>>> + .subresourceRange = (VkImageSubresourceRange) {
>>>> + .aspectMask = vp->in.aspect,
>>>> + .layerCount = 1,
>>>> + .levelCount = 1,
>>>> + },
>>>> + };
>>>> + ff_vk_exec_update_frame(&ctx->s, exec, src,
>>>> + &img_bar[nb_img_bar], &nb_img_bar);
>>>> +
>>>> + if (!ctx->common.layered_dpb) {
>>>> + /* Source image's ref slot.
>>>> + * No need to do a layout conversion, since the frames which are allocated
>>>> + * with a DPB usage are automatically converted. */
>>>> + err = ff_vk_exec_add_dep_frame(&ctx->s, exec, base_pic->recon_image,
>>>> + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
>>>> + VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR);
>>>> + if (err < 0)
>>>> + return err;
>>>> +
>>>> + /* All references */
>>>> + for (int i = 0; i < MAX_REFERENCE_LIST_NUM; i++) {
>>>> + for (int j = 0; j < base_pic->nb_refs[i]; j++) {
>>>> + FFHWBaseEncodePicture *ref = base_pic->refs[i][j];
>>>> + err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ref->recon_image,
>>>> + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
>>>> + VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR);
>>>> + if (err < 0)
>>>> + return err;
>>>> + }
>>>> + }
>>>> + } else {
>>>> + err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ctx->common.layered_frame,
>>>> + VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR,
>>>> + VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR);
>>>> + if (err < 0)
>>>> + return err;
>>>> + }
>>>> +
>>>> + /* Change image layout */
>>>> + vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
>>>> + .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
>>>> + .pImageMemoryBarriers = img_bar,
>>>> + .imageMemoryBarrierCount = nb_img_bar,
>>>> + });
>>>> +
>>>> + /* Start, use parameters */
>>>> + vk->CmdBeginVideoCodingKHR(cmd_buf, &encode_start);
>>>> +
>>>> + /* Send control data */
>>>> + if (base_pic->force_idr)
>>>> + vk->CmdControlVideoCodingKHR(cmd_buf, &encode_ctrl);
>>>> +
>>>> + /* Encode */
>>>> + vk->CmdBeginQuery(cmd_buf, ctx->enc_pool.query_pool, exec->query_idx + 0, 0);
>>>> + vk->CmdEncodeVideoKHR(cmd_buf, &encode_info);
>>>> + vk->CmdEndQuery(cmd_buf, ctx->enc_pool.query_pool, exec->query_idx + 0);
>>>> +
>>>> + /* End encoding */
>>>> + vk->CmdEndVideoCodingKHR(cmd_buf, &encode_end);
>>>> +
>>>> + /* End recording and submit for execution */
>>>> + err = ff_vk_exec_submit(&ctx->s, vp->exec);
>>>> + if (err < 0)
>>>> + goto fail;
>>>> +
>>>> + /* We don't need to keep the input image any longer, its already ref'd */
>>>> + av_frame_free(&base_pic->input_image);
>>>> +
>>>> + return 0;
>>>> +
>>>> +fail:
>>>> + return err;
>>>> +}
>>>> +
>>>> +static void vulkan_encode_wait(AVCodecContext *avctx,
>>>> + FFHWBaseEncodePicture *base_pic)
>>>> +{
>>>> + FFVulkanEncodeContext *ctx = avctx->priv_data;
>>>> + FFVulkanEncodePicture *vp = base_pic->priv;
>>>> +
>>>> + av_assert0(base_pic->encode_issued);
>>>> +
>>>> + if (base_pic->encode_complete)
>>>> + return;
>>>> +
>>>> + ff_vk_exec_wait(&ctx->s, vp->exec);
>>>> + base_pic->encode_complete = 1;
>>>> +}
>>>> +
>>>> +static int vulkan_encode_output(AVCodecContext *avctx,
>>>> + FFHWBaseEncodePicture *base_pic, AVPacket *pkt)
>>>> +{
>>>> + VkResult ret;
>>>> + FFVulkanEncodePicture *vp = base_pic->priv;
>>>> + FFVulkanEncodeContext *ctx = avctx->priv_data;
>>>> + FFVkBuffer *sd_buf = (FFVkBuffer *)vp->pkt_buf->data;
>>>> + uint32_t *query_data;
>>>> +
>>>> + vulkan_encode_wait(avctx, base_pic);
>>>> +
>>>> + ret = ff_vk_exec_get_query(&ctx->s, vp->exec, (void **)&query_data, 0);
>>>> + if (ret == VK_NOT_READY) {
>>>> + av_log(avctx, AV_LOG_ERROR, "Unable to perform query: %s!\n",
>>>> + ff_vk_ret2str(ret));
>>>> + return AVERROR(EINVAL);
>>>> + }
>>>> +
>>>> + if (ret != VK_NOT_READY && ret != VK_SUCCESS) {
>>>> + av_log(avctx, AV_LOG_ERROR, "Unable to perform query: %s!\n",
>>>> + ff_vk_ret2str(ret));
>>>> + return AVERROR_EXTERNAL;
>>>> + }
>>>>
>>>> + if (query_data[2] != VK_QUERY_RESULT_STATUS_COMPLETE_KHR) {
>>>> + av_log(avctx, AV_LOG_ERROR, "Unable to encode: %u\n", query_data[2]);
>>>> + return AVERROR_EXTERNAL;
>>>> + }
>>>
>>> The query pool is created with ctx->enc_caps.supportedEncodeFeedbackFlags. If
>>> the implementation reports all feedback flags defined by the spec,
>>> query_data[2] would refer to the HAS_OVERRIDES field of the feedback. In
>>> this case, the status would live in query_data[3].
>>
>>
>> v2 sent to the ML which fixes both issues you mentioned, plus other bugs
>> I and others encountered.
>>
>> The spec text says:
>> > When retrieving the results of video encode feedback queries, the
>> > values corresponding to each enabled video encode feedback are written
>> > in the order of the bits defined above
>>
>> The order they're given in is:
>> VK_VIDEO_ENCODE_FEEDBACK_BITSTREAM_BUFFER_OFFSET_BIT_KHR = 0x00000001,
>> VK_VIDEO_ENCODE_FEEDBACK_BITSTREAM_BYTES_WRITTEN_BIT_KHR = 0x00000002,
>> VK_VIDEO_ENCODE_FEEDBACK_BITSTREAM_HAS_OVERRIDES_BIT_KHR = 0x00000004,
>
> Yes, so if all these feedback bits are reported as supported the query
> feedback buffer would look like:
>
> - query_data[0]: BUFFER_OFFSET
> - query_data[1]: BYTES_WRITTEN
> - query_data[2]: HAS_OVERRIDES
>
> Then, the result status appears at query_data[3] (if
> VK_QUERY_RESULT_WITH_STATUS_BIT_KHR is set).
>
>>
>> So I think the code is correct on this.
>>
>> Thanks for reviewing/testing
Fixed this in my branch, thanks
-------------- next part --------------
A non-text attachment was scrubbed...
Name: OpenPGP_0xA2FEA5F03F034464.asc
Type: application/pgp-keys
Size: 624 bytes
Desc: OpenPGP public key
URL: <https://ffmpeg.org/pipermail/ffmpeg-devel/attachments/20240915/e29ecc78/attachment.key>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: OpenPGP_signature.asc
Type: application/pgp-signature
Size: 236 bytes
Desc: OpenPGP digital signature
URL: <https://ffmpeg.org/pipermail/ffmpeg-devel/attachments/20240915/e29ecc78/attachment.sig>
More information about the ffmpeg-devel
mailing list