FFmpeg
All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Modules Pages
vulkan_decode.c
Go to the documentation of this file.
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 #include "libavutil/refstruct.h"
20 #include "vulkan_video.h"
21 #include "vulkan_decode.h"
22 #include "config_components.h"
23 #include "libavutil/avassert.h"
24 #include "libavutil/mem.h"
26 
27 #define DECODER_IS_SDR(codec_id) \
28  ((codec_id) == AV_CODEC_ID_FFV1)
29 
30 #if CONFIG_H264_VULKAN_HWACCEL
32 #endif
33 #if CONFIG_HEVC_VULKAN_HWACCEL
35 #endif
36 #if CONFIG_AV1_VULKAN_HWACCEL
38 #endif
39 #if CONFIG_FFV1_VULKAN_HWACCEL
41 #endif
42 
44 #if CONFIG_H264_VULKAN_HWACCEL
46 #endif
47 #if CONFIG_HEVC_VULKAN_HWACCEL
49 #endif
50 #if CONFIG_AV1_VULKAN_HWACCEL
52 #endif
53 #if CONFIG_FFV1_VULKAN_HWACCEL
55 #endif
56 };
57 
59 {
60  for (size_t i = 0; i < FF_ARRAY_ELEMS(dec_descs); i++)
61  if (dec_descs[i]->codec_id == codec_id)
62  return dec_descs[i];
63  av_assert1(!"no codec descriptor");
64  return NULL;
65 }
66 
67 static const VkVideoProfileInfoKHR *get_video_profile(FFVulkanDecodeShared *ctx, enum AVCodecID codec_id)
68 {
69  const VkVideoProfileListInfoKHR *profile_list;
70 
71  VkStructureType profile_struct_type =
72  codec_id == AV_CODEC_ID_H264 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR :
73  codec_id == AV_CODEC_ID_HEVC ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR :
74  codec_id == AV_CODEC_ID_AV1 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_KHR :
75  VK_STRUCTURE_TYPE_MAX_ENUM;
76  if (profile_struct_type == VK_STRUCTURE_TYPE_MAX_ENUM)
77  return NULL;
78 
79  profile_list = ff_vk_find_struct(ctx->s.hwfc->create_pnext,
80  VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
81  if (!profile_list)
82  return NULL;
83 
84  for (int i = 0; i < profile_list->profileCount; i++)
85  if (ff_vk_find_struct(profile_list->pProfiles[i].pNext, profile_struct_type))
86  return &profile_list->pProfiles[i];
87 
88  return NULL;
89 }
90 
92 {
93  int err;
94  FFVulkanDecodeContext *src_ctx = src->internal->hwaccel_priv_data;
95  FFVulkanDecodeContext *dst_ctx = dst->internal->hwaccel_priv_data;
96 
97  av_refstruct_replace(&dst_ctx->shared_ctx, src_ctx->shared_ctx);
98 
99  err = av_buffer_replace(&dst_ctx->session_params, src_ctx->session_params);
100  if (err < 0)
101  return err;
102 
103  dst_ctx->dedicated_dpb = src_ctx->dedicated_dpb;
104  dst_ctx->external_fg = src_ctx->external_fg;
105  dst_ctx->frame_id_alloc_mask = src_ctx->frame_id_alloc_mask;
106 
107  return 0;
108 }
109 
110 int ff_vk_params_invalidate(AVCodecContext *avctx, int t, const uint8_t *b, uint32_t s)
111 {
114  return 0;
115 }
116 
118 {
119  int err;
120  AVFrame *avf = av_frame_alloc();
121  if (!avf)
122  return NULL;
123 
124  err = av_hwframe_get_buffer(ctx->common.dpb_hwfc_ref, avf, 0x0);
125  if (err < 0)
126  av_frame_free(&avf);
127 
128  return avf;
129 }
130 
132 {
134  FFVulkanFunctions *vk = &ctx->s.vkfn;
135 
136  vkpic->dpb_frame = NULL;
137  for (int i = 0; i < AV_NUM_DATA_POINTERS; i++) {
138  vkpic->view.ref[i] = VK_NULL_HANDLE;
139  vkpic->view.out[i] = VK_NULL_HANDLE;
140  vkpic->view.dst[i] = VK_NULL_HANDLE;
141  }
142 
143  vkpic->destroy_image_view = vk->DestroyImageView;
144  vkpic->wait_semaphores = vk->WaitSemaphores;
145 }
146 
148  FFVulkanDecodePicture *vkpic, int is_current,
149  int alloc_dpb)
150 {
151  int err;
153 
154  vkpic->slices_size = 0;
155 
156  /* If the decoder made a blank frame to make up for a missing ref, or the
157  * frame is the current frame so it's missing one, create a re-representation */
158  if (vkpic->view.ref[0])
159  return 0;
160 
161  init_frame(dec, vkpic);
162 
163  if (ctx->common.layered_dpb && alloc_dpb) {
164  vkpic->view.ref[0] = ctx->common.layered_view;
165  vkpic->view.aspect_ref[0] = ctx->common.layered_aspect;
166  } else if (alloc_dpb) {
167  AVHWFramesContext *dpb_frames = (AVHWFramesContext *)ctx->common.dpb_hwfc_ref->data;
168  AVVulkanFramesContext *dpb_hwfc = dpb_frames->hwctx;
169 
170  vkpic->dpb_frame = vk_get_dpb_pool(ctx);
171  if (!vkpic->dpb_frame)
172  return AVERROR(ENOMEM);
173 
174  err = ff_vk_create_view(&ctx->s, &ctx->common,
175  &vkpic->view.ref[0], &vkpic->view.aspect_ref[0],
176  (AVVkFrame *)vkpic->dpb_frame->data[0],
177  dpb_hwfc->format[0], !is_current);
178  if (err < 0)
179  return err;
180 
181  vkpic->view.dst[0] = vkpic->view.ref[0];
182  }
183 
184  if (!alloc_dpb || is_current) {
186  AVVulkanFramesContext *hwfc = frames->hwctx;
187 
188  err = ff_vk_create_view(&ctx->s, &ctx->common,
189  &vkpic->view.out[0], &vkpic->view.aspect[0],
190  (AVVkFrame *)pic->data[0],
191  hwfc->format[0], !is_current);
192  if (err < 0)
193  return err;
194 
195  if (!alloc_dpb) {
196  vkpic->view.ref[0] = vkpic->view.out[0];
197  vkpic->view.aspect_ref[0] = vkpic->view.aspect[0];
198  }
199  }
200 
201  return 0;
202 }
203 
205  FFVulkanDecodePicture *vkpic, int is_current,
206  enum FFVkShaderRepFormat rep_fmt, int alloc_dpb)
207 {
208  int err;
211 
212  vkpic->slices_size = 0;
213 
214  if (vkpic->view.ref[0])
215  return 0;
216 
217  init_frame(dec, vkpic);
218 
219  for (int i = 0; i < av_pix_fmt_count_planes(frames->sw_format); i++) {
220  if (alloc_dpb) {
221  vkpic->dpb_frame = vk_get_dpb_pool(ctx);
222  if (!vkpic->dpb_frame)
223  return AVERROR(ENOMEM);
224 
225  err = ff_vk_create_imageview(&ctx->s,
226  &vkpic->view.ref[i], &vkpic->view.aspect_ref[i],
227  vkpic->dpb_frame, i, rep_fmt);
228  if (err < 0)
229  return err;
230 
231  vkpic->view.dst[i] = vkpic->view.ref[i];
232  }
233 
234  if (!alloc_dpb || is_current) {
235  err = ff_vk_create_imageview(&ctx->s,
236  &vkpic->view.out[i], &vkpic->view.aspect[i],
237  pic, i, rep_fmt);
238  if (err < 0)
239  return err;
240 
241  if (!alloc_dpb) {
242  vkpic->view.ref[i] = vkpic->view.out[i];
243  vkpic->view.aspect_ref[i] = vkpic->view.aspect[i];
244  }
245  }
246  }
247 
248  return 0;
249 }
250 
252  const uint8_t *data, size_t size, int add_startcode,
253  uint32_t *nb_slices, const uint32_t **offsets)
254 {
257 
258  static const uint8_t startcode_prefix[3] = { 0x0, 0x0, 0x1 };
259  const size_t startcode_len = add_startcode ? sizeof(startcode_prefix) : 0;
260  const int nb = nb_slices ? *nb_slices : 0;
261  uint8_t *slices;
262  uint32_t *slice_off;
263  FFVkBuffer *vkbuf;
264 
265  size_t new_size = vp->slices_size + startcode_len + size +
266  ctx->caps.minBitstreamBufferSizeAlignment;
267  new_size = FFALIGN(new_size, ctx->caps.minBitstreamBufferSizeAlignment);
268 
269  if (offsets) {
270  slice_off = av_fast_realloc(dec->slice_off, &dec->slice_off_max,
271  (nb + 1)*sizeof(slice_off));
272  if (!slice_off)
273  return AVERROR(ENOMEM);
274 
275  *offsets = dec->slice_off = slice_off;
276 
277  slice_off[nb] = vp->slices_size;
278  }
279 
280  vkbuf = vp->slices_buf ? (FFVkBuffer *)vp->slices_buf->data : NULL;
281  if (!vkbuf || vkbuf->size < new_size) {
282  int err;
283  AVBufferRef *new_ref;
284  FFVkBuffer *new_buf;
285 
286  /* No point in requesting anything smaller. */
287  size_t buf_size = FFMAX(new_size, 1024*1024);
288 
289  /* Align buffer to nearest power of two. Makes fragmentation management
290  * easier, and gives us ample headroom. */
291  buf_size = 2 << av_log2(buf_size);
292 
293  err = ff_vk_get_pooled_buffer(&ctx->s, &ctx->buf_pool, &new_ref,
294  DECODER_IS_SDR(avctx->codec_id) ?
295  (VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
296  VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) :
297  VK_BUFFER_USAGE_VIDEO_DECODE_SRC_BIT_KHR,
298  ctx->s.hwfc->create_pnext, buf_size,
299  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
300  (DECODER_IS_SDR(avctx->codec_id) ?
301  VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT : 0x0));
302  if (err < 0)
303  return err;
304 
305  new_buf = (FFVkBuffer *)new_ref->data;
306 
307  /* Copy data from the old buffer */
308  if (vkbuf) {
309  memcpy(new_buf->mapped_mem, vkbuf->mapped_mem, vp->slices_size);
311  }
312 
313  vp->slices_buf = new_ref;
314  vkbuf = new_buf;
315  }
316  slices = vkbuf->mapped_mem;
317 
318  /* Startcode */
319  memcpy(slices + vp->slices_size, startcode_prefix, startcode_len);
320 
321  /* Slice data */
322  memcpy(slices + vp->slices_size + startcode_len, data, size);
323 
324  if (nb_slices)
325  *nb_slices = nb + 1;
326 
327  vp->slices_size += startcode_len + size;
328 
329  return 0;
330 }
331 
333 {
336 
337  FFVulkanFunctions *vk = &ctx->s.vkfn;
338  VkVideoBeginCodingInfoKHR decode_start = {
339  .sType = VK_STRUCTURE_TYPE_VIDEO_BEGIN_CODING_INFO_KHR,
340  .videoSession = ctx->common.session,
341  .videoSessionParameters = ctx->empty_session_params,
342  };
343  VkVideoCodingControlInfoKHR decode_ctrl = {
344  .sType = VK_STRUCTURE_TYPE_VIDEO_CODING_CONTROL_INFO_KHR,
345  .flags = VK_VIDEO_CODING_CONTROL_RESET_BIT_KHR,
346  };
347  VkVideoEndCodingInfoKHR decode_end = {
348  .sType = VK_STRUCTURE_TYPE_VIDEO_END_CODING_INFO_KHR,
349  };
350 
351  VkCommandBuffer cmd_buf;
352  FFVkExecContext *exec;
353 
354  /* Non-video queues do not need to be reset */
355  if (!(get_codecdesc(avctx->codec_id)->decode_op))
356  return;
357 
358  exec = ff_vk_exec_get(&ctx->s, &ctx->exec_pool);
359  ff_vk_exec_start(&ctx->s, exec);
360  cmd_buf = exec->buf;
361 
362  vk->CmdBeginVideoCodingKHR(cmd_buf, &decode_start);
363  vk->CmdControlVideoCodingKHR(cmd_buf, &decode_ctrl);
364  vk->CmdEndVideoCodingKHR(cmd_buf, &decode_end);
365  ff_vk_exec_submit(&ctx->s, exec);
366 }
367 
369  AVFrame *pic, FFVulkanDecodePicture *vp,
370  AVFrame *rpic[], FFVulkanDecodePicture *rvkp[])
371 {
372  int err;
373  VkResult ret;
374  VkCommandBuffer cmd_buf;
375  FFVkBuffer *sd_buf;
376 
379  FFVulkanFunctions *vk = &ctx->s.vkfn;
380 
381  /* Output */
382  AVVkFrame *vkf = (AVVkFrame *)pic->buf[0]->data;
383 
384  /* Quirks */
385  const int layered_dpb = ctx->common.layered_dpb;
386 
387  VkVideoBeginCodingInfoKHR decode_start = {
388  .sType = VK_STRUCTURE_TYPE_VIDEO_BEGIN_CODING_INFO_KHR,
389  .videoSession = ctx->common.session,
390  .videoSessionParameters = dec->session_params ?
391  *((VkVideoSessionParametersKHR *)dec->session_params->data) :
392  VK_NULL_HANDLE,
393  .referenceSlotCount = vp->decode_info.referenceSlotCount,
394  .pReferenceSlots = vp->decode_info.pReferenceSlots,
395  };
396  VkVideoEndCodingInfoKHR decode_end = {
397  .sType = VK_STRUCTURE_TYPE_VIDEO_END_CODING_INFO_KHR,
398  };
399 
400  VkImageMemoryBarrier2 img_bar[37];
401  int nb_img_bar = 0;
402  size_t data_size = FFALIGN(vp->slices_size,
403  ctx->caps.minBitstreamBufferSizeAlignment);
404 
405  FFVkExecContext *exec = ff_vk_exec_get(&ctx->s, &ctx->exec_pool);
406 
407  /* The current decoding reference has to be bound as an inactive reference */
408  VkVideoReferenceSlotInfoKHR *cur_vk_ref;
409  cur_vk_ref = (void *)&decode_start.pReferenceSlots[decode_start.referenceSlotCount];
410  cur_vk_ref[0] = vp->ref_slot;
411  cur_vk_ref[0].slotIndex = -1;
412  decode_start.referenceSlotCount++;
413 
414  sd_buf = (FFVkBuffer *)vp->slices_buf->data;
415 
416  /* Flush if needed */
417  if (!(sd_buf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) {
418  VkMappedMemoryRange flush_buf = {
419  .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
420  .memory = sd_buf->mem,
421  .offset = 0,
422  .size = FFALIGN(vp->slices_size,
423  ctx->s.props.properties.limits.nonCoherentAtomSize),
424  };
425 
426  ret = vk->FlushMappedMemoryRanges(ctx->s.hwctx->act_dev, 1, &flush_buf);
427  if (ret != VK_SUCCESS) {
428  av_log(avctx, AV_LOG_ERROR, "Failed to flush memory: %s\n",
429  ff_vk_ret2str(ret));
430  return AVERROR_EXTERNAL;
431  }
432  }
433 
434  vp->decode_info.srcBuffer = sd_buf->buf;
435  vp->decode_info.srcBufferOffset = 0;
436  vp->decode_info.srcBufferRange = data_size;
437 
438  /* Start command buffer recording */
439  err = ff_vk_exec_start(&ctx->s, exec);
440  if (err < 0)
441  return err;
442  cmd_buf = exec->buf;
443 
444  /* Slices */
445  err = ff_vk_exec_add_dep_buf(&ctx->s, exec, &vp->slices_buf, 1, 0);
446  if (err < 0)
447  return err;
448  vp->slices_buf = NULL; /* Owned by the exec buffer from now on */
449 
450  /* Parameters */
451  err = ff_vk_exec_add_dep_buf(&ctx->s, exec, &dec->session_params, 1, 1);
452  if (err < 0)
453  return err;
454 
455  err = ff_vk_exec_add_dep_frame(&ctx->s, exec, pic,
456  VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
457  VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR);
458  if (err < 0)
459  return err;
460 
461  err = ff_vk_exec_mirror_sem_value(&ctx->s, exec, &vp->sem, &vp->sem_value,
462  pic);
463  if (err < 0)
464  return err;
465 
466  /* Output image - change layout, as it comes from a pool */
467  img_bar[nb_img_bar] = (VkImageMemoryBarrier2) {
468  .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
469  .pNext = NULL,
470  .srcStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
471  .dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
472  .srcAccessMask = VK_ACCESS_2_NONE,
473  .dstAccessMask = VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR,
474  .oldLayout = vkf->layout[0],
475  .newLayout = (layered_dpb || vp->dpb_frame) ?
476  VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR :
477  VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR, /* Spec, 07252 utter madness */
478  .srcQueueFamilyIndex = vkf->queue_family[0],
479  .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
480  .image = vkf->img[0],
481  .subresourceRange = (VkImageSubresourceRange) {
482  .aspectMask = vp->view.aspect[0],
483  .layerCount = 1,
484  .levelCount = 1,
485  },
486  };
487  ff_vk_exec_update_frame(&ctx->s, exec, pic,
488  &img_bar[nb_img_bar], &nb_img_bar);
489 
490  /* Reference for the current image, if existing and not layered */
491  if (vp->dpb_frame) {
492  err = ff_vk_exec_add_dep_frame(&ctx->s, exec, vp->dpb_frame,
493  VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
494  VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR);
495  if (err < 0)
496  return err;
497  }
498 
499  if (!layered_dpb) {
500  /* All references (apart from the current) for non-layered refs */
501 
502  for (int i = 0; i < vp->decode_info.referenceSlotCount; i++) {
503  AVFrame *ref_frame = rpic[i];
504  FFVulkanDecodePicture *rvp = rvkp[i];
505  AVFrame *ref = rvp->dpb_frame ? rvp->dpb_frame : ref_frame;
506 
507  err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ref,
508  VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
509  VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR);
510  if (err < 0)
511  return err;
512 
513  if (err == 0) {
514  err = ff_vk_exec_mirror_sem_value(&ctx->s, exec,
515  &rvp->sem, &rvp->sem_value,
516  ref);
517  if (err < 0)
518  return err;
519  }
520 
521  if (!rvp->dpb_frame) {
522  AVVkFrame *rvkf = (AVVkFrame *)ref->data[0];
523 
524  img_bar[nb_img_bar] = (VkImageMemoryBarrier2) {
525  .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
526  .pNext = NULL,
527  .srcStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
528  .dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
529  .srcAccessMask = VK_ACCESS_2_NONE,
530  .dstAccessMask = VK_ACCESS_2_VIDEO_DECODE_READ_BIT_KHR |
531  VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR,
532  .oldLayout = rvkf->layout[0],
533  .newLayout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR,
534  .srcQueueFamilyIndex = rvkf->queue_family[0],
535  .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
536  .image = rvkf->img[0],
537  .subresourceRange = (VkImageSubresourceRange) {
538  .aspectMask = rvp->view.aspect_ref[0],
539  .layerCount = 1,
540  .levelCount = 1,
541  },
542  };
543  ff_vk_exec_update_frame(&ctx->s, exec, ref,
544  &img_bar[nb_img_bar], &nb_img_bar);
545  }
546  }
547  } else if (vp->decode_info.referenceSlotCount ||
548  vp->view.out[0] != vp->view.ref[0]) {
549  /* Single barrier for a single layered ref */
550  err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ctx->common.layered_frame,
551  VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
552  VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR);
553  if (err < 0)
554  return err;
555  }
556 
557  /* Change image layout */
558  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
559  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
560  .dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT,
561  .pImageMemoryBarriers = img_bar,
562  .imageMemoryBarrierCount = nb_img_bar,
563  });
564 
565  /* Start, use parameters, decode and end decoding */
566  vk->CmdBeginVideoCodingKHR(cmd_buf, &decode_start);
567  vk->CmdDecodeVideoKHR(cmd_buf, &vp->decode_info);
568  vk->CmdEndVideoCodingKHR(cmd_buf, &decode_end);
569 
570  /* End recording and submit for execution */
571  return ff_vk_exec_submit(&ctx->s, exec);
572 }
573 
575 {
576  AVVulkanDeviceContext *hwctx = dev_ctx->hwctx;
577 
578  VkSemaphoreWaitInfo sem_wait = (VkSemaphoreWaitInfo) {
579  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
580  .pSemaphores = &vp->sem,
581  .pValues = &vp->sem_value,
582  .semaphoreCount = 1,
583  };
584 
585  /* We do not have to lock the frame here because we're not interested
586  * in the actual current semaphore value, but only that it's later than
587  * the time we submitted the image for decoding. */
588  if (vp->sem)
589  vp->wait_semaphores(hwctx->act_dev, &sem_wait, UINT64_MAX);
590 
591  /* Free slices data */
593 
594  /* Destroy image view (out) */
595  for (int i = 0; i < AV_NUM_DATA_POINTERS; i++) {
596  if (vp->view.out[i] && vp->view.out[i] != vp->view.dst[i])
597  vp->destroy_image_view(hwctx->act_dev, vp->view.out[i], hwctx->alloc);
598 
599  /* Destroy image view (ref, unlayered) */
600  if (vp->view.dst[i])
601  vp->destroy_image_view(hwctx->act_dev, vp->view.dst[i], hwctx->alloc);
602  }
603 
604  av_frame_free(&vp->dpb_frame);
605 }
606 
607 static void free_common(AVRefStructOpaque unused, void *obj)
608 {
609  FFVulkanDecodeShared *ctx = obj;
610  FFVulkanContext *s = &ctx->s;
611  FFVulkanFunctions *vk = &ctx->s.vkfn;
612 
613  /* Wait on and free execution pool */
614  ff_vk_exec_pool_free(&ctx->s, &ctx->exec_pool);
615 
616  /* This also frees all references from this pool */
617  av_frame_free(&ctx->common.layered_frame);
618 
619  /* Destroy parameters */
620  if (ctx->empty_session_params)
621  vk->DestroyVideoSessionParametersKHR(s->hwctx->act_dev,
622  ctx->empty_session_params,
623  s->hwctx->alloc);
624 
625  av_buffer_pool_uninit(&ctx->buf_pool);
626 
627  ff_vk_video_common_uninit(s, &ctx->common);
628 
629  if (ctx->sd_ctx_free)
630  ctx->sd_ctx_free(ctx);
631 
632  ff_vk_uninit(s);
633 }
634 
635 static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_ref)
636 {
637  int err;
639  const FFVulkanDecodeDescriptor *vk_desc = get_codecdesc(avctx->codec_id);
641  AVHWDeviceContext *device = (AVHWDeviceContext *)frames->device_ref->data;
642  AVVulkanDeviceContext *hwctx = device->hwctx;
644 
645  if (dec->shared_ctx)
646  return 0;
647 
648  dec->shared_ctx = av_refstruct_alloc_ext(sizeof(*ctx), 0, NULL,
649  free_common);
650  if (!dec->shared_ctx)
651  return AVERROR(ENOMEM);
652 
653  ctx = dec->shared_ctx;
654 
655  ctx->s.extensions = ff_vk_extensions_to_mask(hwctx->enabled_dev_extensions,
657 
658  if (vk_desc->queue_flags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) {
659  if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) {
660  av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",
661  VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME);
663  return AVERROR(ENOSYS);
664  }
665  }
666 
667  err = ff_vk_load_functions(device, &ctx->s.vkfn, ctx->s.extensions, 1, 1);
668  if (err < 0) {
670  return err;
671  }
672 
673  return 0;
674 }
675 
676 static VkResult vulkan_setup_profile(AVCodecContext *avctx,
678  AVVulkanDeviceContext *hwctx,
679  FFVulkanFunctions *vk,
680  const FFVulkanDecodeDescriptor *vk_desc,
681  VkVideoDecodeH264CapabilitiesKHR *h264_caps,
682  VkVideoDecodeH265CapabilitiesKHR *h265_caps,
683  VkVideoDecodeAV1CapabilitiesKHR *av1_caps,
684  VkVideoCapabilitiesKHR *caps,
685  VkVideoDecodeCapabilitiesKHR *dec_caps,
686  int cur_profile)
687 {
688  VkVideoDecodeUsageInfoKHR *usage = &prof->usage;
689  VkVideoProfileInfoKHR *profile = &prof->profile;
690  VkVideoProfileListInfoKHR *profile_list = &prof->profile_list;
691 
692  VkVideoDecodeH264ProfileInfoKHR *h264_profile = &prof->h264_profile;
693  VkVideoDecodeH265ProfileInfoKHR *h265_profile = &prof->h265_profile;
694  VkVideoDecodeAV1ProfileInfoKHR *av1_profile = &prof->av1_profile;
695 
697  if (!desc)
698  return AVERROR(EINVAL);
699 
700  if (avctx->codec_id == AV_CODEC_ID_H264) {
701  dec_caps->pNext = h264_caps;
702  usage->pNext = h264_profile;
703  h264_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR;
704 
705  /* Vulkan transmits all the constrant_set flags, rather than wanting them
706  * merged in the profile IDC */
707  h264_profile->stdProfileIdc = cur_profile & ~(AV_PROFILE_H264_CONSTRAINED |
709 
710  h264_profile->pictureLayout = avctx->field_order == AV_FIELD_UNKNOWN ||
712  VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_PROGRESSIVE_KHR :
713  VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_INTERLACED_INTERLEAVED_LINES_BIT_KHR;
714  } else if (avctx->codec_id == AV_CODEC_ID_H265) {
715  dec_caps->pNext = h265_caps;
716  usage->pNext = h265_profile;
717  h265_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR;
718  h265_profile->stdProfileIdc = cur_profile;
719  } else if (avctx->codec_id == AV_CODEC_ID_AV1) {
720  dec_caps->pNext = av1_caps;
721  usage->pNext = av1_profile;
722  av1_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_KHR;
723  av1_profile->stdProfile = cur_profile;
724  av1_profile->filmGrainSupport = !(avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN);
725  }
726 
727  usage->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_USAGE_INFO_KHR;
728  usage->videoUsageHints = VK_VIDEO_DECODE_USAGE_DEFAULT_KHR;
729 
730  profile->sType = VK_STRUCTURE_TYPE_VIDEO_PROFILE_INFO_KHR;
731  profile->pNext = usage;
732  profile->videoCodecOperation = vk_desc->decode_op;
733  profile->chromaSubsampling = ff_vk_subsampling_from_av_desc(desc);
734  profile->lumaBitDepth = ff_vk_depth_from_av_depth(desc->comp[0].depth);
735  profile->chromaBitDepth = profile->lumaBitDepth;
736 
737  profile_list->sType = VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR;
738  profile_list->profileCount = 1;
739  profile_list->pProfiles = profile;
740 
741  /* Get the capabilities of the decoder for the given profile */
742  caps->sType = VK_STRUCTURE_TYPE_VIDEO_CAPABILITIES_KHR;
743  caps->pNext = dec_caps;
744  dec_caps->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_CAPABILITIES_KHR;
745  /* dec_caps->pNext already filled in */
746 
747  return vk->GetPhysicalDeviceVideoCapabilitiesKHR(hwctx->phys_dev, profile,
748  caps);
749 }
750 
751 static int vulkan_decode_get_profile(AVCodecContext *avctx, AVBufferRef *frames_ref,
752  enum AVPixelFormat *pix_fmt, VkFormat *vk_fmt,
754  int *dpb_dedicate)
755 {
756  VkResult ret;
757  int max_level, base_profile, cur_profile;
758  const FFVulkanDecodeDescriptor *vk_desc = get_codecdesc(avctx->codec_id);
760  AVHWDeviceContext *device = (AVHWDeviceContext *)frames->device_ref->data;
761  AVVulkanDeviceContext *hwctx = device->hwctx;
762  enum AVPixelFormat source_format;
763  enum AVPixelFormat best_format;
764  VkFormat best_vkfmt;
765 
768  FFVulkanFunctions *vk = &ctx->s.vkfn;
769 
770  VkVideoCapabilitiesKHR *caps = &ctx->caps;
771  VkVideoDecodeCapabilitiesKHR *dec_caps = &ctx->dec_caps;
772 
773  VkVideoDecodeH264CapabilitiesKHR h264_caps = {
774  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_CAPABILITIES_KHR,
775  };
776  VkVideoDecodeH265CapabilitiesKHR h265_caps = {
777  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_CAPABILITIES_KHR,
778  };
779  VkVideoDecodeAV1CapabilitiesKHR av1_caps = {
780  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_CAPABILITIES_KHR,
781  };
782 
783  VkPhysicalDeviceVideoFormatInfoKHR fmt_info = {
784  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_FORMAT_INFO_KHR,
785  .pNext = &prof->profile_list,
786  };
787  VkVideoFormatPropertiesKHR *ret_info;
788  uint32_t nb_out_fmts = 0;
789 
790  if (!(vk_desc->decode_extension & ctx->s.extensions)) {
791  av_log(avctx, AV_LOG_ERROR, "Device does not support decoding %s!\n",
792  avcodec_get_name(avctx->codec_id));
793  return AVERROR(ENOSYS);
794  }
795 
796  cur_profile = avctx->profile;
799  avctx->codec_id == AV_CODEC_ID_AV1 ? STD_VIDEO_AV1_PROFILE_MAIN :
800  0;
801 
802  ret = vulkan_setup_profile(avctx, prof, hwctx, vk, vk_desc,
803  &h264_caps,
804  &h265_caps,
805  &av1_caps,
806  caps,
807  dec_caps,
808  cur_profile);
809  if (ret == VK_ERROR_VIDEO_PROFILE_OPERATION_NOT_SUPPORTED_KHR &&
811  avctx->profile != base_profile) {
812  av_log(avctx, AV_LOG_VERBOSE, "%s profile %s not supported, attempting "
813  "again with profile %s\n",
814  avcodec_get_name(avctx->codec_id),
815  avcodec_profile_name(avctx->codec_id, cur_profile),
816  avcodec_profile_name(avctx->codec_id, base_profile));
817  cur_profile = base_profile;
818  ret = vulkan_setup_profile(avctx, prof, hwctx, vk, vk_desc,
819  &h264_caps,
820  &h265_caps,
821  &av1_caps,
822  caps,
823  dec_caps,
824  cur_profile);
825  }
826 
827  if (ret == VK_ERROR_VIDEO_PROFILE_OPERATION_NOT_SUPPORTED_KHR) {
828  av_log(avctx, AV_LOG_VERBOSE, "Unable to initialize video session: "
829  "%s profile \"%s\" not supported!\n",
830  avcodec_get_name(avctx->codec_id),
831  avcodec_profile_name(avctx->codec_id, cur_profile));
832  return AVERROR(EINVAL);
833  } else if (ret == VK_ERROR_VIDEO_PROFILE_FORMAT_NOT_SUPPORTED_KHR) {
834  av_log(avctx, AV_LOG_VERBOSE, "Unable to initialize video session: "
835  "format (%s) not supported!\n",
837  return AVERROR(EINVAL);
838  } else if (ret == VK_ERROR_FEATURE_NOT_PRESENT ||
839  ret == VK_ERROR_FORMAT_NOT_SUPPORTED) {
840  return AVERROR(EINVAL);
841  } else if (ret != VK_SUCCESS) {
842  return AVERROR_EXTERNAL;
843  }
844 
845  max_level = avctx->codec_id == AV_CODEC_ID_H264 ? ff_vk_h264_level_to_av(h264_caps.maxLevelIdc) :
846  avctx->codec_id == AV_CODEC_ID_H265 ? ff_vk_h265_level_to_av(h265_caps.maxLevelIdc) :
847  avctx->codec_id == AV_CODEC_ID_AV1 ? av1_caps.maxLevel :
848  0;
849 
850  av_log(avctx, AV_LOG_VERBOSE, "Decoder capabilities for %s profile \"%s\":\n",
851  avcodec_get_name(avctx->codec_id),
852  avcodec_profile_name(avctx->codec_id, cur_profile));
853  av_log(avctx, AV_LOG_VERBOSE, " Maximum level: %i (stream %i)\n",
854  max_level, avctx->level);
855  av_log(avctx, AV_LOG_VERBOSE, " Width: from %i to %i\n",
856  caps->minCodedExtent.width, caps->maxCodedExtent.width);
857  av_log(avctx, AV_LOG_VERBOSE, " Height: from %i to %i\n",
858  caps->minCodedExtent.height, caps->maxCodedExtent.height);
859  av_log(avctx, AV_LOG_VERBOSE, " Width alignment: %i\n",
860  caps->pictureAccessGranularity.width);
861  av_log(avctx, AV_LOG_VERBOSE, " Height alignment: %i\n",
862  caps->pictureAccessGranularity.height);
863  av_log(avctx, AV_LOG_VERBOSE, " Bitstream offset alignment: %"PRIu64"\n",
864  caps->minBitstreamBufferOffsetAlignment);
865  av_log(avctx, AV_LOG_VERBOSE, " Bitstream size alignment: %"PRIu64"\n",
866  caps->minBitstreamBufferSizeAlignment);
867  av_log(avctx, AV_LOG_VERBOSE, " Maximum references: %u\n",
868  caps->maxDpbSlots);
869  av_log(avctx, AV_LOG_VERBOSE, " Maximum active references: %u\n",
870  caps->maxActiveReferencePictures);
871  av_log(avctx, AV_LOG_VERBOSE, " Codec header name: '%s' (driver), '%s' (compiled)\n",
872  caps->stdHeaderVersion.extensionName,
873  vk_desc->ext_props.extensionName);
874  av_log(avctx, AV_LOG_VERBOSE, " Codec header version: %i.%i.%i (driver), %i.%i.%i (compiled)\n",
875  CODEC_VER(caps->stdHeaderVersion.specVersion),
876  CODEC_VER(vk_desc->ext_props.specVersion));
877  av_log(avctx, AV_LOG_VERBOSE, " Decode modes:%s%s%s\n",
878  dec_caps->flags ? "" :
879  " invalid",
880  dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR ?
881  " reuse_dst_dpb" : "",
882  dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR ?
883  " dedicated_dpb" : "");
884  av_log(avctx, AV_LOG_VERBOSE, " Capability flags:%s%s%s\n",
885  caps->flags ? "" :
886  " none",
887  caps->flags & VK_VIDEO_CAPABILITY_PROTECTED_CONTENT_BIT_KHR ?
888  " protected" : "",
889  caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR ?
890  " separate_references" : "");
891 
892  /* Check if decoding is possible with the given parameters */
893  if (avctx->coded_width < caps->minCodedExtent.width ||
894  avctx->coded_height < caps->minCodedExtent.height ||
895  avctx->coded_width > caps->maxCodedExtent.width ||
896  avctx->coded_height > caps->maxCodedExtent.height)
897  return AVERROR(EINVAL);
898 
900  avctx->level > max_level)
901  return AVERROR(EINVAL);
902 
903  /* Some basic sanity checking */
904  if (!(dec_caps->flags & (VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR |
905  VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR))) {
906  av_log(avctx, AV_LOG_ERROR, "Buggy driver signals invalid decoding mode: neither "
907  "VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR nor "
908  "VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR are set!\n");
909  return AVERROR_EXTERNAL;
910  } else if ((dec_caps->flags & (VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR |
911  VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR) ==
912  VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR) &&
913  !(caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR)) {
914  av_log(avctx, AV_LOG_ERROR, "Cannot initialize Vulkan decoding session, buggy driver: "
915  "VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR set "
916  "but VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR is unset!\n");
917  return AVERROR_EXTERNAL;
918  }
919 
920  dec->dedicated_dpb = !(dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR);
921  ctx->common.layered_dpb = !dec->dedicated_dpb ? 0 :
922  !(caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR);
923 
924  if (dec->dedicated_dpb) {
925  fmt_info.imageUsage = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;
926  } else {
927  fmt_info.imageUsage = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR |
928  VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR |
929  VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
930  VK_IMAGE_USAGE_SAMPLED_BIT;
931 
932  if (ctx->s.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE |
934  fmt_info.imageUsage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
935  }
936 
937  /* Get the format of the images necessary */
938  ret = vk->GetPhysicalDeviceVideoFormatPropertiesKHR(hwctx->phys_dev,
939  &fmt_info,
940  &nb_out_fmts, NULL);
941  if (ret == VK_ERROR_FORMAT_NOT_SUPPORTED ||
942  (!nb_out_fmts && ret == VK_SUCCESS)) {
943  return AVERROR(EINVAL);
944  } else if (ret != VK_SUCCESS) {
945  av_log(avctx, AV_LOG_ERROR, "Unable to get Vulkan format properties: %s!\n",
946  ff_vk_ret2str(ret));
947  return AVERROR_EXTERNAL;
948  }
949 
950  ret_info = av_mallocz(sizeof(*ret_info)*nb_out_fmts);
951  if (!ret_info)
952  return AVERROR(ENOMEM);
953 
954  for (int i = 0; i < nb_out_fmts; i++)
955  ret_info[i].sType = VK_STRUCTURE_TYPE_VIDEO_FORMAT_PROPERTIES_KHR;
956 
957  ret = vk->GetPhysicalDeviceVideoFormatPropertiesKHR(hwctx->phys_dev,
958  &fmt_info,
959  &nb_out_fmts, ret_info);
960  if (ret == VK_ERROR_FORMAT_NOT_SUPPORTED ||
961  (!nb_out_fmts && ret == VK_SUCCESS)) {
962  av_free(ret_info);
963  return AVERROR(EINVAL);
964  } else if (ret != VK_SUCCESS) {
965  av_log(avctx, AV_LOG_ERROR, "Unable to get Vulkan format properties: %s!\n",
966  ff_vk_ret2str(ret));
967  av_free(ret_info);
968  return AVERROR_EXTERNAL;
969  }
970 
971  /* Find a format to use */
972  *pix_fmt = best_format = AV_PIX_FMT_NONE;
973  *vk_fmt = best_vkfmt = VK_FORMAT_UNDEFINED;
974  source_format = avctx->sw_pix_fmt;
975 
976  av_log(avctx, AV_LOG_DEBUG, "Choosing best pixel format for decoding from %i:\n", nb_out_fmts);
977  for (int i = 0; i < nb_out_fmts; i++) {
979  if (tmp == AV_PIX_FMT_NONE) {
980  av_log(avctx, AV_LOG_WARNING, "Invalid/unknown Vulkan format %i!\n", ret_info[i].format);
981  continue;
982  }
983 
984  best_format = av_find_best_pix_fmt_of_2(tmp, best_format, source_format, 0, NULL);
985  if (tmp == best_format)
986  best_vkfmt = ret_info[i].format;
987 
988  av_log(avctx, AV_LOG_DEBUG, " %s%s (Vulkan ID: %i)\n",
989  av_get_pix_fmt_name(tmp), tmp == best_format ? "*" : "",
990  ret_info[i].format);
991  }
992 
993  av_free(ret_info);
994 
995  if (best_format == AV_PIX_FMT_NONE) {
996  av_log(avctx, AV_LOG_ERROR, "No valid/compatible pixel format found for decoding!\n");
997  return AVERROR(EINVAL);
998  } else {
999  av_log(avctx, AV_LOG_VERBOSE, "Chosen frame pixfmt: %s (Vulkan ID: %i)\n",
1000  av_get_pix_fmt_name(best_format), best_vkfmt);
1001  }
1002 
1003  *pix_fmt = best_format;
1004  *vk_fmt = best_vkfmt;
1005 
1006  *dpb_dedicate = dec->dedicated_dpb;
1007 
1008  return 0;
1009 }
1010 
1012 {
1013  av_free(hwfc->user_opaque);
1014 }
1015 
1016 int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx)
1017 {
1018  VkFormat vkfmt = VK_FORMAT_UNDEFINED;
1019  int err, dedicated_dpb;
1020  AVHWFramesContext *frames_ctx = (AVHWFramesContext*)hw_frames_ctx->data;
1021  AVVulkanFramesContext *hwfc = frames_ctx->hwctx;
1024 
1025  err = vulkan_decode_bootstrap(avctx, hw_frames_ctx);
1026  if (err < 0)
1027  return err;
1028 
1029  frames_ctx->sw_format = avctx->sw_pix_fmt;
1030 
1031  if (!DECODER_IS_SDR(avctx->codec_id)) {
1032  prof = av_mallocz(sizeof(FFVulkanDecodeProfileData));
1033  if (!prof)
1034  return AVERROR(ENOMEM);
1035 
1036  err = vulkan_decode_get_profile(avctx, hw_frames_ctx,
1037  &frames_ctx->sw_format, &vkfmt,
1038  prof, &dedicated_dpb);
1039  if (err < 0) {
1040  av_free(prof);
1041  return err;
1042  }
1043 
1044  frames_ctx->user_opaque = prof;
1045  frames_ctx->free = free_profile_data;
1046 
1047  hwfc->create_pnext = &prof->profile_list;
1048  } else {
1049  switch (frames_ctx->sw_format) {
1050  case AV_PIX_FMT_GBRAP16:
1051  /* This should be more efficient for downloading and using */
1052  frames_ctx->sw_format = AV_PIX_FMT_RGBA64;
1053  break;
1054  case AV_PIX_FMT_GBRP10:
1055  /* This saves memory bandwidth when downloading */
1056  frames_ctx->sw_format = AV_PIX_FMT_X2BGR10;
1057  break;
1058  case AV_PIX_FMT_BGR0:
1059  /* mpv has issues with bgr0 mapping, so just remap it */
1060  frames_ctx->sw_format = AV_PIX_FMT_RGB0;
1061  break;
1062  default:
1063  break;
1064  }
1065  }
1066 
1067  frames_ctx->width = avctx->coded_width;
1068  frames_ctx->height = avctx->coded_height;
1069  frames_ctx->format = AV_PIX_FMT_VULKAN;
1070 
1071  hwfc->format[0] = vkfmt;
1072  hwfc->tiling = VK_IMAGE_TILING_OPTIMAL;
1073  hwfc->usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
1074  VK_IMAGE_USAGE_SAMPLED_BIT;
1075 
1076  if (prof) {
1078 
1079  hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR;
1080  if (!dec->dedicated_dpb)
1081  hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;
1082 
1083  ctx = dec->shared_ctx;
1084  if (ctx->s.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE |
1086  hwfc->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
1087  } else if (DECODER_IS_SDR(avctx->codec_id)) {
1088  hwfc->usage |= VK_IMAGE_USAGE_STORAGE_BIT;
1089  }
1090 
1091  return err;
1092 }
1093 
1094 static void vk_decode_free_params(void *opaque, uint8_t *data)
1095 {
1096  FFVulkanDecodeShared *ctx = opaque;
1097  FFVulkanFunctions *vk = &ctx->s.vkfn;
1098  VkVideoSessionParametersKHR *par = (VkVideoSessionParametersKHR *)data;
1099  vk->DestroyVideoSessionParametersKHR(ctx->s.hwctx->act_dev, *par,
1100  ctx->s.hwctx->alloc);
1101  av_free(par);
1102 }
1103 
1105  const VkVideoSessionParametersCreateInfoKHR *session_params_create)
1106 {
1107  VkVideoSessionParametersKHR *par = av_malloc(sizeof(*par));
1108  const FFVulkanFunctions *vk = &ctx->s.vkfn;
1109  VkResult ret;
1110 
1111  if (!par)
1112  return AVERROR(ENOMEM);
1113 
1114  /* Create session parameters */
1115  ret = vk->CreateVideoSessionParametersKHR(ctx->s.hwctx->act_dev, session_params_create,
1116  ctx->s.hwctx->alloc, par);
1117  if (ret != VK_SUCCESS) {
1118  av_log(logctx, AV_LOG_ERROR, "Unable to create Vulkan video session parameters: %s!\n",
1119  ff_vk_ret2str(ret));
1120  av_free(par);
1121  return AVERROR_EXTERNAL;
1122  }
1123  *par_ref = av_buffer_create((uint8_t *)par, sizeof(*par),
1125  if (!*par_ref) {
1126  vk_decode_free_params(ctx, (uint8_t *)par);
1127  return AVERROR(ENOMEM);
1128  }
1129 
1130  return 0;
1131 }
1132 
1134 {
1136 
1137  av_freep(&dec->hevc_headers);
1140  av_freep(&dec->slice_off);
1141  return 0;
1142 }
1143 
1146 {
1147  VkResult ret;
1148  FFVulkanContext *s = &ctx->s;
1149  FFVulkanFunctions *vk = &s->vkfn;
1150 
1151  VkVideoDecodeH264SessionParametersCreateInfoKHR h264_params = {
1152  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_SESSION_PARAMETERS_CREATE_INFO_KHR,
1153  };
1154  VkVideoDecodeH265SessionParametersCreateInfoKHR h265_params = {
1155  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_SESSION_PARAMETERS_CREATE_INFO_KHR,
1156  };
1157  StdVideoAV1SequenceHeader av1_empty_seq = { 0 };
1158  VkVideoDecodeAV1SessionParametersCreateInfoKHR av1_params = {
1159  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_SESSION_PARAMETERS_CREATE_INFO_KHR,
1160  .pStdSequenceHeader = &av1_empty_seq,
1161  };
1162  VkVideoSessionParametersCreateInfoKHR session_params_create = {
1163  .sType = VK_STRUCTURE_TYPE_VIDEO_SESSION_PARAMETERS_CREATE_INFO_KHR,
1164  .pNext = avctx->codec_id == AV_CODEC_ID_H264 ? (void *)&h264_params :
1165  avctx->codec_id == AV_CODEC_ID_HEVC ? (void *)&h265_params :
1166  avctx->codec_id == AV_CODEC_ID_AV1 ? (void *)&av1_params :
1167  NULL,
1168  .videoSession = ctx->common.session,
1169  };
1170 
1171  ret = vk->CreateVideoSessionParametersKHR(s->hwctx->act_dev, &session_params_create,
1172  s->hwctx->alloc, &ctx->empty_session_params);
1173  if (ret != VK_SUCCESS) {
1174  av_log(avctx, AV_LOG_ERROR, "Unable to create empty Vulkan video session parameters: %s!\n",
1175  ff_vk_ret2str(ret));
1176  return AVERROR_EXTERNAL;
1177  }
1178 
1179  return 0;
1180 }
1181 
1183 {
1184  int err;
1187  FFVulkanContext *s;
1188  int async_depth;
1189  const VkVideoProfileInfoKHR *profile;
1190  const FFVulkanDecodeDescriptor *vk_desc;
1191  const VkPhysicalDeviceDriverProperties *driver_props;
1192 
1193  VkVideoSessionCreateInfoKHR session_create = {
1194  .sType = VK_STRUCTURE_TYPE_VIDEO_SESSION_CREATE_INFO_KHR,
1195  };
1196 
1198  if (err < 0)
1199  return err;
1200 
1201  /* Initialize contexts */
1202  ctx = dec->shared_ctx;
1203  s = &ctx->s;
1204 
1205  err = ff_vk_init(s, avctx, NULL, avctx->hw_frames_ctx);
1206  if (err < 0)
1207  return err;
1208 
1209  vk_desc = get_codecdesc(avctx->codec_id);
1210 
1212  if ((vk_desc->queue_flags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) && !profile) {
1213  av_log(avctx, AV_LOG_ERROR, "Video profile missing from frames context!");
1214  return AVERROR(EINVAL);
1215  }
1216 
1217  /* Create queue context */
1218  vk_desc = get_codecdesc(avctx->codec_id);
1219  ctx->qf = ff_vk_qf_find(s, vk_desc->queue_flags, vk_desc->decode_op);
1220  if (!ctx->qf) {
1221  av_log(avctx, AV_LOG_ERROR, "Decoding of %s is not supported by this device\n",
1222  avcodec_get_name(avctx->codec_id));
1223  return err;
1224  }
1225 
1226  session_create.queueFamilyIndex = ctx->qf->idx;
1227  session_create.maxCodedExtent = ctx->caps.maxCodedExtent;
1228  session_create.maxDpbSlots = ctx->caps.maxDpbSlots;
1229  session_create.maxActiveReferencePictures = ctx->caps.maxActiveReferencePictures;
1230  session_create.pictureFormat = s->hwfc->format[0];
1231  session_create.referencePictureFormat = session_create.pictureFormat;
1232  session_create.pStdHeaderVersion = &vk_desc->ext_props;
1233  session_create.pVideoProfile = profile;
1234 #ifdef VK_KHR_video_maintenance2
1235  if (ctx->s.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_2)
1236  session_create.flags = VK_VIDEO_SESSION_CREATE_INLINE_SESSION_PARAMETERS_BIT_KHR;
1237 #endif
1238 
1239  /* Create decode exec context for this specific main thread.
1240  * 2 async contexts per thread was experimentally determined to be optimal
1241  * for a majority of streams. */
1242  async_depth = 2*ctx->qf->num;
1243  /* We don't need more than 2 per thread context */
1244  async_depth = FFMIN(async_depth, 2*avctx->thread_count);
1245  /* Make sure there are enough async contexts for each thread */
1246  async_depth = FFMAX(async_depth, avctx->thread_count);
1247 
1248  err = ff_vk_exec_pool_init(s, ctx->qf, &ctx->exec_pool,
1249  async_depth, 0, 0, 0, profile);
1250  if (err < 0)
1251  goto fail;
1252 
1253  if (!DECODER_IS_SDR(avctx->codec_id)) {
1254  err = ff_vk_video_common_init(avctx, s, &ctx->common, &session_create);
1255  if (err < 0)
1256  goto fail;
1257  }
1258 
1259  /* If doing an out-of-place decoding, create a DPB pool */
1260  if (dec->dedicated_dpb || avctx->codec_id == AV_CODEC_ID_AV1) {
1262  AVVulkanFramesContext *dpb_hwfc;
1263 
1264  ctx->common.dpb_hwfc_ref = av_hwframe_ctx_alloc(s->frames->device_ref);
1265  if (!ctx->common.dpb_hwfc_ref) {
1266  err = AVERROR(ENOMEM);
1267  goto fail;
1268  }
1269 
1270  dpb_frames = (AVHWFramesContext *)ctx->common.dpb_hwfc_ref->data;
1271  dpb_frames->format = s->frames->format;
1272  dpb_frames->sw_format = s->frames->sw_format;
1273  dpb_frames->width = avctx->coded_width;
1274  dpb_frames->height = avctx->coded_height;
1275 
1276  dpb_hwfc = dpb_frames->hwctx;
1277  dpb_hwfc->create_pnext = (void *)ff_vk_find_struct(ctx->s.hwfc->create_pnext,
1278  VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
1279  dpb_hwfc->format[0] = s->hwfc->format[0];
1280  dpb_hwfc->tiling = VK_IMAGE_TILING_OPTIMAL;
1281  dpb_hwfc->usage = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR |
1282  VK_IMAGE_USAGE_SAMPLED_BIT; /* Shuts validator up. */
1283 
1284  if (ctx->common.layered_dpb)
1285  dpb_hwfc->nb_layers = ctx->caps.maxDpbSlots;
1286 
1287  err = av_hwframe_ctx_init(ctx->common.dpb_hwfc_ref);
1288  if (err < 0)
1289  goto fail;
1290 
1291  if (ctx->common.layered_dpb) {
1292  ctx->common.layered_frame = vk_get_dpb_pool(ctx);
1293  if (!ctx->common.layered_frame) {
1294  err = AVERROR(ENOMEM);
1295  goto fail;
1296  }
1297 
1298  err = ff_vk_create_view(&ctx->s, &ctx->common,
1299  &ctx->common.layered_view,
1300  &ctx->common.layered_aspect,
1301  (AVVkFrame *)ctx->common.layered_frame->data[0],
1302  s->hwfc->format[0], 1);
1303  if (err < 0)
1304  goto fail;
1305  }
1306  }
1307 
1308  if (!DECODER_IS_SDR(avctx->codec_id)) {
1309  if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_2)) {
1310  err = create_empty_session_parameters(avctx, ctx);
1311  if (err < 0)
1312  return err;
1313  }
1314  } else {
1315  /* For SDR decoders, this alignment value will be 0. Since this will make
1316  * add_slice() malfunction, set it to a sane default value. */
1317  ctx->caps.minBitstreamBufferSizeAlignment = AV_INPUT_BUFFER_PADDING_SIZE;
1318  }
1319 
1320  driver_props = &dec->shared_ctx->s.driver_props;
1321  if (driver_props->driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY &&
1322  driver_props->conformanceVersion.major == 1 &&
1323  driver_props->conformanceVersion.minor == 3 &&
1324  driver_props->conformanceVersion.subminor == 8 &&
1325  driver_props->conformanceVersion.patch < 3)
1326  dec->quirk_av1_offset = 1;
1327 
1328  ff_vk_decode_flush(avctx);
1329 
1330  av_log(avctx, AV_LOG_VERBOSE, "Vulkan decoder initialization sucessful\n");
1331 
1332  return 0;
1333 
1334 fail:
1335  ff_vk_decode_uninit(avctx);
1336 
1337  return err;
1338 }
vulkan_loader.h
AVHWDeviceContext::hwctx
void * hwctx
The format-specific data, allocated and freed by libavutil along with this context.
Definition: hwcontext.h:86
AV_PIX_FMT_GBRAP16
#define AV_PIX_FMT_GBRAP16
Definition: pixfmt.h:551
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:215
AVVulkanDeviceContext::phys_dev
VkPhysicalDevice phys_dev
Physical device.
Definition: hwcontext_vulkan.h:79
FFVulkanDecodePicture::slices_size
size_t slices_size
Definition: vulkan_decode.h:112
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:71
FFVulkanDecodeShared::s
FFVulkanContext s
Definition: vulkan_decode.h:48
FFVulkanDecodeProfileData::profile
VkVideoProfileInfoKHR profile
Definition: vulkan_decode.h:43
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
AV_PROFILE_H264_INTRA
#define AV_PROFILE_H264_INTRA
Definition: defs.h:108
FFVulkanDecodeProfileData::h265_profile
VkVideoDecodeH265ProfileInfoKHR h265_profile
Definition: vulkan_decode.h:40
ff_vk_decode_prepare_frame_sdr
int ff_vk_decode_prepare_frame_sdr(FFVulkanDecodeContext *dec, AVFrame *pic, FFVulkanDecodePicture *vkpic, int is_current, enum FFVkShaderRepFormat rep_fmt, int alloc_dpb)
Software-defined decoder version of ff_vk_decode_prepare_frame.
Definition: vulkan_decode.c:204
av_pix_fmt_desc_get
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3341
AVBufferRef::data
uint8_t * data
The data buffer.
Definition: buffer.h:90
FFVulkanDecodeContext::shared_ctx
FFVulkanDecodeShared * shared_ctx
Definition: vulkan_decode.h:66
AV_FIELD_PROGRESSIVE
@ AV_FIELD_PROGRESSIVE
Definition: defs.h:202
ff_vk_exec_pool_init
int ff_vk_exec_pool_init(FFVulkanContext *s, AVVulkanDeviceQueueFamily *qf, FFVkExecPool *pool, int nb_contexts, int nb_queries, VkQueryType query_type, int query_64bit, const void *query_create_pnext)
Allocates/frees an execution pool.
Definition: vulkan.c:296
AVRefStructOpaque
RefStruct is an API for creating reference-counted objects with minimal overhead.
Definition: refstruct.h:58
AVHWFramesContext::format
enum AVPixelFormat format
The pixel format identifying the underlying HW surface type.
Definition: hwcontext.h:198
dpb_frames
int dpb_frames
Definition: h264_levels.c:163
FF_VK_EXT_VIDEO_MAINTENANCE_2
#define FF_VK_EXT_VIDEO_MAINTENANCE_2
Definition: vulkan_functions.h:54
AV_PROFILE_HEVC_MAIN
#define AV_PROFILE_HEVC_MAIN
Definition: defs.h:159
ff_vk_exec_update_frame
void ff_vk_exec_update_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, VkImageMemoryBarrier2 *bar, uint32_t *nb_img_bar)
Definition: vulkan.c:789
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:70
av_hwframe_ctx_init
int av_hwframe_ctx_init(AVBufferRef *ref)
Finalize the context before use.
Definition: hwcontext.c:326
ff_vk_depth_from_av_depth
VkVideoComponentBitDepthFlagBitsKHR ff_vk_depth_from_av_depth(int depth)
Get Vulkan's bit depth from an [8:12] integer.
Definition: vulkan_video.c:128
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:410
tmp
static uint8_t tmp[11]
Definition: aes_ctr.c:28
ff_vk_dec_av1_desc
const FFVulkanDecodeDescriptor ff_vk_dec_av1_desc
Definition: vulkan_av1.c:26
av_hwframe_ctx_alloc
AVBufferRef * av_hwframe_ctx_alloc(AVBufferRef *device_ref_in)
Allocate an AVHWFramesContext tied to a given device context.
Definition: hwcontext.c:252
AVHWFramesContext::free
void(* free)(struct AVHWFramesContext *ctx)
This field may be set by the caller before calling av_hwframe_ctx_init().
Definition: hwcontext.h:159
AVCodecContext::field_order
enum AVFieldOrder field_order
Field order.
Definition: avcodec.h:722
AVVulkanFramesContext::create_pnext
void * create_pnext
Extension data for image creation.
Definition: hwcontext_vulkan.h:243
ff_vk_find_struct
static const void * ff_vk_find_struct(const void *chain, VkStructureType stype)
Definition: vulkan.h:331
b
#define b
Definition: input.c:42
create_empty_session_parameters
static int create_empty_session_parameters(AVCodecContext *avctx, FFVulkanDecodeShared *ctx)
Definition: vulkan_decode.c:1144
data
const char data[16]
Definition: mxf.c:149
ff_vk_create_view
int ff_vk_create_view(FFVulkanContext *s, FFVkVideoCommon *common, VkImageView *view, VkImageAspectFlags *aspect, AVVkFrame *src, VkFormat vkf, int is_dpb)
Creates image views for video frames.
Definition: vulkan_video.c:271
FFVulkanDecodeContext::frame_id_alloc_mask
uint32_t frame_id_alloc_mask
Definition: vulkan_decode.h:71
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:225
ff_vk_init
int ff_vk_init(FFVulkanContext *s, void *log_parent, AVBufferRef *device_ref, AVBufferRef *frames_ref)
Initializes the AVClass, in case this context is not used as the main user's context.
Definition: vulkan.c:2904
ff_vk_exec_get
FFVkExecContext * ff_vk_exec_get(FFVulkanContext *s, FFVkExecPool *pool)
Retrieve an execution pool.
Definition: vulkan.c:477
ff_vk_uninit
void ff_vk_uninit(FFVulkanContext *s)
Frees main context.
Definition: vulkan.c:2893
FFMAX
#define FFMAX(a, b)
Definition: macros.h:47
avcodec_profile_name
const char * avcodec_profile_name(enum AVCodecID codec_id, int profile)
Return a name for the specified profile, if available.
Definition: utils.c:439
FFVulkanDecodePicture::dst
VkImageView dst[AV_NUM_DATA_POINTERS]
Definition: vulkan_decode.h:91
AVHWFramesContext::width
int width
The allocated dimensions of the frames in this pool.
Definition: hwcontext.h:218
FFVulkanDecodeProfileData::av1_profile
VkVideoDecodeAV1ProfileInfoKHR av1_profile
Definition: vulkan_decode.h:41
AVFrame::buf
AVBufferRef * buf[AV_NUM_DATA_POINTERS]
AVBuffer references backing the data for this frame.
Definition: frame.h:623
FFVulkanDecodeContext
Definition: vulkan_decode.h:65
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:379
ff_vk_exec_add_dep_frame
int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, VkPipelineStageFlagBits2 wait_stage, VkPipelineStageFlagBits2 signal_stage)
Definition: vulkan.c:709
ff_vk_decode_prepare_frame
int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, FFVulkanDecodePicture *vkpic, int is_current, int alloc_dpb)
Prepare a frame, creates the image view, and sets up the dpb fields.
Definition: vulkan_decode.c:147
AV_HWACCEL_FLAG_IGNORE_LEVEL
#define AV_HWACCEL_FLAG_IGNORE_LEVEL
Hardware acceleration should be used for decoding even if the codec level used is unknown or higher t...
Definition: avcodec.h:2168
FFVkShaderRepFormat
FFVkShaderRepFormat
Returns the format to use for images in shaders.
Definition: vulkan.h:388
AVFrame::data
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:431
av_malloc
#define av_malloc(s)
Definition: tableprint_vlc.h:31
AV_HWDEVICE_TYPE_VULKAN
@ AV_HWDEVICE_TYPE_VULKAN
Definition: hwcontext.h:39
FFVulkanDecodeContext::session_params
AVBufferRef * session_params
Definition: vulkan_decode.h:67
ff_vk_subsampling_from_av_desc
VkVideoChromaSubsamplingFlagBitsKHR ff_vk_subsampling_from_av_desc(const AVPixFmtDescriptor *desc)
Get Vulkan's chroma subsampling from a pixfmt descriptor.
Definition: vulkan_video.c:115
av_pix_fmt_count_planes
int av_pix_fmt_count_planes(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3381
AVVulkanDeviceContext::alloc
const VkAllocationCallbacks * alloc
Custom memory allocator, else NULL.
Definition: hwcontext_vulkan.h:63
AVVkFrame::img
VkImage img[AV_NUM_DATA_POINTERS]
Vulkan images to which the memory is bound to.
Definition: hwcontext_vulkan.h:307
FFVulkanDecodeDescriptor::decode_op
VkVideoCodecOperationFlagBitsKHR decode_op
Definition: vulkan_decode.h:33
fail
#define fail()
Definition: checkasm.h:193
FFVulkanDecodePicture::sem_value
uint64_t sem_value
Definition: vulkan_decode.h:97
AVCodecContext::thread_count
int thread_count
thread count is used to decide how many independent tasks should be passed to execute()
Definition: avcodec.h:1601
AV_PIX_FMT_GBRP10
#define AV_PIX_FMT_GBRP10
Definition: pixfmt.h:544
frames
if it could not because there are no more frames
Definition: filter_design.txt:266
AVVulkanFramesContext
Allocated as AVHWFramesContext.hwctx, used to set pool-specific options.
Definition: hwcontext_vulkan.h:213
AVCodecContext::flags
int flags
AV_CODEC_FLAG_*.
Definition: avcodec.h:508
ff_vk_ret2str
const char * ff_vk_ret2str(VkResult res)
Converts Vulkan return values to strings.
Definition: vulkan.c:35
AVCodecContext::coded_height
int coded_height
Definition: avcodec.h:647
ff_vk_decode_frame
int ff_vk_decode_frame(AVCodecContext *avctx, AVFrame *pic, FFVulkanDecodePicture *vp, AVFrame *rpic[], FFVulkanDecodePicture *rvkp[])
Decode a frame.
Definition: vulkan_decode.c:368
FFVulkanDecodeShared
Definition: vulkan_decode.h:47
vulkan_decode_get_profile
static int vulkan_decode_get_profile(AVCodecContext *avctx, AVBufferRef *frames_ref, enum AVPixelFormat *pix_fmt, VkFormat *vk_fmt, FFVulkanDecodeProfileData *prof, int *dpb_dedicate)
Definition: vulkan_decode.c:751
init_frame
static void init_frame(FFVulkanDecodeContext *dec, FFVulkanDecodePicture *vkpic)
Definition: vulkan_decode.c:131
refstruct.h
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:61
av_frame_alloc
AVFrame * av_frame_alloc(void)
Allocate an AVFrame and set its fields to default values.
Definition: frame.c:58
FFVulkanDecodeContext::slice_off
uint32_t * slice_off
Definition: vulkan_decode.h:81
avassert.h
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:209
FFVulkanDecodeProfileData::h264_profile
VkVideoDecodeH264ProfileInfoKHR h264_profile
Definition: vulkan_decode.h:39
FF_ARRAY_ELEMS
#define FF_ARRAY_ELEMS(a)
Definition: sinewin_tablegen.c:29
AVHWFramesContext::height
int height
Definition: hwcontext.h:218
AV_FIELD_UNKNOWN
@ AV_FIELD_UNKNOWN
Definition: defs.h:201
FFVulkanDecodeProfileData::profile_list
VkVideoProfileListInfoKHR profile_list
Definition: vulkan_decode.h:44
av_fast_realloc
void * av_fast_realloc(void *ptr, unsigned int *size, size_t min_size)
Reallocate the given buffer if it is not large enough, otherwise do nothing.
Definition: mem.c:497
FFVulkanDecodePicture::wait_semaphores
PFN_vkWaitSemaphores wait_semaphores
Definition: vulkan_decode.h:115
s
#define s(width, name)
Definition: cbs_vp9.c:198
FFVulkanDecodePicture
Definition: vulkan_decode.h:85
ff_vk_video_common_init
av_cold int ff_vk_video_common_init(AVCodecContext *avctx, FFVulkanContext *s, FFVkVideoCommon *common, VkVideoSessionCreateInfoKHR *session_create)
Initialize video session, allocating and binding necessary memory.
Definition: vulkan_video.c:345
ff_vk_decode_create_params
int ff_vk_decode_create_params(AVBufferRef **par_ref, void *logctx, FFVulkanDecodeShared *ctx, const VkVideoSessionParametersCreateInfoKHR *session_params_create)
Create VkVideoSessionParametersKHR wrapped in an AVBufferRef.
Definition: vulkan_decode.c:1104
ff_vk_exec_mirror_sem_value
int ff_vk_exec_mirror_sem_value(FFVulkanContext *s, FFVkExecContext *e, VkSemaphore *dst, uint64_t *dst_val, AVFrame *f)
Definition: vulkan.c:808
offsets
static const int offsets[]
Definition: hevc_pel.c:34
FFVulkanContext::driver_props
VkPhysicalDeviceDriverProperties driver_props
Definition: vulkan.h:284
AVFormatContext::flags
int flags
Flags modifying the (de)muxer behaviour.
Definition: avformat.h:1451
format
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample format(the sample packing is implied by the sample format) and sample rate. The lists are not just lists
ff_vk_load_functions
static int ff_vk_load_functions(AVHWDeviceContext *ctx, FFVulkanFunctions *vk, uint64_t extensions_mask, int has_inst, int has_dev)
Function loader.
Definition: vulkan_loader.h:108
pix_fmt
static enum AVPixelFormat pix_fmt
Definition: demux_decode.c:41
av_refstruct_alloc_ext
static void * av_refstruct_alloc_ext(size_t size, unsigned flags, void *opaque, void(*free_cb)(AVRefStructOpaque opaque, void *obj))
A wrapper around av_refstruct_alloc_ext_c() for the common case of a non-const qualified opaque.
Definition: refstruct.h:94
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:230
ctx
AVFormatContext * ctx
Definition: movenc.c:49
ff_vk_exec_add_dep_buf
int ff_vk_exec_add_dep_buf(FFVulkanContext *s, FFVkExecContext *e, AVBufferRef **deps, int nb_deps, int ref)
Execution dependency management.
Definition: vulkan.c:549
codec_id
enum AVCodecID codec_id
Definition: vaapi_decode.c:410
ff_vk_exec_pool_free
void ff_vk_exec_pool_free(FFVulkanContext *s, FFVkExecPool *pool)
Definition: vulkan.c:233
AV_CODEC_ID_H264
@ AV_CODEC_ID_H264
Definition: codec_id.h:79
ff_decode_get_hw_frames_ctx
int ff_decode_get_hw_frames_ctx(AVCodecContext *avctx, enum AVHWDeviceType dev_type)
Make sure avctx.hw_frames_ctx is set.
Definition: decode.c:1114
AVCodecContext::codec_id
enum AVCodecID codec_id
Definition: avcodec.h:461
if
if(ret)
Definition: filter_design.txt:179
get_video_profile
static const VkVideoProfileInfoKHR * get_video_profile(FFVulkanDecodeShared *ctx, enum AVCodecID codec_id)
Definition: vulkan_decode.c:67
AVVulkanDeviceContext
Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_vulkan.h:59
AV_PIX_FMT_RGBA64
#define AV_PIX_FMT_RGBA64
Definition: pixfmt.h:518
NULL
#define NULL
Definition: coverity.c:32
AVHWFramesContext::sw_format
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
Definition: hwcontext.h:211
av_buffer_unref
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it.
Definition: buffer.c:139
AV_CODEC_ID_AV1
@ AV_CODEC_ID_AV1
Definition: codec_id.h:284
AVVulkanDeviceContext::nb_enabled_dev_extensions
int nb_enabled_dev_extensions
Definition: hwcontext_vulkan.h:113
ff_vk_decode_free_frame
void ff_vk_decode_free_frame(AVHWDeviceContext *dev_ctx, FFVulkanDecodePicture *vp)
Free a frame and its state.
Definition: vulkan_decode.c:574
ff_vk_video_common_uninit
av_cold void ff_vk_video_common_uninit(FFVulkanContext *s, FFVkVideoCommon *common)
Free video session and required resources.
Definition: vulkan_video.c:315
FF_VK_EXT_VIDEO_ENCODE_QUEUE
#define FF_VK_EXT_VIDEO_ENCODE_QUEUE
Definition: vulkan_functions.h:61
AVCodecContext::internal
struct AVCodecInternal * internal
Private context used for internal data.
Definition: avcodec.h:486
FFVulkanDecodeProfileData::usage
VkVideoDecodeUsageInfoKHR usage
Definition: vulkan_decode.h:42
FFVulkanDecodeDescriptor::decode_extension
FFVulkanExtensions decode_extension
Definition: vulkan_decode.h:31
AV_HWACCEL_FLAG_ALLOW_PROFILE_MISMATCH
#define AV_HWACCEL_FLAG_ALLOW_PROFILE_MISMATCH
Hardware acceleration should still be attempted for decoding when the codec profile does not match th...
Definition: avcodec.h:2188
av_buffer_pool_uninit
void av_buffer_pool_uninit(AVBufferPool **ppool)
Mark the pool as being available for freeing.
Definition: buffer.c:328
ff_vk_decode_uninit
int ff_vk_decode_uninit(AVCodecContext *avctx)
Free decoder.
Definition: vulkan_decode.c:1133
AVVulkanFramesContext::format
VkFormat format[AV_NUM_DATA_POINTERS]
Vulkan format for each image.
Definition: hwcontext_vulkan.h:273
FFVkBuffer::size
size_t size
Definition: vulkan.h:92
AVVulkanFramesContext::usage
VkImageUsageFlagBits usage
Defines extra usage of output frames.
Definition: hwcontext_vulkan.h:232
AV_PIX_FMT_BGR0
@ AV_PIX_FMT_BGR0
packed BGR 8:8:8, 32bpp, BGRXBGRX... X=unused/undefined
Definition: pixfmt.h:265
FFVulkanDecodePicture::aspect
VkImageAspectFlags aspect[AV_NUM_DATA_POINTERS]
Definition: vulkan_decode.h:92
FFVulkanDecodePicture::aspect_ref
VkImageAspectFlags aspect_ref[AV_NUM_DATA_POINTERS]
Definition: vulkan_decode.h:93
FFVkBuffer::mapped_mem
uint8_t * mapped_mem
Definition: vulkan.h:101
FFVulkanContext
Definition: vulkan.h:276
ff_vk_frame_params
int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx)
Initialize hw_frames_ctx with the parameters needed to decode the stream using the parameters from av...
Definition: vulkan_decode.c:1016
AVCodecContext::level
int level
Encoding level descriptor.
Definition: avcodec.h:1802
av_buffer_create
AVBufferRef * av_buffer_create(uint8_t *data, size_t size, void(*free)(void *opaque, uint8_t *data), void *opaque, int flags)
Create an AVBuffer from an existing array.
Definition: buffer.c:55
AVCodecID
AVCodecID
Identify the syntax and semantics of the bitstream.
Definition: codec_id.h:49
usage
const char * usage
Definition: floatimg_cmp.c:60
ff_vk_create_imageview
int ff_vk_create_imageview(FFVulkanContext *s, VkImageView *img_view, VkImageAspectFlags *aspect, AVFrame *f, int plane, enum FFVkShaderRepFormat rep_fmt)
Create a single imageview for a given plane.
Definition: vulkan.c:1816
AV_PIX_FMT_X2BGR10
#define AV_PIX_FMT_X2BGR10
Definition: pixfmt.h:597
free_common
static void free_common(AVRefStructOpaque unused, void *obj)
Definition: vulkan_decode.c:607
FF_VK_EXT_VIDEO_MAINTENANCE_1
#define FF_VK_EXT_VIDEO_MAINTENANCE_1
Definition: vulkan_functions.h:53
ff_vk_h265_level_to_av
int ff_vk_h265_level_to_av(StdVideoH265LevelIdc level)
Definition: vulkan_video.c:191
sem_wait
#define sem_wait(psem)
Definition: semaphore.h:27
AVCodecInternal::hwaccel_priv_data
void * hwaccel_priv_data
hwaccel-specific private data
Definition: internal.h:130
dst
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t int int16_t * dst
Definition: dsp.h:83
AVVkFrame
Definition: hwcontext_vulkan.h:302
FFVulkanDecodePicture::ref
VkImageView ref[AV_NUM_DATA_POINTERS]
Definition: vulkan_decode.h:89
size
int size
Definition: twinvq_data.h:10344
AV_NUM_DATA_POINTERS
#define AV_NUM_DATA_POINTERS
Definition: frame.h:411
ref_frame
static int ref_frame(VVCFrame *dst, const VVCFrame *src)
Definition: dec.c:593
ff_vk_dec_h264_desc
const FFVulkanDecodeDescriptor ff_vk_dec_h264_desc
Definition: vulkan_h264.c:24
FFVulkanDecodeContext::slice_off_max
unsigned int slice_off_max
Definition: vulkan_decode.h:82
AVVkFrame::queue_family
uint32_t queue_family[AV_NUM_DATA_POINTERS]
Queue family of the images.
Definition: hwcontext_vulkan.h:366
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
FFVkExecContext
Definition: vulkan.h:112
AV_PIX_FMT_RGB0
@ AV_PIX_FMT_RGB0
packed RGB 8:8:8, 32bpp, RGBXRGBX... X=unused/undefined
Definition: pixfmt.h:263
vk_decode_free_params
static void vk_decode_free_params(void *opaque, uint8_t *data)
Definition: vulkan_decode.c:1094
FFVulkanDecodeProfileData
Definition: vulkan_decode.h:38
FF_VK_EXT_VIDEO_DECODE_QUEUE
#define FF_VK_EXT_VIDEO_DECODE_QUEUE
Definition: vulkan_functions.h:56
av_refstruct_unref
void av_refstruct_unref(void *objp)
Decrement the reference count of the underlying object and automatically free the object if there are...
Definition: refstruct.c:120
avcodec_get_name
const char * avcodec_get_name(enum AVCodecID id)
Get the name of a codec.
Definition: utils.c:406
FFVulkanDecodePicture::out
VkImageView out[AV_NUM_DATA_POINTERS]
Definition: vulkan_decode.h:90
ff_vk_exec_start
int ff_vk_exec_start(FFVulkanContext *s, FFVkExecContext *e)
Start/submit/wait an execution.
Definition: vulkan.c:489
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:256
FFVulkanDecodeDescriptor::ext_props
VkExtensionProperties ext_props
Definition: vulkan_decode.h:35
VkFormat
enum VkFormat VkFormat
Definition: hwcontext_stub.c:25
ff_vk_h264_level_to_av
int ff_vk_h264_level_to_av(StdVideoH264LevelIdc level)
Convert level from Vulkan to AV.
Definition: vulkan_video.c:139
AVCodecContext::hwaccel_flags
int hwaccel_flags
Bit set of AV_HWACCEL_FLAG_* flags, which affect hardware accelerated decoding (if active).
Definition: avcodec.h:1524
av_assert1
#define av_assert1(cond)
assert() equivalent, that does not lie in speed critical code.
Definition: avassert.h:56
AV_CODEC_ID_HEVC
@ AV_CODEC_ID_HEVC
Definition: codec_id.h:228
DECODER_IS_SDR
#define DECODER_IS_SDR(codec_id)
Definition: vulkan_decode.c:27
FFMIN
#define FFMIN(a, b)
Definition: macros.h:49
FFVulkanDecodePicture::sem
VkSemaphore sem
Definition: vulkan_decode.h:96
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:256
FFVulkanDecodeContext::external_fg
int external_fg
Definition: vulkan_decode.h:70
av_buffer_replace
int av_buffer_replace(AVBufferRef **pdst, const AVBufferRef *src)
Ensure dst refers to the same data as src.
Definition: buffer.c:233
profile
int profile
Definition: mxfenc.c:2250
AVCodecContext::hw_frames_ctx
AVBufferRef * hw_frames_ctx
A reference to the AVHWFramesContext describing the input (for encoding) or output (decoding) frames.
Definition: avcodec.h:1493
CODEC_VER
#define CODEC_VER(ver)
Definition: vulkan_video.h:30
AVHWFramesContext
This struct describes a set or pool of "hardware" frames (i.e.
Definition: hwcontext.h:116
ff_vk_decode_flush
void ff_vk_decode_flush(AVCodecContext *avctx)
Flush decoder.
Definition: vulkan_decode.c:332
ret
ret
Definition: filter_design.txt:187
AVHWFramesContext::hwctx
void * hwctx
The format-specific data, allocated and freed automatically along with this context.
Definition: hwcontext.h:151
AVHWFramesContext::user_opaque
void * user_opaque
Arbitrary user data, to be used e.g.
Definition: hwcontext.h:164
ff_vk_decode_add_slice
int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp, const uint8_t *data, size_t size, int add_startcode, uint32_t *nb_slices, const uint32_t **offsets)
Add slice data to frame.
Definition: vulkan_decode.c:251
AV_PROFILE_H264_CONSTRAINED
#define AV_PROFILE_H264_CONSTRAINED
Definition: defs.h:107
dec_descs
static const FFVulkanDecodeDescriptor * dec_descs[]
Definition: vulkan_decode.c:43
FFVulkanDecodeContext::hevc_headers
struct HEVCHeaderSet * hevc_headers
Definition: vulkan_decode.h:78
ff_vk_qf_find
AVVulkanDeviceQueueFamily * ff_vk_qf_find(FFVulkanContext *s, VkQueueFlagBits dev_family, VkVideoCodecOperationFlagBitsKHR vid_ops)
Chooses an appropriate QF.
Definition: vulkan.c:220
AV_INPUT_BUFFER_PADDING_SIZE
#define AV_INPUT_BUFFER_PADDING_SIZE
Definition: defs.h:40
FFVkExecContext::buf
VkCommandBuffer buf
Definition: vulkan.h:124
AVFrame::hw_frames_ctx
AVBufferRef * hw_frames_ctx
For hwaccel-format frames, this should be a reference to the AVHWFramesContext describing the frame.
Definition: frame.h:769
ff_vk_dec_hevc_desc
const FFVulkanDecodeDescriptor ff_vk_dec_hevc_desc
Definition: vulkan_hevc.c:26
FFVulkanDecodePicture::view
struct FFVulkanDecodePicture::@297 view
get_codecdesc
static const FFVulkanDecodeDescriptor * get_codecdesc(enum AVCodecID codec_id)
Definition: vulkan_decode.c:58
AVCodecContext
main external API structure.
Definition: avcodec.h:451
FFVulkanDecodeContext::dedicated_dpb
int dedicated_dpb
Definition: vulkan_decode.h:69
av_find_best_pix_fmt_of_2
enum AVPixelFormat av_find_best_pix_fmt_of_2(enum AVPixelFormat dst_pix_fmt1, enum AVPixelFormat dst_pix_fmt2, enum AVPixelFormat src_pix_fmt, int has_alpha, int *loss_ptr)
Compute what kind of losses will occur when converting from one specific pixel format to another.
Definition: pixdesc.c:3620
ff_vk_dec_ffv1_desc
const FFVulkanDecodeDescriptor ff_vk_dec_ffv1_desc
Definition: vulkan_ffv1.c:38
av_refstruct_replace
void av_refstruct_replace(void *dstp, const void *src)
Ensure *dstp refers to the same object as src.
Definition: refstruct.c:160
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:72
AVCodecContext::profile
int profile
profile
Definition: avcodec.h:1658
FFVulkanDecodeDescriptor
Definition: vulkan_decode.h:29
AV_CODEC_ID_H265
#define AV_CODEC_ID_H265
Definition: codec_id.h:229
ref
static int ref[MAX_W *MAX_W]
Definition: jpeg2000dwt.c:117
AVCodecContext::export_side_data
int export_side_data
Bit set of AV_CODEC_EXPORT_DATA_* flags, which affects the kind of metadata exported in frame,...
Definition: avcodec.h:1956
ff_vk_params_invalidate
int ff_vk_params_invalidate(AVCodecContext *avctx, int t, const uint8_t *b, uint32_t s)
Removes current session parameters to recreate them.
Definition: vulkan_decode.c:110
FFVulkanDecodePicture::dpb_frame
AVFrame * dpb_frame
Definition: vulkan_decode.h:86
AV_PROFILE_H264_CONSTRAINED_BASELINE
#define AV_PROFILE_H264_CONSTRAINED_BASELINE
Definition: defs.h:111
ff_vk_update_thread_context
int ff_vk_update_thread_context(AVCodecContext *dst, const AVCodecContext *src)
Synchronize the contexts between 2 threads.
Definition: vulkan_decode.c:91
AVVulkanFramesContext::tiling
VkImageTiling tiling
Controls the tiling of allocated frames.
Definition: hwcontext_vulkan.h:222
vulkan_video.h
AVCodecContext::coded_width
int coded_width
Bitstream width / height, may be different from width/height e.g.
Definition: avcodec.h:647
desc
const char * desc
Definition: libsvtav1.c:79
AVVulkanDeviceContext::enabled_dev_extensions
const char *const * enabled_dev_extensions
Enabled device extensions.
Definition: hwcontext_vulkan.h:112
AVVulkanFramesContext::nb_layers
int nb_layers
Number of layers each image will have.
Definition: hwcontext_vulkan.h:278
FFVulkanDecodePicture::slices_buf
AVBufferRef * slices_buf
Definition: vulkan_decode.h:111
mem.h
AVVkFrame::layout
VkImageLayout layout[AV_NUM_DATA_POINTERS]
Definition: hwcontext_vulkan.h:332
AVBufferRef
A reference to a data buffer.
Definition: buffer.h:82
AVPixFmtDescriptor
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:69
FFVulkanDecodeDescriptor::queue_flags
VkQueueFlagBits queue_flags
Definition: vulkan_decode.h:32
AVVulkanDeviceContext::act_dev
VkDevice act_dev
Active device.
Definition: hwcontext_vulkan.h:84
vulkan_decode.h
av_free
#define av_free(p)
Definition: tableprint_vlc.h:34
decode_end
static av_cold int decode_end(AVCodecContext *avctx)
Definition: 4xm.c:976
FFALIGN
#define FFALIGN(x, a)
Definition: macros.h:78
vulkan_setup_profile
static VkResult vulkan_setup_profile(AVCodecContext *avctx, FFVulkanDecodeProfileData *prof, AVVulkanDeviceContext *hwctx, FFVulkanFunctions *vk, const FFVulkanDecodeDescriptor *vk_desc, VkVideoDecodeH264CapabilitiesKHR *h264_caps, VkVideoDecodeH265CapabilitiesKHR *h265_caps, VkVideoDecodeAV1CapabilitiesKHR *av1_caps, VkVideoCapabilitiesKHR *caps, VkVideoDecodeCapabilitiesKHR *dec_caps, int cur_profile)
Definition: vulkan_decode.c:676
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:35
FFVkBuffer
Definition: vulkan.h:88
vk_get_dpb_pool
static AVFrame * vk_get_dpb_pool(FFVulkanDecodeShared *ctx)
Definition: vulkan_decode.c:117
ff_vk_exec_submit
int ff_vk_exec_submit(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:834
FFVulkanDecodePicture::destroy_image_view
PFN_vkDestroyImageView destroy_image_view
Definition: vulkan_decode.h:116
ff_vk_extensions_to_mask
static uint64_t ff_vk_extensions_to_mask(const char *const *extensions, int nb_extensions)
Definition: vulkan_loader.h:36
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
vulkan_decode_bootstrap
static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_ref)
Definition: vulkan_decode.c:635
ff_vk_pix_fmt_from_vkfmt
enum AVPixelFormat ff_vk_pix_fmt_from_vkfmt(VkFormat vkf)
Get pixfmt from a Vulkan format.
Definition: vulkan_video.c:99
ff_vk_decode_init
int ff_vk_decode_init(AVCodecContext *avctx)
Initialize decoder.
Definition: vulkan_decode.c:1182
FFVulkanDecodePicture::decode_info
VkVideoDecodeInfoKHR decode_info
Definition: vulkan_decode.h:108
AVCodecContext::sw_pix_fmt
enum AVPixelFormat sw_pix_fmt
Nominal unaccelerated pixel format, see AV_PIX_FMT_xxx.
Definition: avcodec.h:678
av_hwframe_get_buffer
int av_hwframe_get_buffer(AVBufferRef *hwframe_ref, AVFrame *frame, int flags)
Allocate a new frame attached to the given AVHWFramesContext.
Definition: hwcontext.c:495
av_log2
int av_log2(unsigned v)
Definition: intmath.c:26
FFVulkanFunctions
Definition: vulkan_functions.h:265
FFVulkanDecodeContext::quirk_av1_offset
int quirk_av1_offset
Definition: vulkan_decode.h:75
ff_vk_get_pooled_buffer
int ff_vk_get_pooled_buffer(FFVulkanContext *ctx, AVBufferPool **buf_pool, AVBufferRef **buf, VkBufferUsageFlags usage, void *create_pNext, size_t size, VkMemoryPropertyFlagBits mem_props)
Initialize a pool and create AVBufferRefs containing FFVkBuffer.
Definition: vulkan.c:1174
src
#define src
Definition: vp8dsp.c:248
free_profile_data
static void free_profile_data(AVHWFramesContext *hwfc)
Definition: vulkan_decode.c:1011
AV_CODEC_EXPORT_DATA_FILM_GRAIN
#define AV_CODEC_EXPORT_DATA_FILM_GRAIN
Decoding only.
Definition: avcodec.h:420
av_get_pix_fmt_name
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:3261