FFmpeg
ffplay_renderer.c
Go to the documentation of this file.
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 #define VK_NO_PROTOTYPES
20 #define VK_ENABLE_BETA_EXTENSIONS
21 
22 #include "config.h"
23 #include "ffplay_renderer.h"
24 
25 #if (SDL_VERSION_ATLEAST(2, 0, 6) && CONFIG_LIBPLACEBO)
26 /* Get PL_API_VER */
27 #include <libplacebo/config.h>
28 #define HAVE_VULKAN_RENDERER (PL_API_VER >= 278)
29 #else
30 #define HAVE_VULKAN_RENDERER 0
31 #endif
32 
33 #if HAVE_VULKAN_RENDERER
34 
35 #if defined(_WIN32) && !defined(VK_USE_PLATFORM_WIN32_KHR)
36 #define VK_USE_PLATFORM_WIN32_KHR
37 #endif
38 
39 #include <libplacebo/vulkan.h>
40 #include <libplacebo/utils/frame_queue.h>
41 #include <libplacebo/utils/libav.h>
42 #include <SDL_vulkan.h>
43 
44 #include "libavutil/bprint.h"
45 
46 #endif
47 
48 struct VkRenderer {
49  const AVClass *class;
50 
51  int (*create)(VkRenderer *renderer, SDL_Window *window, AVDictionary *dict);
52 
54 
56 
58 
60 };
61 
62 #if HAVE_VULKAN_RENDERER
63 
64 typedef struct RendererContext {
65  VkRenderer api;
66 
67  // Can be NULL when vulkan instance is created by avutil
68  pl_vk_inst placebo_instance;
69  pl_vulkan placebo_vulkan;
70  pl_swapchain swapchain;
71  VkSurfaceKHR vk_surface;
72  pl_renderer renderer;
73  pl_tex tex[4];
74 
75  pl_log vk_log;
76 
77  AVBufferRef *hw_device_ref;
78  AVBufferRef *hw_frame_ref;
79  enum AVPixelFormat *transfer_formats;
80  AVHWFramesConstraints *constraints;
81 
82  PFN_vkGetInstanceProcAddr get_proc_addr;
83  // This field is a copy from pl_vk_inst->instance or hw_device_ref instance.
84  VkInstance inst;
85 
86  AVFrame *vk_frame;
87 } RendererContext;
88 
89 static void vk_log_cb(void *log_priv, enum pl_log_level level,
90  const char *msg)
91 {
92  static const int level_map[] = {
100  };
101 
102  if (level > 0 && level < FF_ARRAY_ELEMS(level_map))
103  av_log(log_priv, level_map[level], "%s\n", msg);
104 }
105 
106 // Should keep sync with optional_device_exts inside hwcontext_vulkan.c
107 static const char *optional_device_exts[] = {
108  /* Misc or required by other extensions */
109  VK_KHR_PORTABILITY_SUBSET_EXTENSION_NAME,
110  VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME,
111  VK_KHR_SAMPLER_YCBCR_CONVERSION_EXTENSION_NAME,
112  VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME,
113  VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME,
114  VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME,
115  VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME,
116 
117  /* Imports/exports */
118  VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME,
119  VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME,
120  VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME,
121  VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME,
122  VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME,
123 #ifdef _WIN32
124  VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME,
125  VK_KHR_EXTERNAL_SEMAPHORE_WIN32_EXTENSION_NAME,
126 #endif
127 
128  /* Video encoding/decoding */
129  VK_KHR_VIDEO_QUEUE_EXTENSION_NAME,
130  VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME,
131  VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME,
132  VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME,
133  "VK_MESA_video_decode_av1",
134 };
135 
136 static inline int enable_debug(const AVDictionary *opt)
137 {
138  AVDictionaryEntry *entry = av_dict_get(opt, "debug", NULL, 0);
139  int debug = entry && strtol(entry->value, NULL, 10);
140  return debug;
141 }
142 
143 static void hwctx_lock_queue(void *priv, uint32_t qf, uint32_t qidx)
144 {
145  AVHWDeviceContext *avhwctx = priv;
146  const AVVulkanDeviceContext *hwctx = avhwctx->hwctx;
147  hwctx->lock_queue(avhwctx, qf, qidx);
148 }
149 
150 static void hwctx_unlock_queue(void *priv, uint32_t qf, uint32_t qidx)
151 {
152  AVHWDeviceContext *avhwctx = priv;
153  const AVVulkanDeviceContext *hwctx = avhwctx->hwctx;
154  hwctx->unlock_queue(avhwctx, qf, qidx);
155 }
156 
157 static int add_instance_extension(const char **ext, unsigned num_ext,
158  const AVDictionary *opt,
159  AVDictionary **dict)
160 {
161  const char *inst_ext_key = "instance_extensions";
162  AVDictionaryEntry *entry;
163  AVBPrint buf;
164  char *ext_list = NULL;
165  int ret;
166 
168  for (int i = 0; i < num_ext; i++) {
169  if (i)
170  av_bprintf(&buf, "+%s", ext[i]);
171  else
172  av_bprintf(&buf, "%s", ext[i]);
173  }
174 
175  entry = av_dict_get(opt, inst_ext_key, NULL, 0);
176  if (entry && entry->value && entry->value[0]) {
177  if (num_ext)
178  av_bprintf(&buf, "+");
179  av_bprintf(&buf, "%s", entry->value);
180  }
181 
182  ret = av_bprint_finalize(&buf, &ext_list);
183  if (ret < 0)
184  return ret;
185  return av_dict_set(dict, inst_ext_key, ext_list, AV_DICT_DONT_STRDUP_VAL);
186 }
187 
188 static int add_device_extension(const AVDictionary *opt,
189  AVDictionary **dict)
190 {
191  const char *dev_ext_key = "device_extensions";
192  AVDictionaryEntry *entry;
193  AVBPrint buf;
194  char *ext_list = NULL;
195  int ret;
196 
198  av_bprintf(&buf, "%s", VK_KHR_SWAPCHAIN_EXTENSION_NAME);
199  for (int i = 0; i < pl_vulkan_num_recommended_extensions; i++)
200  av_bprintf(&buf, "+%s", pl_vulkan_recommended_extensions[i]);
201 
202  entry = av_dict_get(opt, dev_ext_key, NULL, 0);
203  if (entry && entry->value && entry->value[0])
204  av_bprintf(&buf, "+%s", entry->value);
205 
206  ret = av_bprint_finalize(&buf, &ext_list);
207  if (ret < 0)
208  return ret;
209  return av_dict_set(dict, dev_ext_key, ext_list, AV_DICT_DONT_STRDUP_VAL);
210 }
211 
212 static const char *select_device(const AVDictionary *opt)
213 {
214  const AVDictionaryEntry *entry;
215 
216  entry = av_dict_get(opt, "device", NULL, 0);
217  if (entry)
218  return entry->value;
219  return NULL;
220 }
221 
222 static int create_vk_by_hwcontext(VkRenderer *renderer,
223  const char **ext, unsigned num_ext,
224  const AVDictionary *opt)
225 {
226  RendererContext *ctx = (RendererContext *) renderer;
227  AVHWDeviceContext *dev;
228  AVVulkanDeviceContext *hwctx;
229  AVDictionary *dict = NULL;
230  int ret;
231 
232  ret = add_instance_extension(ext, num_ext, opt, &dict);
233  if (ret < 0)
234  return ret;
235  ret = add_device_extension(opt, &dict);
236  if (ret) {
237  av_dict_free(&dict);
238  return ret;
239  }
240 
242  select_device(opt), dict, 0);
243  av_dict_free(&dict);
244  if (ret < 0)
245  return ret;
246 
247  dev = (AVHWDeviceContext *) ctx->hw_device_ref->data;
248  hwctx = dev->hwctx;
249 
250  // There is no way to pass SDL GetInstanceProcAddr to hwdevice.
251  // Check the result and return error if they don't match.
252  if (hwctx->get_proc_addr != SDL_Vulkan_GetVkGetInstanceProcAddr()) {
254  "hwdevice and SDL use different get_proc_addr. "
255  "Try -vulkan_params create_by_placebo=1\n");
256  return AVERROR_PATCHWELCOME;
257  }
258 
259  ctx->get_proc_addr = hwctx->get_proc_addr;
260  ctx->inst = hwctx->inst;
261  ctx->placebo_vulkan = pl_vulkan_import(ctx->vk_log,
262  pl_vulkan_import_params(
263  .instance = hwctx->inst,
264  .get_proc_addr = hwctx->get_proc_addr,
265  .phys_device = hwctx->phys_dev,
266  .device = hwctx->act_dev,
267  .extensions = hwctx->enabled_dev_extensions,
268  .num_extensions = hwctx->nb_enabled_dev_extensions,
269  .features = &hwctx->device_features,
270  .lock_queue = hwctx_lock_queue,
271  .unlock_queue = hwctx_unlock_queue,
272  .queue_ctx = dev,
273  .queue_graphics = {
274  .index = hwctx->queue_family_index,
275  .count = hwctx->nb_graphics_queues,
276  },
277  .queue_compute = {
278  .index = hwctx->queue_family_comp_index,
279  .count = hwctx->nb_comp_queues,
280  },
281  .queue_transfer = {
282  .index = hwctx->queue_family_tx_index,
283  .count = hwctx->nb_tx_queues,
284  },
285  ));
286  if (!ctx->placebo_vulkan)
287  return AVERROR_EXTERNAL;
288 
289  return 0;
290 }
291 
292 static void placebo_lock_queue(struct AVHWDeviceContext *dev_ctx,
293  uint32_t queue_family, uint32_t index)
294 {
295  RendererContext *ctx = dev_ctx->user_opaque;
296  pl_vulkan vk = ctx->placebo_vulkan;
297  vk->lock_queue(vk, queue_family, index);
298 }
299 
300 static void placebo_unlock_queue(struct AVHWDeviceContext *dev_ctx,
301  uint32_t queue_family,
302  uint32_t index)
303 {
304  RendererContext *ctx = dev_ctx->user_opaque;
305  pl_vulkan vk = ctx->placebo_vulkan;
306  vk->unlock_queue(vk, queue_family, index);
307 }
308 
309 static int get_decode_queue(VkRenderer *renderer, int *index, int *count)
310 {
311  RendererContext *ctx = (RendererContext *) renderer;
312  VkQueueFamilyProperties *queue_family_prop = NULL;
313  uint32_t num_queue_family_prop = 0;
314  PFN_vkGetPhysicalDeviceQueueFamilyProperties get_queue_family_prop;
315  PFN_vkGetInstanceProcAddr get_proc_addr = ctx->get_proc_addr;
316 
317  *index = -1;
318  *count = 0;
319  get_queue_family_prop = (PFN_vkGetPhysicalDeviceQueueFamilyProperties)
320  get_proc_addr(ctx->placebo_instance->instance,
321  "vkGetPhysicalDeviceQueueFamilyProperties");
322  get_queue_family_prop(ctx->placebo_vulkan->phys_device,
323  &num_queue_family_prop, NULL);
324  if (!num_queue_family_prop)
325  return AVERROR_EXTERNAL;
326 
327  queue_family_prop = av_calloc(num_queue_family_prop,
328  sizeof(*queue_family_prop));
329  if (!queue_family_prop)
330  return AVERROR(ENOMEM);
331 
332  get_queue_family_prop(ctx->placebo_vulkan->phys_device,
333  &num_queue_family_prop,
334  queue_family_prop);
335 
336  for (int i = 0; i < num_queue_family_prop; i++) {
337  if (queue_family_prop[i].queueFlags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) {
338  *index = i;
339  *count = queue_family_prop[i].queueCount;
340  break;
341  }
342  }
343  av_free(queue_family_prop);
344 
345  return 0;
346 }
347 
348 static int create_vk_by_placebo(VkRenderer *renderer,
349  const char **ext, unsigned num_ext,
350  const AVDictionary *opt)
351 {
352  RendererContext *ctx = (RendererContext *) renderer;
353  AVHWDeviceContext *device_ctx;
354  AVVulkanDeviceContext *vk_dev_ctx;
355  int decode_index;
356  int decode_count;
357  int ret;
358 
359  ctx->get_proc_addr = SDL_Vulkan_GetVkGetInstanceProcAddr();
360 
361  ctx->placebo_instance = pl_vk_inst_create(ctx->vk_log, pl_vk_inst_params(
362  .get_proc_addr = ctx->get_proc_addr,
363  .debug = enable_debug(opt),
364  .extensions = ext,
365  .num_extensions = num_ext
366  ));
367  if (!ctx->placebo_instance) {
368  return AVERROR_EXTERNAL;
369  }
370  ctx->inst = ctx->placebo_instance->instance;
371 
372  ctx->placebo_vulkan = pl_vulkan_create(ctx->vk_log, pl_vulkan_params(
373  .instance = ctx->placebo_instance->instance,
374  .get_proc_addr = ctx->placebo_instance->get_proc_addr,
375  .surface = ctx->vk_surface,
376  .allow_software = false,
377  .opt_extensions = optional_device_exts,
378  .num_opt_extensions = FF_ARRAY_ELEMS(optional_device_exts),
379  .extra_queues = VK_QUEUE_VIDEO_DECODE_BIT_KHR,
380  .device_name = select_device(opt),
381  ));
382  if (!ctx->placebo_vulkan)
383  return AVERROR_EXTERNAL;
385  if (!ctx->hw_device_ref) {
386  return AVERROR(ENOMEM);
387  }
388 
389  device_ctx = (AVHWDeviceContext *) ctx->hw_device_ref->data;
390  device_ctx->user_opaque = ctx;
391 
392  vk_dev_ctx = device_ctx->hwctx;
393  vk_dev_ctx->lock_queue = placebo_lock_queue,
394  vk_dev_ctx->unlock_queue = placebo_unlock_queue;
395 
396  vk_dev_ctx->get_proc_addr = ctx->placebo_instance->get_proc_addr;
397 
398  vk_dev_ctx->inst = ctx->placebo_instance->instance;
399  vk_dev_ctx->phys_dev = ctx->placebo_vulkan->phys_device;
400  vk_dev_ctx->act_dev = ctx->placebo_vulkan->device;
401 
402  vk_dev_ctx->device_features = *ctx->placebo_vulkan->features;
403 
404  vk_dev_ctx->enabled_inst_extensions = ctx->placebo_instance->extensions;
405  vk_dev_ctx->nb_enabled_inst_extensions = ctx->placebo_instance->num_extensions;
406 
407  vk_dev_ctx->enabled_dev_extensions = ctx->placebo_vulkan->extensions;
408  vk_dev_ctx->nb_enabled_dev_extensions = ctx->placebo_vulkan->num_extensions;
409 
410  vk_dev_ctx->queue_family_index = ctx->placebo_vulkan->queue_graphics.index;
411  vk_dev_ctx->nb_graphics_queues = ctx->placebo_vulkan->queue_graphics.count;
412 
413  vk_dev_ctx->queue_family_tx_index = ctx->placebo_vulkan->queue_transfer.index;
414  vk_dev_ctx->nb_tx_queues = ctx->placebo_vulkan->queue_transfer.count;
415 
416  vk_dev_ctx->queue_family_comp_index = ctx->placebo_vulkan->queue_compute.index;
417  vk_dev_ctx->nb_comp_queues = ctx->placebo_vulkan->queue_compute.count;
418 
419  ret = get_decode_queue(renderer, &decode_index, &decode_count);
420  if (ret < 0)
421  return ret;
422 
424  vk_dev_ctx->nb_decode_queues = decode_count;
425 
426  ret = av_hwdevice_ctx_init(ctx->hw_device_ref);
427  if (ret < 0)
428  return ret;
429 
430  return 0;
431 }
432 
433 static int create(VkRenderer *renderer, SDL_Window *window, AVDictionary *opt)
434 {
435  int ret = 0;
436  unsigned num_ext = 0;
437  const char **ext = NULL;
438  int w, h;
439  struct pl_log_params vk_log_params = {
440  .log_cb = vk_log_cb,
441  .log_level = PL_LOG_DEBUG,
442  .log_priv = renderer,
443  };
444  RendererContext *ctx = (RendererContext *) renderer;
445  AVDictionaryEntry *entry;
446 
447  ctx->vk_log = pl_log_create(PL_API_VER, &vk_log_params);
448 
449  if (!SDL_Vulkan_GetInstanceExtensions(window, &num_ext, NULL)) {
450  av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n",
451  SDL_GetError());
452  return AVERROR_EXTERNAL;
453  }
454 
455  ext = av_calloc(num_ext, sizeof(*ext));
456  if (!ext) {
457  ret = AVERROR(ENOMEM);
458  goto out;
459  }
460 
461  SDL_Vulkan_GetInstanceExtensions(window, &num_ext, ext);
462 
463  entry = av_dict_get(opt, "create_by_placebo", NULL, 0);
464  if (entry && strtol(entry->value, NULL, 10))
465  ret = create_vk_by_placebo(renderer, ext, num_ext, opt);
466  else
467  ret = create_vk_by_hwcontext(renderer, ext, num_ext, opt);
468  if (ret < 0)
469  goto out;
470 
471  if (!SDL_Vulkan_CreateSurface(window, ctx->inst, &ctx->vk_surface)) {
473  goto out;
474  }
475 
476  ctx->swapchain = pl_vulkan_create_swapchain(
477  ctx->placebo_vulkan,
478  pl_vulkan_swapchain_params(
479  .surface = ctx->vk_surface,
480  .present_mode = VK_PRESENT_MODE_FIFO_KHR));
481  if (!ctx->swapchain) {
483  goto out;
484  }
485 
486  SDL_Vulkan_GetDrawableSize(window, &w, &h);
487  pl_swapchain_resize(ctx->swapchain, &w, &h);
488 
489  ctx->renderer = pl_renderer_create(ctx->vk_log, ctx->placebo_vulkan->gpu);
490  if (!ctx->renderer) {
492  goto out;
493  }
494 
495  ctx->vk_frame = av_frame_alloc();
496  if (!ctx->vk_frame) {
497  ret = AVERROR(ENOMEM);
498  goto out;
499  }
500 
501  ret = 0;
502 
503 out:
504  av_free(ext);
505  return ret;
506 }
507 
508 static int get_hw_dev(VkRenderer *renderer, AVBufferRef **dev)
509 {
510  RendererContext *ctx = (RendererContext *) renderer;
511 
512  *dev = ctx->hw_device_ref;
513  return 0;
514 }
515 
516 static int create_hw_frame(VkRenderer *renderer, AVFrame *frame)
517 {
518  RendererContext *ctx = (RendererContext *) renderer;
519  AVHWFramesContext *src_hw_frame = (AVHWFramesContext *)
521  AVHWFramesContext *hw_frame;
522  AVVulkanFramesContext *vk_frame_ctx;
523  int ret;
524 
525  if (ctx->hw_frame_ref) {
526  hw_frame = (AVHWFramesContext *) ctx->hw_frame_ref->data;
527 
528  if (hw_frame->width == frame->width &&
529  hw_frame->height == frame->height &&
530  hw_frame->sw_format == src_hw_frame->sw_format)
531  return 0;
532 
533  av_buffer_unref(&ctx->hw_frame_ref);
534  }
535 
536  if (!ctx->constraints) {
538  ctx->hw_device_ref, NULL);
539  if (!ctx->constraints)
540  return AVERROR(ENOMEM);
541  }
542 
543  // Check constraints and skip create hwframe. Don't take it as error since
544  // we can fallback to memory copy from GPU to CPU.
545  if ((ctx->constraints->max_width &&
546  ctx->constraints->max_width < frame->width) ||
547  (ctx->constraints->max_height &&
548  ctx->constraints->max_height < frame->height) ||
549  (ctx->constraints->min_width &&
550  ctx->constraints->min_width > frame->width) ||
551  (ctx->constraints->min_height &&
552  ctx->constraints->min_height > frame->height))
553  return 0;
554 
555  if (ctx->constraints->valid_sw_formats) {
556  enum AVPixelFormat *sw_formats = ctx->constraints->valid_sw_formats;
557  while (*sw_formats != AV_PIX_FMT_NONE) {
558  if (*sw_formats == src_hw_frame->sw_format)
559  break;
560  sw_formats++;
561  }
562  if (*sw_formats == AV_PIX_FMT_NONE)
563  return 0;
564  }
565 
566  ctx->hw_frame_ref = av_hwframe_ctx_alloc(ctx->hw_device_ref);
567  if (!ctx->hw_frame_ref)
568  return AVERROR(ENOMEM);
569 
570  hw_frame = (AVHWFramesContext *) ctx->hw_frame_ref->data;
571  hw_frame->format = AV_PIX_FMT_VULKAN;
572  hw_frame->sw_format = src_hw_frame->sw_format;
573  hw_frame->width = frame->width;
574  hw_frame->height = frame->height;
575 
577  vk_frame_ctx = hw_frame->hwctx;
578  vk_frame_ctx->flags = AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE;
579  }
580 
581  ret = av_hwframe_ctx_init(ctx->hw_frame_ref);
582  if (ret < 0) {
583  av_log(renderer, AV_LOG_ERROR, "Create hwframe context failed, %s\n",
584  av_err2str(ret));
585  return ret;
586  }
587 
588  av_hwframe_transfer_get_formats(ctx->hw_frame_ref,
590  &ctx->transfer_formats, 0);
591 
592  return 0;
593 }
594 
595 static inline int check_hw_transfer(RendererContext *ctx, AVFrame *frame)
596 {
597  if (!ctx->hw_frame_ref || !ctx->transfer_formats)
598  return 0;
599 
600  for (int i = 0; ctx->transfer_formats[i] != AV_PIX_FMT_NONE; i++)
601  if (ctx->transfer_formats[i] == frame->format)
602  return 1;
603 
604  return 0;
605 }
606 
607 static inline int move_to_output_frame(RendererContext *ctx, AVFrame *frame)
608 {
609  int ret = av_frame_copy_props(ctx->vk_frame, frame);
610  if (ret < 0)
611  return ret;
613  av_frame_move_ref(frame, ctx->vk_frame);
614  return 0;
615 }
616 
617 static int map_frame(VkRenderer *renderer, AVFrame *frame, int use_hw_frame)
618 {
619  RendererContext *ctx = (RendererContext *) renderer;
620  int ret;
621 
622  if (use_hw_frame && !ctx->hw_frame_ref)
623  return AVERROR(ENOSYS);
624 
625  // Try map data first
626  av_frame_unref(ctx->vk_frame);
627  if (use_hw_frame) {
628  ctx->vk_frame->hw_frames_ctx = av_buffer_ref(ctx->hw_frame_ref);
629  ctx->vk_frame->format = AV_PIX_FMT_VULKAN;
630  }
631  ret = av_hwframe_map(ctx->vk_frame, frame, 0);
632  if (!ret)
633  return move_to_output_frame(ctx, frame);
634 
635  if (ret != AVERROR(ENOSYS))
636  av_log(NULL, AV_LOG_FATAL, "Map frame failed: %s\n", av_err2str(ret));
637  return ret;
638 }
639 
640 static int transfer_frame(VkRenderer *renderer, AVFrame *frame, int use_hw_frame)
641 {
642  RendererContext *ctx = (RendererContext *) renderer;
643  int ret;
644 
645  if (use_hw_frame && !check_hw_transfer(ctx, frame))
646  return AVERROR(ENOSYS);
647 
648  av_frame_unref(ctx->vk_frame);
649  if (use_hw_frame)
650  av_hwframe_get_buffer(ctx->hw_frame_ref, ctx->vk_frame, 0);
651  ret = av_hwframe_transfer_data(ctx->vk_frame, frame, 1);
652  if (!ret)
653  return move_to_output_frame(ctx, frame);
654 
655  if (ret != AVERROR(ENOSYS))
656  av_log(NULL, AV_LOG_FATAL, "Transfer frame failed: %s\n",
657  av_err2str(ret));
658  return ret;
659 }
660 
662 {
663  int ret;
664 
665  if (!frame->hw_frames_ctx)
666  return 0;
667 
669  return 0;
670 
671  ret = create_hw_frame(renderer, frame);
672  if (ret < 0)
673  return ret;
674 
675  for (int use_hw = 1; use_hw >=0; use_hw--) {
676  ret = map_frame(renderer, frame, use_hw);
677  if (!ret)
678  return 0;
679  if (ret != AVERROR(ENOSYS))
680  return ret;
681 
682  ret = transfer_frame(renderer, frame, use_hw);
683  if (!ret)
684  return 0;
685  if (ret != AVERROR(ENOSYS))
686  return ret;
687  }
688 
689  return ret;
690 }
691 
692 static int display(VkRenderer *renderer, AVFrame *frame)
693 {
694  struct pl_swapchain_frame swap_frame = {0};
695  struct pl_frame pl_frame = {0};
696  struct pl_frame target = {0};
697  RendererContext *ctx = (RendererContext *) renderer;
698  int ret = 0;
699 
701  if (ret < 0)
702  return ret;
703 
704  if (!pl_map_avframe_ex(ctx->placebo_vulkan->gpu, &pl_frame, pl_avframe_params(
705  .frame = frame,
706  .tex = ctx->tex))) {
707  av_log(NULL, AV_LOG_ERROR, "pl_map_avframe_ex failed\n");
708  return AVERROR_EXTERNAL;
709  }
710 
711  if (!pl_swapchain_start_frame(ctx->swapchain, &swap_frame)) {
712  av_log(NULL, AV_LOG_ERROR, "start frame failed\n");
714  goto out;
715  }
716 
717  pl_frame_from_swapchain(&target, &swap_frame);
718  if (!pl_render_image(ctx->renderer, &pl_frame, &target,
719  &pl_render_default_params)) {
720  av_log(NULL, AV_LOG_ERROR, "pl_render_image failed\n");
722  goto out;
723  }
724 
725  if (!pl_swapchain_submit_frame(ctx->swapchain)) {
726  av_log(NULL, AV_LOG_ERROR, "pl_swapchain_submit_frame failed\n");
728  goto out;
729  }
730  pl_swapchain_swap_buffers(ctx->swapchain);
731 
732 out:
733  pl_unmap_avframe(ctx->placebo_vulkan->gpu, &pl_frame);
734  return ret;
735 }
736 
737 static int resize(VkRenderer *renderer, int width, int height)
738 {
739  RendererContext *ctx = (RendererContext *) renderer;
740 
741  if (!pl_swapchain_resize(ctx->swapchain, &width, &height))
742  return AVERROR_EXTERNAL;
743  return 0;
744 }
745 
746 static void destroy(VkRenderer *renderer)
747 {
748  RendererContext *ctx = (RendererContext *) renderer;
749  PFN_vkDestroySurfaceKHR vkDestroySurfaceKHR;
750 
751  av_frame_free(&ctx->vk_frame);
752  av_freep(&ctx->transfer_formats);
753  av_hwframe_constraints_free(&ctx->constraints);
754  av_buffer_unref(&ctx->hw_frame_ref);
755 
756  if (ctx->placebo_vulkan) {
757  for (int i = 0; i < FF_ARRAY_ELEMS(ctx->tex); i++)
758  pl_tex_destroy(ctx->placebo_vulkan->gpu, &ctx->tex[i]);
759  pl_renderer_destroy(&ctx->renderer);
760  pl_swapchain_destroy(&ctx->swapchain);
761  pl_vulkan_destroy(&ctx->placebo_vulkan);
762  }
763 
764  if (ctx->vk_surface) {
765  vkDestroySurfaceKHR = (PFN_vkDestroySurfaceKHR)
766  ctx->get_proc_addr(ctx->inst, "vkDestroySurfaceKHR");
767  vkDestroySurfaceKHR(ctx->inst, ctx->vk_surface, NULL);
768  ctx->vk_surface = NULL;
769  }
770 
771  av_buffer_unref(&ctx->hw_device_ref);
772  pl_vk_inst_destroy(&ctx->placebo_instance);
773 
774  pl_log_destroy(&ctx->vk_log);
775 }
776 
777 static const AVClass vulkan_renderer_class = {
778  .class_name = "Vulkan Renderer",
779  .item_name = av_default_item_name,
780  .version = LIBAVUTIL_VERSION_INT,
781 };
782 
784 {
785  RendererContext *ctx = av_mallocz(sizeof(*ctx));
787 
788  if (!ctx)
789  return NULL;
790 
791  renderer = &ctx->api;
792  renderer->class = &vulkan_renderer_class;
793  renderer->get_hw_dev = get_hw_dev;
794  renderer->create = create;
795  renderer->display = display;
796  renderer->resize = resize;
797  renderer->destroy = destroy;
798 
799  return renderer;
800 }
801 
802 #else
803 
805 {
806  return NULL;
807 }
808 
809 #endif
810 
812  AVDictionary *opt)
813 {
814  return renderer->create(renderer, window, opt);
815 }
816 
818 {
819  return renderer->get_hw_dev(renderer, dev);
820 }
821 
823 {
824  return renderer->display(renderer, frame);
825 }
826 
828 {
829  return renderer->resize(renderer, width, height);
830 }
831 
833 {
834  renderer->destroy(renderer);
835 }
AVHWDeviceContext::hwctx
void * hwctx
The format-specific data, allocated and freed by libavutil along with this context.
Definition: hwcontext.h:93
VkRenderer::create
int(* create)(VkRenderer *renderer, SDL_Window *window, AVDictionary *dict)
Definition: ffplay_renderer.c:51
AVVulkanDeviceContext::phys_dev
VkPhysicalDevice phys_dev
Physical device.
Definition: hwcontext_vulkan.h:65
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:186
AV_PIX_FMT_CUDA
@ AV_PIX_FMT_CUDA
HW acceleration through CUDA.
Definition: pixfmt.h:260
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:71
convert_frame
static int convert_frame(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
Convert a frame from linear RGB to logspace LAB, and accumulate channel totals for each row Convert f...
Definition: vf_grayworld.c:131
level
uint8_t level
Definition: svq3.c:204
AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
@ AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
Definition: hwcontext_vulkan.h:170
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
out
FILE * out
Definition: movenc.c:54
destroy
static void destroy(struct ResampleContext **c)
Definition: soxr_resample.c:64
av_bprint_init
void av_bprint_init(AVBPrint *buf, unsigned size_init, unsigned size_max)
Definition: bprint.c:69
AV_LOG_QUIET
#define AV_LOG_QUIET
Print no output.
Definition: log.h:162
AVBufferRef::data
uint8_t * data
The data buffer.
Definition: buffer.h:90
ffplay_renderer.h
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:100
av_hwframe_ctx_init
int av_hwframe_ctx_init(AVBufferRef *ref)
Finalize the context before use.
Definition: hwcontext.c:330
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:340
AVVulkanDeviceContext::get_proc_addr
PFN_vkGetInstanceProcAddr get_proc_addr
Pointer to the instance-provided vkGetInstanceProcAddr loading function.
Definition: hwcontext_vulkan.h:55
optional_device_exts
static const VulkanOptExtension optional_device_exts[]
Definition: hwcontext_vulkan.c:423
AVFrame::width
int width
Definition: frame.h:412
w
uint8_t w
Definition: llviddspenc.c:38
av_hwframe_ctx_alloc
AVBufferRef * av_hwframe_ctx_alloc(AVBufferRef *device_ref_in)
Allocate an AVHWFramesContext tied to a given device context.
Definition: hwcontext.c:245
AVHWDeviceContext::user_opaque
void * user_opaque
Arbitrary user data, to be used e.g.
Definition: hwcontext.h:110
av_hwframe_map
int av_hwframe_map(AVFrame *dst, const AVFrame *src, int flags)
Map a hardware frame.
Definition: hwcontext.c:786
vk_renderer_create
int vk_renderer_create(VkRenderer *renderer, SDL_Window *window, AVDictionary *opt)
Definition: ffplay_renderer.c:811
AVVulkanDeviceContext::queue_family_decode_index
int queue_family_decode_index
Queue family index for video decode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:138
AVVulkanDeviceContext::inst
VkInstance inst
Vulkan instance.
Definition: hwcontext_vulkan.h:60
AVDictionary
Definition: dict.c:34
av_buffer_ref
AVBufferRef * av_buffer_ref(const AVBufferRef *buf)
Create a new reference to an AVBuffer.
Definition: buffer.c:103
map_frame
static bool map_frame(pl_gpu gpu, pl_tex *tex, const struct pl_source_frame *src, struct pl_frame *out)
Definition: vf_libplacebo.c:962
vk_renderer_destroy
void vk_renderer_destroy(VkRenderer *renderer)
Definition: ffplay_renderer.c:832
av_hwdevice_get_hwframe_constraints
AVHWFramesConstraints * av_hwdevice_get_hwframe_constraints(AVBufferRef *ref, const void *hwconfig)
Get the constraints on HW frames given a device and the HW-specific configuration to be used with tha...
Definition: hwcontext.c:573
av_hwdevice_ctx_init
int av_hwdevice_ctx_init(AVBufferRef *ref)
Finalize the device context before use.
Definition: hwcontext.c:205
AVVulkanDeviceContext::nb_decode_queues
int nb_decode_queues
Definition: hwcontext_vulkan.h:139
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:383
AVVulkanDeviceContext::queue_family_index
int queue_family_index
Queue family index for graphics operations, and the number of queues enabled for it.
Definition: hwcontext_vulkan.h:108
AV_HWDEVICE_TYPE_VULKAN
@ AV_HWDEVICE_TYPE_VULKAN
Definition: hwcontext.h:39
AVHWFramesConstraints
This struct describes the constraints on hardware frames attached to a given device with a hardware-s...
Definition: hwcontext.h:457
window
static SDL_Window * window
Definition: ffplay.c:364
AVVulkanDeviceContext::queue_family_comp_index
int queue_family_comp_index
Queue family index for compute operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:122
AVVulkanFramesContext
Allocated as AVHWFramesContext.hwctx, used to set pool-specific options.
Definition: hwcontext_vulkan.h:176
AV_BPRINT_SIZE_AUTOMATIC
#define AV_BPRINT_SIZE_AUTOMATIC
AV_DICT_DONT_STRDUP_VAL
#define AV_DICT_DONT_STRDUP_VAL
Take ownership of a value that's been allocated with av_malloc() or another memory allocation functio...
Definition: dict.h:79
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:62
av_frame_alloc
AVFrame * av_frame_alloc(void)
Allocate an AVFrame and set its fields to default values.
Definition: frame.c:88
AV_LOG_TRACE
#define AV_LOG_TRACE
Extremely verbose debugging, useful for libav* development.
Definition: log.h:206
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
vk_get_renderer
VkRenderer * vk_get_renderer(void)
Definition: ffplay_renderer.c:804
FF_ARRAY_ELEMS
#define FF_ARRAY_ELEMS(a)
Definition: sinewin_tablegen.c:29
av_hwdevice_ctx_alloc
AVBufferRef * av_hwdevice_ctx_alloc(enum AVHWDeviceType type)
Allocate an AVHWDeviceContext for a given hardware type.
Definition: hwcontext.c:147
AVVulkanDeviceContext::nb_graphics_queues
int nb_graphics_queues
Definition: hwcontext_vulkan.h:109
av_dict_get
AVDictionaryEntry * av_dict_get(const AVDictionary *m, const char *key, const AVDictionaryEntry *prev, int flags)
Get a dictionary entry with matching key.
Definition: dict.c:62
av_hwframe_constraints_free
void av_hwframe_constraints_free(AVHWFramesConstraints **constraints)
Free an AVHWFrameConstraints structure.
Definition: hwcontext.c:598
width
#define width
decode_index
static int decode_index(SGAVideoContext *s, AVFrame *frame)
Definition: sga.c:180
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:201
ctx
AVFormatContext * ctx
Definition: movenc.c:48
vk_renderer_get_hw_dev
int vk_renderer_get_hw_dev(VkRenderer *renderer, AVBufferRef **dev)
Definition: ffplay_renderer.c:817
vk_renderer_display
int vk_renderer_display(VkRenderer *renderer, AVFrame *frame)
Definition: ffplay_renderer.c:822
renderer
static SDL_Renderer * renderer
Definition: ffplay.c:365
frame
static AVFrame * frame
Definition: demux_decode.c:54
if
if(ret)
Definition: filter_design.txt:179
AVVulkanDeviceContext
Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_vulkan.h:44
LIBAVUTIL_VERSION_INT
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
VkRenderer
Definition: ffplay_renderer.c:48
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
NULL
#define NULL
Definition: coverity.c:32
AVHWFramesContext::sw_format
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
Definition: hwcontext.h:226
AVERROR_PATCHWELCOME
#define AVERROR_PATCHWELCOME
Not yet implemented in FFmpeg, patches welcome.
Definition: error.h:64
av_frame_copy_props
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:736
av_buffer_unref
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it.
Definition: buffer.c:139
AVVulkanDeviceContext::nb_enabled_dev_extensions
int nb_enabled_dev_extensions
Definition: hwcontext_vulkan.h:99
create
static struct ResampleContext * create(struct ResampleContext *c, int out_rate, int in_rate, int filter_size, int phase_shift, int linear, double cutoff, enum AVSampleFormat format, enum SwrFilterType filter_type, double kaiser_beta, double precision, int cheby, int exact_rational)
Definition: soxr_resample.c:32
AVVulkanDeviceContext::unlock_queue
void(* unlock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Similar to lock_queue(), unlocks a queue.
Definition: hwcontext_vulkan.h:152
av_default_item_name
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:237
AVVulkanDeviceContext::enabled_inst_extensions
const char *const * enabled_inst_extensions
Enabled instance extensions.
Definition: hwcontext_vulkan.h:87
AVVulkanDeviceContext::queue_family_tx_index
int queue_family_tx_index
Queue family index for transfer operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:115
index
int index
Definition: gxfenc.c:89
av_bprint_finalize
int av_bprint_finalize(AVBPrint *buf, char **ret_str)
Finalize a print buffer.
Definition: bprint.c:240
av_err2str
#define av_err2str(errnum)
Convenience macro, the return value should be used only directly in function arguments but never stan...
Definition: error.h:121
AVFrame::format
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
Definition: frame.h:427
VkRenderer::get_hw_dev
int(* get_hw_dev)(VkRenderer *renderer, AVBufferRef **dev)
Definition: ffplay_renderer.c:53
height
#define height
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
av_dict_free
void av_dict_free(AVDictionary **pm)
Free all the memory allocated for an AVDictionary struct and all keys and values.
Definition: dict.c:225
AVVulkanDeviceContext::lock_queue
void(* lock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Locks a queue, preventing other threads from submitting any command buffers to this queue.
Definition: hwcontext_vulkan.h:147
AV_LOG_INFO
#define AV_LOG_INFO
Standard information.
Definition: log.h:191
bprint.h
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:255
av_frame_move_ref
void av_frame_move_ref(AVFrame *dst, AVFrame *src)
Move everything contained in src to dst and reset src.
Definition: frame.c:649
vk_renderer_resize
int vk_renderer_resize(VkRenderer *renderer, int width, int height)
Definition: ffplay_renderer.c:827
av_frame_unref
void av_frame_unref(AVFrame *frame)
Unreference all the buffers referenced by frame and reset the frame fields.
Definition: frame.c:622
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:254
av_calloc
void * av_calloc(size_t nmemb, size_t size)
Definition: mem.c:262
AVHWFramesContext
This struct describes a set or pool of "hardware" frames (i.e.
Definition: hwcontext.h:125
ret
ret
Definition: filter_design.txt:187
AV_LOG_FATAL
#define AV_LOG_FATAL
Something went wrong and recovery is not possible.
Definition: log.h:174
AVClass::class_name
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:71
VkRenderer::destroy
void(* destroy)(VkRenderer *renderer)
Definition: ffplay_renderer.c:59
av_hwdevice_ctx_create
int av_hwdevice_ctx_create(AVBufferRef **pdevice_ref, enum AVHWDeviceType type, const char *device, AVDictionary *opts, int flags)
Open a device of the specified type and create an AVHWDeviceContext for it.
Definition: hwcontext.c:607
av_bprintf
void av_bprintf(AVBPrint *buf, const char *fmt,...)
Definition: bprint.c:99
av_hwframe_transfer_data
int av_hwframe_transfer_data(AVFrame *dst, const AVFrame *src, int flags)
Copy data to or from a hw surface.
Definition: hwcontext.c:440
AVFrame::hw_frames_ctx
AVBufferRef * hw_frames_ctx
For hwaccel-format frames, this should be a reference to the AVHWFramesContext describing the frame.
Definition: frame.h:752
AVFormatContext::debug
int debug
Flags to enable debugging.
Definition: avformat.h:1634
AVFrame::height
int height
Definition: frame.h:412
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:72
AVVulkanDeviceContext::nb_comp_queues
int nb_comp_queues
Definition: hwcontext_vulkan.h:123
av_hwframe_transfer_get_formats
int av_hwframe_transfer_get_formats(AVBufferRef *hwframe_ref, enum AVHWFrameTransferDirection dir, enum AVPixelFormat **formats, int flags)
Get a list of possible source or target formats usable in av_hwframe_transfer_data().
Definition: hwcontext.c:378
AVVulkanDeviceContext::enabled_dev_extensions
const char *const * enabled_dev_extensions
Enabled device extensions.
Definition: hwcontext_vulkan.h:98
AVBufferRef
A reference to a data buffer.
Definition: buffer.h:82
AVVulkanDeviceContext::act_dev
VkDevice act_dev
Active device.
Definition: hwcontext_vulkan.h:70
AVVulkanDeviceContext::nb_enabled_inst_extensions
int nb_enabled_inst_extensions
Definition: hwcontext_vulkan.h:88
av_free
#define av_free(p)
Definition: tableprint_vlc.h:33
AVDictionaryEntry
Definition: dict.h:89
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
av_dict_set
int av_dict_set(AVDictionary **pm, const char *key, const char *value, int flags)
Set the given entry in *pm, overwriting an existing entry.
Definition: dict.c:88
AVVulkanDeviceContext::nb_tx_queues
int nb_tx_queues
Definition: hwcontext_vulkan.h:116
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
h
h
Definition: vp9dsp_template.c:2038
AVVulkanDeviceContext::device_features
VkPhysicalDeviceFeatures2 device_features
This structure should be set to the set of features that present and enabled during device creation.
Definition: hwcontext_vulkan.h:78
AVDictionaryEntry::value
char * value
Definition: dict.h:91
int
int
Definition: ffmpeg_filter.c:424
VkRenderer::display
int(* display)(VkRenderer *renderer, AVFrame *frame)
Definition: ffplay_renderer.c:55
av_hwframe_get_buffer
int av_hwframe_get_buffer(AVBufferRef *hwframe_ref, AVFrame *frame, int flags)
Allocate a new frame attached to the given AVHWFramesContext.
Definition: hwcontext.c:499
VkRenderer::resize
int(* resize)(VkRenderer *renderer, int width, int height)
Definition: ffplay_renderer.c:57
AV_HWFRAME_TRANSFER_DIRECTION_TO
@ AV_HWFRAME_TRANSFER_DIRECTION_TO
Transfer the data to the queried hw frame.
Definition: hwcontext.h:428