FFmpeg
All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Modules Pages
hwcontext_vulkan.h
Go to the documentation of this file.
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 #ifndef AVUTIL_HWCONTEXT_VULKAN_H
20 #define AVUTIL_HWCONTEXT_VULKAN_H
21 
22 #if defined(_WIN32) && !defined(VK_USE_PLATFORM_WIN32_KHR)
23 #define VK_USE_PLATFORM_WIN32_KHR
24 #endif
25 #include <vulkan/vulkan.h>
26 
27 #include "pixfmt.h"
28 #include "frame.h"
29 #include "hwcontext.h"
30 
31 typedef struct AVVkFrame AVVkFrame;
32 
33 typedef struct AVVulkanDeviceQueueFamily {
34  /* Queue family index */
35  int idx;
36  /* Number of queues in the queue family in use */
37  int num;
38  /* Queue family capabilities. Must be non-zero.
39  * Flags may be removed to indicate the queue family may not be used
40  * for a given purpose. */
41  VkQueueFlagBits flags;
42  /* Vulkan implementations are allowed to list multiple video queues
43  * which differ in what they can encode or decode. */
44  VkVideoCodecOperationFlagBitsKHR video_caps;
46 
47 /**
48  * @file
49  * API-specific header for AV_HWDEVICE_TYPE_VULKAN.
50  *
51  * For user-allocated pools, AVHWFramesContext.pool must return AVBufferRefs
52  * with the data pointer set to an AVVkFrame.
53  */
54 
55 /**
56  * Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
57  * All of these can be set before init to change what the context uses
58  */
59 typedef struct AVVulkanDeviceContext {
60  /**
61  * Custom memory allocator, else NULL
62  */
63  const VkAllocationCallbacks *alloc;
64 
65  /**
66  * Pointer to a vkGetInstanceProcAddr loading function.
67  * If unset, will dynamically load and use libvulkan.
68  */
69  PFN_vkGetInstanceProcAddr get_proc_addr;
70 
71  /**
72  * Vulkan instance. Must be at least version 1.3.
73  */
74  VkInstance inst;
75 
76  /**
77  * Physical device
78  */
79  VkPhysicalDevice phys_dev;
80 
81  /**
82  * Active device
83  */
84  VkDevice act_dev;
85 
86  /**
87  * This structure should be set to the set of features that present and enabled
88  * during device creation. When a device is created by FFmpeg, it will default to
89  * enabling all that are present of the shaderImageGatherExtended,
90  * fragmentStoresAndAtomics, shaderInt64 and vertexPipelineStoresAndAtomics features.
91  */
92  VkPhysicalDeviceFeatures2 device_features;
93 
94  /**
95  * Enabled instance extensions.
96  * If supplying your own device context, set this to an array of strings, with
97  * each entry containing the specified Vulkan extension string to enable.
98  * Duplicates are possible and accepted.
99  * If no extensions are enabled, set these fields to NULL, and 0 respectively.
100  */
101  const char * const *enabled_inst_extensions;
103 
104  /**
105  * Enabled device extensions. By default, VK_KHR_external_memory_fd,
106  * VK_EXT_external_memory_dma_buf, VK_EXT_image_drm_format_modifier,
107  * VK_KHR_external_semaphore_fd and VK_EXT_external_memory_host are enabled if found.
108  * If supplying your own device context, these fields takes the same format as
109  * the above fields, with the same conditions that duplicates are possible
110  * and accepted, and that NULL and 0 respectively means no extensions are enabled.
111  */
112  const char * const *enabled_dev_extensions;
114 
115 #if FF_API_VULKAN_FIXED_QUEUES
116  /**
117  * Queue family index for graphics operations, and the number of queues
118  * enabled for it. If unavaiable, will be set to -1. Not required.
119  * av_hwdevice_create() will attempt to find a dedicated queue for each
120  * queue family, or pick the one with the least unrelated flags set.
121  * Queue indices here may overlap if a queue has to share capabilities.
122  */
127 
128  /**
129  * Queue family index for transfer operations and the number of queues
130  * enabled. Required.
131  */
136 
137  /**
138  * Queue family index for compute operations and the number of queues
139  * enabled. Required.
140  */
145 
146  /**
147  * Queue family index for video encode ops, and the amount of queues enabled.
148  * If the device doesn't support such, queue_family_encode_index will be -1.
149  * Not required.
150  */
155 
156  /**
157  * Queue family index for video decode ops, and the amount of queues enabled.
158  * If the device doesn't support such, queue_family_decode_index will be -1.
159  * Not required.
160  */
165 #endif
166 
167  /**
168  * Locks a queue, preventing other threads from submitting any command
169  * buffers to this queue.
170  * If set to NULL, will be set to lavu-internal functions that utilize a
171  * mutex.
172  */
173  void (*lock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index);
174 
175  /**
176  * Similar to lock_queue(), unlocks a queue. Must only be called after locking.
177  */
178  void (*unlock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index);
179 
180  /**
181  * Queue families used. Must be preferentially ordered. List may contain
182  * duplicates.
183  *
184  * For compatibility reasons, all the enabled queue families listed above
185  * (queue_family_(tx/comp/encode/decode)_index) must also be included in
186  * this list until they're removed after deprecation.
187  */
189  int nb_qf;
191 
192 /**
193  * Defines the behaviour of frame allocation.
194  */
195 typedef enum AVVkFrameFlags {
196  /* Unless this flag is set, autodetected flags will be OR'd based on the
197  * device and tiling during av_hwframe_ctx_init(). */
198  AV_VK_FRAME_FLAG_NONE = (1ULL << 0),
199 
200  /* Disables multiplane images.
201  * This is required to export/import images from CUDA. */
204 
205 /**
206  * Allocated as AVHWFramesContext.hwctx, used to set pool-specific options
207  */
208 typedef struct AVVulkanFramesContext {
209  /**
210  * Controls the tiling of allocated frames.
211  * If left as VK_IMAGE_TILING_OPTIMAL (0), will use optimal tiling.
212  * Can be set to VK_IMAGE_TILING_LINEAR to force linear images,
213  * or VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT to force DMABUF-backed
214  * images.
215  * @note Imported frames from other APIs ignore this.
216  */
217  VkImageTiling tiling;
218 
219  /**
220  * Defines extra usage of output frames. If non-zero, all flags MUST be
221  * supported by the VkFormat. Otherwise, will use supported flags amongst:
222  * - VK_IMAGE_USAGE_SAMPLED_BIT
223  * - VK_IMAGE_USAGE_STORAGE_BIT
224  * - VK_IMAGE_USAGE_TRANSFER_SRC_BIT
225  * - VK_IMAGE_USAGE_TRANSFER_DST_BIT
226  */
227  VkImageUsageFlagBits usage;
228 
229  /**
230  * Extension data for image creation.
231  * If DRM tiling is used, a VkImageDrmFormatModifierListCreateInfoEXT structure
232  * can be added to specify the exact modifier to use.
233  *
234  * Additional structures may be added at av_hwframe_ctx_init() time,
235  * which will be freed automatically on uninit(), so users must only free
236  * any structures they've allocated themselves.
237  */
239 
240  /**
241  * Extension data for memory allocation. Must have as many entries as
242  * the number of planes of the sw_format.
243  * This will be chained to VkExportMemoryAllocateInfo, which is used
244  * to make all pool images exportable to other APIs if the necessary
245  * extensions are present in enabled_dev_extensions.
246  */
248 
249  /**
250  * A combination of AVVkFrameFlags. Unless AV_VK_FRAME_FLAG_NONE is set,
251  * autodetected flags will be OR'd based on the device and tiling during
252  * av_hwframe_ctx_init().
253  */
255 
256  /**
257  * Flags to set during image creation. If unset, defaults to
258  * VK_IMAGE_CREATE_ALIAS_BIT.
259  */
260  VkImageCreateFlags img_flags;
261 
262  /**
263  * Vulkan format for each image. MUST be compatible with the pixel format.
264  * If unset, will be automatically set.
265  * There are at most two compatible formats for a frame - a multiplane
266  * format, and a single-plane multi-image format.
267  */
269 
270  /**
271  * Number of layers each image will have.
272  */
274 
275  /**
276  * Locks a frame, preventing other threads from changing frame properties.
277  * Users SHOULD only ever lock just before command submission in order
278  * to get accurate frame properties, and unlock immediately after command
279  * submission without waiting for it to finish.
280  *
281  * If unset, will be set to lavu-internal functions that utilize a mutex.
282  */
283  void (*lock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf);
284 
285  /**
286  * Similar to lock_frame(), unlocks a frame. Must only be called after locking.
287  */
288  void (*unlock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf);
290 
291 /*
292  * Frame structure.
293  *
294  * @note the size of this structure is not part of the ABI, to allocate
295  * you must use @av_vk_frame_alloc().
296  */
297 struct AVVkFrame {
298  /**
299  * Vulkan images to which the memory is bound to.
300  * May be one for multiplane formats, or multiple.
301  */
303 
304  /**
305  * Tiling for the frame.
306  */
307  VkImageTiling tiling;
308 
309  /**
310  * Memory backing the images. Either one, or as many as there are planes
311  * in the sw_format.
312  * In case of having multiple VkImages, but one memory, the offset field
313  * will indicate the bound offset for each image.
314  */
315  VkDeviceMemory mem[AV_NUM_DATA_POINTERS];
317 
318  /**
319  * OR'd flags for all memory allocated
320  */
321  VkMemoryPropertyFlagBits flags;
322 
323  /**
324  * Updated after every barrier. One per VkImage.
325  */
326  VkAccessFlagBits access[AV_NUM_DATA_POINTERS];
327  VkImageLayout layout[AV_NUM_DATA_POINTERS];
328 
329  /**
330  * Synchronization timeline semaphores, one for each VkImage.
331  * Must not be freed manually. Must be waited on at every submission using
332  * the value in sem_value, and must be signalled at every submission,
333  * using an incremented value.
334  */
335  VkSemaphore sem[AV_NUM_DATA_POINTERS];
336 
337  /**
338  * Up to date semaphore value at which each image becomes accessible.
339  * One per VkImage.
340  * Clients must wait on this value when submitting a command queue,
341  * and increment it when signalling.
342  */
344 
345  /**
346  * Internal data.
347  */
348  struct AVVkFrameInternal *internal;
349 
350  /**
351  * Describes the binding offset of each image to the VkDeviceMemory.
352  * One per VkImage.
353  */
355 
356  /**
357  * Queue family of the images. Must be VK_QUEUE_FAMILY_IGNORED if
358  * the image was allocated with the CONCURRENT concurrency option.
359  * One per VkImage.
360  */
362 };
363 
364 /**
365  * Allocates a single AVVkFrame and initializes everything as 0.
366  * @note Must be freed via av_free()
367  */
369 
370 /**
371  * Returns the optimal per-plane Vulkan format for a given sw_format,
372  * one for each plane.
373  * Returns NULL on unsupported formats.
374  */
376 
377 #endif /* AVUTIL_HWCONTEXT_VULKAN_H */
AVVulkanDeviceContext::phys_dev
VkPhysicalDevice phys_dev
Physical device.
Definition: hwcontext_vulkan.h:79
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:71
AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
@ AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
Definition: hwcontext_vulkan.h:202
AVVulkanDeviceContext::get_proc_addr
PFN_vkGetInstanceProcAddr get_proc_addr
Pointer to a vkGetInstanceProcAddr loading function.
Definition: hwcontext_vulkan.h:69
AVVulkanFramesContext::create_pnext
void * create_pnext
Extension data for image creation.
Definition: hwcontext_vulkan.h:238
AVVulkanDeviceContext::queue_family_encode_index
attribute_deprecated int queue_family_encode_index
Queue family index for video encode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:152
AVVulkanDeviceContext::inst
VkInstance inst
Vulkan instance.
Definition: hwcontext_vulkan.h:74
AVVulkanFramesContext::lock_frame
void(* lock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf)
Locks a frame, preventing other threads from changing frame properties.
Definition: hwcontext_vulkan.h:283
AVVulkanDeviceContext::alloc
const VkAllocationCallbacks * alloc
Custom memory allocator, else NULL.
Definition: hwcontext_vulkan.h:63
AVVkFrame::img
VkImage img[AV_NUM_DATA_POINTERS]
Vulkan images to which the memory is bound to.
Definition: hwcontext_vulkan.h:302
AVVkFrame::offset
ptrdiff_t offset[AV_NUM_DATA_POINTERS]
Describes the binding offset of each image to the VkDeviceMemory.
Definition: hwcontext_vulkan.h:354
AVVulkanFramesContext::flags
AVVkFrameFlags flags
A combination of AVVkFrameFlags.
Definition: hwcontext_vulkan.h:254
AVVulkanDeviceContext::nb_tx_queues
attribute_deprecated int nb_tx_queues
Definition: hwcontext_vulkan.h:135
AVVkFrame::mem
VkDeviceMemory mem[AV_NUM_DATA_POINTERS]
Memory backing the images.
Definition: hwcontext_vulkan.h:315
AVVulkanFramesContext
Allocated as AVHWFramesContext.hwctx, used to set pool-specific options.
Definition: hwcontext_vulkan.h:208
AVVulkanDeviceContext::queue_family_decode_index
attribute_deprecated int queue_family_decode_index
Queue family index for video decode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:162
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:61
AVVulkanDeviceQueueFamily::num
int num
Definition: hwcontext_vulkan.h:37
AV_VK_FRAME_FLAG_NONE
@ AV_VK_FRAME_FLAG_NONE
Definition: hwcontext_vulkan.h:198
AVVulkanDeviceContext::nb_encode_queues
attribute_deprecated int nb_encode_queues
Definition: hwcontext_vulkan.h:154
fc
#define fc(width, name, range_min, range_max)
Definition: cbs_av1.c:493
AVVulkanFramesContext::unlock_frame
void(* unlock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf)
Similar to lock_frame(), unlocks a frame.
Definition: hwcontext_vulkan.h:288
AVVulkanFramesContext::img_flags
VkImageCreateFlags img_flags
Flags to set during image creation.
Definition: hwcontext_vulkan.h:260
ctx
AVFormatContext * ctx
Definition: movenc.c:49
AVVulkanDeviceContext
Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_vulkan.h:59
AVVulkanDeviceContext::nb_enabled_dev_extensions
int nb_enabled_dev_extensions
Definition: hwcontext_vulkan.h:113
AVVkFrameInternal
Definition: hwcontext_vulkan.c:166
AVVkFrame::flags
VkMemoryPropertyFlagBits flags
OR'd flags for all memory allocated.
Definition: hwcontext_vulkan.h:321
AVVulkanFramesContext::alloc_pnext
void * alloc_pnext[AV_NUM_DATA_POINTERS]
Extension data for memory allocation.
Definition: hwcontext_vulkan.h:247
AVVulkanDeviceContext::unlock_queue
void(* unlock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Similar to lock_queue(), unlocks a queue.
Definition: hwcontext_vulkan.h:178
AVVulkanDeviceContext::nb_decode_queues
attribute_deprecated int nb_decode_queues
Definition: hwcontext_vulkan.h:164
AVVulkanDeviceContext::nb_qf
int nb_qf
Definition: hwcontext_vulkan.h:189
AVVulkanDeviceContext::queue_family_tx_index
attribute_deprecated int queue_family_tx_index
Queue family index for transfer operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:133
AVVulkanDeviceContext::enabled_inst_extensions
const char *const * enabled_inst_extensions
Enabled instance extensions.
Definition: hwcontext_vulkan.h:101
AVVulkanFramesContext::format
VkFormat format[AV_NUM_DATA_POINTERS]
Vulkan format for each image.
Definition: hwcontext_vulkan.h:268
AVVulkanFramesContext::usage
VkImageUsageFlagBits usage
Defines extra usage of output frames.
Definition: hwcontext_vulkan.h:227
AVVulkanDeviceContext::qf
AVVulkanDeviceQueueFamily qf[64]
Queue families used.
Definition: hwcontext_vulkan.h:188
index
int index
Definition: gxfenc.c:90
AVVkFrame::size
size_t size[AV_NUM_DATA_POINTERS]
Definition: hwcontext_vulkan.h:316
AVVkFrame::access
VkAccessFlagBits access[AV_NUM_DATA_POINTERS]
Updated after every barrier.
Definition: hwcontext_vulkan.h:326
av_vkfmt_from_pixfmt
const VkFormat * av_vkfmt_from_pixfmt(enum AVPixelFormat p)
Returns the optimal per-plane Vulkan format for a given sw_format, one for each plane.
Definition: hwcontext_stub.c:30
AVVkFrame
Definition: hwcontext_vulkan.h:297
AV_NUM_DATA_POINTERS
#define AV_NUM_DATA_POINTERS
Definition: frame.h:411
frame.h
attribute_deprecated
#define attribute_deprecated
Definition: attributes.h:104
AVVulkanDeviceContext::queue_family_index
attribute_deprecated int queue_family_index
Queue family index for graphics operations, and the number of queues enabled for it.
Definition: hwcontext_vulkan.h:124
av_vk_frame_alloc
AVVkFrame * av_vk_frame_alloc(void)
Allocates a single AVVkFrame and initializes everything as 0.
Definition: hwcontext_stub.c:35
AVVkFrame::queue_family
uint32_t queue_family[AV_NUM_DATA_POINTERS]
Queue family of the images.
Definition: hwcontext_vulkan.h:361
AVVulkanDeviceQueueFamily::idx
int idx
Definition: hwcontext_vulkan.h:35
AVVulkanDeviceContext::lock_queue
void(* lock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Locks a queue, preventing other threads from submitting any command buffers to this queue.
Definition: hwcontext_vulkan.h:173
AVVulkanDeviceContext::queue_family_comp_index
attribute_deprecated int queue_family_comp_index
Queue family index for compute operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:142
VkFormat
enum VkFormat VkFormat
Definition: hwcontext_stub.c:25
AVVkFrameFlags
AVVkFrameFlags
Defines the behaviour of frame allocation.
Definition: hwcontext_vulkan.h:195
AVVkFrame::sem
VkSemaphore sem[AV_NUM_DATA_POINTERS]
Synchronization timeline semaphores, one for each VkImage.
Definition: hwcontext_vulkan.h:335
AVHWFramesContext
This struct describes a set or pool of "hardware" frames (i.e.
Definition: hwcontext.h:116
AVVkFrame::tiling
VkImageTiling tiling
Tiling for the frame.
Definition: hwcontext_vulkan.h:307
pixfmt.h
AVVulkanDeviceContext::nb_graphics_queues
attribute_deprecated int nb_graphics_queues
Definition: hwcontext_vulkan.h:126
AVVulkanFramesContext::tiling
VkImageTiling tiling
Controls the tiling of allocated frames.
Definition: hwcontext_vulkan.h:217
AVVkFrame::sem_value
uint64_t sem_value[AV_NUM_DATA_POINTERS]
Up to date semaphore value at which each image becomes accessible.
Definition: hwcontext_vulkan.h:343
AVVulkanDeviceContext::enabled_dev_extensions
const char *const * enabled_dev_extensions
Enabled device extensions.
Definition: hwcontext_vulkan.h:112
AVVulkanFramesContext::nb_layers
int nb_layers
Number of layers each image will have.
Definition: hwcontext_vulkan.h:273
AVVkFrame::layout
VkImageLayout layout[AV_NUM_DATA_POINTERS]
Definition: hwcontext_vulkan.h:327
AVVulkanDeviceContext::act_dev
VkDevice act_dev
Active device.
Definition: hwcontext_vulkan.h:84
AVVulkanDeviceContext::nb_enabled_inst_extensions
int nb_enabled_inst_extensions
Definition: hwcontext_vulkan.h:102
hwcontext.h
AVVulkanDeviceQueueFamily
Definition: hwcontext_vulkan.h:33
AVVulkanDeviceContext::device_features
VkPhysicalDeviceFeatures2 device_features
This structure should be set to the set of features that present and enabled during device creation.
Definition: hwcontext_vulkan.h:92
AVVulkanDeviceQueueFamily::flags
VkQueueFlagBits flags
Definition: hwcontext_vulkan.h:41
AVVulkanDeviceQueueFamily::video_caps
VkVideoCodecOperationFlagBitsKHR video_caps
Definition: hwcontext_vulkan.h:44
AVVulkanDeviceContext::nb_comp_queues
attribute_deprecated int nb_comp_queues
Definition: hwcontext_vulkan.h:144