FFmpeg
vf_avgblur_vulkan.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) Lynne
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include "libavutil/random_seed.h"
22 #include "libavutil/vulkan_spirv.h"
23 #include "libavutil/opt.h"
24 #include "vulkan_filter.h"
25 
26 #include "filters.h"
27 #include "video.h"
28 
29 typedef struct AvgBlurVulkanContext {
31 
35  VkSampler sampler;
37 
38  /* Push constants / options */
39  struct {
40  float filter_norm[4];
42  } opts;
43 
44  int size_x;
45  int size_y;
46  int planes;
48 
49 static const char blur_kernel[] = {
50  C(0, void distort(const ivec2 pos, const int idx) )
51  C(0, { )
52  C(1, vec4 sum = vec4(0); )
53  C(1, for (int y = -filter_len.y; y <= filter_len.y; y++) )
54  C(1, for (int x = -filter_len.x; x <= filter_len.x; x++) )
55  C(2, sum += texture(input_img[idx], pos + ivec2(x, y)); )
56  C(0, )
57  C(1, imageStore(output_img[idx], pos, sum * filter_norm); )
58  C(0, } )
59 };
60 
62 {
63  int err;
64  uint8_t *spv_data;
65  size_t spv_len;
66  void *spv_opaque = NULL;
67  AvgBlurVulkanContext *s = ctx->priv;
68  FFVulkanContext *vkctx = &s->vkctx;
69  const int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
70  FFVulkanShader *shd;
71  FFVkSPIRVCompiler *spv;
73 
74  spv = ff_vk_spirv_init();
75  if (!spv) {
76  av_log(ctx, AV_LOG_ERROR, "Unable to initialize SPIR-V compiler!\n");
77  return AVERROR_EXTERNAL;
78  }
79 
80  s->qf = ff_vk_qf_find(vkctx, VK_QUEUE_COMPUTE_BIT, 0);
81  if (!s->qf) {
82  av_log(ctx, AV_LOG_ERROR, "Device has no compute queues\n");
83  err = AVERROR(ENOTSUP);
84  goto fail;
85  }
86 
87  RET(ff_vk_exec_pool_init(vkctx, s->qf, &s->e, s->qf->num*4, 0, 0, 0, NULL));
88  RET(ff_vk_init_sampler(vkctx, &s->sampler, 1, VK_FILTER_LINEAR));
89  RET(ff_vk_shader_init(vkctx, &s->shd, "avgblur",
90  VK_SHADER_STAGE_COMPUTE_BIT,
91  NULL, 0,
92  32, 1, 1,
93  0));
94  shd = &s->shd;
95 
97  {
98  .name = "input_img",
99  .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
100  .dimensions = 2,
101  .elems = planes,
102  .stages = VK_SHADER_STAGE_COMPUTE_BIT,
103  .samplers = DUP_SAMPLER(s->sampler),
104  },
105  {
106  .name = "output_img",
107  .type = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
108  .mem_layout = ff_vk_shader_rep_fmt(s->vkctx.output_format, FF_VK_REP_FLOAT),
109  .mem_quali = "writeonly",
110  .dimensions = 2,
111  .elems = planes,
112  .stages = VK_SHADER_STAGE_COMPUTE_BIT,
113  },
114  };
115 
116  RET(ff_vk_shader_add_descriptor_set(vkctx, shd, desc, 2, 0, 0));
117 
118  GLSLC(0, layout(push_constant, std430) uniform pushConstants { );
119  GLSLC(1, vec4 filter_norm; );
120  GLSLC(1, ivec2 filter_len; );
121  GLSLC(0, }; );
122  GLSLC(0, );
123 
124  ff_vk_shader_add_push_const(&s->shd, 0, sizeof(s->opts),
125  VK_SHADER_STAGE_COMPUTE_BIT);
126 
127  GLSLD( blur_kernel );
128  GLSLC(0, void main() );
129  GLSLC(0, { );
130  GLSLC(1, ivec2 size; );
131  GLSLC(1, const ivec2 pos = ivec2(gl_GlobalInvocationID.xy); );
132  for (int i = 0; i < planes; i++) {
133  GLSLC(0, );
134  GLSLF(1, size = imageSize(output_img[%i]); ,i);
135  GLSLC(1, if (!IS_WITHIN(pos, size)) );
136  GLSLC(2, return; );
137  if (s->planes & (1 << i)) {
138  GLSLF(1, distort(pos, %i); ,i);
139  } else {
140  GLSLF(1, vec4 res = texture(input_img[%i], pos); ,i);
141  GLSLF(1, imageStore(output_img[%i], pos, res); ,i);
142  }
143  }
144  GLSLC(0, } );
145 
146  RET(spv->compile_shader(vkctx, spv, &s->shd, &spv_data, &spv_len, "main",
147  &spv_opaque));
148  RET(ff_vk_shader_link(vkctx, &s->shd, spv_data, spv_len, "main"));
149 
150  RET(ff_vk_shader_register_exec(vkctx, &s->e, &s->shd));
151 
152  s->initialized = 1;
153  s->opts.filter_len[0] = s->size_x - 1;
154  s->opts.filter_len[1] = s->size_y - 1;
155 
156  s->opts.filter_norm[0] = s->opts.filter_len[0]*2 + 1;
157  s->opts.filter_norm[0] = 1.0/(s->opts.filter_norm[0]*s->opts.filter_norm[0]);
158  s->opts.filter_norm[1] = s->opts.filter_norm[0];
159  s->opts.filter_norm[2] = s->opts.filter_norm[0];
160  s->opts.filter_norm[3] = s->opts.filter_norm[0];
161 
162 fail:
163  if (spv_opaque)
164  spv->free_shader(spv, &spv_opaque);
165  if (spv)
166  spv->uninit(&spv);
167 
168  return err;
169 }
170 
172 {
173  int err;
174  AVFrame *out = NULL;
175  AVFilterContext *ctx = link->dst;
176  AvgBlurVulkanContext *s = ctx->priv;
177  AVFilterLink *outlink = ctx->outputs[0];
178 
179  out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
180  if (!out) {
181  err = AVERROR(ENOMEM);
182  goto fail;
183  }
184 
185  if (!s->initialized)
186  RET(init_filter(ctx, in));
187 
188  RET(ff_vk_filter_process_simple(&s->vkctx, &s->e, &s->shd,
189  out, in, s->sampler, &s->opts, sizeof(s->opts)));
190 
191  err = av_frame_copy_props(out, in);
192  if (err < 0)
193  goto fail;
194 
195  av_frame_free(&in);
196 
197  return ff_filter_frame(outlink, out);
198 
199 fail:
200  av_frame_free(&in);
201  av_frame_free(&out);
202  return err;
203 }
204 
206 {
207  AvgBlurVulkanContext *s = avctx->priv;
208  FFVulkanContext *vkctx = &s->vkctx;
209  FFVulkanFunctions *vk = &vkctx->vkfn;
210 
211  ff_vk_exec_pool_free(vkctx, &s->e);
212  ff_vk_shader_free(vkctx, &s->shd);
213 
214  if (s->sampler)
215  vk->DestroySampler(vkctx->hwctx->act_dev, s->sampler,
216  vkctx->hwctx->alloc);
217 
218  ff_vk_uninit(&s->vkctx);
219 
220  s->initialized = 0;
221 }
222 
223 #define OFFSET(x) offsetof(AvgBlurVulkanContext, x)
224 #define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
226  { "sizeX", "Set horizontal radius", OFFSET(size_x), AV_OPT_TYPE_INT, { .i64 = 3 }, 1, 32, .flags = FLAGS },
227  { "sizeY", "Set vertical radius", OFFSET(size_y), AV_OPT_TYPE_INT, { .i64 = 3 }, 1, 32, .flags = FLAGS },
228  { "planes", "Set planes to filter (bitmask)", OFFSET(planes), AV_OPT_TYPE_INT, {.i64 = 0xF}, 0, 0xF, .flags = FLAGS },
229  { NULL },
230 };
231 
232 AVFILTER_DEFINE_CLASS(avgblur_vulkan);
233 
235  {
236  .name = "default",
237  .type = AVMEDIA_TYPE_VIDEO,
238  .filter_frame = &avgblur_vulkan_filter_frame,
239  .config_props = &ff_vk_filter_config_input,
240  },
241 };
242 
244  {
245  .name = "default",
246  .type = AVMEDIA_TYPE_VIDEO,
247  .config_props = &ff_vk_filter_config_output,
248  },
249 };
250 
252  .p.name = "avgblur_vulkan",
253  .p.description = NULL_IF_CONFIG_SMALL("Apply avgblur mask to input video"),
254  .p.priv_class = &avgblur_vulkan_class,
255  .p.flags = AVFILTER_FLAG_HWDEVICE,
256  .priv_size = sizeof(AvgBlurVulkanContext),
262  .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
263 };
ff_get_video_buffer
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:116
AvgBlurVulkanContext
Definition: vf_avgblur_vulkan.c:29
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
FLAGS
#define FLAGS
Definition: vf_avgblur_vulkan.c:224
avgblur_vulkan_outputs
static const AVFilterPad avgblur_vulkan_outputs[]
Definition: vf_avgblur_vulkan.c:243
ff_vk_shader_free
void ff_vk_shader_free(FFVulkanContext *s, FFVulkanShader *shd)
Free a shader.
Definition: vulkan.c:2572
ff_vk_shader_init
int ff_vk_shader_init(FFVulkanContext *s, FFVulkanShader *shd, const char *name, VkPipelineStageFlags stage, const char *extensions[], int nb_extensions, int lg_x, int lg_y, int lg_z, uint32_t required_subgroup_size)
Initialize a shader object, with a specific set of extensions, type+bind, local group size,...
Definition: vulkan.c:1715
out
FILE * out
Definition: movenc.c:55
avgblur_vulkan_inputs
static const AVFilterPad avgblur_vulkan_inputs[]
Definition: vf_avgblur_vulkan.c:234
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1078
RET
#define RET(x)
Definition: vulkan.h:67
ff_vk_filter_process_simple
int ff_vk_filter_process_simple(FFVulkanContext *vkctx, FFVkExecPool *e, FFVulkanShader *shd, AVFrame *out_f, AVFrame *in_f, VkSampler sampler, void *push_src, size_t push_size)
Submit a compute shader with a zero/one input and single out for execution.
Definition: vulkan_filter.c:242
ff_vk_exec_pool_init
int ff_vk_exec_pool_init(FFVulkanContext *s, AVVulkanDeviceQueueFamily *qf, FFVkExecPool *pool, int nb_contexts, int nb_queries, VkQueryType query_type, int query_64bit, const void *query_create_pnext)
Allocates/frees an execution pool.
Definition: vulkan.c:296
planes
static const struct @475 planes[]
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:163
FILTER_INPUTS
#define FILTER_INPUTS(array)
Definition: filters.h:262
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:410
AvgBlurVulkanContext::shd
FFVulkanShader shd
Definition: vf_avgblur_vulkan.c:36
ff_vk_filter_init
int ff_vk_filter_init(AVFilterContext *avctx)
General lavfi IO functions.
Definition: vulkan_filter.c:233
AVOption
AVOption.
Definition: opt.h:429
ff_vk_uninit
void ff_vk_uninit(FFVulkanContext *s)
Frees main context.
Definition: vulkan.c:2611
FFVkSPIRVCompiler::uninit
void(* uninit)(struct FFVkSPIRVCompiler **ctx)
Definition: vulkan_spirv.h:32
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:203
avgblur_vulkan_uninit
static void avgblur_vulkan_uninit(AVFilterContext *avctx)
Definition: vf_avgblur_vulkan.c:205
AvgBlurVulkanContext::initialized
int initialized
Definition: vf_avgblur_vulkan.c:32
AvgBlurVulkanContext::sampler
VkSampler sampler
Definition: vf_avgblur_vulkan.c:35
blur_kernel
static const char blur_kernel[]
Definition: vf_avgblur_vulkan.c:49
video.h
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(avgblur_vulkan)
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:379
av_pix_fmt_count_planes
int av_pix_fmt_count_planes(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3288
AVVulkanDeviceContext::alloc
const VkAllocationCallbacks * alloc
Custom memory allocator, else NULL.
Definition: hwcontext_vulkan.h:63
AvgBlurVulkanContext::filter_norm
float filter_norm[4]
Definition: vf_avgblur_vulkan.c:40
AVFilterContext::priv
void * priv
private data for use by the filter
Definition: avfilter.h:272
fail
#define fail()
Definition: checkasm.h:193
vulkan_filter.h
avgblur_vulkan_options
static const AVOption avgblur_vulkan_options[]
Definition: vf_avgblur_vulkan.c:225
ff_vk_shader_register_exec
int ff_vk_shader_register_exec(FFVulkanContext *s, FFVkExecPool *pool, FFVulkanShader *shd)
Register a shader with an exec pool.
Definition: vulkan.c:2212
ff_vk_shader_add_descriptor_set
int ff_vk_shader_add_descriptor_set(FFVulkanContext *s, FFVulkanShader *shd, FFVulkanDescriptorSetBinding *desc, int nb, int singular, int print_to_shader_only)
Add descriptor to a shader.
Definition: vulkan.c:2079
AVFilterPad
A filter pad used for either input or output.
Definition: filters.h:38
C
s EdgeDetect Foobar g libavfilter vf_edgedetect c libavfilter vf_foobar c edit libavfilter and add an entry for foobar following the pattern of the other filters edit libavfilter allfilters and add an entry for foobar following the pattern of the other filters configure make j< whatever > ffmpeg ffmpeg i you should get a foobar png with Lena edge detected That s your new playground is ready Some little details about what s going which in turn will define variables for the build system and the C
Definition: writing_filters.txt:58
GLSLC
#define GLSLC(N, S)
Definition: vulkan.h:44
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:209
av_cold
#define av_cold
Definition: attributes.h:90
main
int main
Definition: dovi_rpuenc.c:37
FFFilter
Definition: filters.h:265
s
#define s(width, name)
Definition: cbs_vp9.c:198
filters.h
FF_VK_REP_FLOAT
@ FF_VK_REP_FLOAT
Definition: vulkan.h:376
ctx
AVFormatContext * ctx
Definition: movenc.c:49
GLSLD
#define GLSLD(D)
Definition: vulkan.h:59
ff_vk_exec_pool_free
void ff_vk_exec_pool_free(FFVulkanContext *s, FFVkExecPool *pool)
Definition: vulkan.c:233
FILTER_OUTPUTS
#define FILTER_OUTPUTS(array)
Definition: filters.h:263
link
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a link
Definition: filter_design.txt:23
AvgBlurVulkanContext::e
FFVkExecPool e
Definition: vf_avgblur_vulkan.c:33
ff_vk_shader_rep_fmt
const char * ff_vk_shader_rep_fmt(enum AVPixelFormat pix_fmt, enum FFVkShaderRepFormat rep_fmt)
Definition: vulkan.c:1322
NULL
#define NULL
Definition: coverity.c:32
av_frame_copy_props
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:726
DUP_SAMPLER
#define DUP_SAMPLER(x)
Definition: vulkan.h:73
AvgBlurVulkanContext::opts
struct AvgBlurVulkanContext::@334 opts
OFFSET
#define OFFSET(x)
Definition: vf_avgblur_vulkan.c:223
ff_vk_filter_config_output
int ff_vk_filter_config_output(AVFilterLink *outlink)
Definition: vulkan_filter.c:209
AvgBlurVulkanContext::size_y
int size_y
Definition: vf_avgblur_vulkan.c:45
FFVulkanContext
Definition: vulkan.h:266
AvgBlurVulkanContext::filter_len
int32_t filter_len[2]
Definition: vf_avgblur_vulkan.c:41
FF_FILTER_FLAG_HWFRAME_AWARE
#define FF_FILTER_FLAG_HWFRAME_AWARE
The filter is aware of hardware frames, and any hardware frame context should not be automatically pr...
Definition: filters.h:206
init
int(* init)(AVBSFContext *ctx)
Definition: dts2pts.c:368
AvgBlurVulkanContext::planes
int planes
Definition: vf_avgblur_vulkan.c:46
FFVulkanDescriptorSetBinding
Definition: vulkan.h:75
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:94
AVFILTER_FLAG_HWDEVICE
#define AVFILTER_FLAG_HWDEVICE
The filter can create hardware frames using AVFilterContext.hw_device_ctx.
Definition: avfilter.h:171
size
int size
Definition: twinvq_data.h:10344
FFVulkanShader
Definition: vulkan.h:182
AvgBlurVulkanContext::vkctx
FFVulkanContext vkctx
Definition: vf_avgblur_vulkan.c:30
init_filter
static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in)
Definition: vf_avgblur_vulkan.c:61
FFVkSPIRVCompiler::compile_shader
int(* compile_shader)(FFVulkanContext *s, struct FFVkSPIRVCompiler *ctx, FFVulkanShader *shd, uint8_t **data, size_t *size, const char *entrypoint, void **opaque)
Definition: vulkan_spirv.h:28
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
FFVkSPIRVCompiler
Definition: vulkan_spirv.h:26
layout
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel layout
Definition: filter_design.txt:18
uninit
static void uninit(AVBSFContext *ctx)
Definition: pcm_rechunk.c:68
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:256
ff_vk_shader_link
int ff_vk_shader_link(FFVulkanContext *s, FFVulkanShader *shd, uint8_t *spirv, size_t spirv_len, const char *entrypoint)
Link a shader into an executable.
Definition: vulkan.c:2004
AvgBlurVulkanContext::size_x
int size_x
Definition: vf_avgblur_vulkan.c:44
vulkan_spirv.h
AVFilterPad::name
const char * name
Pad name.
Definition: filters.h:44
FFVkSPIRVCompiler::free_shader
void(* free_shader)(struct FFVkSPIRVCompiler *ctx, void **opaque)
Definition: vulkan_spirv.h:31
FFVulkanContext::vkfn
FFVulkanFunctions vkfn
Definition: vulkan.h:270
FFVkExecPool
Definition: vulkan.h:244
pos
unsigned int pos
Definition: spdifenc.c:414
ff_vk_shader_add_push_const
int ff_vk_shader_add_push_const(FFVulkanShader *shd, int offset, int size, VkShaderStageFlagBits stage)
Add/update push constants for execution.
Definition: vulkan.c:1231
ff_vk_qf_find
AVVulkanDeviceQueueFamily * ff_vk_qf_find(FFVulkanContext *s, VkQueueFlagBits dev_family, VkVideoCodecOperationFlagBitsKHR vid_ops)
Chooses an appropriate QF.
Definition: vulkan.c:220
random_seed.h
GLSLF
#define GLSLF(N, S,...)
Definition: vulkan.h:54
AV_OPT_TYPE_INT
@ AV_OPT_TYPE_INT
Underlying C type is int.
Definition: opt.h:259
AVFilterContext
An instance of a filter.
Definition: avfilter.h:257
avgblur_vulkan_filter_frame
static int avgblur_vulkan_filter_frame(AVFilterLink *link, AVFrame *in)
Definition: vf_avgblur_vulkan.c:171
desc
const char * desc
Definition: libsvtav1.c:79
ff_vk_filter_config_input
int ff_vk_filter_config_input(AVFilterLink *inlink)
Definition: vulkan_filter.c:176
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
FFFilter::p
AVFilter p
The public AVFilter.
Definition: filters.h:269
FFVulkanContext::hwctx
AVVulkanDeviceContext * hwctx
Definition: vulkan.h:295
AVVulkanDeviceContext::act_dev
VkDevice act_dev
Active device.
Definition: hwcontext_vulkan.h:84
ff_vk_init_sampler
int ff_vk_init_sampler(FFVulkanContext *s, VkSampler *sampler, int unnorm_coords, VkFilter filt)
Create a sampler.
Definition: vulkan.c:1252
int32_t
int32_t
Definition: audioconvert.c:56
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
AVVulkanDeviceQueueFamily
Definition: hwcontext_vulkan.h:33
FILTER_SINGLE_PIXFMT
#define FILTER_SINGLE_PIXFMT(pix_fmt_)
Definition: filters.h:252
FFVulkanFunctions
Definition: vulkan_functions.h:263
AvgBlurVulkanContext::qf
AVVulkanDeviceQueueFamily * qf
Definition: vf_avgblur_vulkan.c:34
ff_vf_avgblur_vulkan
const FFFilter ff_vf_avgblur_vulkan
Definition: vf_avgblur_vulkan.c:251