FFmpeg
vf_stack_vaapi.c
Go to the documentation of this file.
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 /**
20  * @file
21  * Hardware accelerated hstack, vstack and xstack filters based on VA-API
22  */
23 
24 #include "config_components.h"
25 
26 #include "libavutil/opt.h"
27 #include "libavutil/common.h"
28 #include "libavutil/pixdesc.h"
29 #include "libavutil/eval.h"
30 #include "libavutil/hwcontext.h"
31 #include "libavutil/avstring.h"
32 #include "libavutil/avassert.h"
33 #include "libavutil/imgutils.h"
34 #include "libavutil/mathematics.h"
35 #include "libavutil/parseutils.h"
36 #include "libavutil/mem.h"
37 
38 #include "internal.h"
39 #include "filters.h"
40 #include "formats.h"
41 #include "video.h"
42 #include "framesync.h"
43 #include "vaapi_vpp.h"
44 
45 #define HSTACK_NAME "hstack_vaapi"
46 #define VSTACK_NAME "vstack_vaapi"
47 #define XSTACK_NAME "xstack_vaapi"
48 #define HWContext VAAPIVPPContext
49 #define StackHWContext StackVAAPIContext
50 #include "stack_internal.h"
51 
52 typedef struct StackVAAPIContext {
54 
55  VARectangle *rects;
57 
59 {
60  AVFilterContext *avctx = fs->parent;
61  AVFilterLink *outlink = avctx->outputs[0];
62  StackVAAPIContext *sctx = fs->opaque;
63  VAAPIVPPContext *vppctx = fs->opaque;
64  AVFrame *oframe, *iframe;
65  VAProcPipelineParameterBuffer *params = NULL;
66  VARectangle *irect = NULL;
67  int ret = 0;
68 
69  if (vppctx->va_context == VA_INVALID_ID)
70  return AVERROR(EINVAL);
71 
72  oframe = ff_get_video_buffer(outlink, outlink->w, outlink->h);
73  if (!oframe)
74  return AVERROR(ENOMEM);
75 
76  irect = av_calloc(avctx->nb_inputs, sizeof(*irect));
77  params = av_calloc(avctx->nb_inputs, sizeof(*params));
78  if (!irect || !params) {
79  ret = AVERROR(ENOMEM);
80  goto fail;
81  }
82 
83  for (int i = 0; i < avctx->nb_inputs; i++) {
84  ret = ff_framesync_get_frame(fs, i, &iframe, 0);
85  if (ret)
86  goto fail;
87 
88  if (i == 0) {
89  ret = av_frame_copy_props(oframe, iframe);
90  if (ret < 0)
91  goto fail;
92  }
93 
94  ret = ff_vaapi_vpp_init_params(avctx, &params[i], iframe, oframe);
95  if (ret)
96  goto fail;
97 
98  av_log(avctx, AV_LOG_DEBUG, "stack input %d: %s, %ux%u (%"PRId64").\n",
99  i, av_get_pix_fmt_name(iframe->format),
100  iframe->width, iframe->height, iframe->pts);
101  irect[i].x = 0;
102  irect[i].y = 0;
103  irect[i].width = iframe->width;
104  irect[i].height = iframe->height;
105  params[i].surface_region = &irect[i];
106  params[i].surface = (VASurfaceID)(uintptr_t)iframe->data[3];
107  params[i].output_region = &sctx->rects[i];
108 
109  if (sctx->base.fillcolor_enable)
110  params[i].output_background_color = (sctx->base.fillcolor[3] << 24 |
111  sctx->base.fillcolor[0] << 16 |
112  sctx->base.fillcolor[1] << 8 |
113  sctx->base.fillcolor[2]);
114  }
115 
116  oframe->pts = av_rescale_q(sctx->base.fs.pts, sctx->base.fs.time_base, outlink->time_base);
117  oframe->sample_aspect_ratio = outlink->sample_aspect_ratio;
118 
119  ret = ff_vaapi_vpp_render_pictures(avctx, params, avctx->nb_inputs, oframe);
120  if (ret)
121  goto fail;
122 
123  av_freep(&irect);
124  av_freep(&params);
125  return ff_filter_frame(outlink, oframe);
126 
127 fail:
128  av_freep(&irect);
129  av_freep(&params);
130  av_frame_free(&oframe);
131  return ret;
132 }
133 
134 static int config_output(AVFilterLink *outlink)
135 {
136  AVFilterContext *avctx = outlink->src;
137  StackVAAPIContext *sctx = avctx->priv;
138  VAAPIVPPContext *vppctx = avctx->priv;
139  AVFilterLink *inlink0 = avctx->inputs[0];
140  AVHWFramesContext *hwfc0 = NULL;
141  int ret;
142 
143  if (inlink0->format != AV_PIX_FMT_VAAPI || !inlink0->hw_frames_ctx || !inlink0->hw_frames_ctx->data) {
144  av_log(avctx, AV_LOG_ERROR, "Software pixel format is not supported.\n");
145  return AVERROR(EINVAL);
146  }
147 
148  hwfc0 = (AVHWFramesContext *)inlink0->hw_frames_ctx->data;
149 
150  for (int i = 1; i < sctx->base.nb_inputs; i++) {
151  AVFilterLink *inlink = avctx->inputs[i];
152  AVHWFramesContext *hwfc = NULL;
153 
154  if (inlink->format != AV_PIX_FMT_VAAPI || !inlink->hw_frames_ctx || !inlink->hw_frames_ctx->data) {
155  av_log(avctx, AV_LOG_ERROR, "Software pixel format is not supported.\n");
156  return AVERROR(EINVAL);
157  }
158 
159  hwfc = (AVHWFramesContext *)inlink->hw_frames_ctx->data;
160 
161  if (hwfc0->sw_format != hwfc->sw_format) {
162  av_log(avctx, AV_LOG_ERROR, "All inputs should have the same underlying software pixel format.\n");
163  return AVERROR(EINVAL);
164  }
165 
166  if (hwfc0->device_ctx != hwfc->device_ctx) {
167  av_log(avctx, AV_LOG_ERROR, "All inputs should have the same underlying vaapi devices.\n");
168  return AVERROR(EINVAL);
169  }
170  }
171 
172  ff_vaapi_vpp_config_input(inlink0);
173  vppctx->output_format = hwfc0->sw_format;
174 
175  ret = config_comm_output(outlink);
176  if (ret < 0)
177  return ret;
178 
179  for (int i = 0; i < sctx->base.nb_inputs; i++) {
180  sctx->rects[i].x = sctx->base.regions[i].x;
181  sctx->rects[i].y = sctx->base.regions[i].y;
182  sctx->rects[i].width = sctx->base.regions[i].width;
183  sctx->rects[i].height = sctx->base.regions[i].height;
184  }
185 
186  vppctx->output_width = outlink->w;
187  vppctx->output_height = outlink->h;
188 
189  return ff_vaapi_vpp_config_output(outlink);
190 }
191 
193 {
194  StackVAAPIContext *sctx = avctx->priv;
195  VAAPIVPPContext *vppctx = avctx->priv;
196  int ret;
197 
198  ret = stack_init(avctx);
199  if (ret)
200  return ret;
201 
202  /* stack region */
203  sctx->rects = av_calloc(sctx->base.nb_inputs, sizeof(*sctx->rects));
204  if (!sctx->rects)
205  return AVERROR(ENOMEM);
206 
207  ff_vaapi_vpp_ctx_init(avctx);
208  vppctx->output_format = AV_PIX_FMT_NONE;
209 
210  return 0;
211 }
212 
214 {
215  StackVAAPIContext *sctx = avctx->priv;
216 
217  stack_uninit(avctx);
218 
219  av_freep(&sctx->rects);
220 }
221 
223 {
224  static const enum AVPixelFormat pixel_formats[] = {
227  };
228 
230 }
231 
232 #include "stack_internal.c"
233 
234 #if CONFIG_HSTACK_VAAPI_FILTER
235 
236 DEFINE_HSTACK_OPTIONS(vaapi);
237 DEFINE_STACK_FILTER(hstack, vaapi, "VA-API", 0);
238 
239 #endif
240 
241 #if CONFIG_VSTACK_VAAPI_FILTER
242 
243 DEFINE_VSTACK_OPTIONS(vaapi);
244 DEFINE_STACK_FILTER(vstack, vaapi, "VA-API", 0);
245 
246 #endif
247 
248 #if CONFIG_XSTACK_VAAPI_FILTER
249 
250 DEFINE_XSTACK_OPTIONS(vaapi);
251 DEFINE_STACK_FILTER(xstack, vaapi, "VA-API", 0);
252 
253 #endif
ff_get_video_buffer
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:112
ff_vaapi_vpp_ctx_init
void ff_vaapi_vpp_ctx_init(AVFilterContext *avctx)
Definition: vaapi_vpp.c:715
StackItemRegion::x
int x
Definition: stack_internal.h:29
StackVAAPIContext::rects
VARectangle * rects
Definition: vf_stack_vaapi.c:55
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:71
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
StackItemRegion::y
int y
Definition: stack_internal.h:30
StackVAAPIContext::base
StackBaseContext base
Definition: vf_stack_vaapi.c:53
config_output
static int config_output(AVFilterLink *outlink)
Definition: vf_stack_vaapi.c:134
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1015
AVBufferRef::data
uint8_t * data
The data buffer.
Definition: buffer.h:90
ff_framesync_get_frame
int ff_framesync_get_frame(FFFrameSync *fs, unsigned in, AVFrame **rframe, unsigned get)
Get the current frame in an input.
Definition: framesync.c:270
vaapi_stack_uninit
static av_cold void vaapi_stack_uninit(AVFilterContext *avctx)
Definition: vf_stack_vaapi.c:213
inlink
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
Definition: filter_design.txt:212
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:160
FFFrameSync::time_base
AVRational time_base
Time base for the output events.
Definition: framesync.h:184
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:374
pixdesc.h
AVFrame::pts
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:486
AVFrame::width
int width
Definition: frame.h:446
ff_vaapi_vpp_render_pictures
int ff_vaapi_vpp_render_pictures(AVFilterContext *avctx, VAProcPipelineParameterBuffer *params_list, int cout, AVFrame *output_frame)
Definition: vaapi_vpp.c:640
mathematics.h
FFFrameSync
Frame sync structure.
Definition: framesync.h:168
video.h
StackItemRegion::height
int height
Definition: stack_internal.h:32
AVFrame::data
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:395
formats.h
StackBaseContext::fs
FFFrameSync fs
Definition: stack_internal.h:38
AVFilterContext::priv
void * priv
private data for use by the filter
Definition: avfilter.h:422
fail
#define fail()
Definition: checkasm.h:182
StackItemRegion::width
int width
Definition: stack_internal.h:31
DEFINE_STACK_FILTER
#define DEFINE_STACK_FILTER(category, api, capi, filter_flags)
Definition: stack_internal.c:336
avassert.h
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
av_cold
#define av_cold
Definition: attributes.h:90
stack_uninit
static av_cold void stack_uninit(AVFilterContext *avctx)
Definition: stack_internal.c:286
ff_set_common_formats_from_list
int ff_set_common_formats_from_list(AVFilterContext *ctx, const int *fmts)
Equivalent to ff_set_common_formats(ctx, ff_make_format_list(fmts))
Definition: formats.c:874
filters.h
DEFINE_HSTACK_OPTIONS
#define DEFINE_HSTACK_OPTIONS(api)
Definition: stack_internal.c:312
VAAPIVPPContext::output_width
int output_width
Definition: vaapi_vpp.h:53
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:201
VAAPIVPPContext::output_format
enum AVPixelFormat output_format
Definition: vaapi_vpp.h:52
av_rescale_q
int64_t av_rescale_q(int64_t a, AVRational bq, AVRational cq)
Rescale a 64-bit integer by 2 rational numbers.
Definition: mathematics.c:142
pixel_formats
static enum AVPixelFormat pixel_formats[]
Definition: vf_sr.c:64
StackBaseContext::fillcolor
uint8_t fillcolor[4]
Definition: stack_internal.h:40
if
if(ret)
Definition: filter_design.txt:179
NULL
#define NULL
Definition: coverity.c:32
AVHWFramesContext::sw_format
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
Definition: hwcontext.h:210
av_frame_copy_props
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:709
fs
#define fs(width, name, subs,...)
Definition: cbs_vp9.c:200
vaapi_stack_query_formats
static int vaapi_stack_query_formats(AVFilterContext *avctx)
Definition: vf_stack_vaapi.c:222
AVFilterContext::inputs
AVFilterLink ** inputs
array of pointers to input links
Definition: avfilter.h:415
parseutils.h
ff_vaapi_vpp_config_input
int ff_vaapi_vpp_config_input(AVFilterLink *inlink)
Definition: vaapi_vpp.c:75
DEFINE_XSTACK_OPTIONS
#define DEFINE_XSTACK_OPTIONS(api)
Definition: stack_internal.c:326
FFFrameSync::pts
int64_t pts
Timestamp of the current event.
Definition: framesync.h:189
stack_internal.h
eval.h
vaapi_vpp.h
AVFilterContext::nb_inputs
unsigned nb_inputs
number of input pads
Definition: avfilter.h:416
for
for(k=2;k<=8;++k)
Definition: h264pred_template.c:425
DEFINE_VSTACK_OPTIONS
#define DEFINE_VSTACK_OPTIONS(api)
Definition: stack_internal.c:319
AVFrame::format
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
Definition: frame.h:461
AV_PIX_FMT_VAAPI
@ AV_PIX_FMT_VAAPI
Hardware acceleration through VA-API, data[3] contains a VASurfaceID.
Definition: pixfmt.h:126
StackBaseContext::fillcolor_enable
int fillcolor_enable
Definition: stack_internal.h:41
internal.h
StackBaseContext
Definition: stack_internal.h:35
VAAPIVPPContext::output_height
int output_height
Definition: vaapi_vpp.h:54
StackBaseContext::nb_inputs
int nb_inputs
Definition: stack_internal.h:45
StackBaseContext::regions
StackItemRegion * regions
Definition: stack_internal.h:42
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:256
common.h
config_comm_output
static int config_comm_output(AVFilterLink *outlink)
Definition: stack_internal.c:53
av_calloc
void * av_calloc(size_t nmemb, size_t size)
Definition: mem.c:264
AVHWFramesContext
This struct describes a set or pool of "hardware" frames (i.e.
Definition: hwcontext.h:115
ret
ret
Definition: filter_design.txt:187
VAAPIVPPContext
Definition: vaapi_vpp.h:38
AVHWFramesContext::device_ctx
AVHWDeviceContext * device_ctx
The parent AVHWDeviceContext.
Definition: hwcontext.h:134
VAAPIVPPContext::va_context
VAContextID va_context
Definition: vaapi_vpp.h:46
AVFrame::sample_aspect_ratio
AVRational sample_aspect_ratio
Sample aspect ratio for the video frame, 0/1 if unknown/unspecified.
Definition: frame.h:481
vaapi_stack_init
static int vaapi_stack_init(AVFilterContext *avctx)
Definition: vf_stack_vaapi.c:192
AVFrame::height
int height
Definition: frame.h:446
framesync.h
ff_vaapi_vpp_config_output
int ff_vaapi_vpp_config_output(AVFilterLink *outlink)
Definition: vaapi_vpp.c:100
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:72
stack_init
static int stack_init(AVFilterContext *avctx)
Definition: stack_internal.c:223
AVFilterContext
An instance of a filter.
Definition: avfilter.h:407
mem.h
process_frame
static int process_frame(FFFrameSync *fs)
Definition: vf_stack_vaapi.c:58
stack_internal.c
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
imgutils.h
hwcontext.h
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
StackVAAPIContext
Definition: vf_stack_vaapi.c:52
avstring.h
av_get_pix_fmt_name
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:2885
ff_vaapi_vpp_init_params
int ff_vaapi_vpp_init_params(AVFilterContext *avctx, VAProcPipelineParameterBuffer *params, const AVFrame *input_frame, AVFrame *output_frame)
Definition: vaapi_vpp.c:534
AVFilterContext::outputs
AVFilterLink ** outputs
array of pointers to output links
Definition: avfilter.h:419