[FFmpeg-devel] [PATCH v3 5/5] libavfilter: VAAPI surface converter
wm4
nfxjfg at googlemail.com
Tue Jan 19 13:27:52 CET 2016
On Mon, 18 Jan 2016 22:53:33 +0000
Mark Thompson <sw at jkqxz.net> wrote:
> ---
> configure | 1 +
> libavfilter/Makefile | 1 +
> libavfilter/allfilters.c | 1 +
> libavfilter/vf_vaapi_conv.c | 480 ++++++++++++++++++++++++++++++++++++++++++++
> 4 files changed, 483 insertions(+)
> create mode 100644 libavfilter/vf_vaapi_conv.c
>
> diff --git a/configure b/configure
> index 9da8e8b..71c0bc0 100755
> --- a/configure
> +++ b/configure
> @@ -2913,6 +2913,7 @@ stereo3d_filter_deps="gpl"
> subtitles_filter_deps="avformat avcodec libass"
> super2xsai_filter_deps="gpl"
> tinterlace_filter_deps="gpl"
> +vaapi_conv_filter_deps="vaapi"
> vidstabdetect_filter_deps="libvidstab"
> vidstabtransform_filter_deps="libvidstab"
> pixfmts_super2xsai_test_deps="super2xsai_filter"
> diff --git a/libavfilter/Makefile b/libavfilter/Makefile
> index e3e3561..9a4ca12 100644
> --- a/libavfilter/Makefile
> +++ b/libavfilter/Makefile
> @@ -246,6 +246,7 @@ OBJS-$(CONFIG_TRANSPOSE_FILTER) += vf_transpose.o
> OBJS-$(CONFIG_TRIM_FILTER) += trim.o
> OBJS-$(CONFIG_UNSHARP_FILTER) += vf_unsharp.o
> OBJS-$(CONFIG_USPP_FILTER) += vf_uspp.o
> +OBJS-$(CONFIG_VAAPI) += vf_vaapi_conv.o
> OBJS-$(CONFIG_VECTORSCOPE_FILTER) += vf_vectorscope.o
> OBJS-$(CONFIG_VFLIP_FILTER) += vf_vflip.o
> OBJS-$(CONFIG_VIDSTABDETECT_FILTER) += vidstabutils.o vf_vidstabdetect.o
> diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
> index 1faf393..cfbfdca 100644
> --- a/libavfilter/allfilters.c
> +++ b/libavfilter/allfilters.c
> @@ -266,6 +266,7 @@ void avfilter_register_all(void)
> REGISTER_FILTER(TRIM, trim, vf);
> REGISTER_FILTER(UNSHARP, unsharp, vf);
> REGISTER_FILTER(USPP, uspp, vf);
> + REGISTER_FILTER(VAAPI_CONV, vaapi_conv, vf);
> REGISTER_FILTER(VECTORSCOPE, vectorscope, vf);
> REGISTER_FILTER(VFLIP, vflip, vf);
> REGISTER_FILTER(VIDSTABDETECT, vidstabdetect, vf);
> diff --git a/libavfilter/vf_vaapi_conv.c b/libavfilter/vf_vaapi_conv.c
> new file mode 100644
> index 0000000..5180e7c
> --- /dev/null
> +++ b/libavfilter/vf_vaapi_conv.c
> @@ -0,0 +1,480 @@
> +/*
> + * VAAPI converter (scaling and colour conversion).
> + *
> + * Copyright (C) 2016 Mark Thompson <mrt at jkqxz.net>
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#include "avfilter.h"
> +#include "formats.h"
> +#include "internal.h"
> +
> +#include "libavutil/avassert.h"
> +#include "libavutil/opt.h"
> +#include "libavutil/pixdesc.h"
> +#include "libavutil/vaapi.h"
> +
> +#include <va/va_vpp.h>
> +
> +typedef struct VAAPIConvContext {
> + const AVClass *class;
> +
> + AVVAAPIHardwareContext *hardware_context;
> + AVVAAPIPipelineConfig va_config;
> + AVVAAPIPipelineContext va_context;
> + int pipeline_initialised;
> +
> + int input_is_vaapi;
> + AVVAAPISurfaceConfig input_config;
> + AVVAAPISurfaceConfig output_config;
> +
> + int output_width;
> + int output_height;
> +
> + struct {
> + int64_t hardware_context;
> + int output_size[2];
> + } options;
> +
> +} VAAPIConvContext;
> +
> +
> +static int vaapi_conv_query_formats(AVFilterContext *avctx)
> +{
> + VAAPIConvContext *ctx = avctx->priv;
> + VAStatus vas;
> + VAConfigAttrib rt_format = {
> + .type = VAConfigAttribRTFormat
> + };
> + enum AVPixelFormat pix_fmt_list[16] = {
> + AV_PIX_FMT_VAAPI,
> + };
> + int pix_fmt_count = 1, err;
> +
> +#if 0
> + // The Intel driver doesn't return anything useful here - it only
> + // declares support for YUV 4:2:0 formats, despite working perfectly
> + // with 32-bit RGB ones. Given another usable platform, this will
> + // need to be updated.
> + vas = vaGetConfigAttributes(ctx->hardware_context->display,
> + VAProfileNone, VAEntrypointVideoProc,
> + &rt_format, 1);
> +#else
> + vas = VA_STATUS_SUCCESS;
> + rt_format.value = VA_RT_FORMAT_YUV420 | VA_RT_FORMAT_RGB32;
> +#endif
> + if(vas != VA_STATUS_SUCCESS) {
> + av_log(ctx, AV_LOG_ERROR, "Failed to get config attributes: "
> + "%d (%s).\n", vas, vaErrorStr(vas));
> + } else {
> + if(rt_format.value & VA_RT_FORMAT_YUV420) {
> + av_log(ctx, AV_LOG_DEBUG, "YUV420 formats supported.\n");
> + pix_fmt_list[pix_fmt_count++] = AV_PIX_FMT_YUV420P;
> + pix_fmt_list[pix_fmt_count++] = AV_PIX_FMT_NV12;
> + }
> + if(rt_format.value & VA_RT_FORMAT_YUV422) {
> + av_log(ctx, AV_LOG_DEBUG, "YUV422 formats supported.\n");
> + pix_fmt_list[pix_fmt_count++] = AV_PIX_FMT_YUV422P;
> + pix_fmt_list[pix_fmt_count++] = AV_PIX_FMT_YUYV422;
> + }
> + if(rt_format.value & VA_RT_FORMAT_YUV444) {
> + av_log(ctx, AV_LOG_DEBUG, "YUV444 formats supported.\n");
> + pix_fmt_list[pix_fmt_count++] = AV_PIX_FMT_YUV444P;
> + }
> + if(rt_format.value & VA_RT_FORMAT_YUV400) {
> + av_log(ctx, AV_LOG_DEBUG, "Grayscale formats supported.\n");
> + pix_fmt_list[pix_fmt_count++] = AV_PIX_FMT_GRAY8;
> + }
> + if(rt_format.value & VA_RT_FORMAT_RGB32) {
> + av_log(ctx, AV_LOG_DEBUG, "RGB32 formats supported.\n");
> + pix_fmt_list[pix_fmt_count++] = AV_PIX_FMT_RGBA;
> + pix_fmt_list[pix_fmt_count++] = AV_PIX_FMT_BGRA;
> + pix_fmt_list[pix_fmt_count++] = AV_PIX_FMT_RGB0;
> + pix_fmt_list[pix_fmt_count++] = AV_PIX_FMT_BGR0;
> + }
> + }
> +
> + pix_fmt_list[pix_fmt_count] = AV_PIX_FMT_NONE;
> +
> + if(avctx->inputs[0]) {
> + err = ff_formats_ref(ff_make_format_list(pix_fmt_list),
> + &avctx->inputs[0]->out_formats);
> + if(err < 0)
> + return err;
> + }
> +
> + if(avctx->outputs[0]) {
> + // Truncate the list: no support for normal output yet.
> + pix_fmt_list[1] = AV_PIX_FMT_NONE;
> +
> + err = ff_formats_ref(ff_make_format_list(pix_fmt_list),
> + &avctx->outputs[0]->in_formats);
> + if(err < 0)
> + return err;
> + }
> +
> + return 0;
> +}
> +
> +static int vaapi_conv_config_pipeline(VAAPIConvContext *ctx)
> +{
> + AVVAAPIPipelineConfig *config = &ctx->va_config;
> + int err;
> +
> + config->profile = VAProfileNone;
> + config->entrypoint = VAEntrypointVideoProc;
> +
> + config->attribute_count = 0;
> +
> + av_vaapi_lock_hardware_context(ctx->hardware_context);
> +
> + err = ff_vaapi_pipeline_init(&ctx->va_context, ctx->hardware_context,
> + &ctx->va_config, &ctx->input_config,
> + &ctx->output_config);
> + if(err) {
> + av_log(ctx, AV_LOG_ERROR, "Failed to create video processing "
> + "pipeline: " "%d (%s).\n", err, av_err2str(err));
> + }
> +
> + av_vaapi_unlock_hardware_context(ctx->hardware_context);
> +
> + return err;
> +}
> +
> +static int vaapi_conv_config_input(AVFilterLink *inlink)
> +{
> + AVFilterContext *avctx = inlink->dst;
> + VAAPIConvContext *ctx = avctx->priv;
> + AVVAAPISurfaceConfig *config = &ctx->input_config;
> +
> + if(inlink->format == AV_PIX_FMT_VAAPI) {
> + av_log(ctx, AV_LOG_INFO, "Input is VAAPI (using incoming surfaces).\n");
> + ctx->input_is_vaapi = 1;
> + return 0;
> + }
> + ctx->input_is_vaapi = 0;
> +
> + config->rt_format = VA_RT_FORMAT_YUV420;
> + config->av_format = AV_PIX_FMT_VAAPI;
> +
> + switch(inlink->format) {
> + case AV_PIX_FMT_BGR0:
> + case AV_PIX_FMT_BGRA:
> + config->image_format.fourcc = VA_FOURCC_BGRX;
> + config->image_format.byte_order = VA_LSB_FIRST;
> + config->image_format.bits_per_pixel = 32;
> + config->image_format.depth = 8;
> + config->image_format.red_mask = 0x00ff0000;
> + config->image_format.green_mask = 0x0000ff00;
> + config->image_format.blue_mask = 0x000000ff;
> + config->image_format.alpha_mask = 0x00000000;
> + break;
> +
> + case AV_PIX_FMT_RGB0:
> + case AV_PIX_FMT_RGBA:
> + config->image_format.fourcc = VA_FOURCC_RGBX;
> + config->image_format.byte_order = VA_LSB_FIRST;
> + config->image_format.bits_per_pixel = 32;
> + config->image_format.depth = 8;
> + config->image_format.red_mask = 0x000000ff;
> + config->image_format.green_mask = 0x0000ff00;
> + config->image_format.blue_mask = 0x00ff0000;
> + config->image_format.alpha_mask = 0x00000000;
> + break;
> +
> + case AV_PIX_FMT_NV12:
> + config->image_format.fourcc = VA_FOURCC_NV12;
> + config->image_format.bits_per_pixel = 12;
> + break;
> + case AV_PIX_FMT_YUV420P:
> + config->image_format.fourcc = VA_FOURCC_YV12;
> + config->image_format.bits_per_pixel = 12;
> + break;
Doesn't this duplicate what vaQueryImageFormats() returns?
Also I think your AV_PIX_FMT <-> VA_FOURCC mappings are duplicated
somewhere else to a degree.
> +
> + default:
> + av_log(ctx, AV_LOG_ERROR, "Tried to configure with invalid input "
> + "format %s.\n", av_get_pix_fmt_name(inlink->format));
> + return AVERROR(EINVAL);
> + }
> +
> + config->count = 4;
> + config->width = inlink->w;
> + config->height = inlink->h;
> +
> + config->attribute_count = 0;
> +
> + if(ctx->output_width == 0)
> + ctx->output_width = inlink->w;
> + if(ctx->output_height == 0)
> + ctx->output_height = inlink->h;
> +
> + return 0;
> +}
> +
> +static int vaapi_conv_config_output(AVFilterLink *outlink)
> +{
> + AVFilterContext *avctx = outlink->src;
> + VAAPIConvContext *ctx = avctx->priv;
> + AVVAAPISurfaceConfig *config = &ctx->output_config;
> +
> + av_assert0(outlink->format == AV_PIX_FMT_VAAPI);
> + outlink->w = ctx->output_width;
> + outlink->h = ctx->output_height;
> +
> + config->rt_format = VA_RT_FORMAT_YUV420;
> + config->av_format = AV_PIX_FMT_VAAPI;
> +
> + config->image_format.fourcc = VA_FOURCC_NV12;
> + config->image_format.bits_per_pixel = 12;
> +
> + config->count = 4;
> + config->width = outlink->w;
> + config->height = outlink->h;
> +
> + config->attribute_count = 0;
> +
> + return vaapi_conv_config_pipeline(ctx);
> +}
> +
> +static int vaapi_conv_filter_frame(AVFilterLink *inlink, AVFrame *pic)
> +{
> + AVFilterContext *avctx = inlink->dst;
> + AVFilterLink *outlink = avctx->outputs[0];
> + VAAPIConvContext *ctx = avctx->priv;
> + AVVAAPISurface *input, *output;
> + AVFrame *input_image, *output_image;
> + VAProcPipelineParameterBuffer params;
> + VABufferID params_id;
> + VAStatus vas;
> + int err;
> +
> + av_log(ctx, AV_LOG_DEBUG, "Filter frame: %s, %ux%u.\n",
> + av_get_pix_fmt_name(pic->format), pic->width, pic->height);
> +
> + av_vaapi_lock_hardware_context(ctx->hardware_context);
> +
> + if(pic->data[3]) {
> + input_image = pic;
> + input = (AVVAAPISurface*)pic->buf[0]->data;
> +
> + } else {
> + input_image = av_frame_alloc();
> +
> + err = ff_vaapi_get_input_surface(&ctx->va_context, input_image);
> + if(err) {
> + av_log(ctx, AV_LOG_ERROR, "Failed to allocate surface to "
> + "copy input frame: %d (%s).\n", err, av_err2str(err));
> + goto fail;
> + }
> +
> + input = (AVVAAPISurface*)input_image->buf[0]->data;
> +
> + err = ff_vaapi_map_surface(input, 0);
> + if(err) {
> + av_log(ctx, AV_LOG_ERROR, "Failed to map input surface: "
> + "%d (%s).\n", err, av_err2str(err));
> + goto fail;
> + }
> +
> + err = ff_vaapi_copy_to_surface(pic, input);
> + if(err) {
> + av_log(ctx, AV_LOG_ERROR, "Failed to copy to input surface: "
> + "%d (%s).\n", err, av_err2str(err));
> + goto fail;
> + }
> +
> + err = ff_vaapi_unmap_surface(input, 1);
> + if(err) {
> + av_log(ctx, AV_LOG_ERROR, "Failed to unmap input surface: "
> + "%d (%s).\n", err, av_err2str(err));
> + goto fail;
> + }
So why is there not a simple upload function that just takes two
AVFrames?
> + }
> + av_log(ctx, AV_LOG_DEBUG, "Using surface %#x for input image.\n",
> + input->id);
> +
> + output_image = av_frame_alloc();
> + if(!output_image) {
> + err = AVERROR(ENOMEM);
> + goto fail;
> + }
> + av_frame_copy_props(output_image, pic);
> +
> + err = ff_vaapi_get_output_surface(&ctx->va_context, output_image);
> + if(err) {
> + av_log(ctx, AV_LOG_ERROR, "Failed to allocate surface for "
> + "output frame: %d (%s).\n", err, av_err2str(err));
> + goto fail;
> + }
> + output = (AVVAAPISurface*)output_image->buf[0]->data;
> + av_log(ctx, AV_LOG_DEBUG, "Using surface %#x for output image.\n",
> + output->id);
> +
> + memset(¶ms, 0, sizeof(params));
> +
> + params.surface = input->id;
> + params.surface_region = 0;
> + params.surface_color_standard = VAProcColorStandardNone;
Why not set it to the correct value? AVFrame.color_space or so.
> +
> + params.output_region = 0;
> + params.output_background_color = 0xff000000;
> + params.output_color_standard = VAProcColorStandardNone;
> +
> + params.pipeline_flags = 0;
> + params.filter_flags = VA_FILTER_SCALING_HQ;
> +
> + vas = vaBeginPicture(ctx->hardware_context->display,
> + ctx->va_context.context_id, output->id);
> + if(vas != VA_STATUS_SUCCESS) {
> + av_log(ctx, AV_LOG_ERROR, "Failed to attach new picture: "
> + "%d (%s).\n", vas, vaErrorStr(vas));
> + err = AVERROR_EXTERNAL;
> + goto fail;
> + }
> +
> + vas = vaCreateBuffer(ctx->hardware_context->display,
> + ctx->va_context.context_id,
> + VAProcPipelineParameterBufferType,
> + sizeof(params), 1, ¶ms, ¶ms_id);
> + if(vas != VA_STATUS_SUCCESS) {
> + av_log(ctx, AV_LOG_ERROR, "Failed to create parameter buffer: "
> + "%d (%s).\n", vas, vaErrorStr(vas));
> + err = AVERROR_EXTERNAL;
> + goto fail;
> + }
> + av_log(ctx, AV_LOG_DEBUG, "Pipeline parameter buffer is %#x.\n",
> + params_id);
> +
> + vas = vaRenderPicture(ctx->hardware_context->display,
> + ctx->va_context.context_id, ¶ms_id, 1);
> + if(vas != VA_STATUS_SUCCESS) {
> + av_log(ctx, AV_LOG_ERROR, "Failed to render parameter buffer: "
> + "%d (%s).\n", vas, vaErrorStr(vas));
> + err = AVERROR_EXTERNAL;
> + goto fail;
> + }
> +
> + vas = vaEndPicture(ctx->hardware_context->display,
> + ctx->va_context.context_id);
> + if(vas != VA_STATUS_SUCCESS) {
> + av_log(ctx, AV_LOG_ERROR, "Failed to start picture processing: "
> + "%d (%s).\n", vas, vaErrorStr(vas));
> + err = AVERROR_EXTERNAL;
> + goto fail;
> + }
> +
> + vas = vaSyncSurface(ctx->hardware_context->display, output->id);
> + if(vas != VA_STATUS_SUCCESS) {
> + av_log(ctx, AV_LOG_ERROR, "Failed to sync picture completion: "
> + "%d (%s).\n", vas, vaErrorStr(vas));
> + err = AVERROR_EXTERNAL;
> + goto fail;
> + }
> +
> + av_frame_free(&input_image);
> + if(pic->format != AV_PIX_FMT_VAAPI)
> + av_frame_free(&pic);
> +
> + av_vaapi_unlock_hardware_context(ctx->hardware_context);
> +
> + return ff_filter_frame(outlink, output_image);
> +
> + fail:
> + av_vaapi_unlock_hardware_context(ctx->hardware_context);
> + return err;
> +}
> +
> +static av_cold int vaapi_conv_init(AVFilterContext *avctx)
> +{
> + VAAPIConvContext *ctx = avctx->priv;
> +
> + if(ctx->options.hardware_context == 0) {
> + av_log(ctx, AV_LOG_ERROR, "VAAPI encode requires hardware context.\n");
> + av_assert0(0);
Uh, what? Crash (sometimes)?
> + return AVERROR(EINVAL);
> + }
> + ctx->hardware_context =
> + (AVVAAPIHardwareContext*)ctx->options.hardware_context;
> +
> + ctx->output_width = ctx->options.output_size[0];
> + ctx->output_height = ctx->options.output_size[1];
> +
> + return 0;
> +}
> +
> +static av_cold void vaapi_conv_uninit(AVFilterContext *avctx)
> +{
> + VAAPIConvContext *ctx = avctx->priv;
> +
> + if(ctx->pipeline_initialised) {
> + av_vaapi_lock_hardware_context(ctx->hardware_context);
> + ff_vaapi_pipeline_uninit(&ctx->va_context);
> + av_vaapi_unlock_hardware_context(ctx->hardware_context);
> + }
> +}
> +
> +
> +#define OFFSET(member) offsetof(VAAPIConvContext, options.member)
> +#define FLAGS (AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_FILTERING_PARAM)
> +static const AVOption vaapi_conv_options[] = {
> + { "hardware_context", "VAAPI hardware context",
> + OFFSET(hardware_context), AV_OPT_TYPE_INT64,
> + { .i64 = 0 }, INT64_MIN, INT64_MAX, AV_OPT_FLAG_VIDEO_PARAM },
Setting it this way is not ideal, but I guess there's no proper way yet.
> + { "size", "Set output size",
> + OFFSET(output_size), AV_OPT_TYPE_IMAGE_SIZE,
> + { 0 }, 0, 0, FLAGS },
> + { 0 },
> +};
> +
> +static const AVClass vaapi_conv_class = {
> + .class_name = "VAAPI/conv",
> + .item_name = av_default_item_name,
> + .option = vaapi_conv_options,
> + .version = LIBAVUTIL_VERSION_INT,
> +};
> +
> +static const AVFilterPad vaapi_conv_inputs[] = {
> + {
> + .name = "default",
> + .type = AVMEDIA_TYPE_VIDEO,
> + .filter_frame = &vaapi_conv_filter_frame,
> + .config_props = &vaapi_conv_config_input,
> + },
> + { 0 }
> +};
> +
> +static const AVFilterPad vaapi_conv_outputs[] = {
> + {
> + .name = "default",
> + .type = AVMEDIA_TYPE_VIDEO,
> + .config_props = &vaapi_conv_config_output,
> + },
> + { 0 }
> +};
> +
> +AVFilter ff_vf_vaapi_conv = {
> + .name = "vaapi_conv",
> + .description = NULL_IF_CONFIG_SMALL("Convert to/from VAAPI surfaces."),
> + .priv_size = sizeof(VAAPIConvContext),
> + .init = &vaapi_conv_init,
> + .uninit = &vaapi_conv_uninit,
> + .query_formats = &vaapi_conv_query_formats,
> + .inputs = vaapi_conv_inputs,
> + .outputs = vaapi_conv_outputs,
> + .priv_class = &vaapi_conv_class,
> +};
More information about the ffmpeg-devel
mailing list