[FFmpeg-devel] [PATCH v6 1/6] libavcodec: VAAPI support infrastructure

[Mark Thompson]

On 08/02/16 12:08, wm4 wrote:
> On Sun, 7 Feb 2016 21:53:39 +0000
> Mark Thompson <sw at jkqxz.net> wrote:
>> diff --git a/libavcodec/vaapi_support.h b/libavcodec/vaapi_support.h
>> new file mode 100644
>> index 0000000..c159ee7
>> --- /dev/null
>> +++ b/libavcodec/vaapi_support.h
>> @@ -0,0 +1,284 @@
>> +/*
>> + * VAAPI helper functions.
>> + *
>> + * Copyright (C) 2016 Mark Thompson <mrt at jkqxz.net>
>> + *
>> + * This file is part of FFmpeg.
>> + *
>> + * FFmpeg is free software; you can redistribute it and/or
>> + * modify it under the terms of the GNU Lesser General Public
>> + * License as published by the Free Software Foundation; either
>> + * version 2.1 of the License, or (at your option) any later version.
>> + *
>> + * FFmpeg is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
>> + * Lesser General Public License for more details.
>> + *
>> + * You should have received a copy of the GNU Lesser General Public
>> + * License along with FFmpeg; if not, write to the Free Software
>> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
>> + */
>> +
>> +#ifndef AVCODEC_VAAPI_SUPPORT_H
>> +#define AVCODEC_VAAPI_SUPPORT_H
>> +
>> +/**
>> + * @defgroup lavc_vaapi VAAPI support
>> + * @ingroup lavc_misc
>> + * @{
>> + */
>> +
>> +#include <va/va.h>
>> +
>> +#include "libavutil/pixfmt.h"
>> +#include "libavutil/frame.h"
>> +
>> +#include "vaapi.h"
>> +
>> +
>> +/**
>> + * Structure defining a VAAPI hardware context.
>> + *
>> + * This is managed by the user, and must be passed to all components which
>> + * require it.
>> + *
>> + * Allocated using @ref av_vaapi_alloc_hardware_context().
>> + */
>> +typedef struct AVVAAPIHardwareContext {
>> +    /** Logging class.
>> +     * @ref av_vaapi_alloc_hardware_context sets this to a default value,
>> +     * and the user can override it if desired.
>> +     */
>> +    const AVClass *class;
>> +    /** libva display handle.
>> +     * Initialised by the user.
>> +     */
>> +    VADisplay display;
>> +    /** Context lock function.
>> +     * This should be set by the user to allow suitable mutual exclusion if
>> +     * multiple components might use this context simultaneously.
>> +     * Note that recursive locking must be supported.
>> +     */
>> +    void (*lock)(void *user_context);
>> +    /** Context unlock function. */
>> +    void (*unlock)(void *user_context);
>> +    /** User data passed as argument to lock and unlock. */
>> +    void *lock_user_context;
>> +} AVVAAPIHardwareContext;
>> +
>> +/** Allocate a new @ref AVVAAPIHardwareContext.
>> + *
>> + * Also fills in a default logging class.
>> + * Use @ref av_free() to free it after use.
>> + *
>> + * @return Newly-allocated context.
>> + */
>> +AVVAAPIHardwareContext *av_vaapi_alloc_hardware_context(void);
>> +
>> +/** Lock the context, using the user-supplied lock function.
>> + */
>> +void av_vaapi_lock_hardware_context(AVVAAPIHardwareContext *ctx);
>> +/** Unlock the context, using the user-supplied lock function.
>> + */
>> +void av_vaapi_unlock_hardware_context(AVVAAPIHardwareContext *ctx);
>> +
>> +
>> +/** Surface configuration structure.
>> + *
>> + * Defines the properties of surfaces to be allocated in an
>> + * @ref AVVAAPISurfacePool.
>> + */
>> +typedef struct AVVAAPISurfaceConfig {
>> +    /** Pixel format of the surfaces. */
>> +    enum AVPixelFormat av_format;
>> +
>> +    /** libva image format of the surfaces.
>> +     *
>> +     * In most cases, this is obtainable by @ref av_vaapi_get_image_format()
>> +     * on the av_format.
>> +     */
>> +    VAImageFormat va_format;
>> +
>> +    /** Width of the surface. */
>> +    unsigned int width;
>> +    /** Height of the surface. */
>> +    unsigned int height;
>> +
>> +    /** Number of extra attributes to be passed to vaCreateSurfaces(). */
>> +    unsigned int attribute_count;
>> +    /** Pointer to extra attributes (or null if attribute_count is zero). */
>> +    VASurfaceAttrib *attributes;
>> +} AVVAAPISurfaceConfig;
>
> No alloc function for this struct? (Although I can't think of anything
> that could possibly added to it in the future. Maybe not needed?)

Yeah.  I was thinking that unless libva changes very significantly, any 
different features are going to apply to the pool (which can be changed, see 
below) rather than the surfaces.

>> +
>> +/** Opaque surface pool structure.
>> + *
>> + * Manages a set of VAAPI surfaces contained in AVFrames.  Individual frames
>> + * use reference-counting to track lifetime.
>> + */
>> +typedef struct AVVAAPISurfacePool AVVAAPISurfacePool;
>> +
>> +/** Allocate a new @ref AVVAAPISurfacePool.
>> + *
>> + * @param hw_ctx VAAPI hardware context this pool will create surfaces in.
>> + * @return Newly-allocated surface pool.
>> + */
>> +AVVAAPISurfacePool *av_vaapi_alloc_surface_pool(AVVAAPIHardwareContext *hw_ctx);
>> +
>> +/** Initialise an @ref AVVAAPISurfacePool and allocate the surfaces in it.
>> + *
>> + * @param pool Pool to initialise (allocated by
>> + *     @ref av_vaapi_alloc_surface_pool()).
>> + * @param config Surface configuration for the allocated surfaces.  It need
>> + *     not last beyond this call (can be on the stack).
>> + * @param surface_count Number of surfaces to allocate in the pool.
>> + * @return Zero on success, or negative error code.
>> + */
>> +int av_vaapi_surface_pool_init(AVVAAPISurfacePool *pool,
>> +                               AVVAAPISurfaceConfig *config,
>> +                               int surface_count);
>> +
>
> Any particular reason why alloc and init are separate functions?

If adding new features later, you may need a way to get some additional option 
into the pool before init runs and allocates the surfaces.  Hence alloc is 
separate, so that you can add another function 
av_vaapi_surface_pool_change_behaviour_somehow() between the two.

>> +/** Uninitialise an @ref AVVAAPISurfacePool.
>> + *
>> + * Frees all surfaces which are not currently referenced.  Surfaces which are
>> + * still referenced elsewhere by AVFrames will be freed when their last
>> + * reference disappears.
>> + */
>> +int av_vaapi_surface_pool_uninit(AVVAAPISurfacePool *pool);
>> +
>> +/** Allocates a free surface from the surface pool.
>> + *
>> + * @param pool Pool to allocate from.
>> + * @param target AVFrame to attach references to.  Most properties of the frame
>> + *     itself will not be changed.
>> + * @return Zero on success, or negative error code.
>> + */
>> +int av_vaapi_surface_pool_get(AVVAAPISurfacePool *pool, AVFrame *target);
>> +
>> +
>> +/** Pipeline configuration structure.
>> + *
>> + * Defines the properties of a pipeline, to be passed as an argument to
>> + * @ref av_vaapi_pipeline_init().
>> + */
>
> Doxygen should make explicit that av_vaapi_alloc_pipeline_context must
> be used to allocate the struct.

Yes.

>> +typedef struct AVVAAPIPipelineConfig {
>> +    /** Codec profile to use (or VAProfileNone for non-codec operations). */
>> +    VAProfile profile;
>> +    /** Processing entry point. */
>> +    VAEntrypoint entrypoint;
>> +
>> +    /** Width of the output surfaces of this pipeline. */
>> +    unsigned int width;
>> +    /** Height of the output surfaces of this pipeline. */
>> +    unsigned int height;
>> +
>> +    /** Number of extra attributes to be passed to vaCreateContext(). */
>> +    unsigned int attribute_count;
>> +    /** Pointer to extra attribues (or null if attribute_count is zero). */
>> +    VAConfigAttrib *attributes;
>> +} AVVAAPIPipelineConfig;

Unlike surface config above, this is really just compressing the argument list 
to the init function.  Hence no alloc function.

>> +/** Opaque pipeline context structure.
>> + *
>> + * Manages a VAAPI pipeline context with an associated output surface pool.
>> + */
>> +typedef struct AVVAAPIPipelineContext AVVAAPIPipelineContext;
>> +
>> +/** Allocate a new @ref AVVAAPIPipelineContext.
>> + *
>> + * @param hw_ctx VAAPI hardware context this pipeline will be created in.
>> + * @return Newly-allocated pipeline context.
>> + */
>> +AVVAAPIPipelineContext *av_vaapi_alloc_pipeline_context(AVVAAPIHardwareContext *hw_ctx);
>> +
>> +/** Initialise an @ref AVVAAPIPipelineContext to be ready to process surfaces.
>> + *
>> + * @param ctx Pipeline context to initialised (allocated by
>> + *     @ref av_vaapi_alloc_pipeline_context()).
>> + * @param config Pipeline configuration.
>> + * @param pool Output surface pool.  If the pipeline has no output surfaces
>> + *     (for example, an intra-only encoder), this can be null.
>> + * @return Zero on success, or negative error code.
>> + */
>
> Does the function preallocate the pool itself? If so, how can you set
> the number of additional surfaces? (The user might need them for
> whatever reasons.)

No, the user has to supply the output pool themselves, already initialised with 
appropriate configuration.  (Multiple pipelines can target the same set of 
surfaces as output.)

>> +int av_vaapi_pipeline_init(AVVAAPIPipelineContext *ctx,
>> +                           AVVAAPIPipelineConfig *config,
>> +                           AVVAAPISurfacePool *pool);
>> +
>> +/** Uninitialise an @ref AVVAAPIPipelineContext.
>> + *
>> + * No operations should be outstanding on the pipeline when this is called.
>> + * The output surface pool is not affected.
>> + */
>> +int av_vaapi_pipeline_uninit(AVVAAPIPipelineContext *ctx);
>> +
>> +/** Fill a @ref vaapi_context for use with a VAAPI hwaccel decoder. */
>> +int av_vaapi_fill_vaapi_context(AVVAAPIPipelineContext *ctx,
>> +                                struct vaapi_context *vaapi_context);
>> +
>> +/** Return the context ID of the pipeline, for use in va*Picture calls. */
>> +VAContextID av_vaapi_get_pipeline_context_id(AVVAAPIPipelineContext *ctx);
>> +
>> +
>> +enum {
>> +    /** Map surface for read access. */
>> +    AV_VAAPI_MAP_READ   = 0x01,
>> +    /** Map surface for write access. */
>> +    AV_VAAPI_MAP_WRITE  = 0x02,
>> +    /** Map surface directly.  Fail if direct access is not possible. */
>> +    AV_VAAPI_MAP_DIRECT = 0x04,
>> +};
>> +
>> +/** Map a surface so that it can be operated on by the CPU.
>> + *
>> + * This may need to copy the frame data between some sort of external memory
>> + * and normal CPU-addressable memory.  The copy-in will only take place if
>> + * read access was requested.
>> + *
>> + * @param frame AVFrame to map.  Must have been allocated in an
>> + *     @ref AVVAAPISurfacePool
>> + * @param flags A set of AV_VAAPI_MAP flags, ored together.
>> + * @return Zero on success, or negative error code.
>> + */
>> +int av_vaapi_map_frame(AVFrame *frame, int flags);
>
> I know I've stared at the implementation for this in the few previous
> patch iterations, but just looking at this it's not clear to me at all
> what the input and output is.
>
> So the input is a PIX_FMT_VAAPI AVFrame. And on output, this is
> replaced with a "software" AVFrame or what? What happens on unmap? Is
> it magically undone? What happens with the vaapi frame reference?
>
> Maybe it would be better to add a separate AVFrame argument, into which
> the surface is mapped.

This function is the lowest level mapping for people who know exactly what they 
are doing (especially wrt memory types).  The input is the AV_PIX_FMT_VAAPI 
AVFrame with buf[0,1] giving the mapping to a surface.  The output is the same 
frame with data[0,1,2] and linesize[0,1,2] filled in to use, but nothing else 
about the frame changed.  Unmap removes the mapping along with the data and 
linesize values.  I'll clarify how it works in the comment.

If you think there should be a pair of functions mapping/unmapping to/from a 
distinct AVFrame (which would call these two) then I would prefer to add that 
separately.

>> +
>> +/** Unmap a surface after operating on it with the CPU.
>> + *
>> + * If the frame data was copied and write access was requested, this will copy
>> + * back to external memory.
>> + */
>> +int av_vaapi_unmap_frame(AVFrame *frame);
>> +
>> +/** Copy data from memory to a surface.
>> + *
>> + * The size and format of the source must match the underlying format of the
>> + * surface.
>> + */
>> +int av_vaapi_copy_to_surface(AVFrame *dst, const AVFrame *src);
>> +
>> +/** Copy data from a surface to memory.
>> + *
>> + * The size and format of the destination must match the underlying format of
>> + * the surface.
>> + */
>> +int av_vaapi_copy_from_surface(AVFrame *dst, AVFrame *src);
>> +
>> +
>> +/** Given a libva fourcc, return the corresponding pixel format. */
>> +enum AVPixelFormat av_vaapi_pix_fmt(unsigned int fourcc);
>> +/** Given a pixel format, return the corresponding libva fourcc. */
>> +unsigned int av_vaapi_fourcc(enum AVPixelFormat pix_fmt);
>> +/** Given a libva fourcc, return the associated pipeline chroma format. */
>> +unsigned int av_vaapi_rt_format(unsigned int fourcc);
>> +
>> +/** Given a pixel format, return the corresponding libva image format.
>> + *
>> + * This uses vaQueryImageFormats(), so results need not be exactly equivalent
>> + * between different drivers.
>> + */
>> +int av_vaapi_get_image_format(AVVAAPIHardwareContext *hw_ctx,
>> +                              enum AVPixelFormat pix_fmt,
>> +                              VAImageFormat *image_format);
>> +
>> +/** @} */
>> +
>> +#endif /* AVCODEC_VAAPI_SUPPORT_H */