[FFmpeg-devel] [RFC] amfenc: Add support for OpenCL input

Mark Thompson sw at jkqxz.net
Tue Jan 23 17:21:29 EET 2018


On 23/01/18 15:04, Mironov, Mikhail wrote:
>> -----Original Message-----
>> From: ffmpeg-devel [mailto:ffmpeg-devel-bounces at ffmpeg.org] On Behalf
>> Of Mark Thompson
>> Sent: January 22, 2018 6:57 PM
>> To: FFmpeg development discussions and patches <ffmpeg-
>> devel at ffmpeg.org>
>> Subject: [FFmpeg-devel] [RFC] amfenc: Add support for OpenCL input
>>
>> ---
>> This allows passing OpenCL frames to AMF without a download/upload step
>> to get around AMD's lack of support for D3D11 mapping.
>>
>> For example:
>>
>> ./ffmpeg -hwaccel dxva2 -hwaccel_output_format dxva2_vld -i input.mp4 -an
>> -vf
>> 'hwmap=derive_device=opencl,program_opencl=source=examples.cl:kernel=
>> rotate_image' -c:v h264_amf output.mp4
>>
>> * I can't find any documentation or examples for these functions, so I'm
>> guessing a bit exactly how they are meant to work.  In particular, there are
>> some locking functions which I have ignored because I have no idea under
>> what circumstances something might want to be locked.
>> * I tried to write common parts with D3D11, but I might well have broken
>> D3D11 support in the process - it doesn't work at all for me so I can't test it.
>> * Not sure how to get non-NV12 to work.  I may be missing something, or it
>> may just not be there - the trace messages suggest it doesn't like the width of
>> RGB0 or the second plane of GRAY8.
>>
>> - Mark
>>
>>
>>  libavcodec/amfenc.c | 178 +++++++++++++++++++++++++++++++++++---------
>> --------
>>  libavcodec/amfenc.h |   1 +
>>  2 files changed, 123 insertions(+), 56 deletions(-)
>>
>> diff --git a/libavcodec/amfenc.c b/libavcodec/amfenc.c index
>> 89a10ff253..220cdd278f 100644
>> --- a/libavcodec/amfenc.c
>> +++ b/libavcodec/amfenc.c
>> @@ -24,6 +24,9 @@
>>  #if CONFIG_D3D11VA
>>  #include "libavutil/hwcontext_d3d11va.h"
>>  #endif
>> +#if CONFIG_OPENCL
>> +#include "libavutil/hwcontext_opencl.h"
>> +#endif
>>  #include "libavutil/mem.h"
>>  #include "libavutil/pixdesc.h"
>>  #include "libavutil/time.h"
>> @@ -51,6 +54,9 @@ const enum AVPixelFormat ff_amf_pix_fmts[] = {  #if
>> CONFIG_D3D11VA
>>      AV_PIX_FMT_D3D11,
>>  #endif
>> +#if CONFIG_OPENCL
>> +    AV_PIX_FMT_OPENCL,
>> +#endif
>>      AV_PIX_FMT_NONE
>>  };
>>
>> @@ -69,6 +75,7 @@ static const FormatMap format_map[] =
>>      { AV_PIX_FMT_YUV420P,    AMF_SURFACE_YUV420P },
>>      { AV_PIX_FMT_YUYV422,    AMF_SURFACE_YUY2 },
>>      { AV_PIX_FMT_D3D11,      AMF_SURFACE_NV12 },
>> +    { AV_PIX_FMT_OPENCL,     AMF_SURFACE_NV12 },
>>  };
>>
>>
>> @@ -154,8 +161,9 @@ static int amf_load_library(AVCodecContext *avctx)
>>
>>  static int amf_init_context(AVCodecContext *avctx)  {
>> -    AmfContext         *ctx = avctx->priv_data;
>> -    AMF_RESULT          res = AMF_OK;
>> +    AmfContext *ctx = avctx->priv_data;
>> +    AMF_RESULT res;
>> +    AVHWDeviceContext *hwdev = NULL;
>>
>>      // configure AMF logger
>>      // the return of these functions indicates old state and do not affect
>> behaviour @@ -173,59 +181,91 @@ static int
>> amf_init_context(AVCodecContext *avctx)
>>
>>      res = ctx->factory->pVtbl->CreateContext(ctx->factory, &ctx->context);
>>      AMF_RETURN_IF_FALSE(ctx, res == AMF_OK, AVERROR_UNKNOWN,
>> "CreateContext() failed with error %d\n", res);
>> -    // try to reuse existing DX device
>> -#if CONFIG_D3D11VA
>> +
>> +    // Attempt to initialise from an existing D3D11 or OpenCL device.
>>      if (avctx->hw_frames_ctx) {
>> -        AVHWFramesContext *device_ctx = (AVHWFramesContext*)avctx-
>>> hw_frames_ctx->data;
>> -        if (device_ctx->device_ctx->type == AV_HWDEVICE_TYPE_D3D11VA) {
>> -            if (amf_av_to_amf_format(device_ctx->sw_format) !=
>> AMF_SURFACE_UNKNOWN) {
>> -                if (device_ctx->device_ctx->hwctx) {
>> -                    AVD3D11VADeviceContext *device_d3d11 =
>> (AVD3D11VADeviceContext *)device_ctx->device_ctx->hwctx;
>> -                    res = ctx->context->pVtbl->InitDX11(ctx->context, device_d3d11-
>>> device, AMF_DX11_1);
>> -                    if (res == AMF_OK) {
>> -                        ctx->hw_frames_ctx = av_buffer_ref(avctx->hw_frames_ctx);
>> -                        if (!ctx->hw_frames_ctx) {
>> -                            return AVERROR(ENOMEM);
>> -                        }
>> -                    } else {
>> -                        if(res == AMF_NOT_SUPPORTED)
>> -                            av_log(avctx, AV_LOG_INFO, "avctx->hw_frames_ctx has
>> D3D11 device which doesn't have D3D11VA interface, switching to
>> default\n");
>> -                        else
>> -                            av_log(avctx, AV_LOG_INFO, "avctx->hw_frames_ctx has
>> non-AMD device, switching to default\n");
>> -                    }
>> -                }
>> -            } else {
>> -                av_log(avctx, AV_LOG_INFO, "avctx->hw_frames_ctx has format
>> not uspported by AMF, switching to default\n");
>> -            }
>> +        AVHWFramesContext *hwfc =
>> + (AVHWFramesContext*)avctx->hw_frames_ctx->data;
>> +
>> +        if (amf_av_to_amf_format(hwfc->sw_format) ==
>> AMF_SURFACE_UNKNOWN) {
>> +            av_log(avctx, AV_LOG_VERBOSE, "Input hardware frame format (%s)
>> is not supported.\n",
>> +                   av_get_pix_fmt_name(hwfc->sw_format));
>> +        } else {
>> +            hwdev = hwfc->device_ctx;
>> +
>> +            ctx->hw_frames_ctx = av_buffer_ref(avctx->hw_frames_ctx);
>> +            if (!ctx->hw_frames_ctx)
>> +                return AVERROR(ENOMEM);
>>          }
>> -    } else if (avctx->hw_device_ctx) {
>> -        AVHWDeviceContext *device_ctx = (AVHWDeviceContext*)(avctx-
>>> hw_device_ctx->data);
>> -        if (device_ctx->type == AV_HWDEVICE_TYPE_D3D11VA) {
>> -            if (device_ctx->hwctx) {
>> -                AVD3D11VADeviceContext *device_d3d11 =
>> (AVD3D11VADeviceContext *)device_ctx->hwctx;
>> -                res = ctx->context->pVtbl->InitDX11(ctx->context, device_d3d11-
>>> device, AMF_DX11_1);
>> +    }
>> +    if (!hwdev && avctx->hw_device_ctx) {
>> +        hwdev = (AVHWDeviceContext*)avctx->hw_device_ctx->data;
>> +
>> +        ctx->hw_device_ctx = av_buffer_ref(avctx->hw_device_ctx);
>> +        if (!ctx->hw_device_ctx)
>> +            return AVERROR(ENOMEM);
>> +    }
>> +    if (hwdev) {
>> +#if CONFIG_D3D11VA
>> +        if (hwdev->type == AV_HWDEVICE_TYPE_D3D11VA) {
>> +            AVD3D11VADeviceContext *d3d11dev = hwdev->hwctx;
>> +
>> +            res = ctx->context->pVtbl->InitDX11(ctx->context,
>> +                                                d3d11dev->device, AMF_DX11_1);
>> +            if (res == AMF_OK) {
>> +                av_log(avctx, AV_LOG_VERBOSE, "Initialised from "
>> +                       "external D3D11 device.\n");
>> +                return 0;
>> +            }
>> +
>> +            av_log(avctx, AV_LOG_INFO, "Failed to initialise from "
>> +                   "external D3D11 device: %d.\n", res);
>> +        } else
>> +#endif
>> +#if CONFIG_OPENCL
>> +        if (hwdev->type == AV_HWDEVICE_TYPE_OPENCL) {
>> +            AVOpenCLDeviceContext *cldev = hwdev->hwctx;
>> +            cl_int cle;
>> +
>> +            ctx->cl_command_queue =
>> +                clCreateCommandQueue(cldev->context, cldev->device_id, 0,
>> &cle);
>> +            if (!ctx->cl_command_queue) {
>> +                av_log(avctx, AV_LOG_INFO, "Failed to create OpenCL "
>> +                       "command queue: %d.\n", cle);
>> +            } else {
>> +                res = ctx->context->pVtbl->InitOpenCL(ctx->context,
>> +
>> + ctx->cl_command_queue);
>>                  if (res == AMF_OK) {
>> -                    ctx->hw_device_ctx = av_buffer_ref(avctx->hw_device_ctx);
>> -                    if (!ctx->hw_device_ctx) {
>> -                        return AVERROR(ENOMEM);
>> -                    }
>> -                } else {
>> -                    if (res == AMF_NOT_SUPPORTED)
>> -                        av_log(avctx, AV_LOG_INFO, "avctx->hw_device_ctx has D3D11
>> device which doesn't have D3D11VA interface, switching to default\n");
>> -                    else
>> -                        av_log(avctx, AV_LOG_INFO, "avctx->hw_device_ctx has non-
>> AMD device, switching to default\n");
>> +                    av_log(avctx, AV_LOG_VERBOSE, "Initialised from "
>> +                           "external OpenCL device.\n");
>> +                    return 0;
>>                  }
>> +                av_log(avctx, AV_LOG_INFO, "Failed to initialise from "
>> +                       "external OpenCL device: %d.\n", res);
>>              }
>> +        } else
>> +#endif
>> +        {
>> +            av_log(avctx, AV_LOG_INFO, "Input device type %s is not
>> supported.\n",
>> +                   av_hwdevice_get_type_name(hwdev->type));
>>          }
>>      }
>> -#endif
>> -    if (!ctx->hw_frames_ctx && !ctx->hw_device_ctx) {
>> -        res = ctx->context->pVtbl->InitDX11(ctx->context, NULL, AMF_DX11_1);
>> -        if (res != AMF_OK) {
>> -            res = ctx->context->pVtbl->InitDX9(ctx->context, NULL);
>> -            AMF_RETURN_IF_FALSE(ctx, res == AMF_OK, AVERROR_UNKNOWN,
>> "InitDX9() failed with error %d\n", res);
>> +
>> +    // Initialise from a new D3D11 device, or D3D9 if D3D11 is not available.
>> +    res = ctx->context->pVtbl->InitDX11(ctx->context, NULL, AMF_DX11_1);
>> +    if (res == AMF_OK) {
>> +        av_log(avctx, AV_LOG_VERBOSE, "Initialised from internal D3D11
>> device.\n");
>> +    } else {
>> +        av_log(avctx, AV_LOG_VERBOSE, "Failed to initialise from internal
>> D3D11 device: %d.\n", res);
>> +        res = ctx->context->pVtbl->InitDX9(ctx->context, NULL);
>> +        if (res == AMF_OK) {
>> +            av_log(avctx, AV_LOG_VERBOSE, "Initialised from internal D3D9
>> device.\n");
>> +        } else {
>> +            av_log(avctx, AV_LOG_VERBOSE, "Failed to initialise from internal
>> D3D9 device: %d.\n", res);
>> +            av_log(avctx, AV_LOG_ERROR, "Unable to initialise AMF.\n");
>> +            return AVERROR_UNKNOWN;
>>          }
>>      }
>> +
>>      return 0;
>>  }
>>
>> @@ -279,6 +319,11 @@ int av_cold ff_amf_encode_close(AVCodecContext
>> *avctx)
>>      av_buffer_unref(&ctx->hw_device_ctx);
>>      av_buffer_unref(&ctx->hw_frames_ctx);
>>
>> +#if CONFIG_OPENCL
>> +    if (ctx->cl_command_queue)
>> +        clReleaseCommandQueue(ctx->cl_command_queue);
>> +#endif
>> +
>>      if (ctx->trace) {
>>          ctx->trace->pVtbl->UnregisterWriter(ctx->trace,
>> FFMPEG_AMF_WRITER_ID);
>>      }
>> @@ -485,17 +530,38 @@ int ff_amf_send_frame(AVCodecContext *avctx,
>> const AVFrame *frame)
>>              (AVHWDeviceContext*)ctx->hw_device_ctx->data)
>>          )) {
>>  #if CONFIG_D3D11VA
>> -            static const GUID AMFTextureArrayIndexGUID = { 0x28115527,
>> 0xe7c3, 0x4b66, { 0x99, 0xd3, 0x4f, 0x2a, 0xe6, 0xb4, 0x7f, 0xaf } };
>> -            ID3D11Texture2D *texture = (ID3D11Texture2D*)frame->data[0]; //
>> actual texture
>> -            int index = (int)(size_t)frame->data[1]; // index is a slice in texture
>> array is - set to tell AMF which slice to use
>> -            texture->lpVtbl->SetPrivateData(texture,
>> &AMFTextureArrayIndexGUID, sizeof(index), &index);
>> -
>> -            res = ctx->context->pVtbl->CreateSurfaceFromDX11Native(ctx-
>>> context, texture, &surface, NULL); // wrap to AMF surface
>> -            AMF_RETURN_IF_FALSE(ctx, res == AMF_OK, AVERROR(ENOMEM),
>> "CreateSurfaceFromDX11Native() failed  with error %d\n", res);
>> -
>> -            // input HW surfaces can be vertically aligned by 16; tell AMF the real
>> size
>> -            surface->pVtbl->SetCrop(surface, 0, 0, frame->width, frame->height);
>> +            if (frame->format == AV_PIX_FMT_D3D11) {
>> +                static const GUID AMFTextureArrayIndexGUID = { 0x28115527,
>> 0xe7c3, 0x4b66, { 0x99, 0xd3, 0x4f, 0x2a, 0xe6, 0xb4, 0x7f, 0xaf } };
>> +                ID3D11Texture2D *texture = (ID3D11Texture2D*)frame->data[0];
>> // actual texture
>> +                int index = (int)(size_t)frame->data[1]; // index is a slice in texture
>> array is - set to tell AMF which slice to use
>> +                texture->lpVtbl->SetPrivateData(texture,
>> + &AMFTextureArrayIndexGUID, sizeof(index), &index);
>> +
>> +                res = ctx->context->pVtbl->CreateSurfaceFromDX11Native(ctx-
>>> context, texture, &surface, NULL); // wrap to AMF surface
>> +                AMF_RETURN_IF_FALSE(ctx, res == AMF_OK,
>> + AVERROR(ENOMEM), "CreateSurfaceFromDX11Native() failed  with error
>> + %d\n", res);
>> +
>> +                // input HW surfaces can be vertically aligned by 16; tell AMF the
>> real size
>> +                surface->pVtbl->SetCrop(surface, 0, 0, frame->width, frame-
>>> height);
>> +            } else
>> +#endif
>> +#if CONFIG_OPENCL
>> +            if (frame->format == AV_PIX_FMT_OPENCL) {
>> +                void *planes[AV_NUM_DATA_POINTERS];
>> +                AMF_SURFACE_FORMAT format;
>> +                int i;
>> +
>> +                for (i = 0; i < AV_NUM_DATA_POINTERS; i++)
>> +                    planes[i] = frame->data[i];
>> +
>> +                format = amf_av_to_amf_format(frame->format);
>> +
>> +                res = ctx->context->pVtbl->CreateSurfaceFromOpenCLNative(ctx-
>>> context, format,
>> +                                                                         frame->width, frame->height,
>> +                                                                         planes, &surface, NULL);
>> +                AMF_RETURN_IF_FALSE(ctx, res == AMF_OK,
>> AVERROR_UNKNOWN,
>> +                                    "CreateSurfaceFromOpenCLNative() failed with error
>> %d\n", res);
>> +            } else
>>  #endif
>> +            av_assert0(0 && "Invalid hardware input format.");
>>          } else {
>>              res = ctx->context->pVtbl->AllocSurface(ctx->context,
>> AMF_MEMORY_HOST, ctx->format, avctx->width, avctx->height, &surface);
>>              AMF_RETURN_IF_FALSE(ctx, res == AMF_OK, AVERROR(ENOMEM),
>> "AllocSurface() failed  with error %d\n", res); diff --git a/libavcodec/amfenc.h
>> b/libavcodec/amfenc.h index 84f0aad2fa..bb8fd1807a 100644
>> --- a/libavcodec/amfenc.h
>> +++ b/libavcodec/amfenc.h
>> @@ -61,6 +61,7 @@ typedef struct AmfContext {
>>
>>      AVBufferRef        *hw_device_ctx; ///< pointer to HW accelerator
>> (decoder)
>>      AVBufferRef        *hw_frames_ctx; ///< pointer to HW accelerator (frame
>> allocator)
>> +    void               *cl_command_queue; ///< Command queue for use with
>> OpenCL input
>>
>>      // helpers to handle async calls
>>      int                 delayed_drain;
>> --
>> 2.11.0
> 
> AMF encoder works via D3D9 or D3D11 only. AMF OpenCL support is done for possible integration with 
> external image processing. Passing regular OpenCL 2D images will cause mapping to system memory and copy. 
> The fast way is to use interop: 
> - Allocate last processing NV12 surface as D3D11 texture
> - iterop it into OpenCL

This step doesn't work - AMD has no support for mapping NV12 D3D11 textures to OpenCL.  (That's why I wrote the patch at all.)

Does D3D9 also work?  That does have the necessary support, but would admittedly end up with the same multiple-mapping ugliness as does D3D9 -> OpenCL -> D3D9 -> libmfx on Intel.

> - use as output for the last OCL kernel
> - un-interop back to D3D11
> - submit to AMF.
> There is not much value to initialize AMF with OpenCL unless AMF color space converter is used. 
> The converter would do the sequence described above.
> 
> If AMF CSC is used few things has to be done:
> 1. Device should be created by passing D3D11 device as a parameter. It is done in hwcontext_opencl.c clGetDeviceIDsFromD3D11KNR(). 
> 2. The D3D11 device used there should be passed to AMF via InitDX11() preferably before InitOpenCL() call.
> 3. Add RGB formats for submission.

What is the OpenCL doing in this case?  I may be missing something, but from what you've described there doesn't appear to be any OpenCL interaction with AMF at all, it's just being given D3D11 textures.

- Mark


More information about the ffmpeg-devel mailing list