[FFmpeg-cvslog] hwcontext_opencl: QSV to OpenCL mapping for Intel Media SDK
Mark Thompson
git at videolan.org
Thu Nov 23 01:22:48 EET 2017
ffmpeg | branch: master | Mark Thompson <sw at jkqxz.net> | Sat Mar 4 23:57:49 2017 +0000| [403cde8e85101c8d4d99a3df4bde51e5a24cfded] | committer: Mark Thompson
hwcontext_opencl: QSV to OpenCL mapping for Intel Media SDK
Uses the cl_intel_va_api_media_sharing extension, which supports only
NV12 surfaces and only mapping from QSV to OpenCL.
> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=403cde8e85101c8d4d99a3df4bde51e5a24cfded
---
configure | 5 +
libavutil/hwcontext_opencl.c | 365 +++++++++++++++++++++++++++++++++++++++++++
2 files changed, 370 insertions(+)
diff --git a/configure b/configure
index a05c5cf960..34fe16f80e 100755
--- a/configure
+++ b/configure
@@ -2121,6 +2121,7 @@ HAVE_LIST="
makeinfo
makeinfo_html
opencl_vaapi_beignet
+ opencl_vaapi_intel_media
perl
pod2man
texi2html
@@ -6159,6 +6160,10 @@ enabled vaapi &&
if enabled_all opencl vaapi ; then
check_type "CL/cl_intel.h" "clCreateImageFromFdINTEL_fn" &&
enable opencl_vaapi_beignet
+ if enabled libmfx ; then
+ check_type "CL/cl.h CL/va_ext.h" "clCreateFromVA_APIMediaSurfaceINTEL_fn" &&
+ enable opencl_vaapi_intel_media
+ fi
fi
enabled vdpau &&
diff --git a/libavutil/hwcontext_opencl.c b/libavutil/hwcontext_opencl.c
index 7f99f5af3f..ee6ebd18d1 100644
--- a/libavutil/hwcontext_opencl.c
+++ b/libavutil/hwcontext_opencl.c
@@ -37,6 +37,13 @@
#include "hwcontext_vaapi.h"
#endif
+#if HAVE_OPENCL_VAAPI_INTEL_MEDIA
+#include <mfx/mfxstructures.h>
+#include <va/va.h>
+#include <CL/va_ext.h>
+#include "hwcontext_vaapi.h"
+#endif
+
typedef struct OpenCLDeviceContext {
// Default command queue to use for transfer/mapping operations on
@@ -53,6 +60,16 @@ typedef struct OpenCLDeviceContext {
int vaapi_mapping_usable;
clCreateImageFromFdINTEL_fn clCreateImageFromFdINTEL;
#endif
+
+#if HAVE_OPENCL_VAAPI_INTEL_MEDIA
+ int qsv_mapping_usable;
+ clCreateFromVA_APIMediaSurfaceINTEL_fn
+ clCreateFromVA_APIMediaSurfaceINTEL;
+ clEnqueueAcquireVA_APIMediaSurfacesINTEL_fn
+ clEnqueueAcquireVA_APIMediaSurfacesINTEL;
+ clEnqueueReleaseVA_APIMediaSurfacesINTEL_fn
+ clEnqueueReleaseVA_APIMediaSurfacesINTEL;
+#endif
} OpenCLDeviceContext;
typedef struct OpenCLFramesContext {
@@ -633,6 +650,85 @@ static int opencl_device_init(AVHWDeviceContext *hwdev)
}
#endif
+#if HAVE_OPENCL_VAAPI_INTEL_MEDIA
+ {
+ size_t props_size;
+ cl_context_properties *props = NULL;
+ VADisplay va_display;
+ const char *va_ext = "cl_intel_va_api_media_sharing";
+ int i, fail = 0;
+
+ if (!opencl_check_extension(hwdev, va_ext)) {
+ av_log(hwdev, AV_LOG_VERBOSE, "The %s extension is "
+ "required for QSV to OpenCL mapping.\n", va_ext);
+ goto no_qsv;
+ }
+
+ cle = clGetContextInfo(hwctx->context, CL_CONTEXT_PROPERTIES,
+ 0, NULL, &props_size);
+ if (cle != CL_SUCCESS) {
+ av_log(hwdev, AV_LOG_VERBOSE, "Failed to get context "
+ "properties: %d.\n", cle);
+ goto no_qsv;
+ }
+ if (props_size == 0) {
+ av_log(hwdev, AV_LOG_VERBOSE, "Media sharing must be "
+ "enabled on context creation to use QSV to "
+ "OpenCL mapping.\n");
+ goto no_qsv;
+ }
+
+ props = av_malloc(props_size);
+ if (!props)
+ return AVERROR(ENOMEM);
+
+ cle = clGetContextInfo(hwctx->context, CL_CONTEXT_PROPERTIES,
+ props_size, props, NULL);
+ if (cle != CL_SUCCESS) {
+ av_log(hwdev, AV_LOG_VERBOSE, "Failed to get context "
+ "properties: %d.\n", cle);
+ goto no_qsv;
+ }
+
+ va_display = NULL;
+ for (i = 0; i < (props_size / sizeof(*props) - 1); i++) {
+ if (props[i] == CL_CONTEXT_VA_API_DISPLAY_INTEL) {
+ va_display = (VADisplay)(intptr_t)props[i+1];
+ break;
+ }
+ }
+ if (!va_display) {
+ av_log(hwdev, AV_LOG_VERBOSE, "Media sharing must be "
+ "enabled on context creation to use QSV to "
+ "OpenCL mapping.\n");
+ goto no_qsv;
+ }
+ if (!vaDisplayIsValid(va_display)) {
+ av_log(hwdev, AV_LOG_VERBOSE, "A valid VADisplay is "
+ "required on context creation to use QSV to "
+ "OpenCL mapping.\n");
+ goto no_qsv;
+ }
+
+ CL_FUNC(clCreateFromVA_APIMediaSurfaceINTEL,
+ "Intel QSV to OpenCL mapping");
+ CL_FUNC(clEnqueueAcquireVA_APIMediaSurfacesINTEL,
+ "Intel QSV in OpenCL acquire");
+ CL_FUNC(clEnqueueReleaseVA_APIMediaSurfacesINTEL,
+ "Intel QSV in OpenCL release");
+
+ if (fail) {
+ no_qsv:
+ av_log(hwdev, AV_LOG_WARNING, "QSV to OpenCL mapping "
+ "not usable.\n");
+ priv->qsv_mapping_usable = 0;
+ } else {
+ priv->qsv_mapping_usable = 1;
+ }
+ av_free(props);
+ }
+#endif
+
#undef CL_FUNC
return 0;
@@ -652,6 +748,95 @@ static void opencl_device_uninit(AVHWDeviceContext *hwdev)
}
}
+#if HAVE_OPENCL_VAAPI_INTEL_MEDIA
+static int opencl_filter_intel_media_vaapi_platform(AVHWDeviceContext *hwdev,
+ cl_platform_id platform_id,
+ const char *platform_name,
+ void *context)
+{
+ // This doesn't exist as a platform extension, so just test whether
+ // the function we will use for device enumeration exists.
+
+ if (!clGetExtensionFunctionAddressForPlatform(platform_id,
+ "clGetDeviceIDsFromVA_APIMediaAdapterINTEL")) {
+ av_log(hwdev, AV_LOG_DEBUG, "Platform %s does not export the "
+ "VAAPI device enumeration function.\n", platform_name);
+ return 1;
+ } else {
+ return 0;
+ }
+}
+
+static int opencl_enumerate_intel_media_vaapi_devices(AVHWDeviceContext *hwdev,
+ cl_platform_id platform_id,
+ const char *platform_name,
+ cl_uint *nb_devices,
+ cl_device_id **devices,
+ void *context)
+{
+ VADisplay va_display = context;
+ clGetDeviceIDsFromVA_APIMediaAdapterINTEL_fn
+ clGetDeviceIDsFromVA_APIMediaAdapterINTEL;
+ cl_int cle;
+ int err;
+
+ clGetDeviceIDsFromVA_APIMediaAdapterINTEL =
+ clGetExtensionFunctionAddressForPlatform(platform_id,
+ "clGetDeviceIDsFromVA_APIMediaAdapterINTEL");
+ if (!clGetDeviceIDsFromVA_APIMediaAdapterINTEL) {
+ av_log(hwdev, AV_LOG_ERROR, "Failed to get address of "
+ "clGetDeviceIDsFromVA_APIMediaAdapterINTEL().\n");
+ return AVERROR_UNKNOWN;
+ }
+
+ cle = clGetDeviceIDsFromVA_APIMediaAdapterINTEL(
+ platform_id, CL_VA_API_DISPLAY_INTEL, va_display,
+ CL_PREFERRED_DEVICES_FOR_VA_API_INTEL, 0, NULL, nb_devices);
+ if (cle == CL_DEVICE_NOT_FOUND) {
+ av_log(hwdev, AV_LOG_DEBUG, "No VAAPI-supporting devices found "
+ "on platform \"%s\".\n", platform_name);
+ *nb_devices = 0;
+ return 0;
+ } else if (cle != CL_SUCCESS) {
+ av_log(hwdev, AV_LOG_ERROR, "Failed to get number of devices "
+ "on platform \"%s\": %d.\n", platform_name, cle);
+ return AVERROR_UNKNOWN;
+ }
+
+ *devices = av_malloc_array(*nb_devices, sizeof(**devices));
+ if (!*devices)
+ return AVERROR(ENOMEM);
+
+ cle = clGetDeviceIDsFromVA_APIMediaAdapterINTEL(
+ platform_id, CL_VA_API_DISPLAY_INTEL, va_display,
+ CL_PREFERRED_DEVICES_FOR_VA_API_INTEL, *nb_devices, *devices, NULL);
+ if (cle != CL_SUCCESS) {
+ av_log(hwdev, AV_LOG_ERROR, "Failed to get list of VAAPI-supporting "
+ "devices on platform \"%s\": %d.\n", platform_name, cle);
+ av_freep(devices);
+ return AVERROR_UNKNOWN;
+ }
+
+ return 0;
+}
+
+static int opencl_filter_intel_media_vaapi_device(AVHWDeviceContext *hwdev,
+ cl_device_id device_id,
+ const char *device_name,
+ void *context)
+{
+ const char *va_ext = "cl_intel_va_api_media_sharing";
+
+ if (opencl_check_device_extension(device_id, va_ext)) {
+ return 0;
+ } else {
+ av_log(hwdev, AV_LOG_DEBUG, "Device %s does not support the "
+ "%s extension.\n", device_name, va_ext);
+ return 1;
+ }
+}
+#endif
+
static int opencl_device_derive(AVHWDeviceContext *hwdev,
AVHWDeviceContext *src_ctx,
int flags)
@@ -687,6 +872,37 @@ static int opencl_device_derive(AVHWDeviceContext *hwdev,
break;
#endif
+#if HAVE_OPENCL_VAAPI_INTEL_MEDIA
+ // The generic code automatically attempts to derive from all
+ // ancestors of the given device, so we can ignore QSV devices here
+ // and just consider the inner VAAPI device it was derived from.
+ case AV_HWDEVICE_TYPE_VAAPI:
+ {
+ AVVAAPIDeviceContext *src_hwctx = src_ctx->hwctx;
+ cl_context_properties props[7] = {
+ CL_CONTEXT_PLATFORM,
+ 0,
+ CL_CONTEXT_VA_API_DISPLAY_INTEL,
+ (intptr_t)src_hwctx->display,
+ CL_CONTEXT_INTEROP_USER_SYNC,
+ CL_FALSE,
+ 0,
+ };
+ OpenCLDeviceSelector selector = {
+ .platform_index = -1,
+ .device_index = -1,
+ .context = src_hwctx->display,
+ .enumerate_platforms = &opencl_enumerate_platforms,
+ .filter_platform = &opencl_filter_intel_media_vaapi_platform,
+ .enumerate_devices = &opencl_enumerate_intel_media_vaapi_devices,
+ .filter_device = &opencl_filter_intel_media_vaapi_device,
+ };
+
+ err = opencl_device_create_internal(hwdev, &selector, props);
+ }
+ break;
+#endif
+
default:
err = AVERROR(ENOSYS);
break;
@@ -1525,6 +1741,142 @@ fail:
#endif
+static inline cl_mem_flags opencl_mem_flags_for_mapping(int map_flags)
+{
+ if ((map_flags & AV_HWFRAME_MAP_READ) &&
+ (map_flags & AV_HWFRAME_MAP_WRITE))
+ return CL_MEM_READ_WRITE;
+ else if (map_flags & AV_HWFRAME_MAP_READ)
+ return CL_MEM_READ_ONLY;
+ else if (map_flags & AV_HWFRAME_MAP_WRITE)
+ return CL_MEM_WRITE_ONLY;
+ else
+ return 0;
+}
+
+#if HAVE_OPENCL_VAAPI_INTEL_MEDIA
+
+static void opencl_unmap_from_qsv(AVHWFramesContext *dst_fc,
+ HWMapDescriptor *hwmap)
+{
+ AVOpenCLFrameDescriptor *desc = hwmap->priv;
+ OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
+ OpenCLFramesContext *frames_priv = dst_fc->internal->priv;
+ cl_event event;
+ cl_int cle;
+ int p;
+
+ av_log(dst_fc, AV_LOG_DEBUG, "Unmap QSV/VAAPI surface from OpenCL.\n");
+
+ cle = device_priv->clEnqueueReleaseVA_APIMediaSurfacesINTEL(
+ frames_priv->command_queue, desc->nb_planes, desc->planes,
+ 0, NULL, &event);
+ if (cle != CL_SUCCESS) {
+ av_log(dst_fc, AV_LOG_ERROR, "Failed to release surface "
+ "handles: %d.\n", cle);
+ }
+
+ opencl_wait_events(dst_fc, &event, 1);
+
+ for (p = 0; p < desc->nb_planes; p++) {
+ cle = clReleaseMemObject(desc->planes[p]);
+ if (cle != CL_SUCCESS) {
+ av_log(dst_fc, AV_LOG_ERROR, "Failed to release CL "
+ "image of plane %d of QSV/VAAPI surface: %d\n",
+ p, cle);
+ }
+ }
+
+ av_free(desc);
+}
+
+static int opencl_map_from_qsv(AVHWFramesContext *dst_fc, AVFrame *dst,
+ const AVFrame *src, int flags)
+{
+ AVHWFramesContext *src_fc =
+ (AVHWFramesContext*)src->hw_frames_ctx->data;
+ AVOpenCLDeviceContext *dst_dev = dst_fc->device_ctx->hwctx;
+ OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
+ OpenCLFramesContext *frames_priv = dst_fc->internal->priv;
+ AVOpenCLFrameDescriptor *desc;
+ VASurfaceID va_surface;
+ cl_mem_flags cl_flags;
+ cl_event event;
+ cl_int cle;
+ int err, p;
+
+ if (src->format == AV_PIX_FMT_QSV) {
+ mfxFrameSurface1 *mfx_surface = (mfxFrameSurface1*)src->data[3];
+ va_surface = *(VASurfaceID*)mfx_surface->Data.MemId;
+ } else if (src->format == AV_PIX_FMT_VAAPI) {
+ va_surface = (VASurfaceID)(uintptr_t)src->data[3];
+ } else {
+ return AVERROR(ENOSYS);
+ }
+
+ cl_flags = opencl_mem_flags_for_mapping(flags);
+ if (!cl_flags)
+ return AVERROR(EINVAL);
+
+ av_log(src_fc, AV_LOG_DEBUG, "Map QSV/VAAPI surface %#x to "
+ "OpenCL.\n", va_surface);
+
+ desc = av_mallocz(sizeof(*desc));
+ if (!desc)
+ return AVERROR(ENOMEM);
+
+ // The cl_intel_va_api_media_sharing extension only supports NV12
+ // surfaces, so for now there are always exactly two planes.
+ desc->nb_planes = 2;
+
+ for (p = 0; p < desc->nb_planes; p++) {
+ desc->planes[p] =
+ device_priv->clCreateFromVA_APIMediaSurfaceINTEL(
+ dst_dev->context, cl_flags, &va_surface, p, &cle);
+ if (!desc->planes[p]) {
+ av_log(dst_fc, AV_LOG_ERROR, "Failed to create CL "
+ "image from plane %d of QSV/VAAPI surface "
+ "%#x: %d.\n", p, va_surface, cle);
+ err = AVERROR(EIO);
+ goto fail;
+ }
+
+ dst->data[p] = (uint8_t*)desc->planes[p];
+ }
+
+ cle = device_priv->clEnqueueAcquireVA_APIMediaSurfacesINTEL(
+ frames_priv->command_queue, desc->nb_planes, desc->planes,
+ 0, NULL, &event);
+ if (cle != CL_SUCCESS) {
+ av_log(dst_fc, AV_LOG_ERROR, "Failed to acquire surface "
+ "handles: %d.\n", cle);
+ err = AVERROR(EIO);
+ goto fail;
+ }
+
+ err = opencl_wait_events(dst_fc, &event, 1);
+ if (err < 0)
+ goto fail;
+
+ err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
+ &opencl_unmap_from_qsv, desc);
+ if (err < 0)
+ goto fail;
+
+ dst->width = src->width;
+ dst->height = src->height;
+
+ return 0;
+
+fail:
+ for (p = 0; p < desc->nb_planes; p++)
+ if (desc->planes[p])
+ clReleaseMemObject(desc->planes[p]);
+ av_freep(&desc);
+ return err;
+}
+
+#endif
static int opencl_map_from(AVHWFramesContext *hwfc, AVFrame *dst,
const AVFrame *src, int flags)
@@ -1546,6 +1898,12 @@ static int opencl_map_to(AVHWFramesContext *hwfc, AVFrame *dst,
if (priv->vaapi_mapping_usable)
return opencl_map_from_vaapi(hwfc, dst, src, flags);
#endif
+#if HAVE_OPENCL_VAAPI_INTEL_MEDIA
+ case AV_PIX_FMT_QSV:
+ case AV_PIX_FMT_VAAPI:
+ if (priv->qsv_mapping_usable)
+ return opencl_map_from_qsv(hwfc, dst, src, flags);
+#endif
}
return AVERROR(ENOSYS);
}
@@ -1561,6 +1919,13 @@ static int opencl_frames_derive_to(AVHWFramesContext *dst_fc,
return AVERROR(ENOSYS);
break;
#endif
+#if HAVE_OPENCL_VAAPI_INTEL_MEDIA
+ case AV_HWDEVICE_TYPE_QSV:
+ case AV_HWDEVICE_TYPE_VAAPI:
+ if (!priv->qsv_mapping_usable)
+ return AVERROR(ENOSYS);
+ break;
+#endif
default:
return AVERROR(ENOSYS);
}
More information about the ffmpeg-cvslog
mailing list