[FFmpeg-devel] [PATCH 2/2] avutil/hwcontext_opencl: Add map from Videotoolbox to OpenCL

[Zhao Zhili]

From: Zhao Zhili <zhilizhao at tencent.com>

For example:
./ffmpeg -hwaccel videotoolbox \
	-hwaccel_output_format videotoolbox_vld \
	-i foo.mp4 \
	-vf hwmap=derive_device=opencl,transpose_opencl=dir=clock,hwmap,format=nv12 \
	-c:v hevc_videotoolbox \
	-c:a copy \
	-b:v 2M -tag:v hvc1 bar.mp4
---
 configure                    |  6 +++
 libavutil/hwcontext_opencl.c | 97 ++++++++++++++++++++++++++++++++++++
 2 files changed, 103 insertions(+)

diff --git a/configure b/configure
index 197f762b58..efcb4c4e49 100755
--- a/configure
+++ b/configure
@@ -2460,6 +2460,7 @@ HAVE_LIST="
     opencl_dxva2
     opencl_vaapi_beignet
     opencl_vaapi_intel_media
+    opencl_videotoolbox
     perl
     pod2man
     texi2html
@@ -7217,6 +7218,11 @@ if enabled_all opencl d3d11va ; then
         enable opencl_d3d11
 fi
 
+if enabled_all opencl videotoolbox ; then
+    check_func_headers OpenCL/cl_gl_ext.h clCreateImageFromIOSurfaceWithPropertiesAPPLE -framework VideoToolbox -framework OpenCL &&
+        enable opencl_videotoolbox
+fi
+
 enabled vdpau &&
     check_cpp_condition vdpau vdpau/vdpau.h "defined VDP_DECODER_PROFILE_MPEG4_PART2_ASP"
 
diff --git a/libavutil/hwcontext_opencl.c b/libavutil/hwcontext_opencl.c
index de093fffb1..efd8d0e094 100644
--- a/libavutil/hwcontext_opencl.c
+++ b/libavutil/hwcontext_opencl.c
@@ -72,6 +72,11 @@
 #include "hwcontext_drm.h"
 #endif
 
+#if HAVE_OPENCL_VIDEOTOOLBOX
+#include <OpenCL/cl_gl_ext.h>
+#include <VideoToolbox/VideoToolbox.h>
+#endif
+
 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA && CONFIG_LIBMFX
 extern int ff_qsv_get_surface_base_handle(mfxFrameSurface1 *surf,
                                           enum AVHWDeviceType base_dev_typ,
@@ -1364,6 +1369,12 @@ static int opencl_device_derive(AVHWDeviceContext *hwdev,
         break;
 #endif
 
+#if HAVE_OPENCL_VIDEOTOOLBOX
+    case AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
+        err = opencl_device_create(hwdev, NULL, NULL, 0);
+        break;
+#endif
+
     default:
         err = AVERROR(ENOSYS);
         break;
@@ -2819,6 +2830,84 @@ fail:
 
 #endif
 
+#if HAVE_OPENCL_VIDEOTOOLBOX
+
+static void opencl_unmap_from_vt(AVHWFramesContext *hwfc,
+                                 HWMapDescriptor *hwmap)
+{
+    uint8_t *desc = hwmap->priv;
+    opencl_pool_free(hwfc, desc);
+}
+
+static int opencl_map_from_vt(AVHWFramesContext *dst_fc, AVFrame *dst,
+                              const AVFrame *src, int flags)
+{
+    CVPixelBufferRef pixbuf = (CVPixelBufferRef) src->data[3];
+    IOSurfaceRef io_surface_ref = CVPixelBufferGetIOSurface(pixbuf);
+    cl_int err = 0;
+    AVOpenCLFrameDescriptor *desc = NULL;
+    AVOpenCLDeviceContext *dst_dev = dst_fc->device_ctx->hwctx;
+
+    if (!io_surface_ref) {
+        av_log(dst_fc, AV_LOG_ERROR, "Failed to get IOSurfaceRef\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    desc = av_mallocz(sizeof(*desc));
+    if (!desc)
+        return AVERROR(ENOMEM);
+
+    for (int p = 0;; p++) {
+        cl_image_format image_format;
+        cl_image_desc image_desc;
+        cl_iosurface_properties_APPLE props[] = {
+                CL_IOSURFACE_REF_APPLE, (cl_iosurface_properties_APPLE) io_surface_ref,
+                CL_IOSURFACE_PLANE_APPLE, p,
+                0
+        };
+
+        err = opencl_get_plane_format(dst_fc->sw_format, p,
+                                      src->width, src->height,
+                                      &image_format, &image_desc);
+        if (err == AVERROR(ENOENT))
+            break;
+        if (err < 0)
+            goto fail;
+
+        desc->planes[p] = clCreateImageFromIOSurfaceWithPropertiesAPPLE(dst_dev->context,
+                                                    opencl_mem_flags_for_mapping(flags),
+                                                    &image_format, &image_desc,
+                                                    props, &err);
+        if (!desc->planes[p]) {
+            av_log(dst_fc, AV_LOG_ERROR, "Failed to create image from IOSurfaceRef\n");
+            err = AVERROR(EIO);
+            goto fail;
+        }
+        desc->nb_planes++;
+    }
+
+    for (int i = 0; i < desc->nb_planes; i++)
+        dst->data[i] = (uint8_t *) desc->planes[i];
+
+    err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
+                                opencl_unmap_from_vt, desc);
+    if (err < 0)
+        goto fail;
+
+    dst->width = src->width;
+    dst->height = src->height;
+
+    return 0;
+
+fail:
+    for (int i = 0; i < desc->nb_planes; i++)
+        clReleaseMemObject(desc->planes[i]);
+    av_freep(&desc);
+    return err;
+}
+
+#endif
+
 static int opencl_map_from(AVHWFramesContext *hwfc, AVFrame *dst,
                            const AVFrame *src, int flags)
 {
@@ -2864,6 +2953,10 @@ static int opencl_map_to(AVHWFramesContext *hwfc, AVFrame *dst,
     case AV_PIX_FMT_DRM_PRIME:
         if (priv->drm_arm_mapping_usable)
             return opencl_map_from_drm_arm(hwfc, dst, src, flags);
+#endif
+#if HAVE_OPENCL_VIDEOTOOLBOX
+    case AV_PIX_FMT_VIDEOTOOLBOX:
+        return opencl_map_from_vt(hwfc, dst, src, flags);
 #endif
     }
     return AVERROR(ENOSYS);
@@ -2922,6 +3015,10 @@ static int opencl_frames_derive_to(AVHWFramesContext *dst_fc,
         if (!priv->drm_arm_mapping_usable)
             return AVERROR(ENOSYS);
         break;
+#endif
+#if HAVE_OPENCL_VIDEOTOOLBOX
+    case AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
+        break;
 #endif
     default:
         return AVERROR(ENOSYS);
-- 
2.42.0