[FFmpeg-devel] [PATCH v2] hwcontext_vaapi: implement auto connecting to a wayland display

[Rostislav Pehlivanov]

Allows Wayland users to use vaapi without having to specify a
vaapi drm device path.
"wayland-0" is the default display name (like ":0") under Wayland,
and given there's no way to get the display name from a connection,
use it.

Signed-off-by: Rostislav Pehlivanov <atomnuker at gmail.com>
---
 configure                   |  6 +++++-
 libavutil/hwcontext_vaapi.c | 31 +++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/configure b/configure
index 1d2fffa132..1a785ce0af 100755
--- a/configure
+++ b/configure
@@ -2062,6 +2062,7 @@ SYSTEM_FUNCS="
 
 SYSTEM_LIBRARIES="
     vaapi_drm
+    vaapi_wayland
     vaapi_x11
     vdpau_x11
     wincrypt
@@ -3347,7 +3348,7 @@ swscale_suggest="libm"
 
 avcodec_extralibs="pthreads_extralibs iconv_extralibs"
 avfilter_extralibs="pthreads_extralibs"
-avutil_extralibs="d3d11va_extralibs nanosleep_extralibs pthreads_extralibs vaapi_drm_extralibs vaapi_x11_extralibs vdpau_x11_extralibs"
+avutil_extralibs="d3d11va_extralibs nanosleep_extralibs pthreads_extralibs vaapi_drm_extralibs vaapi_x11_extralibs vaapi_wayland_extralibs vdpau_x11_extralibs"
 
 # programs
 ffmpeg_deps="avcodec avfilter avformat swresample"
@@ -6162,6 +6163,9 @@ enabled vaapi &&
 enabled vaapi &&
     check_lib vaapi_x11 "va/va.h va/va_x11.h" vaGetDisplay -lva -lva-x11 -lX11
 
+enabled vaapi &&
+    check_lib vaapi_wayland "va/va.h va/va_wayland.h" vaGetDisplayWl -lva -lva-wayland -lwayland-client
+
 enabled vaapi &&
     check_cpp_condition "va/va.h" "VA_CHECK_VERSION(1, 0, 0)" &&
     enable vaapi_1
diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 29698d1b27..02437806ad 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -24,6 +24,9 @@
 #if HAVE_VAAPI_DRM
 #   include <va/va_drm.h>
 #endif
+#if HAVE_VAAPI_WAYLAND
+#   include <va/va_wayland.h>
+#endif
 
 #if CONFIG_LIBDRM
 #   include <va/va_drmcommon.h>
@@ -51,6 +54,9 @@
 #endif
 
 typedef struct VAAPIDevicePriv {
+#if HAVE_VAAPI_WAYLAND
+    struct wl_display *wayland_display;
+#endif
 #if HAVE_VAAPI_X11
     Display *x11_display;
 #endif
@@ -1196,6 +1202,10 @@ static void vaapi_device_free(AVHWDeviceContext *ctx)
     if (hwctx->display)
         vaTerminate(hwctx->display);
 
+#if HAVE_VAAPI_WAYLAND
+    if (priv->wayland_display)
+        wl_display_disconnect(priv->wayland_display);
+#endif
 #if HAVE_VAAPI_X11
     if (priv->x11_display)
         XCloseDisplay(priv->x11_display);
@@ -1264,6 +1274,27 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
     ctx->user_opaque = priv;
     ctx->free        = vaapi_device_free;
 
+#if HAVE_VAAPI_WAYLAND
+    if (!display && !(device && device[0] == '/')) {
+        const char *disp_name = display ? display : "wayland-0";
+        priv->wayland_display = wl_display_connect(disp_name);
+        if (!priv->wayland_display) {
+            av_log(ctx, AV_LOG_VERBOSE, "Cannot connect to Wayland display "
+                   "%s.\n", disp_name);
+        } else {
+            display = vaGetDisplayWl(priv->wayland_display);
+            if (!display) {
+                av_log(ctx, AV_LOG_ERROR, "Cannot open a VA display "
+                       "from Wayland display %s.\n", disp_name);
+                return AVERROR_UNKNOWN;
+            }
+
+            av_log(ctx, AV_LOG_VERBOSE, "Opened VA display via "
+                   "Wayland display %s.\n", disp_name);
+        }
+    }
+#endif
+
 #if HAVE_VAAPI_X11
     if (!display && !(device && device[0] == '/')) {
         // Try to open the device as an X11 display.
-- 
2.16.0.rc0.223.g4a4ac83678