[FFmpeg-devel] [PATCH v2 2/3] hwcontext_vaapi: do not link against libva-drm.so

[Emil Velikov]

From: Emil Velikov <emil.velikov at collabora.com>

There is an internal ABI between libva.so and libva-drm.so. So having
mismatched versions can cause all sorts of issues.

We had the breakage between libva.so and libva-x11.so addressed with
earlier commit. There's no point in waiting for things to break wrt
libva-drm.so so pre-emptively, switch to dlopen()-ing the library.

Signed-off-by: Emil Velikov <emil.velikov at collabora.com>
---
v2: rebase against the libdl fixup
---
 configure                   |  2 +-
 libavutil/hwcontext_vaapi.c | 48 +++++++++++++++++++++++++++++++++++--
 2 files changed, 47 insertions(+), 3 deletions(-)

diff --git a/configure b/configure
index 605afd58a7..a941d4b927 100755
--- a/configure
+++ b/configure
@@ -3817,7 +3817,7 @@ swscale_suggest="libm stdatomic"
 avcodec_extralibs="pthreads_extralibs iconv_extralibs dxva2_extralibs"
 avfilter_extralibs="pthreads_extralibs"
 avutil_deps="libdl"
-avutil_extralibs="d3d11va_extralibs nanosleep_extralibs pthreads_extralibs vaapi_drm_extralibs vdpau_x11_extralibs"
+avutil_extralibs="d3d11va_extralibs nanosleep_extralibs pthreads_extralibs vdpau_x11_extralibs"
 
 # programs
 ffmpeg_deps="avcodec avfilter avformat"
diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index e44d324928..7734a50fc0 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -31,6 +31,8 @@
 #endif
 #if HAVE_VAAPI_DRM
 #   include <va/va_drm.h>
+#   include <dlfcn.h>
+#   define VA_DRM_LIB "libva-drm.so" VA_ABI
 #endif
 
 #if CONFIG_LIBDRM
@@ -66,6 +68,7 @@ typedef struct VAAPIDevicePriv {
     Display *x11_display;
 #endif
 
+    void *libva_drm;
     int drm_fd;
 } VAAPIDevicePriv;
 
@@ -1582,6 +1585,8 @@ static void vaapi_device_free(AVHWDeviceContext *ctx)
 
     if (priv->drm_fd >= 0)
         close(priv->drm_fd);
+    if (priv->libva_drm)
+        dlclose(priv->libva_drm);
 
     av_freep(&priv);
 }
@@ -1665,6 +1670,8 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
 
 #if HAVE_VAAPI_DRM
     while (!display && try_drm) {
+        VADisplay (*GetDisplayDRM)(int fd);
+
         // If the device is specified, try to open it as a DRM device node.
         // If not, look for a usable render node, possibly restricted to those
         // using a specified kernel driver.
@@ -1722,8 +1729,26 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
                 break;
         }
 
-        display = vaGetDisplayDRM(priv->drm_fd);
+        priv->libva_drm = dlopen(VA_DRM_LIB, RTLD_NOW | RTLD_LOCAL);
+        if (!priv->libva_drm) {
+            av_log(ctx, AV_LOG_ERROR, "Cannot open %s library %s.\n",
+                   VA_DRM_LIB, dlerror());
+            return AVERROR_UNKNOWN;
+        }
+
+        GetDisplayDRM = dlsym(priv->libva_drm, "vaGetDisplayDRM");
+        if (!GetDisplayDRM) {
+            av_log(ctx, AV_LOG_ERROR, "Cannot retrieve %s entrypoint %s.\n",
+                   "vaGetDisplayDRM", dlerror());
+            // Always dlclose after the dlerror(). The former can alter the
+            // error string returned by the latter.
+            dlclose(priv->libva_drm);
+            return AVERROR_UNKNOWN;
+        }
+
+        display = GetDisplayDRM(priv->drm_fd);
         if (!display) {
+            dlclose(priv->libva_drm);
             av_log(ctx, AV_LOG_VERBOSE, "Cannot open a VA display "
                    "from DRM device %s.\n", device);
             return AVERROR_EXTERNAL;
@@ -1811,6 +1836,7 @@ static int vaapi_device_derive(AVHWDeviceContext *ctx,
 #if HAVE_VAAPI_DRM
     if (src_ctx->type == AV_HWDEVICE_TYPE_DRM) {
         AVDRMDeviceContext *src_hwctx = src_ctx->hwctx;
+        VADisplay (*GetDisplayDRM)(int fd);
         VADisplay *display;
         VAAPIDevicePriv *priv;
         int fd;
@@ -1879,8 +1905,26 @@ static int vaapi_device_derive(AVHWDeviceContext *ctx,
         ctx->user_opaque = priv;
         ctx->free        = &vaapi_device_free;
 
-        display = vaGetDisplayDRM(fd);
+        priv->libva_drm = dlopen(VA_DRM_LIB, RTLD_NOW | RTLD_LOCAL);
+        if (!priv->libva_drm) {
+            av_log(ctx, AV_LOG_ERROR, "Cannot open %s library %s.\n",
+                   VA_DRM_LIB, dlerror());
+            return AVERROR_UNKNOWN;
+        }
+
+        GetDisplayDRM = dlsym(priv->libva_drm, "vaGetDisplayDRM");
+        if (!GetDisplayDRM) {
+            av_log(ctx, AV_LOG_ERROR, "Cannot retrieve %s entrypoint %s.\n",
+                   "vaGetDisplayDRM", dlerror());
+            // Always dlclose after the dlerror(). The former can alter the
+            // error string returned by the latter.
+            dlclose(priv->libva_drm);
+            return AVERROR_UNKNOWN;
+        }
+
+        display = GetDisplayDRM(fd);
         if (!display) {
+            dlclose(priv->libva_drm);
             av_log(ctx, AV_LOG_ERROR, "Failed to open a VA display from "
                    "DRM device.\n");
             return AVERROR(EIO);
-- 
2.37.0