Cross-device DMA-BUF sharing for VAAPI with KMS grab

docs/source/about/advanced_usage.rst

@@ -568,6 +568,8 @@ keybindings
       **Linux + VA-API**
          Unlike with `amdvce` and `nvenc`, it doesn't matter if video encoding is done on a different GPU.
 
+         .. note:: This MUST be a render node (renderD*) not a primary node (card*).
+
          .. code-block:: bash
 
             ls /dev/dri/renderD*  # to find all devices capable of VAAPI

src/platform/linux/kmsgrab.cpp

@@ -14,6 +14,7 @@
 
 #include <filesystem>
 
+#include "src/config.h"
 #include "src/main.h"
 #include "src/platform/common.h"
 #include "src/round_robin.h"
@@ -292,21 +293,30 @@ namespace platf {
           return -1;
         }
 
-        // Open the render node for this card to share with libva.
-        // If it fails, we'll just share the primary node instead.
-        char *rendernode_path = drmGetRenderDeviceNameFromFd(fd.el);
-        if (rendernode_path) {
+        // The render node will be used for color conversion and encoding operations
+        std::string rendernode_path = config::video.adapter_name;
+        if (rendernode_path.empty()) {
+          char *rendernode_cstr = drmGetRenderDeviceNameFromFd(fd.el);
+          if (rendernode_cstr) {
+            rendernode_path.assign(rendernode_cstr);
+            free(rendernode_cstr);
+
+            BOOST_LOG(info) << "Using capture adapter for encoding: "sv << path << " -> "sv << rendernode_path;
+          }
+          else {
+            BOOST_LOG(info) << "Capture adapter has no render node: "sv << path;
+          }
+        }
+        else {
+          BOOST_LOG(info) << "Using manually selected adapter for encoding: "sv << path << " -> "sv << rendernode_path;
+        }
+        if (!rendernode_path.empty()) {
           BOOST_LOG(debug) << "Opening render node: "sv << rendernode_path;
-          render_fd.el = open(rendernode_path, O_RDWR);
+          render_fd.el = open(rendernode_path.c_str(), O_RDWR);
           if (render_fd.el < 0) {
             BOOST_LOG(warning) << "Couldn't open render node: "sv << rendernode_path << ": "sv << strerror(errno);
             render_fd.el = dup(fd.el);
           }
-          free(rendernode_path);
-        }
-        else {
-          BOOST_LOG(warning) << "No render device name for: "sv << path;
-          render_fd.el = dup(fd.el);
         }
 
         if (drmSetClientCap(fd.el, DRM_CLIENT_CAP_UNIVERSAL_PLANES, 1)) {
@@ -1427,15 +1437,44 @@ namespace platf {
         }
 
 #ifdef SUNSHINE_BUILD_VAAPI
-        if (!va::validate(card.render_fd.el)) {
-#else
-        if (true) {
-#endif
-          BOOST_LOG(warning) << "Monitor "sv << display_name << " doesn't support hardware encoding. Reverting back to GPU -> RAM -> GPU"sv;
-          return -1;
-        }
+        if (card.render_fd.el < 0 || !va::validate(card.render_fd.el)) {
+          BOOST_LOG(warning) << "Monitor "sv << display_name << " doesn't support hardware encoding. Testing other available GPUs."sv;
 
-        return 0;
+          fs::path card_dir { "/dev/dri"sv };
+          for (auto &entry : fs::directory_iterator { card_dir }) {
+            auto file = entry.path().filename();
+
+            auto filestring = file.generic_u8string();
+            if (std::string_view { filestring }.substr(0, 7) != "renderD"sv) {
+              continue;
+            }
+
+            // Check if this render device has support for VAAPI and supports EGL import of framebuffers from this GPU
+            file_t compatible_render_fd;
+            compatible_render_fd.el = open(entry.path().c_str(), O_RDWR);
+            if (compatible_render_fd.el >= 0 && va::validate(compatible_render_fd.el)) {
+              file_t fb_fd[4];
+              egl::surface_descriptor_t sd;
+              if (refresh(fb_fd, &sd) == capture_e::ok) {
+                gbm::gbm_t gbm { gbm::create_device(compatible_render_fd.el) };
+                if (gbm) {
+                  egl::display_t display = egl::make_display(gbm.get());
+                  if (display && egl::import_source(display.get(), sd)) {
+                    BOOST_LOG(info) << "EGL import and hardware encoding test was successful on GPU: "sv << entry.path().c_str();
+                    card.render_fd = std::move(compatible_render_fd);
+                    return 0;
+                  }
+                }
+              }
+            }
+          }
+        }
+        else {
+          return 0;
+        }
+#endif
+        BOOST_LOG(warning) << "Reverting back to GPU -> RAM for monitor "sv << display_name;
+        return -1;
       }
 
       std::uint64_t sequence {};

src/platform/linux/vaapi.cpp

@@ -467,6 +467,10 @@ namespace va {
       return false;
     }
 
+    if (video::active_av1_mode > 1 && !query(display.get(), VAProfileAV1Profile0)) {
+      return false;
+    }
+
     return true;
   }
 

src_assets/common/assets/web/config.html

@@ -455,7 +455,7 @@ <h2 class="accordion-header">
           <label for="adapter_name" class="form-label">Adapter Name</label>
           <input class="form-control" id="adapter_name" placeholder="/dev/dri/renderD128" v-model="config.adapter_name" />
           <div class="form-text">
-            Manually specify a GPU to use for capture.<br>
+            Manually specify a GPU to use for encoding. This MUST be a render node (renderD*) not a primary node (card*).<br>
             <pre>ls /dev/dri/renderD*  # to find all devices capable of VAAPI</pre>
             <pre>
               vainfo --display drm --device /dev/dri/renderD129 | \