ImGui integration with Gstreamer

[zgjja]

Version/Branch of Dear ImGui:

docking

Back-ends:

imgui_impl_glfw.cpp + imgui_impl_opengl3.cpp

Compiler, OS:

Linux + GCC11

Full config/build information:

CMake

cmake_minimum_required(VERSION 3.27)

project(imgui)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)

find_package(PkgConfig REQUIRED)
pkg_check_modules(glib REQUIRED IMPORTED_TARGET glib-2.0)
pkg_check_modules(
  gst
  REQUIRED
  IMPORTED_TARGET
  gstreamer-1.0
  gstreamer-rtsp-server-1.0
  gstreamer-video-1.0
  gstreamer-base-1.0
  gstreamer-app-1.0
  gstreamer-gl-egl-1.0)
pkg_check_modules(glfw REQUIRED IMPORTED_TARGET glfw3)
pkg_check_modules(opengl REQUIRED IMPORTED_TARGET glesv2 egl)

set(imgui_src
    imgui/imgui.cpp
    imgui/imgui_draw.cpp
    imgui/imgui_tables.cpp
    imgui/imgui_widgets.cpp
    imgui/imgui_demo.cpp
    imgui/backends/imgui_impl_glfw.cpp
    imgui/backends/imgui_impl_opengl3.cpp)

set(src main.cpp)

add_executable(${PROJECT_NAME} ${src} ${imgui_src})

target_include_directories(
  ${PROJECT_NAME}
  PUBLIC "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/imgui>"
         "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/imgui/backends>"
         "$<INSTALL_INTERFACE:${CMAKE_BINARY_DIR}/include>"
         "$<INSTALL_INTERFACE:${CMAKE_BINARY_DIR}/include/backends>")

target_link_libraries(${PROJECT_NAME} PUBLIC PkgConfig::opengl PkgConfig::glfw
                                             PkgConfig::gst)

target_compile_definitions(
  ${PROJECT_NAME}
  PUBLIC IMGUI_IMPL_OPENGL_ES2
  PUBLIC GLFW_EXPOSE_NATIVE_X11)

Details:

I want to integrate ImGui with deepstream(it uses gstreamer -- a video processing library as backbone). I will summarize some details for better understanding:

• a "sink" element from gstreamer can output a video to a window(X11 for my case)
• a sink element window can be redirected to your own window ID (use GstVideoOverlay)
• glfw has ability to get window ID created by ImGui with glfwGetX11Window

From the info above, i want to use ImGui to show this video content, so my demo code would be similar to examples/example_glfw_opengl3

Result:

1. If my code do glfwMakeContextCurrent(gl_window);, the gstreamer will complain:

ERROR egladaption ext/eglgles/gstegladaptation_egl.c:210:gst_egl_adaptation_context_make_current:<sink> Couldn't bind context

in this situation, the ImGui elements can be rendered but my video cannot. I checked the code, here is the link: gstegladaptation_egl.c(this may not be the exact version of my library use)
2. If I remove that line of code, the video can be rendered but ImGui elements cannot, and ImGui complains:

ERROR: ImGui_ImplOpenGL3_CreateDeviceObjects: failed to compile fragment shader! With GLSL: #version 100

ERROR: ImGui_ImplOpenGL3_CreateDeviceObjects: failed to link shader program! With GLSL #version 100

Analyse

• I check the code of gstegladaptation_egl.c, it use eglMakeCurrent() to use its internal context, maybe glfwMakeContextCurrent conflict with it (NOT sure)??
• In github, there are a few trying to do this kind of integration, but they grab the buffer from gstreamer directly and sent to ImGui rather than using GstVideoOverlay concept, which require more work to do.
• X11 and GTK+ can create window directly and pass the window handle, this works fine with gstreamer, but they are not easy to use like ImGui
• if I want to do something like this, how can i make it??

Screenshots/Video:

• Case 1
  Can render video, but no ImGui elements
  
• Case 2
  Can render ImGui elements, but no video
  

Minimal, Complete and Verifiable Example code:

Generate the test video

  auto *pipeline =
      gst_parse_launch("nvvideotestsrc ! nvvideoconvert ! nveglglessink name=sink", nullptr);
  auto *bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline));
  gst_bus_set_sync_handler(bus, (GstBusSyncHandler)dispatch_window_handle, (gpointer)pipeline, NULL);
  gst_object_unref(bus);
  gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_PLAYING);

Dispatch ImGui window

static GstBusSyncReply dispatch_window_handle(GstBus *bus, GstMessage *message, GstPipeline *pipeline) {
  // ignore anything but 'prepare-window-handle' element messages
  if (!gst_is_video_overlay_prepare_window_handle_message(message)) {
    return GST_BUS_PASS;
  }
  auto *sink    = GST_ELEMENT(gst_bin_get_by_name(GST_BIN(pipeline), "sink"));
  auto *overlay = GST_VIDEO_OVERLAY(sink);
  
  glfwSetErrorCallback(glfw_error_callback);
  if (!glfwInit()) {
    raise(SIGINT);
  }
  glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 2);
  glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 0);
  glfwWindowHint(GLFW_CLIENT_API, GLFW_OPENGL_ES_API);
  IMGUI_CHECKVERSION();
  
  gl_window     = glfwCreateWindow(1280, 720, "Hello, world!", nullptr, nullptr);
  if (!gl_window) {
    glfwTerminate();
    raise(SIGINT);
  }

  // glfwMakeContextCurrent(gl_window);  <-- This code cause my problem!
  // glfwSwapInterval(1);
  auto a = gst_gl_context_get_current_gl_context(GST_GL_PLATFORM_ANY);
  
  auto  x11_window = glfwGetX11Window(gl_window);
// These 2 lines are not necessary
//  auto  x11_display = glfwGetX11Display();
//  g_object_set(G_OBJECT(sink), "display", (gpointer)x11_display, NULL);

  auto  imgui_ctx  = ImGui::CreateContext();
  auto &io         = ImGui::GetIO(); (void)io;
  io.ConfigFlags |= ImGuiConfigFlags_NavEnableKeyboard;
  io.ConfigFlags |= ImGuiConfigFlags_NavEnableSetMousePos;
  io.ConfigFlags |= ImGuiConfigFlags_ViewportsEnable;

  ImGui::SetCurrentContext(imgui_ctx);
  ImGui::StyleColorsDark();

  ImGui_ImplGlfw_InitForOpenGL(gl_window, true);
  ImGui_ImplOpenGL3_Init(nullptr);

  gst_video_overlay_set_window_handle(overlay, (guintptr)x11_window);
  gst_video_overlay_set_render_rectangle(overlay, 0, 0, 512, 256);
  gst_video_overlay_expose(overlay);
  gst_message_unref(message);
  return GST_BUS_DROP;
}

ImGui render loop

while (!glfwWindowShouldClose(gl_window)) {
glfwPollEvents();

ImGui_ImplOpenGL3_NewFrame();
ImGui_ImplGlfw_NewFrame();
ImGui::NewFrame();

if (!ImGui::Begin("DearImGuiDemo", nullptr, 0)) {
  ImGui::End();
  return 1;
}

ImGui::Text("Hello, world!");

ImGui::End();

ImGui::Render();
glClear(GL_COLOR_BUFFER_BIT);
ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());

auto &io = ImGui::GetIO();
if (io.ConfigFlags & ImGuiConfigFlags_ViewportsEnable) {
  ImGui::UpdatePlatformWindows();
  ImGui::RenderPlatformWindowsDefault();
}

glfwSwapBuffers(gl_window);

}

[GamingMinds-DanielC]

You will most likely have to juggle contexts in your render loop. Make your context current only for the portion of the frame where you need it to be, then restore the context required for video rendering.

Btw, your early out in your render loop is a potential hazard. If ImGui::Begin() returns false, you should only skip submitting items to it. You are skipping the entire rest of the render loop and are ending the loop as well, that seems dangerous.