[SYCL][UR] Add Windows debug libs to install target (#17512)

1. Add install target for Windows debug libs called
`install-unified-runtime-libraries` which installs only the runtime
library subset of targets. This can be used to provide `ur_loaderd.dll`
and `ur_adatper_<name>d.dll` which link against the multithreaded debug
DLL C runtime by using the `DEBUG_POSTFIX` target property to append `d`
to the library names.
2. Add `UR_USE_DEBUG_POSTFIX` option to enable adding the `d` suffix to
library names.
3. Remove setting redundant `OUTPUT_NAME`s as combining
`DEBUG_POSTFIX=d` and then setting
`LIBRARY_OUTPUT_NAME`/`RUNTIME_OUTPUT_NAME` exposes a bug in CMake where
the `DEBUG_POSTFIX` is ignored for the `ur_loader.dll`. Removing these
redundant property setters fixed the previous issue.
4. Add debug UR subbuild on Windows `ExternalProject` compiled in debug
mode and with the `UR_USE_DEBUG_POSTFIX` option enabled.
* The resulting libraries are then copied to the
`<build>/bin`/`<build>/lib` directories so they can be used in testing
(not yet implemented).
* Additionally, they are also included in the `deploy-sycl-toolchain`
install target alongside the normally named runtime libraries.
5. Update `ur_proxy_loaderd.dll` to use Windows debug libs

Author: kbenzie

sycl/CMakeLists.txt

@@ -402,7 +402,7 @@ add_custom_target( sycl-toolchain ALL
 )
 
 if (WIN32)
-  add_dependencies(sycl-toolchain ur_win_proxy_loader)
+  add_dependencies(sycl-toolchain ur_win_proxy_loader unified-runtimed-build)
 endif()
 
 if("cuda" IN_LIST SYCL_ENABLE_BACKENDS)

sycl/cmake/modules/FetchUnifiedRuntime.cmake

@@ -317,6 +317,113 @@ if("native_cpu" IN_LIST SYCL_ENABLE_BACKENDS)
   endif()
 endif()
 
+if(CMAKE_SYSTEM_NAME STREQUAL Windows)
+  # On Windows, also build/install debug libraries with the d suffix that are
+  # compiled with /MDd so users can link against these in debug builds.
+  include(ExternalProject)
+  set(URD_BINARY_DIR ${CMAKE_BINARY_DIR}/unified-runtimed)
+  set(URD_INSTALL_DIR ${URD_BINARY_DIR}/install)
+
+  # This creates a subbuild which can be used in dependencies with the
+  # unified-runtimed target. It invokes the install-unified-runtime-libraries
+  # target to install the UR runtime libraries.
+  ExternalProject_Add(unified-runtimed
+    SOURCE_DIR ${UNIFIED_RUNTIME_SOURCE_DIR}
+    BINARY_DIR ${URD_BINARY_DIR}
+    INSTALL_DIR ${URD_INSTALL_DIR}
+    INSTALL_COMMAND ${CMAKE_COMMAND}
+      --build <BINARY_DIR> --config Debug
+      --target install-unified-runtime-libraries
+    CMAKE_CACHE_ARGS
+      -DCMAKE_BUILD_TYPE:STRING=Debug
+      -DCMAKE_INSTALL_PREFIX:STRING=<INSTALL_DIR>
+      # Enable d suffix on libraries
+      -DUR_USE_DEBUG_POSTFIX:BOOL=ON
+      # Don't build unnecessary targets in subbuild.
+      -DUR_BUILD_EXAMPLES:BOOL=OFF
+      -DUR_BUILD_TESTS:BOOL=OFF
+      -DUR_BUILD_TOOLS:BOOL=OFF
+      # Sanitizer layer is not supported on Windows.
+      -DUR_ENABLE_SYMBOLIZER:BOOL=OFF
+      # Inherit settings from parent build.
+      -DUR_ENABLE_TRACING:BOOL=${UR_ENABLE_TRACING}
+      -DUR_ENABLE_COMGR:BOOL=${UR_ENABLE_COMGR}
+      -DUR_BUILD_ADAPTER_L0:BOOL=${UR_BUILD_ADAPTER_L0}
+      -DUR_BUILD_ADAPTER_L0_V2:BOOL=${UR_BUILD_ADAPTER_L0_V2}
+      -DUR_BUILD_ADAPTER_OPENCL:BOOL=${UR_BUILD_ADAPTER_OPENCL}
+      -DUR_BUILD_ADAPTER_CUDA:BOOL=${UR_BUILD_ADAPTER_CUDA}
+      -DUR_BUILD_ADAPTER_HIP:BOOL=${UR_BUILD_ADAPTER_HIP}
+      -DUR_BUILD_ADAPTER_NATIVE_CPU:BOOL=${UR_BUILD_ADAPTER_NATIVE_CPU}
+      -DUMF_BUILD_EXAMPLES:BOOL=${UMF_BUILD_EXAMPLES}
+      -DUMF_BUILD_SHARED_LIBRARY:BOOL=${UMF_BUILD_SHARED_LIBRARY}
+      -DUMF_LINK_HWLOC_STATICALLY:BOOL=${UMF_LINK_HWLOC_STATICALLY}
+      -DUMF_DISABLE_HWLOC:BOOL=${UMF_DISABLE_HWLOC}
+      # Enable d suffix in UMF
+      -DUMF_USE_DEBUG_POSTFIX:BOOL=ON
+  )
+
+  # Copy the debug UR runtime libraries to <build>/bin & <build>/lib for use in
+  # the parent build, e.g. integration testing.
+  set(URD_COPY_FILES)
+  macro(urd_copy_library_to_build library shared)
+    if(${shared})
+      list(APPEND URD_COPY_FILES
+        ${LLVM_BINARY_DIR}/bin/${library}.dll
+      )
+      add_custom_command(
+        OUTPUT
+          ${LLVM_BINARY_DIR}/bin/${library}.dll
+        COMMAND ${CMAKE_COMMAND} -E copy
+          ${URD_INSTALL_DIR}/bin/${library}.dll
+          ${LLVM_BINARY_DIR}/bin/${library}.dll
+      )
+    endif()
+
+    list(APPEND URD_COPY_FILES
+      ${LLVM_BINARY_DIR}/lib/${library}.lib
+    )
+    add_custom_command(
+      OUTPUT
+        ${LLVM_BINARY_DIR}/lib/${library}.lib
+      COMMAND ${CMAKE_COMMAND} -E copy
+        ${URD_INSTALL_DIR}/lib/${library}.lib
+        ${LLVM_BINARY_DIR}/lib/${library}.lib
+    )
+  endmacro()
+
+  urd_copy_library_to_build(ur_loaderd "NOT;${UR_STATIC_LOADER}")
+  foreach(adatper ${SYCL_ENABLE_BACKENDS})
+    if(adapter MATCHES "level_zero")
+      set(shared "NOT;${UR_STATIC_ADAPTER_L0}")
+    else()
+      set(shared TRUE)
+    endif()
+    urd_copy_library_to_build(ur_adapter_${adatper}d "${shared}")
+  endforeach()
+  # Also copy umfd.dll/umfd.lib
+  urd_copy_library_to_build(umfd ${UMF_BUILD_SHARED_LIBRARY})
+
+  add_custom_target(unified-runtimed-build ALL DEPENDS ${URD_COPY_FILES})
+  add_dependencies(unified-runtimed-build unified-runtimed)
+
+  # Add the debug UR runtime libraries to the parent install.
+  install(
+    FILES ${URD_INSTALL_DIR}/bin/ur_loaderd.dll
+    DESTINATION "bin" COMPONENT unified-runtime-loader)
+  foreach(adapter ${SYCL_ENABLE_BACKENDS})
+    install(
+      FILES ${URD_INSTALL_DIR}/bin/ur_adapter_${adapter}d.dll
+      DESTINATION "bin" COMPONENT ur_adapter_${adapter})
+    add_dependencies(install-sycl-ur-adapter-${adapter} unified-runtimed)
+  endforeach()
+  if(UMF_BUILD_SHARED_LIBRARY)
+    # Also install umfd.dll
+    install(
+      FILES ${URD_INSTALL_DIR}/bin/umfd.dll
+      DESTINATION "bin" COMPONENT unified-memory-framework)
+  endif()
+endif()
+
 install(TARGETS umf
   LIBRARY DESTINATION "lib${LLVM_LIBDIR_SUFFIX}" COMPONENT unified-memory-framework
   ARCHIVE DESTINATION "lib${LLVM_LIBDIR_SUFFIX}" COMPONENT unified-memory-framework

sycl/ur_win_proxy_loader/CMakeLists.txt

@@ -57,6 +57,7 @@ if (MSVC)
   add_library(ur_win_proxy_loaderd SHARED ur_win_proxy_loader.cpp ${CMAKE_CURRENT_BINARY_DIR}/versioninfo.rc)
   target_compile_options(ur_win_proxy_loaderd PRIVATE ${WINUNLOAD_CXX_FLAGS_DEBUG})
   target_compile_options(ur_win_proxy_loader PRIVATE ${WINUNLOAD_CXX_FLAGS_RELEASE})
+  target_compile_definitions(ur_win_proxy_loaderd PRIVATE UR_WIN_PROXY_LOADER_DEBUG)
   target_link_libraries(ur_win_proxy_loaderd PRIVATE shlwapi)
   target_link_libraries(ur_win_proxy_loader PRIVATE shlwapi)
   # 0x2000: LOAD_LIBRARY_SAFE_CURRENT_DIRS flag. Using this flag means that loading dependency DLLs

sycl/ur_win_proxy_loader/ur_win_proxy_loader.cpp

@@ -82,24 +82,22 @@ static std::wstring getCurrentDSODir() {
   return Path;
 }
 
-// these are cribbed from include/sycl/detail/ur.hpp
-// a new adapter must be added to both places.
 #ifdef _MSC_VER
-#define __SYCL_UNIFIED_RUNTIME_LOADER_NAME "ur_loader.dll"
-#define __SYCL_OPENCL_ADAPTER_NAME "ur_adapter_opencl.dll"
-#define __SYCL_LEVEL_ZERO_ADAPTER_NAME "ur_adapter_level_zero.dll"
-#define __SYCL_LEVEL_ZERO_V2_ADAPTER_NAME "ur_adapter_level_zero_v2.dll"
-#define __SYCL_CUDA_ADAPTER_NAME "ur_adapter_cuda.dll"
-#define __SYCL_HIP_ADAPTER_NAME "ur_adapter_hip.dll"
-#define __SYCL_NATIVE_CPU_ADAPTER_NAME "ur_adapter_native_cpu.dll"
+
+#ifdef UR_WIN_PROXY_LOADER_DEBUG_POSTFIX
+#define UR_LIBRARY_NAME(NAME) "ur_" #NAME "d.dll"
+#else
+#define UR_LIBRARY_NAME(NAME) "ur_" #NAME ".dll"
+#endif
+
 #else // llvm-mingw
-#define __SYCL_UNIFIED_RUNTIME_LOADER_NAME "libur_loader.dll"
-#define __SYCL_OPENCL_ADAPTER_NAME "libur_adapter_opencl.dll"
-#define __SYCL_LEVEL_ZERO_ADAPTER_NAME "libur_adapter_level_zero.dll"
-#define __SYCL_LEVEL_ZERO_V2_ADAPTER_NAME "libur_adapter_level_zero_v2.dll"
-#define __SYCL_CUDA_ADAPTER_NAME "libur_adapter_cuda.dll"
-#define __SYCL_HIP_ADAPTER_NAME "libur_adapter_hip.dll"
-#define __SYCL_NATIVE_CPU_ADAPTER_NAME "libur_adapter_native_cpu.dll"
+
+#ifdef UR_WIN_PROXY_LOADER_DEBUG_POSTFIX
+#define UR_LIBRARY_NAME(NAME) "libur" #NAME "d.dll"
+#else
+#define UR_LIBRARY_NAME(NAME) "libur" #NAME ".dll"
+#endif
+
 #endif
 
 // ------------------------------------
@@ -141,13 +139,13 @@ void preloadLibraries() {
   };
   // We keep the UR Loader handle so it can be fetched by the runtime, but the
   // adapter libraries themselves won't be used.
-  getDllHandle() = loadAdapter(__SYCL_UNIFIED_RUNTIME_LOADER_NAME);
-  loadAdapter(__SYCL_OPENCL_ADAPTER_NAME);
-  loadAdapter(__SYCL_LEVEL_ZERO_ADAPTER_NAME);
-  loadAdapter(__SYCL_LEVEL_ZERO_V2_ADAPTER_NAME);
-  loadAdapter(__SYCL_CUDA_ADAPTER_NAME);
-  loadAdapter(__SYCL_HIP_ADAPTER_NAME);
-  loadAdapter(__SYCL_NATIVE_CPU_ADAPTER_NAME);
+  getDllHandle() = loadAdapter(UR_LIBRARY_NAME(loader));
+  loadAdapter(UR_LIBRARY_NAME(adapter_opencl));
+  loadAdapter(UR_LIBRARY_NAME(adapter_level_zero));
+  loadAdapter(UR_LIBRARY_NAME(adapter_level_zero_v2));
+  loadAdapter(UR_LIBRARY_NAME(adapter_cuda));
+  loadAdapter(UR_LIBRARY_NAME(adapter_hip));
+  loadAdapter(UR_LIBRARY_NAME(adapter_native_cpu));
 
   // Restore system error handling.
   (void)SetErrorMode(SavedMode);

unified-runtime/CMakeLists.txt

@@ -66,6 +66,7 @@ set(UR_CONFORMANCE_TARGET_TRIPLES "" CACHE STRING
 set(UR_CONFORMANCE_AMD_ARCH "" CACHE STRING "AMD device target ID to build CTS binaries for")
 option(UR_CONFORMANCE_ENABLE_MATCH_FILES "Enable CTS match files" ON)
 option(UR_CONFORMANCE_TEST_LOADER "Also test the loader in the conformance tests" OFF)
+option(UR_USE_DEBUG_POSTFIX "Enable debug postfix 'd' for libraries" OFF)
 set(UR_ADAPTER_LEVEL_ZERO_SOURCE_DIR "" CACHE PATH
     "Path to external 'level_zero' adapter source dir")
 set(UR_ADAPTER_OPENCL_SOURCE_DIR "" CACHE PATH

unified-runtime/cmake/helpers.cmake

@@ -197,20 +197,44 @@ function(add_ur_library name)
             $<$<STREQUAL:$<TARGET_LINKER_FILE_NAME:${name}>,link.exe>:LINKER:/DEPENDENTLOADFLAG:0x2000>
         )
     endif()
+    if(UR_USE_DEBUG_POSTFIX)
+        set_target_properties(${name} PROPERTIES DEBUG_POSTFIX d)
+    endif()
     if(UR_EXTERNAL_DEPENDENCIES)
         add_dependencies(${name} ${UR_EXTERNAL_DEPENDENCIES})
     endif()
+    add_dependencies(unified-runtime-libraries ${name})
 endfunction()
 
+if(NOT TARGET unified-runtime-libraries)
+    add_custom_target(unified-runtime-libraries)
+endif()
+
 function(install_ur_library name)
     install(TARGETS ${name}
+            COMPONENT unified-runtime
             EXPORT ${PROJECT_NAME}-targets
             ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
             RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-            LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT unified-runtime
+            LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
     )
 endfunction()
 
+if(UR_USE_DEBUG_POSTFIX AND NOT TARGET install-unified-runtime-libraries)
+    add_custom_target(install-unified-runtime-libraries
+        COMMAND ${CMAKE_COMMAND}
+            -DCOMPONENT=unified-runtime
+            -P ${CMAKE_BINARY_DIR}/cmake_install.cmake
+        COMMAND ${CMAKE_COMMAND}
+            -DCOMPONENT=umfd
+            -P ${CMAKE_BINARY_DIR}/cmake_install.cmake
+        DEPENDS unified-runtime-libraries
+    )
+    if(TARGET build_umfd)
+        add_dependencies(install-unified-runtime-libraries build_umfd)
+    endif()
+endif()
+
 include(FetchContent)
 
 function(FetchSource GIT_REPOSITORY GIT_TAG GIT_DIR DEST)

unified-runtime/source/common/ur_util.hpp

@@ -59,15 +59,23 @@ int ur_duplicate_fd(int pid, int fd_in);
     defined(SANITIZER_THREAD)
 #define SANITIZER_ANY
 #endif
+
 ///////////////////////////////////////////////////////////////////////////////
+#if UR_USE_DEBUG_POSTFIX
+#define LIBRARY_NAME(NAME) NAME "d"
+#else
+#define LIBRARY_NAME(NAME) NAME
+#endif
+
 #if defined(_WIN32)
-#define MAKE_LIBRARY_NAME(NAME, VERSION) NAME ".dll"
+#define MAKE_LIBRARY_NAME(NAME, VERSION) LIBRARY_NAME(NAME) ".dll"
 #define STATIC_LIBRARY_EXTENSION ".lib"
 #else
 #if defined(__APPLE__)
-#define MAKE_LIBRARY_NAME(NAME, VERSION) "lib" NAME "." VERSION ".dylib"
+#define MAKE_LIBRARY_NAME(NAME, VERSION)                                       \
+  "lib" LIBRARY_NAME(NAME) "." VERSION ".dylib"
 #else
-#define MAKE_LIBRARY_NAME(NAME, VERSION) "lib" NAME ".so." VERSION
+#define MAKE_LIBRARY_NAME(NAME, VERSION) "lib" LIBRARY_NAME(NAME) ".so." VERSION
 #endif
 #define STATIC_LIBRARY_EXTENSION ".a"
 #endif

unified-runtime/source/loader/CMakeLists.txt

@@ -21,6 +21,10 @@ add_ur_library(ur_loader
 )
 install_ur_library(ur_loader)
 
+target_compile_definitions(ur_loader PRIVATE
+    UR_USE_DEBUG_POSTFIX=$<BOOL:${UR_USE_DEBUG_POSTFIX}>
+)
+
 if (MSVC)
     set(TARGET_LIBNAME ur_loader)
     string(TOUPPER ${TARGET_LIBNAME} TARGET_LIBNAME)
@@ -39,11 +43,6 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
     target_link_options(ur_loader PRIVATE "-Wl,--version-script=${LOADER_VERSION_SCRIPT}")
 endif()
 
-set_target_properties(ur_loader PROPERTIES
-    LIBRARY_OUTPUT_NAME ur_loader
-    RUNTIME_OUTPUT_NAME ur_loader
-)
-
 add_library(${PROJECT_NAME}::loader ALIAS ur_loader)
 
 target_include_directories(ur_loader PRIVATE