feature: add wrappers to waitpkg intrinsics

Related-To: NEO-9737

Signed-off-by: Zbigniew Zdanowicz <[email protected]>

Author: zzdanowicz

CMakeLists.txt

@@ -726,15 +726,23 @@ if(NOT MSVC)
   SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftemplate-depth=1024")
 endif()
 
-# _mm_clflushopt support
+# intrinsics (_mm_clflushopt and waitpkg) support
 if(NOT MSVC)
   check_cxx_compiler_flag(-mclflushopt SUPPORTS_CLFLUSHOPT)
+  check_cxx_compiler_flag(-mwaitpkg SUPPORTS_WAITPKG)
   if(SUPPORTS_CLFLUSHOPT)
     add_compile_definitions(SUPPORTS_CLFLUSHOPT)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mclflushopt")
   endif()
+  if(SUPPORTS_WAITPKG)
+    add_compile_definitions(SUPPORTS_WAITPKG)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mwaitpkg")
+  else()
+    message(WARNING "-mwaitpkg flag is not supported by the compiler")
+  endif()
 else()
   add_compile_definitions(SUPPORTS_CLFLUSHOPT)
+  add_compile_definitions(SUPPORTS_WAITPKG)
 endif()
 
 # Compiler warning flags

shared/source/utilities/cpuintrinsics.cpp

@@ -8,8 +8,11 @@
 #include "shared/source/utilities/cpuintrinsics.h"
 
 #if defined(_WIN32)
+#include <immintrin.h>
 #include <intrin.h>
+#pragma intrinsic(__rdtsc)
 #else
+#include <immintrin.h>
 #include <x86intrin.h>
 #endif
 
@@ -42,5 +45,23 @@ void pause() {
     _mm_pause();
 }
 
+unsigned char umwait(unsigned int ctrl, uint64_t counter) {
+#ifdef SUPPORTS_WAITPKG
+    return _umwait(ctrl, counter);
+#else
+    return 0;
+#endif
+}
+
+void umonitor(void *a) {
+#ifdef SUPPORTS_WAITPKG
+    _umonitor(a);
+#endif
+}
+
+uint64_t rdtsc() {
+    return __rdtsc();
+}
+
 } // namespace CpuIntrinsics
 } // namespace NEO

shared/source/utilities/cpuintrinsics.h

@@ -7,6 +7,8 @@
 
 #pragma once
 
+#include <cstdint>
+
 namespace NEO {
 namespace CpuIntrinsics {
 
@@ -18,5 +20,11 @@ void clFlushOpt(void *ptr);
 
 void pause();
 
+unsigned char umwait(unsigned int ctrl, uint64_t counter);
+
+void umonitor(void *a);
+
+uint64_t rdtsc();
+
 } // namespace CpuIntrinsics
 } // namespace NEO

shared/test/common/utilities/cpuintrinsics.cpp

@@ -21,11 +21,23 @@ std::atomic<uint32_t> clFlushCounter(0u);
 std::atomic<uint32_t> pauseCounter(0u);
 std::atomic<uint32_t> sfenceCounter(0u);
 
+std::atomic<uint64_t> lastUmwaitCounter(0u);
+std::atomic<unsigned int> lastUmwaitControl(0u);
+std::atomic<uint32_t> umwaitCounter(0u);
+
+std::atomic<uintptr_t> lastUmonitorPtr(0u);
+std::atomic<uint32_t> umonitorCounter(0u);
+
+std::atomic<uint32_t> rdtscCounter(0u);
+
 volatile TagAddressType *pauseAddress = nullptr;
 TaskCountType pauseValue = 0u;
 uint32_t pauseOffset = 0u;
+uint64_t rdtscRetValue = 0;
+unsigned char umwaitRetValue = 0;
 
 std::function<void()> setupPauseAddress;
+std::function<unsigned char()> controlUmwait;
 } // namespace CpuIntrinsicsTests
 
 namespace NEO {
@@ -56,5 +68,26 @@ void pause() {
     }
 }
 
+unsigned char umwait(unsigned int ctrl, uint64_t counter) {
+    CpuIntrinsicsTests::lastUmwaitControl = ctrl;
+    CpuIntrinsicsTests::lastUmwaitCounter = counter;
+    CpuIntrinsicsTests::umwaitCounter++;
+    if (CpuIntrinsicsTests::controlUmwait) {
+        return CpuIntrinsicsTests::controlUmwait();
+    } else {
+        return CpuIntrinsicsTests::umwaitRetValue;
+    }
+}
+
+void umonitor(void *a) {
+    CpuIntrinsicsTests::lastUmonitorPtr = reinterpret_cast<uintptr_t>(a);
+    CpuIntrinsicsTests::umonitorCounter++;
+}
+
+uint64_t rdtsc() {
+    CpuIntrinsicsTests::rdtscCounter++;
+    return CpuIntrinsicsTests::rdtscRetValue;
+}
+
 } // namespace CpuIntrinsics
 } // namespace NEO