diff --git a/unified-runtime/source/adapters/level_zero/command_buffer.cpp b/unified-runtime/source/adapters/level_zero/command_buffer.cpp index 45ab74c52a969..eb1e9b7cda468 100644 --- a/unified-runtime/source/adapters/level_zero/command_buffer.cpp +++ b/unified-runtime/source/adapters/level_zero/command_buffer.cpp @@ -76,7 +76,9 @@ bool checkCounterBasedEventsSupport(ur_device_handle_t Device) { return std::atoi(UrRet) != 0; }(); - return Device->ImmCommandListUsed && Device->useDriverInOrderLists() && + return Device->ImmCommandListUsed && + Device->Platform->allowDriverInOrderLists( + true /*Only Allow Driver In Order List if requested*/) && useDriverCounterBasedEvents && Device->Platform->ZeDriverEventPoolCountingEventsExtensionFound; } @@ -647,12 +649,9 @@ ur_result_t createMainCommandList(ur_context_handle_t Context, */ bool canBeInOrder(ur_context_handle_t Context, const ur_exp_command_buffer_desc_t *CommandBufferDesc) { - const char *UrRet = std::getenv("UR_L0_USE_DRIVER_INORDER_LISTS"); - // In-order command-lists are not available in old driver version. - bool DriverInOrderRequested = UrRet ? std::atoi(UrRet) != 0 : false; - bool CompatibleDriver = Context->getPlatform()->isDriverVersionNewerOrSimilar( - 1, 3, L0_DRIVER_INORDER_MIN_VERSION); - bool CanUseDriverInOrderLists = CompatibleDriver && DriverInOrderRequested; + bool CanUseDriverInOrderLists = + Context->getPlatform()->allowDriverInOrderLists( + true /*Only Allow Driver In Order List if requested*/); return CanUseDriverInOrderLists ? CommandBufferDesc->isInOrder : false; } diff --git a/unified-runtime/source/adapters/level_zero/common.hpp b/unified-runtime/source/adapters/level_zero/common.hpp index 4ddd1c49a493c..c970daa4c2585 100644 --- a/unified-runtime/source/adapters/level_zero/common.hpp +++ b/unified-runtime/source/adapters/level_zero/common.hpp @@ -607,6 +607,4 @@ extern thread_local int32_t ErrorAdapterNativeCode; // Utility function for setting a message and warning [[maybe_unused]] void setErrorMessage(const char *pMessage, ur_result_t ErrorCode, - int32_t AdapterErrorCode); - -#define L0_DRIVER_INORDER_MIN_VERSION 29534 + int32_t AdapterErrorCode); \ No newline at end of file diff --git a/unified-runtime/source/adapters/level_zero/context.cpp b/unified-runtime/source/adapters/level_zero/context.cpp index de1ca00f9f8fe..e3d6551177bef 100644 --- a/unified-runtime/source/adapters/level_zero/context.cpp +++ b/unified-runtime/source/adapters/level_zero/context.cpp @@ -207,7 +207,8 @@ ur_result_t ur_context_handle_t_::initialize() { ZeCommandQueueDesc.index = 0; ZeCommandQueueDesc.mode = ZE_COMMAND_QUEUE_MODE_SYNCHRONOUS; - if (Device->useDriverInOrderLists() && + if (Device->Platform->allowDriverInOrderLists( + true /*Only Allow Driver In Order List if requested*/) && Device->useDriverCounterBasedEvents()) { logger::debug( "L0 Synchronous Immediate Command List needed with In Order property."); @@ -673,8 +674,9 @@ ur_result_t ur_context_handle_t_::getAvailableCommandList( for (auto ZeCommandListIt = ZeCommandListCache.begin(); ZeCommandListIt != ZeCommandListCache.end(); ++ZeCommandListIt) { // If this is an InOrder Queue, then only allow lists which are in order. - if (Queue->Device->useDriverInOrderLists() && Queue->isInOrderQueue() && - !(ZeCommandListIt->second.InOrderList)) { + if (Queue->Device->Platform->allowDriverInOrderLists( + true /*Only Allow Driver In Order List if requested*/) && + Queue->isInOrderQueue() && !(ZeCommandListIt->second.InOrderList)) { continue; } // Only allow to reuse Regular Command Lists @@ -740,8 +742,9 @@ ur_result_t ur_context_handle_t_::getAvailableCommandList( continue; // If this is an InOrder Queue, then only allow lists which are in order. - if (Queue->Device->useDriverInOrderLists() && Queue->isInOrderQueue() && - !(it->second.IsInOrderList)) { + if (Queue->Device->Platform->allowDriverInOrderLists( + true /*Only Allow Driver In Order List if requested*/) && + Queue->isInOrderQueue() && !(it->second.IsInOrderList)) { continue; } diff --git a/unified-runtime/source/adapters/level_zero/device.cpp b/unified-runtime/source/adapters/level_zero/device.cpp index 429a041787639..9401bacdfafa7 100644 --- a/unified-runtime/source/adapters/level_zero/device.cpp +++ b/unified-runtime/source/adapters/level_zero/device.cpp @@ -1668,22 +1668,6 @@ bool ur_device_handle_t_::useRelaxedAllocationLimits() { return EnableRelaxedAllocationLimits; } -bool ur_device_handle_t_::useDriverInOrderLists() { - // Use in-order lists implementation from L0 driver instead - // of adapter's implementation. - - static const bool UseDriverInOrderLists = [&] { - const char *UrRet = std::getenv("UR_L0_USE_DRIVER_INORDER_LISTS"); - // bool CompatibleDriver = this->Platform->isDriverVersionNewerOrSimilar( - // 1, 3, L0_DRIVER_INORDER_MIN_VERSION); - if (!UrRet) - return false; - return std::atoi(UrRet) != 0; - }(); - - return UseDriverInOrderLists; -} - bool ur_device_handle_t_::useDriverCounterBasedEvents() { // Use counter-based events implementation from L0 driver. diff --git a/unified-runtime/source/adapters/level_zero/device.hpp b/unified-runtime/source/adapters/level_zero/device.hpp index 7520d33c70e68..d6149125fd8f7 100644 --- a/unified-runtime/source/adapters/level_zero/device.hpp +++ b/unified-runtime/source/adapters/level_zero/device.hpp @@ -155,9 +155,6 @@ struct ur_device_handle_t_ : _ur_object { // Read env settings to select immediate commandlist mode. ImmCmdlistMode useImmediateCommandLists(); - // Whether Adapter uses driver's implementation of in-order lists or not - bool useDriverInOrderLists(); - // Whether Adapter uses driver's implementation of counter-based events or not bool useDriverCounterBasedEvents(); diff --git a/unified-runtime/source/adapters/level_zero/event.cpp b/unified-runtime/source/adapters/level_zero/event.cpp index 11c0502a55939..7c372a7970993 100644 --- a/unified-runtime/source/adapters/level_zero/event.cpp +++ b/unified-runtime/source/adapters/level_zero/event.cpp @@ -223,22 +223,6 @@ ur_result_t urEnqueueEventsWaitWithBarrierExt( if (Queue->isInOrderQueue() && InOrderBarrierBySignal && !Queue->isProfilingEnabled()) { if (EventWaitList.Length) { - if (CmdList->second.IsInOrderList) { - for (unsigned i = EventWaitList.Length; i-- > 0;) { - // If the event is a multidevice event, then given driver in - // order lists, we cannot include this into the wait event list - // due to driver limitations. - if (EventWaitList.UrEventList[i]->IsMultiDevice) { - EventWaitList.Length--; - if (EventWaitList.Length != i) { - std::swap(EventWaitList.UrEventList[i], - EventWaitList.UrEventList[EventWaitList.Length]); - std::swap(EventWaitList.ZeEventList[i], - EventWaitList.ZeEventList[EventWaitList.Length]); - } - } - } - } ZE2UR_CALL(zeCommandListAppendWaitOnEvents, (CmdList->first, EventWaitList.Length, EventWaitList.ZeEventList)); @@ -1505,8 +1489,9 @@ ur_result_t _ur_ze_event_list_t::createAndRetainUrZeEventList( // the native driver implementation will already ensure in-order semantics. // The only exception is when a different immediate command was last used on // the same UR Queue. - if (CurQueue->Device->useDriverInOrderLists() && CurQueue->isInOrderQueue() && - CurQueue->UsingImmCmdLists) { + if (CurQueue->Device->Platform->allowDriverInOrderLists( + true /*Only Allow Driver In Order List if requested*/) && + CurQueue->isInOrderQueue() && CurQueue->UsingImmCmdLists) { auto QueueGroup = CurQueue->getQueueGroup(UseCopyEngine); uint32_t QueueGroupOrdinal, QueueIndex; auto NextIndex = QueueGroup.getQueueIndex(&QueueGroupOrdinal, &QueueIndex, @@ -1535,7 +1520,8 @@ ur_result_t _ur_ze_event_list_t::createAndRetainUrZeEventList( // For in-order queue and wait-list which is empty or has events only from // the same queue then we don't need to wait on any other additional events - if (CurQueue->Device->useDriverInOrderLists() && + if (CurQueue->Device->Platform->allowDriverInOrderLists( + true /*Only Allow Driver In Order List if requested*/) && CurQueue->isInOrderQueue() && WaitListEmptyOrAllEventsFromSameQueue(CurQueue, EventListLength, EventList)) { diff --git a/unified-runtime/source/adapters/level_zero/platform.cpp b/unified-runtime/source/adapters/level_zero/platform.cpp index 7694044de51d6..ba8c18cc82110 100644 --- a/unified-runtime/source/adapters/level_zero/platform.cpp +++ b/unified-runtime/source/adapters/level_zero/platform.cpp @@ -494,6 +494,28 @@ ur_result_t ur_platform_handle_t_::initialize() { return UR_RESULT_SUCCESS; } +bool ur_platform_handle_t_::allowDriverInOrderLists(bool OnlyIfRequested) { + // Use in-order lists implementation from L0 driver instead + // of adapter's implementation. + + // The following driver version is known to be passing and only this or newer + // drivers should be allowed by default for in order lists. +#define L0_DRIVER_INORDER_MINOR_VERSION 6 +#define L0_DRIVER_INORDER_PATCH_VERSION 32149 + + static const bool UseEnvVarDriverInOrderLists = [&] { + const char *UrRet = std::getenv("UR_L0_USE_DRIVER_INORDER_LISTS"); + return UrRet ? std::atoi(UrRet) != 0 : false; + }(); + static const bool UseDriverInOrderLists = [this] { + bool CompatibleDriver = this->isDriverVersionNewerOrSimilar( + 1, L0_DRIVER_INORDER_MINOR_VERSION, L0_DRIVER_INORDER_PATCH_VERSION); + return CompatibleDriver || UseEnvVarDriverInOrderLists; + }(); + + return OnlyIfRequested ? UseEnvVarDriverInOrderLists : UseDriverInOrderLists; +} + /// Checks the version of the level-zero driver. /// @param VersionMajor Major verion number to compare to. /// @param VersionMinor Minor verion number to compare to. diff --git a/unified-runtime/source/adapters/level_zero/platform.hpp b/unified-runtime/source/adapters/level_zero/platform.hpp index b34aef34eca35..0cb64361dc29a 100644 --- a/unified-runtime/source/adapters/level_zero/platform.hpp +++ b/unified-runtime/source/adapters/level_zero/platform.hpp @@ -48,6 +48,10 @@ struct ur_platform_handle_t_ : public _ur_platform { // Zero. ZeDriverVersionStringExtension ZeDriverVersionString; + // Helper function to check if the driver supports Driver In Order Lists or + // the User has Requested this support. + bool allowDriverInOrderLists(bool OnlyIfRequested = false); + // Cache versions info from zeDriverGetProperties. std::string ZeDriverVersion; std::string ZeDriverApiVersion; diff --git a/unified-runtime/source/adapters/level_zero/queue.cpp b/unified-runtime/source/adapters/level_zero/queue.cpp index d4cea7faa9d6e..ce03a96a49214 100644 --- a/unified-runtime/source/adapters/level_zero/queue.cpp +++ b/unified-runtime/source/adapters/level_zero/queue.cpp @@ -1198,11 +1198,15 @@ ur_queue_handle_t_::ur_queue_handle_t_( CopyCommandBatch.QueueBatchSize = ZeCommandListBatchCopyConfig.startSize(); this->CounterBasedEventsEnabled = - UsingImmCmdLists && isInOrderQueue() && Device->useDriverInOrderLists() && + UsingImmCmdLists && isInOrderQueue() && + Device->Platform->allowDriverInOrderLists( + true /*Only Allow Driver In Order List if requested*/) && Device->useDriverCounterBasedEvents() && Device->Platform->ZeDriverEventPoolCountingEventsExtensionFound; this->InterruptBasedEventsEnabled = - isLowPowerEvents() && isInOrderQueue() && Device->useDriverInOrderLists(); + isLowPowerEvents() && isInOrderQueue() && + Device->Platform->allowDriverInOrderLists( + true /*Only Allow Driver In Order List if requested*/); } void ur_queue_handle_t_::adjustBatchSizeForFullBatch(bool IsCopy) { @@ -2297,7 +2301,9 @@ ur_result_t ur_queue_handle_t_::createCommandList( ZeCommandListDesc.commandQueueGroupOrdinal = QueueGroupOrdinal; bool IsInOrderList = false; - if (Device->useDriverInOrderLists() && isInOrderQueue()) { + if (Device->Platform->allowDriverInOrderLists( + true /*Only Allow Driver In Order List if requested*/) && + isInOrderQueue()) { ZeCommandListDesc.flags = ZE_COMMAND_LIST_FLAG_IN_ORDER; IsInOrderList = true; } @@ -2434,7 +2440,9 @@ ur_command_list_ptr_t &ur_queue_handle_t_::ur_queue_group_t::getImmCmdList() { ZeCommandQueueDesc.flags |= ZE_COMMAND_QUEUE_FLAG_EXPLICIT_ONLY; } - if (Queue->Device->useDriverInOrderLists() && Queue->isInOrderQueue()) { + if (Queue->Device->Platform->allowDriverInOrderLists( + true /*Only Allow Driver In Order List if requested*/) && + Queue->isInOrderQueue()) { isInOrderList = true; ZeCommandQueueDesc.flags |= ZE_COMMAND_QUEUE_FLAG_IN_ORDER; }