Skip to content

Commit 97799b3

Browse files
johntfalkCompute-Runtime-Automation
johntfalk
authored and
Compute-Runtime-Automation
committed
feature: Implement appendMemoryPrefetch for Shared System USM Allocations
Related-To: NEO-12989 Signed-off-by: John Falkowski <[email protected]>
1 parent 642bdd1 commit 97799b3

29 files changed

+579
-139
lines changed

‎level_zero/core/source/cmdlist/cmdlist_hw.inl

+38-5
Original file line numberDiff line numberDiff line change
@@ -1997,13 +1997,46 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendMemoryCopyKernel2d(Align
19971997
}
19981998

19991999
template <GFXCORE_FAMILY gfxCoreFamily>
2000-
ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendMemoryPrefetch(const void *ptr,
2001-
size_t count) {
2002-
auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr);
2003-
if (allocData) {
2000+
ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendMemoryPrefetch(const void *ptr, size_t size) {
2001+
auto svmAllocMgr = device->getDriverHandle()->getSvmAllocsManager();
2002+
auto allocData = svmAllocMgr->getSVMAlloc(ptr);
2003+
2004+
if (!allocData) {
2005+
if (device->getNEODevice()->areSharedSystemAllocationsAllowed()) {
2006+
this->performMemoryPrefetch = true;
2007+
auto prefetchManager = device->getDriverHandle()->getMemoryManager()->getPrefetchManager();
2008+
if (prefetchManager) {
2009+
prefetchManager->insertAllocation(this->prefetchContext, *device->getDriverHandle()->getSvmAllocsManager(), *device->getNEODevice(), ptr, size);
2010+
}
2011+
return ZE_RESULT_SUCCESS;
2012+
} else {
2013+
return ZE_RESULT_ERROR_INVALID_ARGUMENT;
2014+
}
2015+
}
2016+
2017+
if (NEO::debugManager.flags.AppendMemoryPrefetchForKmdMigratedSharedAllocations.get() == true) {
2018+
this->performMemoryPrefetch = true;
2019+
auto prefetchManager = device->getDriverHandle()->getMemoryManager()->getPrefetchManager();
2020+
if (prefetchManager) {
2021+
prefetchManager->insertAllocation(this->prefetchContext, *device->getDriverHandle()->getSvmAllocsManager(), *device->getNEODevice(), ptr, size);
2022+
}
2023+
}
2024+
2025+
if (NEO::debugManager.flags.AddStatePrefetchCmdToMemoryPrefetchAPI.get() != 1) {
20042026
return ZE_RESULT_SUCCESS;
20052027
}
2006-
return ZE_RESULT_ERROR_INVALID_ARGUMENT;
2028+
2029+
auto gpuAlloc = allocData->gpuAllocations.getGraphicsAllocation(device->getRootDeviceIndex());
2030+
2031+
commandContainer.addToResidencyContainer(gpuAlloc);
2032+
2033+
size_t offset = ptrDiff(ptr, gpuAlloc->getGpuAddress());
2034+
2035+
NEO::LinearStream &cmdStream = *commandContainer.getCommandStream();
2036+
2037+
NEO::EncodeMemoryPrefetch<GfxFamily>::programMemoryPrefetch(cmdStream, *gpuAlloc, static_cast<uint32_t>(size), offset, device->getNEODevice()->getRootDeviceEnvironment());
2038+
2039+
return ZE_RESULT_SUCCESS;
20072040
}
20082041

20092042
template <GFXCORE_FAMILY gfxCoreFamily>

‎level_zero/core/source/xe2_hpg_core/cmdlist_xe2_hpg_core.cpp

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
/*
2-
* Copyright (C) 2024 Intel Corporation
2+
* Copyright (C) 2024-2025 Intel Corporation
33
*
44
* SPDX-License-Identifier: MIT
55
*
66
*/
77

8+
#include "shared/source/memory_manager/prefetch_manager.h"
89
#include "shared/source/xe2_hpg_core/hw_cmds_base.h"
910
#include "shared/source/xe2_hpg_core/hw_info.h"
1011

‎level_zero/core/source/xe_hpc_core/cmdlist_xe_hpc_core.cpp

+1-35
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (C) 2021-2023 Intel Corporation
2+
* Copyright (C) 2021-2025 Intel Corporation
33
*
44
* SPDX-License-Identifier: MIT
55
*
@@ -20,40 +20,6 @@
2020

2121
namespace L0 {
2222

23-
template <>
24-
ze_result_t CommandListCoreFamily<IGFX_XE_HPC_CORE>::appendMemoryPrefetch(const void *ptr, size_t size) {
25-
auto svmAllocMgr = device->getDriverHandle()->getSvmAllocsManager();
26-
auto allocData = svmAllocMgr->getSVMAlloc(ptr);
27-
28-
if (!allocData) {
29-
return ZE_RESULT_ERROR_INVALID_ARGUMENT;
30-
}
31-
32-
if (NEO::debugManager.flags.AppendMemoryPrefetchForKmdMigratedSharedAllocations.get() == true) {
33-
this->performMemoryPrefetch = true;
34-
auto prefetchManager = device->getDriverHandle()->getMemoryManager()->getPrefetchManager();
35-
if (prefetchManager) {
36-
prefetchManager->insertAllocation(this->prefetchContext, ptr, *allocData);
37-
}
38-
}
39-
40-
if (NEO::debugManager.flags.AddStatePrefetchCmdToMemoryPrefetchAPI.get() != 1) {
41-
return ZE_RESULT_SUCCESS;
42-
}
43-
44-
auto gpuAlloc = allocData->gpuAllocations.getGraphicsAllocation(device->getRootDeviceIndex());
45-
46-
commandContainer.addToResidencyContainer(gpuAlloc);
47-
48-
size_t offset = ptrDiff(ptr, gpuAlloc->getGpuAddress());
49-
50-
NEO::LinearStream &cmdStream = *commandContainer.getCommandStream();
51-
52-
NEO::EncodeMemoryPrefetch<GfxFamily>::programMemoryPrefetch(cmdStream, *gpuAlloc, static_cast<uint32_t>(size), offset, device->getNEODevice()->getRootDeviceEnvironment());
53-
54-
return ZE_RESULT_SUCCESS;
55-
}
56-
5723
template struct CommandListCoreFamily<IGFX_XE_HPC_CORE>;
5824
template struct CommandListCoreFamilyImmediate<IGFX_XE_HPC_CORE>;
5925

‎level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp

+2
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,8 @@ TEST_F(CommandListCreateTests, givenNonExistingPtrThenAppendMemoryPrefetchReturn
177177
ze_result_t returnValue;
178178
std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false));
179179
ASSERT_NE(nullptr, commandList);
180+
DebugManagerStateRestore restorer;
181+
debugManager.flags.EnableSharedSystemUsmSupport.set(0);
180182

181183
auto res = commandList->appendMemoryPrefetch(nullptr, 0);
182184
EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res);

‎level_zero/core/test/unit_tests/xe2_hpg_core/test_cmdlist_xe2_hpg_core.cpp

+158-1
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
/*
2-
* Copyright (C) 2024 Intel Corporation
2+
* Copyright (C) 2024-2025 Intel Corporation
33
*
44
* SPDX-License-Identifier: MIT
55
*
66
*/
77

88
#include "shared/source/os_interface/product_helper.h"
9+
#include "shared/source/unified_memory/usm_memory_support.h"
910
#include "shared/test/common/cmd_parse/gen_cmd_parse.h"
1011
#include "shared/test/common/helpers/debug_manager_state_restore.h"
12+
#include "shared/test/common/memory_manager/mock_prefetch_manager.h"
1113
#include "shared/test/common/test_macros/hw_test.h"
1214

1315
#include "level_zero/core/source/event/event.h"
@@ -456,5 +458,160 @@ HWTEST2_F(CmdListLargeGrfTestXe2Hpg,
456458
testBody<FamilyType>();
457459
}
458460

461+
using CommandListStatePrefetchXe2HpgCore = Test<ModuleFixture>;
462+
463+
HWTEST2_F(CommandListStatePrefetchXe2HpgCore, givenUnifiedSharedMemoryWhenPrefetchApiIsCalledThenRequestMemoryPrefetchByDefault, IsXe2HpgCore) {
464+
auto memoryManager = static_cast<MockMemoryManager *>(device->getDriverHandle()->getMemoryManager());
465+
memoryManager->prefetchManager.reset(new MockPrefetchManager());
466+
auto pCommandList = std::make_unique<WhiteBox<::L0::CommandListCoreFamily<gfxCoreFamily>>>();
467+
auto result = pCommandList->initialize(device, NEO::EngineGroupType::compute, 0u);
468+
ASSERT_EQ(ZE_RESULT_SUCCESS, result);
469+
470+
DebugManagerStateRestore restore;
471+
debugManager.flags.AppendMemoryPrefetchForKmdMigratedSharedAllocations.set(true);
472+
473+
size_t size = 10;
474+
size_t alignment = 1u;
475+
void *ptr = nullptr;
476+
477+
ze_device_mem_alloc_desc_t deviceDesc = {};
478+
ze_host_mem_alloc_desc_t hostDesc = {};
479+
result = context->allocSharedMem(device->toHandle(), &deviceDesc, &hostDesc, size, alignment, &ptr);
480+
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
481+
EXPECT_NE(nullptr, ptr);
482+
483+
result = pCommandList->appendMemoryPrefetch(ptr, size);
484+
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
485+
486+
EXPECT_TRUE(pCommandList->isMemoryPrefetchRequested());
487+
488+
context->freeMem(ptr);
489+
}
490+
491+
HWTEST2_F(CommandListStatePrefetchXe2HpgCore, givenUnifiedSharedMemoryWhenPrefetchApiIsCalledThenRequestMemoryPrefetchByDefaultWithNoPrefetchManager, IsXe2HpgCore) {
492+
auto memoryManager = static_cast<MockMemoryManager *>(device->getDriverHandle()->getMemoryManager());
493+
memoryManager->prefetchManager.reset(new MockPrefetchManager());
494+
auto pCommandList = std::make_unique<WhiteBox<::L0::CommandListCoreFamily<gfxCoreFamily>>>();
495+
auto result = pCommandList->initialize(device, NEO::EngineGroupType::compute, 0u);
496+
ASSERT_EQ(ZE_RESULT_SUCCESS, result);
497+
498+
DebugManagerStateRestore restore;
499+
debugManager.flags.AppendMemoryPrefetchForKmdMigratedSharedAllocations.set(true);
500+
501+
size_t size = 10;
502+
size_t alignment = 1u;
503+
void *ptr = nullptr;
504+
505+
ze_device_mem_alloc_desc_t deviceDesc = {};
506+
ze_host_mem_alloc_desc_t hostDesc = {};
507+
result = context->allocSharedMem(device->toHandle(), &deviceDesc, &hostDesc, size, alignment, &ptr);
508+
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
509+
EXPECT_NE(nullptr, ptr);
510+
511+
result = pCommandList->appendMemoryPrefetch(ptr, size);
512+
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
513+
514+
EXPECT_TRUE(pCommandList->isMemoryPrefetchRequested());
515+
516+
context->freeMem(ptr);
517+
}
518+
519+
HWTEST2_F(CommandListStatePrefetchXe2HpgCore, givenUnifiedSharedMemoryWhenPrefetchApiCalledAndDebugFlagFalseThenRequestMemoryPrefetchNotCalled, IsXe2HpgCore) {
520+
auto pCommandList = std::make_unique<WhiteBox<::L0::CommandListCoreFamily<gfxCoreFamily>>>();
521+
auto result = pCommandList->initialize(device, NEO::EngineGroupType::compute, 0u);
522+
ASSERT_EQ(ZE_RESULT_SUCCESS, result);
523+
524+
DebugManagerStateRestore restore;
525+
debugManager.flags.AppendMemoryPrefetchForKmdMigratedSharedAllocations.set(false);
526+
527+
size_t size = 10;
528+
size_t alignment = 1u;
529+
void *ptr = nullptr;
530+
531+
ze_device_mem_alloc_desc_t deviceDesc = {};
532+
ze_host_mem_alloc_desc_t hostDesc = {};
533+
result = context->allocSharedMem(device->toHandle(), &deviceDesc, &hostDesc, size, alignment, &ptr);
534+
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
535+
EXPECT_NE(nullptr, ptr);
536+
537+
result = pCommandList->appendMemoryPrefetch(ptr, size);
538+
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
539+
540+
EXPECT_FALSE(pCommandList->isMemoryPrefetchRequested());
541+
542+
context->freeMem(ptr);
543+
}
544+
545+
HWTEST2_F(CommandListStatePrefetchXe2HpgCore, givenSharedSystemAllocationOnSupportedDeviceWhenPrefetchApiIsCalledThenRequestMemoryPrefetchCalled, IsXe2HpgCore) {
546+
auto memoryManager = static_cast<MockMemoryManager *>(device->getDriverHandle()->getMemoryManager());
547+
memoryManager->prefetchManager.reset(new MockPrefetchManager());
548+
auto pCommandList = std::make_unique<WhiteBox<::L0::CommandListCoreFamily<gfxCoreFamily>>>();
549+
auto result = pCommandList->initialize(device, NEO::EngineGroupType::compute, 0u);
550+
ASSERT_EQ(ZE_RESULT_SUCCESS, result);
551+
552+
auto &hwInfo = *device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo();
553+
554+
VariableBackup<uint64_t> sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities};
555+
sharedSystemMemCapabilities = UnifiedSharedMemoryFlags::access | UnifiedSharedMemoryFlags::sharedSystemPageFaultEnabled;
556+
557+
size_t size = 10;
558+
void *ptr = malloc(size);
559+
560+
EXPECT_NE(nullptr, ptr);
561+
562+
result = pCommandList->appendMemoryPrefetch(ptr, size);
563+
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
564+
565+
EXPECT_TRUE(pCommandList->isMemoryPrefetchRequested());
566+
567+
free(ptr);
568+
}
569+
570+
HWTEST2_F(CommandListStatePrefetchXe2HpgCore, givenSharedSystemAllocationOnSupportedDeviceWhenPrefetchApiIsCalledThenRequestMemoryPrefetchCalledWithNoPrefetchManager, IsXe2HpgCore) {
571+
auto pCommandList = std::make_unique<WhiteBox<::L0::CommandListCoreFamily<gfxCoreFamily>>>();
572+
auto result = pCommandList->initialize(device, NEO::EngineGroupType::compute, 0u);
573+
ASSERT_EQ(ZE_RESULT_SUCCESS, result);
574+
575+
auto &hwInfo = *device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo();
576+
577+
VariableBackup<uint64_t> sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities};
578+
sharedSystemMemCapabilities = UnifiedSharedMemoryFlags::access | UnifiedSharedMemoryFlags::sharedSystemPageFaultEnabled;
579+
580+
size_t size = 10;
581+
void *ptr = malloc(size);
582+
583+
EXPECT_NE(nullptr, ptr);
584+
585+
result = pCommandList->appendMemoryPrefetch(ptr, size);
586+
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
587+
588+
EXPECT_TRUE(pCommandList->isMemoryPrefetchRequested());
589+
590+
free(ptr);
591+
}
592+
593+
HWTEST2_F(CommandListStatePrefetchXe2HpgCore, givenSharedSystemAllocationOnUnSupportedDeviceWhenPrefetchApiIsCalledThenRequestMemoryPrefetchNotCalled, IsXe2HpgCore) {
594+
auto memoryManager = static_cast<MockMemoryManager *>(device->getDriverHandle()->getMemoryManager());
595+
memoryManager->prefetchManager.reset(new MockPrefetchManager());
596+
auto pCommandList = std::make_unique<WhiteBox<::L0::CommandListCoreFamily<gfxCoreFamily>>>();
597+
auto result = pCommandList->initialize(device, NEO::EngineGroupType::compute, 0u);
598+
ASSERT_EQ(ZE_RESULT_SUCCESS, result);
599+
600+
DebugManagerStateRestore restore;
601+
debugManager.flags.EnableSharedSystemUsmSupport.set(0);
602+
603+
size_t size = 10;
604+
void *ptr = malloc(size);
605+
606+
EXPECT_NE(nullptr, ptr);
607+
608+
result = pCommandList->appendMemoryPrefetch(ptr, size);
609+
EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, result);
610+
611+
EXPECT_FALSE(pCommandList->isMemoryPrefetchRequested());
612+
613+
free(ptr);
614+
}
615+
459616
} // namespace ult
460617
} // namespace L0

‎level_zero/core/test/unit_tests/xe_hpc_core/test_cmdlist_xe_hpc_core.cpp

+71-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
#include "shared/source/helpers/gfx_core_helper.h"
99
#include "shared/source/os_interface/product_helper.h"
10+
#include "shared/source/unified_memory/usm_memory_support.h"
1011
#include "shared/test/common/cmd_parse/gen_cmd_parse.h"
1112
#include "shared/test/common/helpers/debug_manager_state_restore.h"
1213
#include "shared/test/common/memory_manager/mock_prefetch_manager.h"
@@ -91,7 +92,7 @@ HWTEST2_F(CommandListStatePrefetchXeHpcCore, givenUnifiedSharedMemoryWhenPrefetc
9192
context->freeMem(ptr);
9293
}
9394

94-
HWTEST2_F(CommandListStatePrefetchXeHpcCore, givenUnifiedSharedMemoryWhenPrefetchApiAndDebuKeyDisabledIsCalledThenRequestMemoryPrefetchIsNotPerformed, IsXeHpcCore) {
95+
HWTEST2_F(CommandListStatePrefetchXeHpcCore, givenUnifiedSharedMemoryWhenPrefetchApiAndDebugKeyDisabledIsCalledThenRequestMemoryPrefetchIsNotPerformed, IsXeHpcCore) {
9596
DebugManagerStateRestore restore;
9697
debugManager.flags.AppendMemoryPrefetchForKmdMigratedSharedAllocations.set(0);
9798

@@ -117,6 +118,75 @@ HWTEST2_F(CommandListStatePrefetchXeHpcCore, givenUnifiedSharedMemoryWhenPrefetc
117118
context->freeMem(ptr);
118119
}
119120

121+
HWTEST2_F(CommandListStatePrefetchXeHpcCore, givenSharedSystemAllocationOnSupportedDeviceWhenPrefetchApiIsCalledThenRequestMemoryPrefetchCalled, IsXeHpcCore) {
122+
auto memoryManager = static_cast<MockMemoryManager *>(device->getDriverHandle()->getMemoryManager());
123+
memoryManager->prefetchManager.reset(new MockPrefetchManager());
124+
auto pCommandList = std::make_unique<WhiteBox<::L0::CommandListCoreFamily<gfxCoreFamily>>>();
125+
auto result = pCommandList->initialize(device, NEO::EngineGroupType::compute, 0u);
126+
ASSERT_EQ(ZE_RESULT_SUCCESS, result);
127+
128+
auto &hwInfo = *device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo();
129+
130+
VariableBackup<uint64_t> sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities};
131+
sharedSystemMemCapabilities = UnifiedSharedMemoryFlags::access | UnifiedSharedMemoryFlags::sharedSystemPageFaultEnabled;
132+
133+
size_t size = 10;
134+
void *ptr = malloc(size);
135+
136+
EXPECT_NE(nullptr, ptr);
137+
138+
result = pCommandList->appendMemoryPrefetch(ptr, size);
139+
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
140+
141+
EXPECT_TRUE(pCommandList->isMemoryPrefetchRequested());
142+
143+
free(ptr);
144+
}
145+
146+
HWTEST2_F(CommandListStatePrefetchXeHpcCore, givenSharedSystemAllocationOnSupportedDeviceWhenPrefetchApiIsCalledThenRequestMemoryPrefetchCalledWithNoPrefetchManager, IsXeHpcCore) {
147+
auto pCommandList = std::make_unique<WhiteBox<::L0::CommandListCoreFamily<gfxCoreFamily>>>();
148+
auto result = pCommandList->initialize(device, NEO::EngineGroupType::compute, 0u);
149+
ASSERT_EQ(ZE_RESULT_SUCCESS, result);
150+
151+
auto &hwInfo = *device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo();
152+
153+
VariableBackup<uint64_t> sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities};
154+
sharedSystemMemCapabilities = UnifiedSharedMemoryFlags::access | UnifiedSharedMemoryFlags::sharedSystemPageFaultEnabled;
155+
156+
size_t size = 10;
157+
void *ptr = malloc(size);
158+
159+
EXPECT_NE(nullptr, ptr);
160+
161+
result = pCommandList->appendMemoryPrefetch(ptr, size);
162+
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
163+
164+
EXPECT_TRUE(pCommandList->isMemoryPrefetchRequested());
165+
166+
free(ptr);
167+
}
168+
169+
HWTEST2_F(CommandListStatePrefetchXeHpcCore, givenSharedSystemAllocationOnUnSupportedDeviceWhenPrefetchApiIsCalledThenRequestMemoryPrefetchNotCalled, IsXeHpcCore) {
170+
auto pCommandList = std::make_unique<WhiteBox<::L0::CommandListCoreFamily<gfxCoreFamily>>>();
171+
auto result = pCommandList->initialize(device, NEO::EngineGroupType::compute, 0u);
172+
ASSERT_EQ(ZE_RESULT_SUCCESS, result);
173+
174+
DebugManagerStateRestore restore;
175+
debugManager.flags.EnableSharedSystemUsmSupport.set(0);
176+
177+
size_t size = 10;
178+
void *ptr = malloc(size);
179+
180+
EXPECT_NE(nullptr, ptr);
181+
182+
result = pCommandList->appendMemoryPrefetch(ptr, size);
183+
EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, result);
184+
185+
EXPECT_FALSE(pCommandList->isMemoryPrefetchRequested());
186+
187+
free(ptr);
188+
}
189+
120190
HWTEST2_F(CommandListStatePrefetchXeHpcCore, givenForceMemoryPrefetchForKmdMigratedSharedAllocationsWhenExecutingCommandListsOnCommandQueueThenMemoryPrefetchIsCalled, IsXeHpcCore) {
121191
DebugManagerStateRestore restore;
122192
debugManager.flags.UseKmdMigration.set(true);

0 commit comments

Comments
 (0)