Skip to content

Commit 9cf3821

Browse files
authored
Add roctracer support (LLNL#404)
* Separate roctx and roctracer support * Begin roctracer service (untested) * roctracer updates (compiles, doesn't link) * Activity tracing prototype. Still does not link * Update roctracer * Some more roctracer fixes * Implement roctracer activity correlation mapping * Roctracer updates. Add profile.hip option. * roctracer: Make kernel name resolution optional * Add rocm ConfigManager controllers and options
1 parent d2a5051 commit 9cf3821

14 files changed

+956
-32
lines changed

CMakeLists.txt

+22-8
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,8 @@ add_caliper_option(WITH_OMPT "Enable OMPT" FALSE)
7171
add_caliper_option(WITH_SAMPLER "Enable Linux sampler (x86 and PPC Linux only)" FALSE)
7272
add_caliper_option(WITH_DYNINST "Enable dyninst (for symbollookup service" FALSE)
7373
add_caliper_option(WITH_GOTCHA "Enable GOTCHA wrapping" ${CALIPER_HAVE_LINUX})
74-
add_caliper_option(WITH_ROCM "Enable AMD ROCtracer/RocTX support" FALSE)
74+
add_caliper_option(WITH_ROCTX "Enable AMD RocTX support" FALSE)
75+
add_caliper_option(WITH_ROCTRACER "Enable AMD RocTracer support" FALSE)
7576
add_caliper_option(WITH_SOS "Enable SOSFlow data management" FALSE)
7677
add_caliper_option(WITH_TAU "Enable TAU service (TAU Performance System)" FALSE)
7778
add_caliper_option(WITH_VTUNE "Enable Intel(R) VTune(tm) annotation bindings" FALSE)
@@ -358,14 +359,26 @@ if (WITH_MPI)
358359
endif()
359360
endif()
360361

361-
if (WITH_ROCM)
362+
if (WITH_ROCTX)
363+
include(FindRocTX)
364+
if (ROCTX_FOUND)
365+
set(CALIPER_HAVE_ROCTX TRUE)
366+
set(CALIPER_RocTX_CMAKE_MSG "Yes, using ${ROCTX_LIBRARIES}")
367+
list(APPEND CALIPER_EXTERNAL_LIBS ${ROCTX_LIBRARIES})
368+
else()
369+
message(WARNING "RocTX support requested but ROCm was not found!\n"
370+
"Set ROCM_PREFIX to installation path and re-run cmake.")
371+
endif()
372+
endif()
373+
374+
if (WITH_ROCTRACER)
362375
include(FindRoctracer)
363-
if (ROCM_FOUND)
364-
set(CALIPER_HAVE_ROCM TRUE)
365-
set(CALIPER_ROCm_CMAKE_MSG "Yes, using ${ROCM_LIBRARIES}")
366-
list(APPEND CALIPER_EXTERNAL_LIBS ${ROCM_LIBRARIES})
376+
if (ROCTRACER_FOUND)
377+
set(CALIPER_HAVE_ROCTRACER TRUE)
378+
set(CALIPER_RocTracer_CMAKE_MSG "Yes, using ${ROCTRACER_LIBRARIES}")
379+
list(APPEND CALIPER_EXTERNAL_LIBS ${ROCTRACER_LIBRARIES})
367380
else()
368-
message(WARNING "ROCm support requested but ROCm was not found!\n"
381+
message(WARNING "RocTracer support requested but ROCm was not found!\n"
369382
"Set ROCM_PREFIX to installation path and re-run cmake.")
370383
endif()
371384
endif()
@@ -548,7 +561,8 @@ set(CALIPER_MODULES
548561
Nvtx
549562
CUpti
550563
Kokkos
551-
ROCm
564+
RocTracer
565+
RocTX
552566
TAU
553567
VTune
554568
PCP

caliper-config.h.in

+2-1
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,9 @@
1919
#cmakedefine CALIPER_HAVE_VTUNE
2020
#cmakedefine CALIPER_HAVE_ADIAK
2121
#cmakedefine CALIPER_HAVE_PCP
22-
#cmakedefine CALIPER_HAVE_ROCM
2322
#cmakedefine CALIPER_HAVE_VARIORUM
23+
#cmakedefine CALIPER_HAVE_ROCTX
24+
#cmakedefine CALIPER_HAVE_ROCTRACER
2425

2526
#cmakedefine CALIPER_REDUCED_CONSTEXPR_USAGE
2627

cmake/FindRocTX.cmake

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Find RocTX libraries/headers
2+
3+
find_path(ROCM_PREFIX
4+
NAMES include/roctracer/roctx.h
5+
)
6+
7+
find_library(ROCTX_LIBRARIES
8+
NAMES roctx64
9+
HINTS ${ROCM_PREFIX}/lib
10+
)
11+
12+
find_path(ROCTX_INCLUDE_DIRS
13+
NAMES roctx.h
14+
HINTS ${ROCM_PREFIX}/include/roctracer
15+
)
16+
17+
include(FindPackageHandleStandardArgs)
18+
19+
find_package_handle_standard_args(ROCTX
20+
DEFAULT_MSG
21+
ROCTX_LIBRARIES
22+
ROCTX_INCLUDE_DIRS
23+
)
24+
25+
mark_as_advanced(
26+
ROCTX_INCLUDE_DIRS
27+
ROCTX_LIBRARIES
28+
)

cmake/FindRoctracer.cmake

+33-10
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,51 @@
11
# Find ROCTracer libraries/headers
22

33
find_path(ROCM_PREFIX
4-
NAMES include/roctracer/roctx.h
4+
NAMES include/roctracer/roctracer.h
55
)
66

7-
find_library(ROCM_LIBRARIES
8-
NAMES roctx64
7+
find_library(ROCTRACER_LIBROCTRACER
8+
NAMES roctracer64
9+
HINTS ${ROCM_PREFIX}/lib
10+
)
11+
find_library(ROCTRACER_LIBHSARUNTIME
12+
NAMES hsa-runtime64
13+
HINTS ${ROCM_PREFIX}/lib
14+
)
15+
find_library(ROCTRACER_LIBHSAKMT
16+
NAMES hsakmt
917
HINTS ${ROCM_PREFIX}/lib
1018
)
1119

12-
find_path(ROCM_INCLUDE_DIRS
13-
NAMES roctx.h
20+
find_path(ROCTRACER_INCLUDE_DIR
21+
NAMES roctracer.h
1422
HINTS ${ROCM_PREFIX}/include/roctracer
1523
)
1624

25+
find_path(HIP_INCLUDE_DIR
26+
NAMES hip/hip_runtime.h
27+
HINTS ${ROCM_PREFIX}/include)
28+
29+
set(ROCTRACER_INCLUDE_DIRS
30+
${ROCTRACER_INCLUDE_DIR}
31+
${HIP_INCLUDE_DIR}
32+
)
33+
set(ROCTRACER_LIBRARIES
34+
${ROCTRACER_LIBROCTRACER}
35+
${ROCTRACER_LIBHSARUNTIME}
36+
${ROCTRACER_LIBHSAKMT})
37+
1738
include(FindPackageHandleStandardArgs)
1839

19-
find_package_handle_standard_args(ROCM
40+
find_package_handle_standard_args(ROCTRACER
2041
DEFAULT_MSG
21-
ROCM_LIBRARIES
22-
ROCM_INCLUDE_DIRS
42+
ROCTRACER_LIBROCTRACER
43+
ROCTRACER_LIBHSARUNTIME
44+
ROCTRACER_LIBHSAKMT
45+
ROCTRACER_INCLUDE_DIRS
2346
)
2447

2548
mark_as_advanced(
26-
ROCM_INCLUDE_DIRS
27-
ROCM_LIBRARIES
49+
ROCTRACER_INCLUDE_DIRS
50+
ROCTRACER_LIBRARIES
2851
)

src/caliper/controllers/CMakeLists.txt

+2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ set(CALIPER_CONTROLLERS_SOURCES
66
controllers/HatchetSampleProfileController.cpp
77
controllers/LoopReportController.cpp
88
controllers/OpenMPReportController.cpp
9+
controllers/ROCmActivityProfileController.cpp
10+
controllers/ROCmActivityReportController.cpp
911
controllers/RuntimeReportController.cpp
1012
controllers/SpotController.cpp
1113
controllers/controllers.cpp
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
// Copyright (c) 2019, Lawrence Livermore National Security, LLC.
2+
// See top-level LICENSE file for details.
3+
4+
#include "caliper/caliper-config.h"
5+
6+
#include "caliper/ChannelController.h"
7+
#include "caliper/ConfigManager.h"
8+
9+
#include "caliper/common/Log.h"
10+
#include "caliper/common/StringConverter.h"
11+
12+
#include "../../services/Services.h"
13+
14+
#include <algorithm>
15+
#include <set>
16+
#include <tuple>
17+
18+
using namespace cali;
19+
20+
namespace
21+
{
22+
23+
class RocmActivityProfileController : public cali::ChannelController
24+
{
25+
public:
26+
27+
RocmActivityProfileController(const char* name, const config_map_t& initial_cfg, const ConfigManager::Options& opts, const std::string& format)
28+
: ChannelController(name, 0, initial_cfg)
29+
{
30+
std::string output(opts.get("output", "rocm_profile").to_string());
31+
32+
if (output != "stdout" && output != "stderr") {
33+
auto pos = output.find_last_of('.');
34+
std::string ext = (format == "cali" ? ".cali" : ".json");
35+
36+
if (pos == std::string::npos || output.substr(pos) != ext)
37+
output.append(ext);
38+
}
39+
40+
auto avail_services = services::get_available_services();
41+
bool have_mpi =
42+
std::find(avail_services.begin(), avail_services.end(), "mpireport") != avail_services.end();
43+
bool have_adiak =
44+
std::find(avail_services.begin(), avail_services.end(), "adiak_import") != avail_services.end();
45+
46+
bool use_mpi = have_mpi;
47+
48+
if (opts.is_set("use.mpi"))
49+
use_mpi = have_mpi && opts.is_enabled("use.mpi");
50+
51+
if (have_adiak) {
52+
config()["CALI_SERVICES_ENABLE"].append(",adiak_import");
53+
config()["CALI_ADIAK_IMPORT_CATEGORIES"] =
54+
opts.get("adiak.import_categories", "2,3").to_string();
55+
}
56+
57+
if (use_mpi) {
58+
config()["CALI_SERVICES_ENABLE" ].append(",mpi,mpireport");
59+
config()["CALI_AGGREGATE_KEY" ] = "*,mpi.rank";
60+
config()["CALI_MPIREPORT_FILENAME"] = output;
61+
config()["CALI_MPIREPORT_WRITE_ON_FINALIZE"] = "false";
62+
config()["CALI_MPIREPORT_CONFIG" ] =
63+
opts.build_query("local", {
64+
{ "select",
65+
"*,scale(rocm.activity.duration,1e-9) as \"time (gpu)\" unit sec"
66+
" ,sum(sum#time.duration) as \"time\" unit sec"
67+
},
68+
{ "group by", "prop:nested,rocm.kernel.name,rocm.activity.kind,mpi.rank" },
69+
{ "format", format }
70+
});
71+
} else {
72+
config()["CALI_SERVICES_ENABLE" ].append(",report");
73+
config()["CALI_REPORT_FILENAME" ] = output;
74+
config()["CALI_REPORT_CONFIG" ] =
75+
opts.build_query("local", {
76+
{ "select",
77+
"*,scale(rocm.activity.duration,1e-9) as \"time (gpu)\" unit sec"
78+
" ,sum(sum#time.duration) as \"time\" unit sec" },
79+
{ "group by", "prop:nested,rocm.kernel.name,rocm.activity.kind" },
80+
{ "format", format }
81+
});
82+
}
83+
84+
opts.update_channel_config(config());
85+
}
86+
};
87+
88+
std::string
89+
check_args(const cali::ConfigManager::Options& opts) {
90+
// Check if output.format is valid
91+
92+
std::string format = opts.get("output.format", "json-split").to_string();
93+
std::set<std::string> allowed_formats = { "cali", "json", "json-split", "hatchet" };
94+
95+
if (allowed_formats.find(format) == allowed_formats.end())
96+
return std::string("rocm-activity-profile: Invalid output format \"") + format + "\"";
97+
98+
return "";
99+
}
100+
101+
cali::ChannelController*
102+
make_controller(const char* name, const config_map_t& initial_cfg, const cali::ConfigManager::Options& opts)
103+
{
104+
std::string format = opts.get("output.format", "json-split").to_string();
105+
106+
if (format == "hatchet")
107+
format = "json-split";
108+
109+
if (!(format == "json-split" || format == "json" || format == "cali")) {
110+
format = "json-split";
111+
Log(0).stream() << "hatchet-region-profile: Unknown output format \"" << format
112+
<< "\". Using json-split."
113+
<< std::endl;
114+
}
115+
116+
return new RocmActivityProfileController(name, initial_cfg, opts, format);
117+
}
118+
119+
const char* controller_spec =
120+
"{"
121+
" \"name\" : \"rocm-activity-profile\","
122+
" \"description\" : \"Record AMD ROCm activities and a write profile\","
123+
" \"categories\" : [ \"adiak\", \"metric\", \"output\", \"region\" ],"
124+
" \"services\" : [ \"aggregate\", \"roctracer\", \"event\", \"timestamp\" ],"
125+
" \"config\" : "
126+
" { \"CALI_CHANNEL_FLUSH_ON_EXIT\" : \"false\","
127+
" \"CALI_EVENT_ENABLE_SNAPSHOT_INFO\" : \"false\","
128+
" \"CALI_ROCTRACER_TRACE_ACTIVITIES\" : \"true\","
129+
" \"CALI_ROCTRACER_RECORD_KERNEL_NAMES\": \"true\""
130+
" },"
131+
" \"options\": "
132+
" ["
133+
" { "
134+
" \"name\": \"output.format\","
135+
" \"type\": \"string\","
136+
" \"description\": \"Output format ('hatchet', 'cali', 'json')\""
137+
" },"
138+
" { "
139+
" \"name\": \"use.mpi\","
140+
" \"type\": \"bool\","
141+
" \"description\": \"Merge results into a single output stream in MPI programs\""
142+
" }"
143+
" ]"
144+
"}";
145+
146+
} // namespace [anonymous]
147+
148+
namespace cali
149+
{
150+
151+
ConfigManager::ConfigInfo rocm_activity_profile_controller_info
152+
{
153+
::controller_spec, ::make_controller, ::check_args
154+
};
155+
156+
}

0 commit comments

Comments
 (0)