Skip to content

rocprofiler: Add basic rccl callbacks #732

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
daboehme merged 4 commits into from
Mar 24, 2026
Merged

rocprofiler: Add basic rccl callbacks #732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
eb6bb9a
Initial rccl profiling support
daboehme Mar 16, 2026
f5119cb
Add rocm.agent as thread id in trace converter
daboehme Mar 16, 2026
e0ee276
Subscribe rocm.api attribute for rccl tracing
daboehme Mar 16, 2026
a23782b
Add profile.rccl option
daboehme Mar 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion python/cali2traceevent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ class CaliTraceEventConverter:
BUILTIN_TID_ATTRIBUTES = [
'omp.thread.id',
'pthread.id',
'rocm.agent',
'rocm.dst.agent'
]

def __init__(self, cfg):
Expand Down Expand Up @@ -294,7 +296,7 @@ def _process_roctracer_activity_rec(self, rec, trec):
dur = float(rec["rocm.activity.duration"])*1e-3
name = rec.get("rocm.kernel.name", cat)

trec.update(ph="X", name=name, cat=cat, ts=tst, dur=dur, tid="rocm")
trec.update(ph="X", name=name, cat=cat, ts=tst, dur=dur)

def _process_sample_rec(self, rec, trec):
trec.update(name="sampler", weight=1, ts=_get_timestamp(rec))
Expand Down
8 changes: 8 additions & 0 deletions src/caliper/controllers/controllers.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -718,6 +718,14 @@ const char* builtin_rocm_option_specs = R"json(
"services": [ "rocprofiler" ],
"config": { "CALI_ROCPROFILER_ENABLE_MARKER_CALLBACKS": "true" }
},
{
"name": "profile.rccl",
"type": "bool",
"description": "Profile rccl API functions",
"category": "region",
"services": [ "rocprofiler" ],
"config": { "CALI_ROCPROFILER_ENABLE_RCCL_CALLBACKS": "true" }
},
{
"name": "rocm.gputime",
"description": "Report GPU time in AMD ROCm activities",
Expand Down
29 changes: 27 additions & 2 deletions src/services/rocprofiler/RocProfiler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ class RocProfilerService

bool m_enable_api_callbacks = false;
bool m_enable_marker_callbacks = false;
bool m_enable_rccl_callbacks = false;
bool m_enable_activity_tracing = false;
bool m_enable_snapshot_timestamps = false;
bool m_enable_allocation_tracing = false;
Expand Down Expand Up @@ -132,6 +133,7 @@ class RocProfilerService

static rocprofiler_context_id_t s_hip_api_ctx;
static rocprofiler_context_id_t s_marker_ctx;
static rocprofiler_context_id_t s_rccl_ctx;
static rocprofiler_context_id_t s_activity_ctx;
static rocprofiler_context_id_t s_rocprofiler_ctx;
static rocprofiler_context_id_t s_alloc_tracing_ctx;
Expand All @@ -144,7 +146,7 @@ class RocProfilerService
Attribute subs_attr = c->get_attribute("subscription_event");
Variant v_true(true);

m_api_attr = c->create_attribute("rocm.api", CALI_TYPE_STRING, CALI_ATTR_NESTED, 1, &subs_attr, &v_true);
m_api_attr = c->create_attribute("rocm.api", CALI_TYPE_STRING, CALI_ATTR_NESTED | CALI_ATTR_SCOPE_THREAD, 1, &subs_attr, &v_true);
m_marker_attr = c->create_attribute("rocm.marker", CALI_TYPE_STRING, CALI_ATTR_NESTED, 1, &subs_attr, &v_true);

m_activity_start_attr =
Expand Down Expand Up @@ -574,6 +576,12 @@ class RocProfilerService
Log(2).stream() << channel->name() << ": rocprofiler: HIP API callbacks activated\n";
}

if (m_enable_rccl_callbacks) {
channel->events().subscribe_attribute(c, m_api_attr);
ROCPROFILER_CALL(rocprofiler_start_context(s_rccl_ctx));
Log(2).stream() << channel->name() << ": rocprofiler: RCCL callbacks activated\n";
}

if (m_enable_marker_callbacks) {
channel->events().subscribe_attribute(c, m_marker_attr);
ROCPROFILER_CALL(rocprofiler_start_context(s_marker_ctx));
Expand Down Expand Up @@ -622,7 +630,8 @@ class RocProfilerService

void pre_finish_cb(Caliper* c, Channel* channel)
{
auto contexts = make_array(s_hip_api_ctx, s_marker_ctx, s_rocprofiler_ctx, s_activity_ctx, s_alloc_tracing_ctx, s_counter_ctx);
auto contexts = make_array(s_hip_api_ctx, s_marker_ctx, s_rccl_ctx,
s_rocprofiler_ctx, s_activity_ctx, s_alloc_tracing_ctx, s_counter_ctx);

for (auto &ctx : contexts) {
int status = 0;
Expand Down Expand Up @@ -722,6 +731,7 @@ class RocProfilerService

m_enable_api_callbacks = config.get("enable_api_callbacks").to_bool();
m_enable_marker_callbacks = config.get("enable_marker_callbacks").to_bool();
m_enable_rccl_callbacks = config.get("enable_rccl_callbacks").to_bool();
m_enable_activity_tracing = config.get("enable_activity_tracing").to_bool();
m_enable_snapshot_timestamps = config.get("enable_snapshot_timestamps").to_bool();
m_enable_allocation_tracing = config.get("enable_allocation_tracing").to_bool();
Expand Down Expand Up @@ -766,6 +776,7 @@ class RocProfilerService
{
ROCPROFILER_CALL(rocprofiler_create_context(&s_hip_api_ctx));
ROCPROFILER_CALL(rocprofiler_create_context(&s_marker_ctx));
ROCPROFILER_CALL(rocprofiler_create_context(&s_rccl_ctx));
ROCPROFILER_CALL(rocprofiler_create_context(&s_activity_ctx));
ROCPROFILER_CALL(rocprofiler_create_context(&s_rocprofiler_ctx));
ROCPROFILER_CALL(rocprofiler_create_context(&s_alloc_tracing_ctx));
Expand All @@ -787,6 +798,14 @@ class RocProfilerService
marker_callback,
nullptr
));
ROCPROFILER_CALL(rocprofiler_configure_callback_tracing_service(
s_rccl_ctx,
ROCPROFILER_CALLBACK_TRACING_RCCL_API,
nullptr,
0,
hip_api_callback,
nullptr
));
ROCPROFILER_CALL(rocprofiler_configure_callback_tracing_service(
s_rocprofiler_ctx,
ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT,
Expand Down Expand Up @@ -894,6 +913,7 @@ RocProfilerService* RocProfilerService::s_instance = nullptr;

rocprofiler_context_id_t RocProfilerService::s_hip_api_ctx = {};
rocprofiler_context_id_t RocProfilerService::s_marker_ctx = {};
rocprofiler_context_id_t RocProfilerService::s_rccl_ctx = {};
rocprofiler_context_id_t RocProfilerService::s_activity_ctx = {};
rocprofiler_context_id_t RocProfilerService::s_rocprofiler_ctx = {};
rocprofiler_context_id_t RocProfilerService::s_alloc_tracing_ctx = {};
Expand All @@ -917,6 +937,11 @@ const char* RocProfilerService::s_spec = R"json(
"description": "Enable roctx marker callbacks",
"value": "false"
},
{ "name": "enable_rccl_callbacks",
"type": "bool",
"description": "Enable rccl callbacks",
"value": "false"
},
{ "name": "enable_activity_tracing",
"type": "bool",
"description": "Enable ROCm GPU activity tracing",
Expand Down
Loading