Skip to content

Commit 6618e84

Browse files
authoredAug 27, 2024
Add lcnodeinfo service (LLNL#586)
* Add lcnodeinfo service * Add lcnodeinfo config options * Fix event-trace config * Add more lcnodeinfo default keys * Add test for lcnodeinfo * Add example_node_info.json file

9 files changed

+245
-5
lines changed
 

‎src/caliper/controllers/HatchetRegionProfileController.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ const char* controller_spec = R"json(
118118
{
119119
"name" : "hatchet-region-profile",
120120
"description" : "Record a region time profile for processing with hatchet",
121-
"categories" : [ "adiak", "metric", "output", "region", "event" ],
121+
"categories" : [ "adiak", "metadata", "metric", "output", "region", "event" ],
122122
"services" : [ "aggregate", "event", "timer" ],
123123
"config" :
124124
{ "CALI_CHANNEL_FLUSH_ON_EXIT" : "false",

‎src/caliper/controllers/HatchetSampleProfileController.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ const char* controller_spec = R"json(
146146
"name" : "hatchet-sample-profile",
147147
"description" : "Record a sampling profile for processing with hatchet",
148148
"services" : [ "sampler", "trace" ],
149-
"categories" : [ "adiak", "sampling", "output" ],
149+
"categories" : [ "adiak", "metadata", "sampling", "output" ],
150150
"config" : { "CALI_CHANNEL_FLUSH_ON_EXIT": "false" },
151151
"defaults" : { "callpath": "true", "source.module": "true" },
152152
"options":

‎src/caliper/controllers/SpotController.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -544,7 +544,7 @@ const char* spot_controller_spec = R"json(
544544
{
545545
"name" : "spot",
546546
"description" : "Record a time profile for the Spot web visualization framework",
547-
"categories" : [ "adiak", "metric", "output", "region", "event" ],
547+
"categories" : [ "adiak", "metadata", "metric", "output", "region", "event" ],
548548
"services" : [ "aggregate", "event", "timer" ],
549549
"config" :
550550
{ "CALI_CHANNEL_FLUSH_ON_EXIT" : "false",

‎src/caliper/controllers/controllers.cpp

+16-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ const char* event_trace_spec = R"json(
1313
"name" : "event-trace",
1414
"description" : "Record a trace of region enter/exit events in .cali format",
1515
"services" : [ "event", "recorder", "timer", "trace" ],
16-
"categories" : [ "output", "event" ],
16+
"categories" : [ "output", "metadata", "event" ],
1717
"config" : { "CALI_CHANNEL_FLUSH_ON_EXIT" : "false" },
1818
"options":
1919
[
@@ -1126,6 +1126,21 @@ const char* builtin_option_specs = R"json(
11261126
"type" : "string",
11271127
"category" : "output"
11281128
},
1129+
{
1130+
"name" : "lcnodeinfo",
1131+
"description" : "Read node info from /etc/node_info.json on LC systems",
1132+
"type" : "bool",
1133+
"category" : "metadata",
1134+
"services" : [ "lcnodeinfo" ]
1135+
},
1136+
{
1137+
"name" : "lcnodeinfo.keys",
1138+
"description" : "Keys to read from /etc/node_info.json file",
1139+
"type" : "bool",
1140+
"category" : "metadata",
1141+
"services" : [ "lcnodeinfo" ],
1142+
"config" : { "CALI_LCNODEINFO_KEYS": "{}" }
1143+
},
11291144
{
11301145
"name" : "adiak.import_categories",
11311146
"services" : [ "adiak_import" ],

‎src/services/CMakeLists.txt

+5
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,11 @@ if (CALIPER_HAVE_MPI)
124124
endif()
125125
endif()
126126

127+
list(APPEND CALIPER_SERVICES_SOURCES
128+
LCNodeInfoService.cpp)
129+
list(APPEND CALIPER_SERVICE_NAMES
130+
"lcnodeinfo")
131+
127132
if (CALIPER_BUILD_TESTING)
128133
add_subdirectory(templates)
129134
endif()

‎src/services/LCNodeInfoService.cpp

+188
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
// Copyright (c) 2015-2024, Lawrence Livermore National Security, LLC.
2+
// See top-level LICENSE file for details.
3+
4+
// LCNodeInfo.cpp
5+
// A Caliper service that collects information from the /etc/node_info.json
6+
// file on TOSS4 systems
7+
8+
#include "caliper/CaliperService.h"
9+
10+
#include "Services.h"
11+
12+
#include "caliper/Caliper.h"
13+
14+
#include "caliper/common/Log.h"
15+
#include "caliper/common/RuntimeConfig.h"
16+
#include "caliper/common/StringConverter.h"
17+
#include "caliper/common/Variant.h"
18+
19+
#include <fstream>
20+
#include <iostream>
21+
#include <string>
22+
#include <vector>
23+
24+
using namespace cali;
25+
26+
namespace
27+
{
28+
29+
const char* lcnodeinfo_service_spec = R"json(
30+
{
31+
"name" : "lcnodeinfo",
32+
"description" : "Read node information from /etc/node_info.json",
33+
"config" :
34+
[
35+
{ "name" : "filename",
36+
"description" : "The JSON file to read",
37+
"type" : "string",
38+
"value" : "/etc/node_info.json"
39+
},
40+
{ "name" : "keys",
41+
"description" : "List of JSON dict keys to read",
42+
"type" : "string",
43+
"value" : "host.name,host.cluster,host.os"
44+
}
45+
]
46+
}
47+
)json";
48+
49+
std::pair<bool, StringConverter>
50+
find_key(const std::vector<std::string>& path, const std::map<std::string, StringConverter>& dict)
51+
{
52+
if (path.empty())
53+
return std::make_pair(false, StringConverter());
54+
55+
auto it = dict.find(path.front());
56+
if (it == dict.end())
57+
return std::make_pair(false, StringConverter());
58+
59+
if (path.size() == 1)
60+
return std::make_pair(true, it->second);
61+
62+
StringConverter ret = it->second;
63+
for (auto path_it = path.begin()+1; path_it != path.end(); ++path_it) {
64+
auto sub_dict = ret.rec_dict();
65+
it = sub_dict.find(*path_it);
66+
if (it == sub_dict.end())
67+
return std::make_pair(false, StringConverter());
68+
ret = it->second;
69+
}
70+
71+
return std::make_pair(true, ret);
72+
}
73+
74+
void
75+
add_entry(Caliper* c, Channel* channel, const std::string& name, const std::string& val)
76+
{
77+
Attribute attr =
78+
c->create_attribute(name.c_str(), CALI_TYPE_STRING,
79+
CALI_ATTR_UNALIGNED | CALI_ATTR_GLOBAL | CALI_ATTR_SKIP_EVENTS);
80+
81+
c->set(channel, attr, Variant(CALI_TYPE_STRING, val.data(), val.size()));
82+
}
83+
84+
unsigned
85+
add_entries(Caliper* c, Channel* channel, const std::string& key, const StringConverter& val)
86+
{
87+
unsigned num_entries = 0;
88+
89+
std::string name("nodeinfo.");
90+
name.append(key);
91+
92+
bool is_dict = false;
93+
auto dict = val.rec_dict(&is_dict);
94+
95+
if (is_dict) {
96+
std::string prefix = name.append(".");
97+
for (const auto& p : dict) {
98+
name = prefix;
99+
name.append(p.first);
100+
add_entry(c, channel, name, p.second.to_string());
101+
++num_entries;
102+
}
103+
} else {
104+
add_entry(c, channel, name, val.to_string());
105+
++num_entries;
106+
}
107+
108+
return num_entries;
109+
}
110+
111+
void
112+
read_nodeinfo(Caliper* c, Channel* channel, const std::vector<std::string>& keys, const std::string& filename)
113+
{
114+
std::ifstream is(filename.c_str(), std::ios::ate);
115+
116+
if (!is) {
117+
Log(1).stream() << channel->name() << ": lcnodeinfo: Cannot open "
118+
<< filename << ", quitting\n";
119+
return;
120+
}
121+
122+
auto size = is.tellg();
123+
std::string str(size, '\0');
124+
is.seekg(0);
125+
if (!is.read(&str[0], size)) {
126+
Log(0).stream() << channel->name() << ": lcnodeinfo: Cannot read "
127+
<< filename << ", quitting\n";
128+
return;
129+
}
130+
131+
if (Log::verbosity() >= 2) {
132+
Log(2).stream() << channel->name() << ": lcnodeinfo: "
133+
<< size << " bytes read from " << filename << std::endl;
134+
}
135+
136+
bool ok = false;
137+
auto top = StringConverter(str).rec_dict(&ok);
138+
139+
if (!ok) {
140+
Log(0).stream() << channel->name() << ": lcnodeinfo: Cannot parse top-level dict in "
141+
<< filename << ", quitting\n";
142+
return;
143+
}
144+
145+
unsigned num_entries = 0;
146+
147+
for (const std::string& key : keys) {
148+
std::vector<std::string> path = StringConverter(key).to_stringlist(".");
149+
auto ret = find_key(path, top);
150+
if (ret.first)
151+
num_entries += add_entries(c, channel, key, ret.second);
152+
else
153+
Log(1).stream() << channel->name() << ": lcnodeinfo: Key "
154+
<< key << " not found\n";
155+
}
156+
157+
Log(1).stream() << channel->name() << ": lcnodeinfo: Added "
158+
<< num_entries << " entries\n";
159+
}
160+
161+
void
162+
lcnodeinfo_register(Caliper*, Channel* channel)
163+
{
164+
ConfigSet cfg =
165+
services::init_config_from_spec(channel->config(), lcnodeinfo_service_spec);
166+
167+
std::vector<std::string> keys = cfg.get("keys").to_stringlist();
168+
std::string filename = cfg.get("filename").to_string();
169+
170+
if (keys.empty()) {
171+
Log(1).stream() << channel->name() << ": lcnodeinfo: "
172+
<< "No keys provided, quitting\n";
173+
return;
174+
}
175+
176+
channel->events().post_init_evt.connect([keys,filename](Caliper* c, Channel* channel){
177+
read_nodeinfo(c, channel, keys, filename);
178+
});
179+
}
180+
181+
}
182+
183+
namespace cali
184+
{
185+
186+
CaliperService lcnodeinfo_service { ::lcnodeinfo_service_spec, ::lcnodeinfo_register };
187+
188+
}

‎test/ci_app_tests/CMakeLists.txt

+4-1
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,10 @@ if (WITH_FORTRAN)
130130
list(APPEND PYTHON_SCRIPTS test_fortran_api.py)
131131
endif()
132132

133-
foreach(file ${PYTHON_SCRIPTS})
133+
set(DATA_FILES
134+
example_node_info.json)
135+
136+
foreach(file ${PYTHON_SCRIPTS} ${DATA_FILES})
134137
add_custom_target(${file} ALL
135138
COMMAND ${CMAKE_COMMAND} -E create_symlink
136139
${CMAKE_CURRENT_SOURCE_DIR}/${file}
+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"host":
3+
{
4+
"os": { "name": "TestOS", "version": "3.11", "kernel": "1.2.3-42" },
5+
"name": "test42",
6+
"cluster": "test",
7+
"do_not_include": "me"
8+
},
9+
"other": "nothing to see here"
10+
}

‎test/ci_app_tests/test_basictrace.py

+19
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,25 @@ def test_globals_selection(self):
151151
self.assertTrue(cat.has_snapshot_with_keys(
152152
snapshots, { 'cali.caliper.version' } ) )
153153

154+
def test_lcnodeinfo(self):
155+
target_cmd = [ './ci_test_basic' ]
156+
query_cmd = [ '../../src/tools/cali-query/cali-query', '-G', '-e' ]
157+
158+
caliper_config = {
159+
'CALI_SERVICES_ENABLE': 'event,lcnodeinfo,aggregate,recorder',
160+
'CALI_LCNODEINFO_FILENAME': 'example_node_info.json',
161+
'CALI_RECORDER_FILENAME': 'stdout'
162+
}
163+
164+
query_output = cat.run_test_with_query(target_cmd, query_cmd, caliper_config)
165+
snapshots = cat.get_snapshots_from_text(query_output)
166+
167+
self.assertTrue(cat.has_snapshot_with_attributes(
168+
snapshots, {
169+
'nodeinfo.host.os.name': 'TestOS',
170+
'nodeinfo.host.os.version': '3.11',
171+
'nodeinfo.host.cluster': 'test' }))
172+
154173
def test_esc(self):
155174
target_cmd = [ './ci_test_basic', 'newline' ]
156175
query_cmd = [ '../../src/tools/cali-query/cali-query', '-j', '-s', 'cali.event.set' ]

0 commit comments

Comments
 (0)
Please sign in to comment.