Skip to content

Commit 421c667

Browse files
authored
Save attribute aliases as metadata (LLNL#301)
* Save some attribute aliases in .cali output * Update reader test case * Add aliases for SPOT time metrics
1 parent b225f95 commit 421c667

File tree

9 files changed

+76
-14
lines changed

9 files changed

+76
-14
lines changed

include/caliper/reader/CaliperMetadataDB.h

+9
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,15 @@ class CaliperMetadataDB : public CaliperMetadataAccessInterface
120120
/// metadata DB
121121
std::vector<Entry> import_globals(CaliperMetadataAccessInterface& db, const std::vector<Entry>& globals);
122122

123+
/// \brief Add a set of attribute aliases
124+
///
125+
/// This adds a "attribute.alias" meta-attribute for the aliased attribute
126+
/// to export alias information in a cali data stream.
127+
/// Currently this is limited to new attributes created with
128+
/// create_attribute() in this database. It does not apply to imported
129+
/// attributes.
130+
void add_attribute_aliases(const std::map<std::string, std::string>& aliases);
131+
123132
/// \brief print usage statistics
124133
std::ostream&
125134
print_statistics(std::ostream& os);

src/mpi/controllers/SpotController.cpp

+7-3
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,8 @@ class SpotController : public cali::ChannelController
359359
QuerySpec spec = tsc->timeseries_spec();
360360
Aggregator cross_agg(spec);
361361

362+
m_db.add_attribute_aliases(spec.aliases);
363+
362364
tsc->timeseries_local_aggregation(c, m_db, namebuf, std::max(blocksize, 1), cross_agg);
363365
cross_aggregate(cross_agg);
364366

@@ -424,9 +426,9 @@ class SpotController : public cali::ChannelController
424426
// --- Setup output reduction aggregator (final cross-process aggregation)
425427
const char* cross_select =
426428
" *"
427-
",min(inclusive#sum#time.duration)"
428-
",max(inclusive#sum#time.duration)"
429-
",avg(inclusive#sum#time.duration)";
429+
",min(inclusive#sum#time.duration) as \"Min time/rank\""
430+
",max(inclusive#sum#time.duration) as \"Max time/rank\""
431+
",avg(inclusive#sum#time.duration) as \"Avg time/rank\"";
430432
std::string cross_query =
431433
std::string("select ")
432434
+ m_opts.query_select("cross", cross_select, false)
@@ -436,6 +438,8 @@ class SpotController : public cali::ChannelController
436438
QuerySpec output_spec(CalQLParser(cross_query.c_str()).spec());
437439
Aggregator output_agg(output_spec);
438440

441+
m_db.add_attribute_aliases(output_spec.aliases);
442+
439443
// --- Flush Caliper buffers into intermediate aggregator to calculate
440444
// region profile inclusive times
441445
{

src/mpi/services/mpireport/MpiReport.cpp

+2
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ class MpiReport
5151

5252
CaliperMetadataDB db;
5353

54+
db.add_attribute_aliases(m_cross_spec.aliases);
55+
5456
Aggregator cross_agg(m_cross_spec);
5557
Aggregator local_agg(m_local_spec);
5658

src/reader/CaliperMetadataDB.cpp

+20
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ struct CaliperMetadataDB::CaliperMetadataDBImpl
4747
vector<Entry> m_globals;
4848
mutex m_globals_lock;
4949

50+
std::map<std::string, std::string> m_attr_aliases;
51+
Attribute m_alias_attr;
52+
5053
inline Node* node(cali_id_t id) const {
5154
std::lock_guard<std::mutex>
5255
g(m_node_lock);
@@ -398,6 +401,12 @@ struct CaliperMetadataDB::CaliperMetadataDBImpl
398401
if (meta > 0)
399402
parent = make_tree_entry(meta, meta_attr, meta_data, parent);
400403

404+
auto alias_it = m_attr_aliases.find(name);
405+
if (alias_it != m_attr_aliases.end()) {
406+
Variant v_alias(static_cast<const char*>(alias_it->second.c_str()));
407+
parent = make_tree_entry(1, &m_alias_attr, &v_alias, parent);
408+
}
409+
401410
Attribute n_attr[2] = { attribute(Attribute::meta_attribute_keys().prop_attr_id),
402411
attribute(Attribute::meta_attribute_keys().name_attr_id) };
403412
Variant n_data[2] = { Variant(prop),
@@ -476,6 +485,10 @@ struct CaliperMetadataDB::CaliperMetadataDBImpl
476485
: m_root { CALI_INV_ID, CALI_INV_ID, { } }
477486
{
478487
setup_bootstrap_nodes();
488+
489+
m_alias_attr =
490+
create_attribute("attribute.alias", CALI_TYPE_STRING, CALI_ATTR_SKIP_EVENTS,
491+
0, nullptr, nullptr);
479492
}
480493

481494
~CaliperMetadataDBImpl() {
@@ -636,6 +649,13 @@ CaliperMetadataDB::import_globals(CaliperMetadataAccessInterface& db, const std:
636649
return mP->import_globals(db, globals);
637650
}
638651

652+
void
653+
CaliperMetadataDB::add_attribute_aliases(const std::map<std::string, std::string>& aliases)
654+
{
655+
for (const auto &p: aliases)
656+
mP->m_attr_aliases[p.first] = p.second;
657+
}
658+
639659
std::ostream&
640660
CaliperMetadataDB::print_statistics(std::ostream& os)
641661
{

src/reader/JsonFormatter.cpp

+7-7
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,7 @@ struct JsonFormatter::JsonFormatterImpl
188188
if (noquote_kvs.size() + quote_kvs.size() > 0) {
189189
if (m_num_recs == 0)
190190
begin_records_section(*real_os);
191-
191+
192192
*real_os << (m_num_recs > 0 ? ",\n" : "") << "{";
193193

194194
int count = 0;
@@ -213,7 +213,7 @@ struct JsonFormatter::JsonFormatterImpl
213213

214214
std::ostream& write_attributes(CaliperMetadataAccessInterface& db, std::ostream& os) {
215215
std::vector<Attribute> attrs;
216-
216+
217217
if (m_selected.empty())
218218
attrs = db.get_all_attributes();
219219
else
@@ -229,14 +229,14 @@ struct JsonFormatter::JsonFormatterImpl
229229
attrs.erase(std::remove_if(attrs.begin(), attrs.end(),
230230
[](const Attribute& a){ return a.is_hidden(); }),
231231
attrs.end());
232-
232+
233233
// write "attr1": { "prop": "value", ... } , "attr2" : ...
234234

235235
int count = 0;
236236
for (const Attribute& a : attrs) {
237237
os << (count++ > 0 ? ",\n" : "\n") << (m_opt_pretty ? "\t" : "")
238238
<< '\"' << a.name() << "\": {";
239-
239+
240240
// encode some properties
241241
os << "\"is_global\": " << (a.is_global() ? "true" : "false")
242242
<< ",\"is_nested\": " << (a.is_nested() ? "true" : "false");
@@ -246,13 +246,13 @@ struct JsonFormatter::JsonFormatterImpl
246246
util::write_esc_string(os << ",\"", db.get_attribute(node->attribute()).name()) << "\": ";
247247
util::write_esc_string(os << "\"", node->data().to_string()) << '\"';
248248
}
249-
249+
250250
os << "}";
251251
}
252252

253253
return os;
254254
}
255-
255+
256256
std::ostream& write_globals(CaliperMetadataAccessInterface& db, std::ostream& os) {
257257
std::vector<Entry> globals = db.get_globals();
258258
std::map<cali_id_t, std::string> global_vals;
@@ -286,7 +286,7 @@ struct JsonFormatter::JsonFormatterImpl
286286

287287
void flush(CaliperMetadataAccessInterface& db) {
288288
std::ostream* real_os = m_os.stream();
289-
289+
290290
// close records section
291291
if (m_num_recs == 0)
292292
begin_records_section(*real_os);

src/reader/test/test_metadb.cpp

+3-3
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ TEST(MetadataDBTest, StringDB) {
156156
std::ostringstream os;
157157
db.print_statistics(os);
158158

159-
// 18 nodes: 12 default nodes + 2 attribute nodes + 4 test nodes
160-
// 4 strings: 1 attribute name + 3 different test node strings
161-
EXPECT_EQ(os.str(), std::string("CaliperMetadataDB: stored 18 nodes, 4 strings.\n"));
159+
// 20 nodes: 14 default nodes + 2 attribute nodes + 4 test nodes
160+
// 5 strings: 2 attribute name + 3 different test node strings
161+
EXPECT_EQ(os.str(), std::string("CaliperMetadataDB: stored 20 nodes, 5 strings.\n"));
162162
}

src/services/report/Report.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ class Report {
6161
CaliperMetadataDB db;
6262
QueryProcessor queryP(spec, stream);
6363

64+
db.add_attribute_aliases(spec.aliases);
6465
db.import_globals(*c, c->get_globals(channel));
6566

6667
c->flush(channel, flush_info, [&queryP,&db](CaliperMetadataAccessInterface& in_db, const std::vector<Entry>& rec){

test/ci_app_tests/test_report.py

+23-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# report / C config test
22

3-
import unittest
3+
import json,unittest
44

55
import calipertest as cat
66

@@ -183,6 +183,28 @@ def test_report_attribute_aliases(self):
183183

184184
self.assertTrue(cat.has_snapshot_with_keys(snapshots, { 'my function alias', 'count' }))
185185

186+
def test_report_attribute_aliases_in_cali(self):
187+
""" Test json object layout """
188+
189+
target_cmd = [ './ci_test_macros' ]
190+
query_cmd = [ '../../src/tools/cali-query/cali-query', '-q', 'format json(object)' ]
191+
192+
caliper_config = {
193+
'CALI_SERVICES_ENABLE' : 'event,trace,report',
194+
'CALI_REPORT_FILENAME' : 'stdout',
195+
'CALI_REPORT_CONFIG' : 'select *,count() as CountAlias group by prop:nested format cali',
196+
'CALI_LOG_VERBOSITY' : '0'
197+
}
198+
199+
obj = json.loads( cat.run_test_with_query(target_cmd, query_cmd, caliper_config ) )
200+
201+
self.assertTrue( { 'records', 'globals', 'attributes' }.issubset(set(obj.keys())) )
202+
203+
self.assertIn('count', obj['attributes'])
204+
self.assertIn('attribute.alias', obj['attributes']['count'])
205+
206+
self.assertEqual('CountAlias', obj['attributes']['count']['attribute.alias'])
207+
186208
def test_report(self):
187209
target_cmd = [ './ci_test_report' ]
188210
# create some distraction: read-from-env should be disabled

test/ci_app_tests/test_spot.py

+4
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ def test_spot_regionprofile(self):
2424

2525
self.assertIn('spot.format.version', obj['globals'])
2626
self.assertIn('avg#inclusive#sum#time.duration', obj['globals']['spot.metrics'])
27+
self.assertIn('avg#inclusive#sum#time.duration', obj['attributes'])
28+
self.assertIn('attribute.alias', obj['attributes']['avg#inclusive#sum#time.duration'])
29+
30+
self.assertEqual('Avg time/rank', obj['attributes']['avg#inclusive#sum#time.duration']['attribute.alias'])
2731

2832
r = None
2933
for rec in obj['records']:

0 commit comments

Comments
 (0)