Save attribute aliases as metadata (LLNL#301)

* Save some attribute aliases in .cali output

* Update reader test case

* Add aliases for SPOT time metrics

Author: daboehme

include/caliper/reader/CaliperMetadataDB.h

@@ -120,6 +120,15 @@ class CaliperMetadataDB : public CaliperMetadataAccessInterface
     ///   metadata DB
     std::vector<Entry> import_globals(CaliperMetadataAccessInterface& db, const std::vector<Entry>& globals);
 
+    /// \brief Add a set of attribute aliases
+    ///
+    /// This adds a "attribute.alias" meta-attribute for the aliased attribute
+    /// to export alias information in a cali data stream.
+    /// Currently this is limited to new attributes created with
+    /// create_attribute() in this database. It does not apply to imported
+    /// attributes.
+    void add_attribute_aliases(const std::map<std::string, std::string>& aliases);
+
     /// \brief print usage statistics
     std::ostream&
     print_statistics(std::ostream& os);

src/mpi/controllers/SpotController.cpp

@@ -359,6 +359,8 @@ class SpotController : public cali::ChannelController
             QuerySpec  spec = tsc->timeseries_spec();
             Aggregator cross_agg(spec);
 
+            m_db.add_attribute_aliases(spec.aliases);
+
             tsc->timeseries_local_aggregation(c, m_db, namebuf, std::max(blocksize, 1), cross_agg);
             cross_aggregate(cross_agg);
 
@@ -424,9 +426,9 @@ class SpotController : public cali::ChannelController
         // --- Setup output reduction aggregator (final cross-process aggregation)
         const char* cross_select =
             " *"
-            ",min(inclusive#sum#time.duration)"
-            ",max(inclusive#sum#time.duration)"
-            ",avg(inclusive#sum#time.duration)";
+            ",min(inclusive#sum#time.duration) as \"Min time/rank\""
+            ",max(inclusive#sum#time.duration) as \"Max time/rank\""
+            ",avg(inclusive#sum#time.duration) as \"Avg time/rank\"";
         std::string cross_query =
             std::string("select ")
             + m_opts.query_select("cross", cross_select, false)
@@ -436,6 +438,8 @@ class SpotController : public cali::ChannelController
         QuerySpec  output_spec(CalQLParser(cross_query.c_str()).spec());
         Aggregator output_agg(output_spec);
 
+        m_db.add_attribute_aliases(output_spec.aliases);
+
         // ---   Flush Caliper buffers into intermediate aggregator to calculate
         //     region profile inclusive times
         {

src/mpi/services/mpireport/MpiReport.cpp

@@ -51,6 +51,8 @@ class MpiReport
 
         CaliperMetadataDB db;
 
+        db.add_attribute_aliases(m_cross_spec.aliases);
+
         Aggregator        cross_agg(m_cross_spec);
         Aggregator        local_agg(m_local_spec);
 

src/reader/CaliperMetadataDB.cpp

@@ -47,6 +47,9 @@ struct CaliperMetadataDB::CaliperMetadataDBImpl
     vector<Entry>             m_globals;
     mutex                     m_globals_lock;
 
+    std::map<std::string, std::string> m_attr_aliases;
+    Attribute                 m_alias_attr;
+
     inline Node* node(cali_id_t id) const {
         std::lock_guard<std::mutex>
             g(m_node_lock);
@@ -398,6 +401,12 @@ struct CaliperMetadataDB::CaliperMetadataDBImpl
         if (meta > 0)
             parent = make_tree_entry(meta, meta_attr, meta_data, parent);
 
+        auto alias_it = m_attr_aliases.find(name);
+        if (alias_it != m_attr_aliases.end()) {
+            Variant v_alias(static_cast<const char*>(alias_it->second.c_str()));
+            parent = make_tree_entry(1, &m_alias_attr, &v_alias, parent);
+        }
+
         Attribute n_attr[2] = { attribute(Attribute::meta_attribute_keys().prop_attr_id),
                                 attribute(Attribute::meta_attribute_keys().name_attr_id) };
         Variant   n_data[2] = { Variant(prop),
@@ -476,6 +485,10 @@ struct CaliperMetadataDB::CaliperMetadataDBImpl
         : m_root { CALI_INV_ID, CALI_INV_ID, { } }
         {
             setup_bootstrap_nodes();
+
+            m_alias_attr =
+                create_attribute("attribute.alias", CALI_TYPE_STRING, CALI_ATTR_SKIP_EVENTS,
+                                 0, nullptr, nullptr);
         }
 
     ~CaliperMetadataDBImpl() {
@@ -636,6 +649,13 @@ CaliperMetadataDB::import_globals(CaliperMetadataAccessInterface& db, const std:
     return mP->import_globals(db, globals);
 }
 
+void
+CaliperMetadataDB::add_attribute_aliases(const std::map<std::string, std::string>& aliases)
+{
+    for (const auto &p: aliases)
+        mP->m_attr_aliases[p.first] = p.second;
+}
+
 std::ostream&
 CaliperMetadataDB::print_statistics(std::ostream& os)
 {

src/reader/JsonFormatter.cpp

@@ -188,7 +188,7 @@ struct JsonFormatter::JsonFormatterImpl
         if (noquote_kvs.size() + quote_kvs.size() > 0) {
             if (m_num_recs == 0)
                 begin_records_section(*real_os);
-            
+
             *real_os << (m_num_recs > 0 ? ",\n" : "") << "{";
 
             int count = 0;
@@ -213,7 +213,7 @@ struct JsonFormatter::JsonFormatterImpl
 
     std::ostream& write_attributes(CaliperMetadataAccessInterface& db, std::ostream& os) {
         std::vector<Attribute> attrs;
-        
+
         if (m_selected.empty())
             attrs = db.get_all_attributes();
         else
@@ -229,14 +229,14 @@ struct JsonFormatter::JsonFormatterImpl
         attrs.erase(std::remove_if(attrs.begin(), attrs.end(),
                                    [](const Attribute& a){ return a.is_hidden(); }),
                     attrs.end());
-        
+
         // write "attr1": { "prop": "value", ... } , "attr2" : ...
 
         int count = 0;
         for (const Attribute& a : attrs) {
             os << (count++ > 0 ? ",\n" : "\n") << (m_opt_pretty ? "\t" : "")
                << '\"' << a.name() << "\": {";
-            
+
             // encode some properties
             os << "\"is_global\": "  << (a.is_global() ? "true" : "false")
                << ",\"is_nested\": " << (a.is_nested() ? "true" : "false");
@@ -246,13 +246,13 @@ struct JsonFormatter::JsonFormatterImpl
                 util::write_esc_string(os << ",\"", db.get_attribute(node->attribute()).name()) << "\": ";
                 util::write_esc_string(os << "\"", node->data().to_string()) << '\"';
             }
-                                                   
+
             os << "}";
         }
 
         return os;
     }
-    
+
     std::ostream& write_globals(CaliperMetadataAccessInterface& db, std::ostream& os) {
         std::vector<Entry>               globals = db.get_globals();
         std::map<cali_id_t, std::string> global_vals;
@@ -286,7 +286,7 @@ struct JsonFormatter::JsonFormatterImpl
 
     void flush(CaliperMetadataAccessInterface& db) {
         std::ostream* real_os = m_os.stream();
-        
+
         // close records section
         if (m_num_recs == 0)
             begin_records_section(*real_os);

src/reader/test/test_metadb.cpp

@@ -156,7 +156,7 @@ TEST(MetadataDBTest, StringDB) {
     std::ostringstream os;
     db.print_statistics(os);
 
-    // 18 nodes: 12 default nodes + 2 attribute nodes + 4 test nodes
-    // 4 strings: 1 attribute name + 3 different test node strings
-    EXPECT_EQ(os.str(), std::string("CaliperMetadataDB: stored 18 nodes, 4 strings.\n"));
+    // 20 nodes: 14 default nodes + 2 attribute nodes + 4 test nodes
+    // 5 strings: 2 attribute name + 3 different test node strings
+    EXPECT_EQ(os.str(), std::string("CaliperMetadataDB: stored 20 nodes, 5 strings.\n"));
 }

src/services/report/Report.cpp

@@ -61,6 +61,7 @@ class Report {
         CaliperMetadataDB db;
         QueryProcessor queryP(spec, stream);
 
+        db.add_attribute_aliases(spec.aliases);
         db.import_globals(*c, c->get_globals(channel));
 
         c->flush(channel, flush_info, [&queryP,&db](CaliperMetadataAccessInterface& in_db, const std::vector<Entry>& rec){

test/ci_app_tests/test_report.py

@@ -1,6 +1,6 @@
 # report / C config test
 
-import unittest
+import json,unittest
 
 import calipertest as cat
 
@@ -183,6 +183,28 @@ def test_report_attribute_aliases(self):
 
         self.assertTrue(cat.has_snapshot_with_keys(snapshots, { 'my function alias', 'count'  }))
 
+    def test_report_attribute_aliases_in_cali(self):
+        """ Test json object layout """
+
+        target_cmd = [ './ci_test_macros' ]
+        query_cmd  = [ '../../src/tools/cali-query/cali-query', '-q', 'format json(object)' ]
+
+        caliper_config = {
+            'CALI_SERVICES_ENABLE'   : 'event,trace,report',
+            'CALI_REPORT_FILENAME'   : 'stdout',
+            'CALI_REPORT_CONFIG'     : 'select *,count() as CountAlias group by prop:nested format cali',
+            'CALI_LOG_VERBOSITY'     : '0'
+        }
+
+        obj = json.loads( cat.run_test_with_query(target_cmd, query_cmd, caliper_config ) )
+
+        self.assertTrue( { 'records', 'globals', 'attributes' }.issubset(set(obj.keys())) )
+
+        self.assertIn('count',           obj['attributes'])
+        self.assertIn('attribute.alias', obj['attributes']['count'])
+
+        self.assertEqual('CountAlias', obj['attributes']['count']['attribute.alias'])
+
     def test_report(self):
         target_cmd = [ './ci_test_report' ]
         # create some distraction: read-from-env should be disabled

test/ci_app_tests/test_spot.py

@@ -24,6 +24,10 @@ def test_spot_regionprofile(self):
 
         self.assertIn('spot.format.version', obj['globals'])
         self.assertIn('avg#inclusive#sum#time.duration', obj['globals']['spot.metrics'])
+        self.assertIn('avg#inclusive#sum#time.duration', obj['attributes'])
+        self.assertIn('attribute.alias', obj['attributes']['avg#inclusive#sum#time.duration'])
+
+        self.assertEqual('Avg time/rank', obj['attributes']['avg#inclusive#sum#time.duration']['attribute.alias'])
 
         r = None
         for rec in obj['records']: