Simplify handling of path attributes (LLNL#458)

* Merge function/annotation/statement attributes

* Add CalQL path keyword to query spec

* Replace use of prop:nested with path everywhere

* Update documentation

Author: daboehme

doc/sphinx/calql.rst

@@ -28,10 +28,12 @@ This table contains a quick reference of all CalQL statements:
     <a>=scale(<b>,<S>)         # computes <a> = <b>*S
     <a>=truncate(<b>,<S>)      # computes <a> = <b> - mod(<b>, S)
     <a>=first(<a0>,<a1>, ...)  # <a> is the first of <a0>, <a1>, ... found in the input record
+    <a>=sum(<a0>,<a1>,...)     # computes sum of <a0>, <a1>, ... in the input record
     ... IF <condition>         # apply only if input record meets condition
 
   SELECT <list>                # Select attributes and define aggregations (i.e., select columns)
     *                          # select all attributes
+    path                       # select the nested region hierarchy
     <attribute>                # select <attribute>
     count()                    # number of input records in grouping
     sum(<attribute>)           # compute sum of <attribute> for grouping
@@ -45,12 +47,16 @@ This table contains a quick reference of all CalQL statements:
     inclusive_scale(<a>,<S>)   # computes inclusive scaled sum of <a> in a nested hierarchy
     inclusive_percent_total(<a>) # computes inclusive percent-of-total of <a> in a nested hierarchy
     any(<attribute>)           # pick one value for <attribute> out of all records in grouping
+    scale_count(<S>)           # count number of input records for grouping and multiplied by <S>
+    inclusive_ratio(<a>,<b>,<S>) # computes inclusive_sum(<a>)/inclusive_sum(<b>)*<S>
+    inclusive_min(<a>)         # compute inclusive min of <a>
+    inclusive_max(<a>)         # compute inclusive max of <a>
     ... AS <name>              # use <name> as column header in tree or table formatter
     ... UNIT <unit>            # use <unit> as unit name
 
   GROUP BY <list>              # Define aggregation grouping (what to aggregate over, e.g. "function,mpi.rank")
     <attribute>                # include <attribute> in grouping
-    prop:nested                # include default annotation attributes ('function', 'annotation', ...) in grouping
+    path                       # include the nested region hierarchy in grouping
 
   WHERE <list>                 # define filter (i.e., select records/rows)
     <attribute>                # records where <attribute> exists
@@ -86,7 +92,7 @@ subsequent aggregations::
   LET
     sec=scale(time.duration,1e-6)
   SELECT
-    prop:nested,sum(sec)
+    path,sum(sec)
 
 We can use the truncate() operator on an iteration counter to
 aggregate blocks of 10 iterations in a time-series profile::
@@ -106,7 +112,7 @@ names found in an input record. It can also be used to rename attributes::
   SELECT
     sum(time) AS Time
   GROUP BY
-    prop:nested
+    path
 
 LET terms have the general form
 
@@ -131,7 +137,7 @@ efficiency from the total and "work" time:
     sum(work)                 AS Work,
     ratio(work,time.duration) AS Efficiency
   GROUP BY
-    prop:nested
+    path
 
 SELECT
 --------------------------------
@@ -175,7 +181,7 @@ A more complex example::
     scale(time.duration,1e-6) AS Time,
     inclusive_percent_total(time.duration) AS "Time %"
   GROUP BY
-    prop:nested
+    path
   FORMAT
     tree
 

include/caliper/cali.h

@@ -384,19 +384,6 @@ cali_begin_string(cali_id_t attr, const char* val);
 void
 cali_end  (cali_id_t   attr);
 
-/**
- * \brief Remove innermost value for attribute \a attr from the blackboard.
- *
- * Creates a mismatch warning if the current value does not match \a val.
- * This function is primarily used by the high-level annotation API.
- *
- * \param attr Attribute ID
- * \param val  Expected value
- */
-
-void
-cali_safe_end_string(cali_id_t attr, const char* val);
-
 /**
  * \brief Set value for attribute \a attr to \a val on the blackboard.
  */

include/caliper/cali_macros.h

@@ -70,9 +70,7 @@
 
 #endif // __cplusplus
 
-extern cali_id_t cali_function_attr_id;
 extern cali_id_t cali_loop_attr_id;
-extern cali_id_t cali_statement_attr_id;
 extern cali_id_t cali_annotation_attr_id;
 
 /// \brief Mark begin of a function.
@@ -83,17 +81,15 @@ extern cali_id_t cali_annotation_attr_id;
 /// \ref CALI_CXX_MARK_FUNCTION instead.
 /// \sa CALI_MARK_FUNCTION_END, CALI_CXX_MARK_FUNCTION
 #define CALI_MARK_FUNCTION_BEGIN \
-    if (cali_function_attr_id == CALI_INV_ID) \
-        cali_init(); \
-    cali_begin_string(cali_function_attr_id, __func__)
+    cali_begin_region(__func__)
 
 /// \brief Mark end of a function.
 ///
 /// Must be placed at \e all exit points of a function marked with
 /// \ref CALI_MARK_FUNCTION_BEGIN.
 /// \sa CALI_MARK_FUNCTION_BEGIN
 #define CALI_MARK_FUNCTION_END \
-    cali_safe_end_string(cali_function_attr_id, __func__)
+    cali_end_region(__func__)
 
 /// \brief Mark a loop
 ///
@@ -110,7 +106,7 @@ extern cali_id_t cali_annotation_attr_id;
         cali_init(); \
     cali_begin_string(cali_loop_attr_id, (name));       \
     cali_id_t __cali_iter_##loop_id = \
-        cali_make_loop_iteration_attribute(name);
+        cali_make_loop_iteration_attribute(name)
 
 /// \brief Mark a loop
 ///
@@ -175,11 +171,9 @@ extern cali_id_t cali_annotation_attr_id;
 ///   \c break, or \c return).
 ///
 #define CALI_WRAP_STATEMENT(name, statement)     \
-    if (cali_statement_attr_id == CALI_INV_ID) \
-        cali_init(); \
-    cali_begin_string(cali_statement_attr_id, (name));  \
+    cali_begin_region((name));  \
     statement; \
-    cali_end(cali_statement_attr_id);
+    cali_end_region((name))
 
 /// \brief Mark begin of a user-defined code region.
 ///

include/caliper/reader/QuerySpec.h

@@ -33,6 +33,9 @@ struct QuerySpec
 
         /// \brief User-defined list
         std::vector<T> list;
+
+        /// \brief Use path (main nested region hierarchy) attributes
+        bool           use_path;
     };
 
     /// \brief Describe function signatures in query specs
@@ -124,12 +127,12 @@ struct QuerySpec
     typedef SelectionList<SortSpec>      SortSelection;
 
     /// \brief List of aggregations to be performed.
-    AggregationSelection         aggregation_ops;
+    AggregationSelection         aggregate;
     /// \brief List of attribute names that form the aggregation key (i.e., GROUP BY spec).
-    AttributeSelection           aggregation_key;
+    AttributeSelection           groupby;
 
     /// \brief List of attributes to print in output
-    AttributeSelection           attribute_selection;
+    AttributeSelection           select;
 
     /// \brief List of filter clauses (filters will be combined with AND)
     FilterSelection              filter;

src/caliper/Annotation.cpp

@@ -22,34 +22,33 @@ namespace cali
 {
 
 extern Attribute class_iteration_attr;
-extern Attribute function_attr;
 extern Attribute loop_attr;
-extern Attribute annotation_attr;
+extern Attribute region_attr;
 
 }
 
 // --- Pre-defined Function annotation class
 
 Function::Function(const char* name)
 {
-    Caliper().begin(function_attr, Variant(name));
+    Caliper().begin(region_attr, Variant(name));
 }
 
 Function::~Function()
 {
-    Caliper().end(function_attr);
+    Caliper().end(region_attr);
 }
 
 // --- Pre-defined scope annotation class
 
 ScopeAnnotation::ScopeAnnotation(const char* name)
 {
-    Caliper().begin(annotation_attr, Variant(name));
+    Caliper().begin(region_attr, Variant(name));
 }
 
 ScopeAnnotation::~ScopeAnnotation()
 {
-    Caliper().end(annotation_attr);
+    Caliper().end(region_attr);
 }
 
 // --- Pre-defined loop annotation class

src/caliper/api.cpp

@@ -15,10 +15,8 @@ cali_id_t cali_class_memoryaddress_attr_id = CALI_INV_ID;
 cali_id_t cali_class_iteration_attr_id     = CALI_INV_ID;
 cali_id_t cali_subscription_event_attr_id  = CALI_INV_ID;
 
-cali_id_t cali_function_attr_id     = CALI_INV_ID;
-cali_id_t cali_loop_attr_id         = CALI_INV_ID;
-cali_id_t cali_statement_attr_id    = CALI_INV_ID;
-cali_id_t cali_annotation_attr_id   = CALI_INV_ID;
+cali_id_t cali_loop_attr_id   = CALI_INV_ID;
+cali_id_t cali_region_attr_id = CALI_INV_ID;
 
 cali_id_t cali_alloc_fn_attr_id                 = CALI_INV_ID;
 cali_id_t cali_alloc_label_attr_id              = CALI_INV_ID;
@@ -36,75 +34,35 @@ namespace cali
     Attribute class_memoryaddress_attr;
     Attribute class_iteration_attr;
     Attribute subscription_event_attr;
-    
-    Attribute function_attr;
+
+    Attribute region_attr;
     Attribute loop_attr;
-    Attribute statement_attr;
-    Attribute annotation_attr;
 
     void init_attribute_classes(Caliper* c) {
-        struct attr_info_t {
-            const char*    name;
-            cali_attr_type type;
-            int            prop;
-            Attribute*     attr;
-            cali_id_t*     attr_id;
-        } attr_info[] = {
-            { "class.aggregatable", CALI_TYPE_BOOL, CALI_ATTR_SKIP_EVENTS,
-              &class_aggregatable_attr, &cali_class_aggregatable_attr_id
-            },
-            { "class.symboladdress", CALI_TYPE_BOOL, CALI_ATTR_SKIP_EVENTS,
-              &class_symboladdress_attr, &cali_class_symboladdress_attr_id
-            },
-            { "class.memoryaddress", CALI_TYPE_BOOL, CALI_ATTR_SKIP_EVENTS,
-              &class_memoryaddress_attr, &cali_class_memoryaddress_attr_id
-            },
-            { "class.iteration",     CALI_TYPE_BOOL, CALI_ATTR_SKIP_EVENTS,
-              &class_iteration_attr, &cali_class_iteration_attr_id
-            },
-            { "subscription_event",  CALI_TYPE_BOOL, CALI_ATTR_SKIP_EVENTS,
-              &subscription_event_attr, &cali_subscription_event_attr_id
-            },
-            { 0, CALI_TYPE_INV, CALI_ATTR_DEFAULT, 0, 0 }
-        };
+        class_aggregatable_attr =
+            c->create_attribute("class.aggregatable", CALI_TYPE_BOOL, CALI_ATTR_SKIP_EVENTS);
+        class_symboladdress_attr =
+            c->create_attribute("class.symboladdress", CALI_TYPE_BOOL, CALI_ATTR_SKIP_EVENTS);
+        class_memoryaddress_attr =
+            c->create_attribute("class.memoryaddress", CALI_TYPE_BOOL, CALI_ATTR_SKIP_EVENTS);
+        class_iteration_attr =
+            c->create_attribute("class.iteration", CALI_TYPE_BOOL, CALI_ATTR_SKIP_EVENTS);
+        subscription_event_attr =
+            c->create_attribute("subscription_event", CALI_TYPE_BOOL, CALI_ATTR_SKIP_EVENTS);
 
-        for (attr_info_t *p = attr_info; p->name; ++p) {
-            *(p->attr) =
-                c->create_attribute(p->name, p->type, p->prop);
-            *(p->attr_id) = (p->attr)->id();
-        }
+        cali_class_aggregatable_attr_id  = class_aggregatable_attr.id();
+        cali_class_symboladdress_attr_id = class_symboladdress_attr.id();
+        cali_class_memoryaddress_attr_id = class_memoryaddress_attr.id();
+        cali_class_iteration_attr_id     = class_iteration_attr.id();
     }
 
     void init_api_attributes(Caliper* c) {
-        // --- code annotation attributes
-
-        struct attr_info_t {
-            const char*    name;
-            cali_attr_type type;
-            int            prop;
-            Attribute*     attr;
-            cali_id_t*     attr_id;
-        } attr_info[] = {
-            { "function",   CALI_TYPE_STRING, CALI_ATTR_NESTED,
-              &function_attr,   &cali_function_attr_id
-            },
-            { "loop",       CALI_TYPE_STRING, CALI_ATTR_NESTED,
-              &loop_attr,       &cali_loop_attr_id
-            },
-            { "statement",  CALI_TYPE_STRING, CALI_ATTR_NESTED,
-              &statement_attr,  &cali_statement_attr_id
-            },
-            { "annotation", CALI_TYPE_STRING, CALI_ATTR_NESTED,
-              &annotation_attr, &cali_annotation_attr_id
-            },
-            { 0, CALI_TYPE_INV, CALI_ATTR_DEFAULT, 0, 0 }
-        };
+        loop_attr =
+            c->create_attribute("loop", CALI_TYPE_STRING, CALI_ATTR_NESTED);
+        region_attr =
+            c->create_attribute("region", CALI_TYPE_STRING, CALI_ATTR_NESTED);
 
-        Variant v_true(true);
-        
-        for (attr_info_t *p = attr_info; p->name; ++p) {
-            *(p->attr)    = c->create_attribute(p->name, p->type, p->prop);
-            *(p->attr_id) = (p->attr)->id();
-        }
+        cali_region_attr_id = region_attr.id();
+        cali_loop_attr_id = loop_attr.id();
     }
 }

src/caliper/cali.cpp

@@ -29,7 +29,7 @@
 namespace cali
 {
 
-extern Attribute annotation_attr;
+extern Attribute region_attr;
 
 }
 
@@ -362,22 +362,22 @@ void
 cali_begin_region(const char* name)
 {
     Caliper c;
-    c.begin(cali::annotation_attr, Variant(name));
+    c.begin(cali::region_attr, Variant(name));
 }
 
 void
 cali_end_region(const char* name)
 {
     Caliper c;
     Variant v_n(name);
-    Variant v_s = c.get(cali::annotation_attr).value();
+    Variant v_s = c.get(cali::region_attr).value();
 
     if (!(v_n == v_s))
         Log(0).stream() << "region nesting error: trying to end \"" << v_n
                         << "\" but current region is \""            << v_s << "\""
                         << std::endl;
 
-    c.end(cali::annotation_attr);
+    c.end(cali::region_attr);
 }
 
 void
@@ -461,28 +461,6 @@ cali_set_string(cali_id_t attr_id, const char* val)
     c.set(attr, Variant(CALI_TYPE_STRING, val, strlen(val)));
 }
 
-void
-cali_safe_end_string(cali_id_t attr_id, const char* val)
-{
-    Caliper   c;
-    Attribute attr = c.get_attribute(attr_id);
-    Variant   v    = c.get(attr).value();
-
-    if (v.type() != CALI_TYPE_STRING)
-        Log(1).stream() << ": Trying to end "
-                        << attr.name() << " which is not a string" << std::endl;
-
-    if (0 != strncmp(static_cast<const char*>(v.data()), val, v.size()))
-        // FIXME: Replace log output with smart error tracker
-        Log(1).stream() << "begin/end marker mismatch: Trying to end "
-                        << attr.name() << "=" << val
-                        << " but current value for "
-                        << attr.name() << " is \"" << v.to_string() << "\""
-                        << std::endl;
-
-    c.end(attr);
-}
-
 //
 // --- By-name annotation interface
 //

src/caliper/controllers/HatchetRegionProfileController.cpp

@@ -62,7 +62,7 @@ class HatchetRegionProfileController : public cali::ChannelController
                 config()["CALI_MPIREPORT_CONFIG"  ] =
                     opts.build_query("local", {
                             { "select",   "*,sum(sum#time.duration) as time unit sec" },
-                            { "group by", "prop:nested,mpi.rank" },
+                            { "group by", "path,mpi.rank" },
                             { "format",   format }
                         });
             } else {
@@ -71,7 +71,7 @@ class HatchetRegionProfileController : public cali::ChannelController
                 config()["CALI_REPORT_CONFIG"     ] =
                     opts.build_query("local", {
                             { "select",   "*,sum(sum#time.duration) as time unit sec" },
-                            { "group by", "prop:nested" },
+                            { "group by", "path" },
                             { "format",   format }
                         });
             }