calql: Add leaf() preprocessor kernel (#525)

* Add preprocessor leaf() kernel

* leaf kernel can use path attributes

* Always apply preprocessor before filter

* Add documentation for leaf kernel

Author: daboehme

doc/sphinx/calql.rst

@@ -29,6 +29,7 @@ This table contains a quick reference of all CalQL statements:
     <a>=truncate(<b>,<S>)      # computes <a> = <b> - mod(<b>, S)
     <a>=first(<a0>,<a1>, ...)  # <a> is the first of <a0>, <a1>, ... found in the input record
     <a>=sum(<a0>,<a1>,...)     # computes sum of <a0>, <a1>, ... in the input record
+    <a>=leaf(<b>)              # Takes the leaf of the region hierarchy <b>
     ... IF <condition>         # apply only if input record meets condition
 
   SELECT <list>                # Select attributes and define aggregations (i.e., select columns)

include/caliper/common/CaliperMetadataAccessInterface.h

@@ -65,4 +65,6 @@ class CaliperMetadataAccessInterface
     get_globals() = 0;
 };
 
+Entry get_path_entry(const CaliperMetadataAccessInterface& db, const Entry& e);
+
 }

src/common/CaliperMetadataAccessInterface.cpp

@@ -32,4 +32,26 @@ CaliperMetadataAccessInterface::find_attributes_with_prop(int prop) const
             ret.push_back(attr);
 
     return ret;
-}
+}
+
+namespace cali
+{
+
+Entry
+get_path_entry(const CaliperMetadataAccessInterface& db, const Entry& e)
+{
+    Entry ret;
+
+    if (e.is_reference()) {
+        for (Node* node = e.node(); node; node = node->parent())
+            if (db.get_attribute(node->attribute()).is_nested()) {
+                ret = Entry(node);
+                break;
+            }
+    }
+
+    return ret;
+}
+
+}
+

src/reader/Preprocessor.cpp

@@ -40,7 +40,6 @@ Variant get_value(const CaliperMetadataAccessInterface& db, const std::string& a
     return Variant();
 }
 
-
 class Kernel {
 public:
 
@@ -273,7 +272,7 @@ class SumKernel : public Kernel
 
             if (v_tgt.empty())
                 continue;
-            
+
             v_sum += v_tgt;
         }
 
@@ -293,12 +292,72 @@ class SumKernel : public Kernel
 
 };
 
+class LeafKernel : public Kernel
+{
+    bool        m_use_path;
+    std::string m_res_attr_name;
+    Attribute   m_res_attr;
+    std::string m_tgt_attr_name;
+    Attribute   m_tgt_attr;
+
+public:
+
+    LeafKernel(const std::string& def)
+        : m_use_path(true),
+          m_res_attr_name(def),
+          m_res_attr(Attribute::invalid),
+          m_tgt_attr(Attribute::invalid)
+        { }
+
+    LeafKernel(const std::string& def, const std::string& tgt)
+        : m_use_path(false),
+          m_res_attr_name(def),
+          m_res_attr(Attribute::invalid),
+          m_tgt_attr_name(tgt),
+          m_tgt_attr(Attribute::invalid)
+        { }
+
+    void process(CaliperMetadataAccessInterface& db, EntryList& rec) {
+        if (m_res_attr == Attribute::invalid) {
+            cali_attr_type type = CALI_TYPE_STRING;
+            int prop = CALI_ATTR_SKIP_EVENTS | CALI_ATTR_ASVALUE;
+
+            if (!m_use_path) {
+                m_tgt_attr = db.get_attribute(m_tgt_attr_name);
+                if (m_tgt_attr == Attribute::invalid)
+                    return;
+                type  = m_tgt_attr.type();
+                prop |= m_tgt_attr.properties();
+                prop &= ~CALI_ATTR_NESTED;
+            }
+
+            m_res_attr = db.create_attribute(m_res_attr_name, type, prop);
+        }
+
+        for (const Entry& e : rec) {
+            Entry e_target = m_use_path ? get_path_entry(db, e) : e.get(m_tgt_attr);
+            if (!e_target.empty() && m_res_attr.type() == e_target.value().type()) {
+                rec.push_back(Entry(m_res_attr, e_target.value()));
+                return;
+            }
+        }
+    }
+
+    static Kernel* create(const std::string& def, const std::vector<std::string>& args) {
+        if (args.empty())
+            return new LeafKernel(def);
+
+        return new LeafKernel(def, args.front());
+    }
+};
+
 enum KernelID {
     ScaledRatio,
     Scale,
     Truncate,
     First,
-    Sum
+    Sum,
+    Leaf
 };
 
 const char* sratio_args[]  = { "numerator", "denominator", "scale" };
@@ -315,6 +374,7 @@ const QuerySpec::FunctionSignature kernel_signatures[] = {
     { KernelID::Truncate,      "truncate",      1, 2, scale_args   },
     { KernelID::First,         "first",         1, 8, first_args   },
     { KernelID::Sum,           "sum",           1, 8, first_args   },
+    { KernelID::Leaf,          "leaf",          0, 1, scale_args   },
 
     QuerySpec::FunctionSignatureTerminator
 };
@@ -326,10 +386,11 @@ const KernelCreateFn kernel_create_fn[] = {
     ScaleKernel::create,
     TruncateKernel::create,
     FirstKernel::create,
-    SumKernel::create
+    SumKernel::create,
+    LeafKernel::create
 };
 
-constexpr int MAX_KERNEL_ID = 4;
+constexpr int MAX_KERNEL_ID = 5;
 
 }
 

src/reader/QueryProcessor.cpp

@@ -23,8 +23,8 @@ struct QueryProcessor::QueryProcessorImpl
 
     void
     process_record(CaliperMetadataAccessInterface& db, const EntryList& in_rec) {
-        if (filter.pass(db, in_rec)) {
-            auto rec = preprocessor.process(db, in_rec);
+        auto rec = preprocessor.process(db, in_rec);
+        if (filter.pass(db, rec)) {
 
             if (do_aggregate)
                 aggregator.add(db, rec);

src/reader/test/test_preprocessor.cpp

@@ -432,4 +432,34 @@ TEST(PreprocessorTest, SumKernel)
 
     EXPECT_EQ(a_it->second.value().to_int(), 42);
     EXPECT_EQ(s_it->second.value().to_int(), 66);
-}
+}
+
+TEST(PreprocessorTest, LeafKernel)
+{
+    CaliperMetadataDB db;
+
+    Attribute ctx =
+        db.create_attribute("ctx", CALI_TYPE_STRING, CALI_ATTR_DEFAULT);
+
+    Attribute a[2] = { ctx, ctx };
+    Variant   v[2] = { Variant("foo"), Variant("bar") };
+
+    Node* n_1 = db.make_tree_entry(2, a, v);
+
+    EntryList rec;
+    rec.push_back(Entry(n_1));
+
+    QuerySpec spec;
+    spec.preprocess_ops.push_back(::make_spec("leaf", ::make_op("leaf", "ctx")));
+
+    Preprocessor pp(spec);
+    EntryList out = pp.process(db, rec);
+
+    Attribute leaf_attr = db.get_attribute("leaf");
+
+    ASSERT_NE(leaf_attr, Attribute::invalid);
+    ASSERT_EQ(out.size(), 2u);
+    EXPECT_EQ(out[1].attribute(), leaf_attr.id());
+    EXPECT_EQ(out[1].value(), n_1->data());
+    EXPECT_NE(out[1].node()->parent(), n_1->parent());
+}

src/tools/cali-query/cali-query.cpp

@@ -302,10 +302,10 @@ int main(int argc, const char* argv[])
         else
             snap_proc = aggregate;
 
-        if (!spec.preprocess_ops.empty())
-            snap_proc = SnapshotFilterStep(Preprocessor(spec),   snap_proc);
         if (spec.filter.selection == QuerySpec::FilterSelection::List)
             snap_proc = SnapshotFilterStep(RecordSelector(spec), snap_proc);
+        if (!spec.preprocess_ops.empty())
+            snap_proc = SnapshotFilterStep(Preprocessor(spec),   snap_proc);
 
         if (args.is_set("list-attributes")) {
             node_proc = AttributeExtract(snap_proc);