Joint-width: compute the essential-variable count e from FDs mined on the gathered data (minimum cover under data FDs, subsuming declared keys on the instance), replacing the all-join-variables placeholder; expose it as ucq_joint_compile_stats.n_enumerating with an H0 x->y test (e=1 vs 2)

Author: PierreSenellart

sql/provsql.common.sql

@@ -4961,6 +4961,10 @@ CREATE OR REPLACE FUNCTION reachability_compile_stats(
  * @param[out] n_bags number of bags in the decomposition
  * @param[out] max_states peak number of DP states at any node
  * @param[out] dd_size number of gates in the emitted d-D
+ * @param[out] n_enumerating maximum number of essential (enumerating) query
+ *        variables over the disjuncts -- the @c e of the @f$2^{O(k^e)}@f$
+ *        bound, with variables functionally determined by others (via FDs
+ *        mined from the data) removed
  */
 CREATE OR REPLACE FUNCTION ucq_joint_compile_stats(
   IN disjunct_nvars INT[],
@@ -4979,7 +4983,8 @@ CREATE OR REPLACE FUNCTION ucq_joint_compile_stats(
   OUT circuit_treewidth_lb INT,
   OUT n_bags BIGINT,
   OUT max_states BIGINT,
-  OUT dd_size BIGINT)
+  OUT dd_size BIGINT,
+  OUT n_enumerating INT)
   AS 'provsql','ucq_joint_compile_stats'
   LANGUAGE C IMMUTABLE PARALLEL SAFE;
 
@@ -5003,7 +5008,8 @@ CREATE OR REPLACE FUNCTION ucq_joint_compile_stats(
   OUT circuit_treewidth_lb INT,
   OUT n_bags BIGINT,
   OUT max_states BIGINT,
-  OUT dd_size BIGINT)
+  OUT dd_size BIGINT,
+  OUT n_enumerating INT)
   AS $$
 DECLARE
   dnv INT[] := '{}'; adisj INT[] := '{}'; arel INT[] := '{}';
@@ -5023,9 +5029,10 @@ BEGIN
     didx := didx + 1;
   END LOOP;
   SELECT s.probability, s.joint_treewidth, s.data_treewidth_lb,
-         s.circuit_treewidth_lb, s.n_bags, s.max_states, s.dd_size
+         s.circuit_treewidth_lb, s.n_bags, s.max_states, s.dd_size,
+         s.n_enumerating
     INTO probability, joint_treewidth, data_treewidth_lb,
-         circuit_treewidth_lb, n_bags, max_states, dd_size
+         circuit_treewidth_lb, n_bags, max_states, dd_size, n_enumerating
     FROM ucq_joint_compile_stats(dnv, adisj, arel, avars, aarity,
       fact_rel, fact_elems, fact_arity, fact_tokens, fact_probs) s;
 END;
@@ -5066,7 +5073,8 @@ CREATE OR REPLACE FUNCTION ucq_joint_compile_stats_tracked(
   OUT circuit_treewidth_lb INT,
   OUT n_bags BIGINT,
   OUT max_states BIGINT,
-  OUT dd_size BIGINT)
+  OUT dd_size BIGINT,
+  OUT n_enumerating INT)
   AS 'provsql','ucq_joint_compile_stats_tracked'
   LANGUAGE C STABLE PARALLEL SAFE;
 
@@ -5086,7 +5094,8 @@ CREATE OR REPLACE FUNCTION ucq_joint_compile_stats_tracked(
   OUT circuit_treewidth_lb INT,
   OUT n_bags BIGINT,
   OUT max_states BIGINT,
-  OUT dd_size BIGINT)
+  OUT dd_size BIGINT,
+  OUT n_enumerating INT)
   AS $$
 DECLARE
   dnv INT[] := '{}'; adisj INT[] := '{}'; arel INT[] := '{}';
@@ -5106,9 +5115,10 @@ BEGIN
     didx := didx + 1;
   END LOOP;
   SELECT s.probability, s.joint_treewidth, s.data_treewidth_lb,
-         s.circuit_treewidth_lb, s.n_bags, s.max_states, s.dd_size
+         s.circuit_treewidth_lb, s.n_bags, s.max_states, s.dd_size,
+         s.n_enumerating
     INTO probability, joint_treewidth, data_treewidth_lb,
-         circuit_treewidth_lb, n_bags, max_states, dd_size
+         circuit_treewidth_lb, n_bags, max_states, dd_size, n_enumerating
     FROM ucq_joint_compile_stats_tracked(dnv, adisj, arel, avars, aarity,
       fact_rel, fact_elems, fact_arity, fact_tokens) s;
 END;

src/UCQJointCompiler.cpp

@@ -37,6 +37,7 @@
 #include <map>
 #include <memory>
 #include <set>
+#include <tuple>
 #include <unordered_map>
 #include <vector>
 
@@ -393,6 +394,118 @@ State join(const QueryCtx &q, const State &s1, const State &s2)
 // ToCS 2020), run over the joint graph (element + gate vertices).
 // ---------------------------------------------------------------------
 
+/**
+ * @brief Number of *essential* (enumerating) variables of a disjunct.
+ *
+ * The exponential parameter of the DP is not every join variable but only
+ * those that must be enumerated: a variable functionally determined by others
+ * does not multiply the partial-homomorphism state.  This returns the size of
+ * a minimum set @c S of join variables whose FD closure covers all of them
+ * (the @c e of the @f$2^{O(k^e)}@f$ bound).
+ *
+ * The FDs are **mined from the gathered tuples** (@p enc.facts, already
+ * post-selection): a column set determines a column when no two tuples of the
+ * relation agree on the former and differ on the latter.  This captures any
+ * declared key (which necessarily holds on the data) along with FDs incidental
+ * to this instance, and needs no catalog lookup.  The closure is sound for the
+ * instance being compiled: if the FD holds on these tuples the variable is
+ * genuinely fixed in every world of this computation.
+ *
+ * The minimum-cover search is @f$2^{|V|}@f$ in the join-variable count @c |V|;
+ * above @c MAX_FD_VARS it is skipped and the full count returned (a sound upper
+ * bound), the design target being a handful of enumerating variables.
+ */
+unsigned essentialVarCount(const CQ &cq, const std::vector<bool> &appears,
+                           const JointEncoding &enc)
+{
+  std::vector<unsigned> V;          // the join variables
+  for (unsigned v = 0; v < cq.n_vars; ++v)
+    if (appears[v])
+      V.push_back(v);
+  const unsigned nV = static_cast<unsigned>(V.size());
+  static constexpr unsigned MAX_FD_VARS = 12;
+  if (nV <= 1 || nV > MAX_FD_VARS)
+    return nV;
+
+  // Tuples per (relation, arity), as pointers into enc.facts.
+  std::map<std::pair<unsigned, std::size_t>,
+           std::vector<const std::vector<unsigned long> *> > byrel;
+  for (const Fact &f : enc.facts)
+    byrel[{f.relation_id, f.elements.size()}].push_back(&f.elements);
+
+  // Memoised data FD: do the columns in bitmask @c known determine column @c c
+  // in the tuples of (@c rel, @c arity)?
+  std::map<std::tuple<unsigned, std::size_t, std::uint64_t, unsigned>, bool> memo;
+  auto dataFD = [&](unsigned rel, std::size_t arity,
+                    std::uint64_t known, unsigned c) -> bool {
+    auto key = std::make_tuple(rel, arity, known, c);
+    auto it = memo.find(key);
+    if (it != memo.end())
+      return it->second;
+    bool holds = true;
+    auto rit = byrel.find({rel, arity});
+    if (rit != byrel.end()) {
+      std::map<std::vector<unsigned long>, unsigned long> grp;
+      for (const std::vector<unsigned long> *t : rit->second) {
+        std::vector<unsigned long> kv;
+        for (std::size_t p = 0; p < arity; ++p)
+          if (known & (std::uint64_t{1} << p))
+            kv.push_back((*t)[p]);
+        auto git = grp.find(kv);
+        if (git == grp.end())
+          grp.emplace(std::move(kv), (*t)[c]);
+        else if (git->second != (*t)[c]) {
+          holds = false;
+          break;
+        }
+      }
+    }
+    memo[key] = holds;
+    return holds;
+  };
+
+  std::unordered_map<unsigned, unsigned> bitOf;
+  for (unsigned i = 0; i < nV; ++i)
+    bitOf[V[i]] = i;
+  const std::uint32_t fullMask = (std::uint32_t{1} << nV) - 1;
+
+  // FD closure of a determined set (bitmask over @c V).
+  auto closure = [&](std::uint32_t det) -> std::uint32_t {
+    bool changed = true;
+    while (changed) {
+      changed = false;
+      for (const Atom &a : cq.atoms) {
+        const std::size_t arity = a.vars.size();
+        std::uint64_t knownPos = 0;
+        for (std::size_t p = 0; p < arity; ++p)
+          if (det & (std::uint32_t{1} << bitOf[a.vars[p]]))
+            knownPos |= (std::uint64_t{1} << p);
+        for (std::size_t p = 0; p < arity; ++p) {
+          if (knownPos & (std::uint64_t{1} << p))
+            continue;                       // this column already determined
+          const unsigned bit = bitOf[a.vars[p]];
+          if (dataFD(a.relation_id, arity, knownPos, static_cast<unsigned>(p))) {
+            det |= (std::uint32_t{1} << bit);
+            knownPos |= (std::uint64_t{1} << p);
+            changed = true;
+          }
+        }
+      }
+    }
+    return det;
+  };
+
+  unsigned best = nV;
+  for (std::uint32_t s = 0; s <= fullMask; ++s) {
+    const unsigned pc = static_cast<unsigned>(__builtin_popcount(s));
+    if (pc >= best)
+      continue;                             // cannot beat the current minimum
+    if (closure(s) == fullMask)
+      best = pc;
+  }
+  return best;
+}
+
 /** @brief Build the per-disjunct query context and the query stats. */
 void buildQueryCtx(const UCQ &ucq, const JointEncoding &enc,
                    QueryCtx &q, UCQJointCompiler::Stats &stats)
@@ -420,14 +533,10 @@ void buildQueryCtx(const UCQ &ucq, const JointEncoding &enc,
         di.atoms_of_var[v] |= (std::uint64_t{1} << ai);
         appears[v] = true;
       }
-    // The exponential parameter is the number of variables that occur in
-    // an atom (the join variables); a variable in no atom never enters a
-    // code.  Key/FD determination (a later milestone) shrinks this.
-    unsigned ne = 0;
-    for (unsigned v = 0; v < di.n_vars; ++v)
-      if (appears[v])
-        ++ne;
-    stats.n_enumerating[d] = ne;
+    // The exponential parameter is the number of *essential* join variables:
+    // those that must be enumerated once the ones functionally determined by
+    // others (via FDs mined from the gathered data) are removed.
+    stats.n_enumerating[d] = essentialVarCount(cq, appears, enc);
   }
   stats.data_treewidth_lb = enc.data_treewidth_lb;
   stats.circuit_treewidth_lb = enc.circuit_treewidth_lb;

src/ucq_joint_evaluate.cpp

@@ -714,15 +714,20 @@ Datum ucq_joint_compile_stats(PG_FUNCTION_ARGS)
       provsql_error("ucq_joint_compile_stats: expected composite return type");
     tupdesc = BlessTupleDesc(tupdesc);
 
-    Datum values[7];
-    bool nulls[7] = {false, false, false, false, false, false, false};
+    Datum values[8];
+    bool nulls[8] = {false, false, false, false, false, false, false, false};
     values[0] = Float8GetDatum(result.dd.probabilityEvaluation());
     values[1] = Int32GetDatum(static_cast<int32>(result.stats.joint_treewidth));
     values[2] = Int32GetDatum(static_cast<int32>(result.stats.data_treewidth_lb));
     values[3] = Int32GetDatum(static_cast<int32>(result.stats.circuit_treewidth_lb));
     values[4] = Int64GetDatum(static_cast<int64>(result.stats.nb_bags));
     values[5] = Int64GetDatum(static_cast<int64>(result.stats.max_states));
     values[6] = Int64GetDatum(static_cast<int64>(result.stats.dd_size));
+    unsigned maxenum = 0;
+    for (unsigned ev : result.stats.n_enumerating)
+      if (ev > maxenum)
+        maxenum = ev;
+    values[7] = Int32GetDatum(static_cast<int32>(maxenum));
 
     PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls)));
   } catch (const std::exception &e) {
@@ -781,15 +786,20 @@ Datum ucq_joint_compile_stats_tracked(PG_FUNCTION_ARGS)
       provsql_error("ucq_joint_compile_stats: expected composite return type");
     tupdesc = BlessTupleDesc(tupdesc);
 
-    Datum values[7];
-    bool nulls[7] = {false, false, false, false, false, false, false};
+    Datum values[8];
+    bool nulls[8] = {false, false, false, false, false, false, false, false};
     values[0] = Float8GetDatum(result.dd.probabilityEvaluation());
     values[1] = Int32GetDatum(static_cast<int32>(result.stats.joint_treewidth));
     values[2] = Int32GetDatum(static_cast<int32>(result.stats.data_treewidth_lb));
     values[3] = Int32GetDatum(static_cast<int32>(result.stats.circuit_treewidth_lb));
     values[4] = Int64GetDatum(static_cast<int64>(result.stats.nb_bags));
     values[5] = Int64GetDatum(static_cast<int64>(result.stats.max_states));
     values[6] = Int64GetDatum(static_cast<int64>(result.stats.dd_size));
+    unsigned maxenum = 0;
+    for (unsigned ev : result.stats.n_enumerating)
+      if (ev > maxenum)
+        maxenum = ev;
+    values[7] = Int32GetDatum(static_cast<int32>(maxenum));
 
     PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls)));
   } catch (const std::exception &e) {

test/expected/ucq_joint.out

@@ -20,9 +20,16 @@ ucq_two_disjuncts
 joint_treewidth|data_treewidth_lb|circuit_treewidth_lb
 1|1|0
 (1 row)
+e_fd_holds
+1
+(1 row)
+e_fd_broken
+2
+(1 row)
 ERROR:  ProvSQL: ucq_joint: joint treewidth exceeds the configured maximum (1); fall back to the standard probability ladder
 CONTEXT:  SQL statement "SELECT s.probability, s.joint_treewidth, s.data_treewidth_lb,
-         s.circuit_treewidth_lb, s.n_bags, s.max_states, s.dd_size
-                                                                                                                         FROM ucq_joint_compile_stats(dnv, adisj, arel, avars, aarity,
+         s.circuit_treewidth_lb, s.n_bags, s.max_states, s.dd_size,
+         s.n_enumerating
+                                                                                                                                        FROM ucq_joint_compile_stats(dnv, adisj, arel, avars, aarity,
       fact_rel, fact_elems, fact_arity, fact_tokens, fact_probs) s"
 PL/pgSQL function ucq_joint_compile_stats(jsonb,integer[],integer[],integer[],uuid[],double precision[]) line 19 at SQL statement

test/sql/ucq_joint.sql

@@ -102,6 +102,28 @@ FROM ucq_joint_compile_stats(
   ARRAY[u('r0'),u('r1'),u('s02'),u('s13'),u('t2'),u('t3')]::uuid[],
   ARRAY[0.5,0.5,0.5,0.5,0.5,0.5]);
 
+-- FD-aware essential-variable count (n_enumerating): a join variable
+-- functionally determined by others -- via an FD *mined from the gathered
+-- tuples* -- is not counted in the exponential parameter e.  H0 =
+-- R(x),S(x,y),T(y): when S carries x -> y in the data, y is determined by x
+-- so e = 1; add a second y for some x and the FD breaks, so e = 2.
+SELECT n_enumerating AS e_fd_holds FROM ucq_joint_compile_stats(
+  '{"disjuncts":[{"n_vars":2,"atoms":[
+      {"rel":0,"vars":[0]},{"rel":1,"vars":[0,1]},{"rel":2,"vars":[1]}]}]}'::jsonb,
+  ARRAY[0,0, 1,1, 2,2],
+  ARRAY[0, 1, 0,2, 1,3, 2, 3],
+  ARRAY[1,1, 2,2, 1,1],
+  ARRAY[u('r0'),u('r1'),u('s02'),u('s13'),u('t2'),u('t3')]::uuid[],
+  ARRAY[0.5,0.5,0.5,0.5,0.5,0.5]);
+SELECT n_enumerating AS e_fd_broken FROM ucq_joint_compile_stats(
+  '{"disjuncts":[{"n_vars":2,"atoms":[
+      {"rel":0,"vars":[0]},{"rel":1,"vars":[0,1]},{"rel":2,"vars":[1]}]}]}'::jsonb,
+  ARRAY[0,0, 1,1,1, 2,2],
+  ARRAY[0, 1, 0,2, 1,3, 0,3, 2, 3],
+  ARRAY[1,1, 2,2,2, 1,1],
+  ARRAY[u('r0'),u('r1'),u('s02'),u('s13'),u('s03'),u('t2'),u('t3')]::uuid[],
+  ARRAY[0.5,0.5,0.5,0.5,0.5,0.5,0.5]);
+
 -- Prop 4.2.11(b): a high-joint-width instance must be rejected (the
 -- caller then falls back to the ladder).  With the joint-width cap
 -- lowered to 1, a triangle over S (Gaifman treewidth 2) is rejected by