Checkpoint first cut at new automatic split policies. (#312)

Provides a first-cut at the following automatic scope splitting
policies: packed and spread.

Signed-off-by: Samuel K. Gutierrez <[email protected]>

Author: samuelkgutierrez

include/quo-vadis-pthread.h

@@ -66,17 +66,17 @@ int
 qv_pthread_scope_split(
     qv_scope_t *scope,
     int npieces,
-    int *color_array,
-    int nthreads,
-    qv_scope_t ***subscope
+    int *kcolors,
+    int k,
+    qv_scope_t ***subscopes
 );
 
 int
 qv_pthread_scope_split_at(
     qv_scope_t *scope,
     qv_hw_obj_type_t type,
-    int *color_array,
-    int nthreads,
+    int *kcolors,
+    int k,
     qv_scope_t ***subscopes
 );
 
@@ -89,18 +89,6 @@ qv_pthread_scopes_free(
     qv_scope_t **scopes
 );
 
-/**
- * Fills color array used in qv_pthread_scope_split*.
- */
-int
-qv_pthread_colors_fill(
-    int *color_array,
-    int array_size,
-    qv_pthread_placement_t policy,
-    int stride,
-    int npieces
-);
-
 #ifdef __cplusplus
 }
 #endif

include/quo-vadis.h

@@ -132,6 +132,10 @@ const int QV_SCOPE_SPLIT_UNDEFINED = -1;
  */
 const int QV_SCOPE_SPLIT_AFFINITY_PRESERVING = -2;
 
+const int QV_SCOPE_SPLIT_PACKED = -3;
+
+const int QV_SCOPE_SPLIT_SPREAD = -4;
+
 /**
  *
  */

src/quo-vadis-pthread.cc

@@ -65,16 +65,16 @@ qv_pthread_scope_split(
     int npieces,
     int *color_array,
     int nthreads,
-    qv_scope_t ***subscope
+    qv_scope_t ***subscopes
 ) {
     const bool invalid_args = !scope || npieces < 0 || !color_array ||
-                              nthreads < 0 || !subscope;
+                              nthreads < 0 || !subscopes;
     if (qvi_unlikely(invalid_args)) {
         return QV_ERR_INVLD_ARG;
     }
     try {
         return scope->thread_split(
-            npieces, color_array, nthreads, QV_HW_OBJ_LAST, subscope
+            npieces, color_array, nthreads, QV_HW_OBJ_LAST, subscopes
         );
     }
     qvi_catch_and_return();
@@ -84,17 +84,15 @@ int
 qv_pthread_scope_split_at(
     qv_scope_t *scope,
     qv_hw_obj_type_t type,
-    int *color_array,
-    int nthreads,
+    int *kcolors,
+    int k,
     qv_scope_t ***subscopes
 ) {
-    if (qvi_unlikely(!scope || !color_array || nthreads < 0 || !subscopes)) {
+    if (qvi_unlikely(!scope || !kcolors || k < 0 || !subscopes)) {
         return QV_ERR_INVLD_ARG;
     }
     try {
-        return scope->thread_split_at(
-            type, color_array, nthreads, subscopes
-        );
+        return scope->thread_split_at(type, kcolors, k, subscopes);
     }
     qvi_catch_and_return();
 }
@@ -145,48 +143,6 @@ qv_pthread_scopes_free(
     qvi_catch_and_return();
 }
 
-int
-qv_pthread_colors_fill(
-   int *color_array,
-   int array_size,
-   qv_pthread_placement_t policy,
-   int stride,
-   int npieces
-) {
-    const bool invalid_args = !color_array || array_size < 0 ||
-                              stride < 1 || npieces < 1;
-    if (qvi_unlikely(invalid_args)) return QV_ERR_INVLD_ARG;
-    // TODO(skg) We should use the mapping algorithms in qvi-map for these. The
-    // problem is that its interfaces aren't yet suited for this type of
-    // mapping.
-    switch(policy) {
-        case QV_POLICY_PACKED: {
-            // TODO(skg) This looks more like spread.
-            for(int idx = 0 ; idx < array_size ; idx++){
-                // color_array[idx] = (idx+idx*(stride-1))%(npieces);
-                color_array[idx] = (idx*stride)%(npieces);
-            }
-            break;
-        }
-        case QV_POLICY_SPREAD: {
-            return QV_ERR_NOT_SUPPORTED;
-        }
-        case QV_POLICY_DISTRIBUTE: {
-            return QV_ERR_NOT_SUPPORTED;
-        }
-        case QV_POLICY_SCATTER: {
-            return QV_ERR_NOT_SUPPORTED;
-        }
-        case QV_POLICY_CHOOSE: {
-            return QV_ERR_NOT_SUPPORTED;
-        }
-        default: {
-            return QV_ERR_INVLD_ARG;
-        }
-    }
-    return QV_SUCCESS;
-}
-
 /*
  * vim: ft=cpp ts=4 sts=4 sw=4 expandtab
  */

src/qvi-hwsplit.cc

@@ -520,6 +520,54 @@ qvi_hwsplit::split_affinity_preserving(void)
     return split_devices_affinity_preserving();
 }
 
+// TODO(skg) Add device splitting.
+int
+qvi_hwsplit::split_packed(void)
+{
+    // cpusets used for mapping.
+    qvi_hwloc_cpusets_t cpusets;
+    // Get the primary cpusets for the mapping.
+    int rc = primary_cpusets(cpusets);
+    if (rc != QV_SUCCESS) return rc;
+    // Maintains the mapping between task (consumer) IDs and resource IDs.
+    qvi_map_t map;
+    rc = qvi_map_packed(map, m_group_size, cpusets);
+    if (rc != QV_SUCCESS) return rc;
+    // Make sure that we mapped all the tasks. If not, this is a bug.
+    if (qvi_map_nfids_mapped(map) != m_group_size) {
+        qvi_abort();
+    }
+    qvi_hwloc_t *const hwloc = m_rmi->hwloc();
+    // Update the hardware pools and colors to reflect the new mapping.
+    return apply_cpuset_mapping(
+        hwloc, map, cpusets, m_hwpools, m_colors
+    );
+}
+
+// TODO(skg) Add device splitting.
+int
+qvi_hwsplit::split_spread(void)
+{
+    // cpusets used for mapping.
+    qvi_hwloc_cpusets_t cpusets;
+    // Get the primary cpusets for the mapping.
+    int rc = primary_cpusets(cpusets);
+    if (rc != QV_SUCCESS) return rc;
+    // Maintains the mapping between task (consumer) IDs and resource IDs.
+    qvi_map_t map;
+    rc = qvi_map_spread(map, m_group_size, cpusets);
+    if (rc != QV_SUCCESS) return rc;
+    // Make sure that we mapped all the tasks. If not, this is a bug.
+    if (qvi_map_nfids_mapped(map) != m_group_size) {
+        qvi_abort();
+    }
+    qvi_hwloc_t *const hwloc = m_rmi->hwloc();
+    // Update the hardware pools and colors to reflect the new mapping.
+    return apply_cpuset_mapping(
+        hwloc, map, cpusets, m_hwpools, m_colors
+    );
+}
+
 /**
  * Takes a vector of colors and clamps their values to [0, ndc)
  * in place, where ndc is the number of distinct numbers found in values.
@@ -590,6 +638,10 @@ qvi_hwsplit::split(void)
     switch (m_colors[0]) {
         case QV_SCOPE_SPLIT_AFFINITY_PRESERVING:
             return split_affinity_preserving();
+        case QV_SCOPE_SPLIT_PACKED:
+            return split_packed();
+        case QV_SCOPE_SPLIT_SPREAD:
+            return split_spread();
         default:
             rc = QV_ERR_INVLD_ARG;
             break;

src/qvi-hwsplit.h

@@ -1,6 +1,6 @@
 /* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2021-2024 Triad National Security, LLC
+ * Copyright (c) 2021-2025 Triad National Security, LLC
  *                         All rights reserved.
  *
  * This file is part of the quo-vadis project. See the LICENSE file at the
@@ -70,7 +70,32 @@ struct qvi_hwsplit {
     std::vector<int> m_colors;
     /** Vector of task affinities. */
     qvi_hwloc_cpusets_t m_affinities;
+    /** */
+    qvi_map_fn_t
+    affinity_preserving_policy(void) const;
+    /** */
+    int
+    split_affinity_preserving_pass1(void);
+    /** User-defined split. */
+    int
+    split_user_defined(void);
+    /** Affinity preserving split. */
+    int
+    split_affinity_preserving(void);
+    /** */
+    int
+    split_packed(void);
+    /** */
+    int
+    split_spread(void);
+    /** Straightforward user-defined device splitting. */
+    int
+    split_devices_user_defined(void);
+    /** Affinity preserving device splitting. */
+    int
+    split_devices_affinity_preserving(void);
 public:
+    // TODO(skg) Cleanup private, protected, public interfaces.
     /** Constructor. */
     qvi_hwsplit(void) = default;
     /** Constructor. */
@@ -124,26 +149,9 @@ struct qvi_hwsplit {
         qvi_hwloc_cpusets_t &result
     ) const;
 
-    qvi_map_fn_t
-    affinity_preserving_policy(void) const;
     /** Releases all devices contained in the hardware split. */
     int
     release_devices(void);
-    /** Straightforward user-defined device splitting. */
-    int
-    split_devices_user_defined(void);
-    /** Affinity preserving device splitting. */
-    int
-    split_devices_affinity_preserving(void);
-    /** User-defined split. */
-    int
-    split_user_defined(void);
-
-    int
-    split_affinity_preserving_pass1(void);
-    /** Affinity preserving split. */
-    int
-    split_affinity_preserving(void);
     /** Splits aggregate scope data. */
     int
     split(void);

tests/test-mpi-scopes.c

@@ -1,14 +1,4 @@
 /* -*- Mode: C; c-basic-offset:4; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2020-2024 Triad National Security, LLC
- *                         All rights reserved.
- *
- * Copyright (c) 2020-2021 Lawrence Livermore National Security, LLC
- *                         All rights reserved.
- *
- * This file is part of the quo-vadis project. See the LICENSE file at the
- * top-level directory of this distribution.
- */
 
 /**
  * @file test-scopes-mpi.c
@@ -134,7 +124,7 @@ main(
     rc = qv_scope_split(
         base_scope,
         npieces,
-        gid,
+        QV_SCOPE_SPLIT_PACKED,
         &sub_scope
     );
     if (rc != QV_SUCCESS) {

tests/test-pthread-split.c

@@ -85,7 +85,6 @@ main(void)
     //
     const int npieces = 2;
     const int nthreads = ncores;
-    int stride = 1;
     int colors[nthreads];
 
     printf(
@@ -103,11 +102,7 @@ main(void)
     }
     printf("\n");
 #endif
-    rc = qv_pthread_colors_fill(colors, nthreads, QV_POLICY_PACKED, stride, npieces);
-    if (rc != QV_SUCCESS) {
-        ers = "qv_pthread_colors_fill() failed";
-        ctu_panic("%s (rc=%s)", ers, qv_strerr(rc));
-    }
+
 #if 0
     fprintf(stdout,"Filled values: ");
     for (int i = 0 ; i < nthreads ; i++) {