Add first cut of Pthread group split support. (#296)

Please note that this is a work in progress, so a lot of cleanup and
fixes are needed.

This adds group splitting capability to Pthreads. That said, there are
still races and memory leaks that I'm working out. In particular, there
appears to be a problem in qvi_hwsplit_coll::split() that results in
inconsistent colorp values (and perhaps other things).

Signed-off-by: Samuel K. Gutierrez <[email protected]>

Author: samuelkgutierrez

src/quo-vadis-pthread.cc

@@ -24,14 +24,14 @@
 #include "qvi-scope.h"
 #include "qvi-utils.h"
 
-struct qvi_pthread_args_s {
+struct qvi_pthread_args {
     qv_scope_t *scope = nullptr;
     qvi_pthread_routine_fun_ptr_t th_routine = nullptr;
     void *th_routine_argp = nullptr;
+    /** Default constructor. */
+    qvi_pthread_args(void) = delete;
     /** Constructor. */
-    qvi_pthread_args_s(void) = delete;
-    /** Constructor. */
-    qvi_pthread_args_s(
+    qvi_pthread_args(
         qv_scope_t *scope_a,
         qvi_pthread_routine_fun_ptr_t th_routine_a,
         void *th_routine_argp_a
@@ -41,15 +41,15 @@ struct qvi_pthread_args_s {
 };
 
 static void *
-qvi_pthread_routine(
+qvi_pthread_start_routine(
     void *arg
 ) {
-    qvi_pthread_args_s *arg_ptr = (qvi_pthread_args_s *)arg;
+    qvi_pthread_args *args = (qvi_pthread_args *)arg;
     // TODO(skg) Check return code.
-    arg_ptr->scope->bind_push();
+    args->scope->bind_push();
 
-    void *const ret = arg_ptr->th_routine(arg_ptr->th_routine_argp);
-    qvi_delete(&arg_ptr);
+    void *const ret = args->th_routine(args->th_routine_argp);
+    qvi_delete(&args);
     pthread_exit(ret);
 }
 
@@ -102,17 +102,21 @@ qv_pthread_create(
     qv_scope_t *scope
 ) {
     // Memory will be freed in qv_pthread_routine to avoid memory leaks.
-    qvi_pthread_args_s *arg_ptr = nullptr;
-    int rc = qvi_new(&arg_ptr, scope, thread_routine, arg);
+    qvi_pthread_args *pthread_start_args = nullptr;
+    int rc = qvi_new(&pthread_start_args, scope, thread_routine, arg);
     // Since this is meant to behave similarly to
     // pthread_create(), return a reasonable errno.
     if (qvi_unlikely(rc != QV_SUCCESS)) return ENOMEM;
-
+    // Note: The provided scope should have been created by
+    // qv_pthread_scope_split*. That is why we can safely cast the scope's
+    // underlying group it to a qvi_group_pthread *.
     auto group = dynamic_cast<qvi_group_pthread *>(scope->group());
-    qvi_pthread_group_pthread_create_args_s *cargs = nullptr;
-    rc = qvi_new(&cargs, group->thgroup, qvi_pthread_routine, arg_ptr);
+    qvi_pthread_group_pthread_create_args *cargs = nullptr;
+    rc = qvi_new(
+        &cargs, group->thgroup, qvi_pthread_start_routine, pthread_start_args
+    );
     if (qvi_unlikely(rc != QV_SUCCESS)) {
-        qvi_delete(&arg_ptr);
+        qvi_delete(&pthread_start_args);
         return ENOMEM;
     }
     return pthread_create(

src/qvi-group-omp.h

@@ -72,7 +72,7 @@ struct qvi_group_omp : public qvi_group {
     );
 
     virtual int
-    thsplit(
+    thread_split(
         int,
         qvi_group **
     ) {

src/qvi-group-pthread.cc

@@ -15,12 +15,36 @@
 #include "qvi-utils.h"
 
 qvi_group_pthread::qvi_group_pthread(
+    qvi_pthread_group_context *ctx,
     int group_size
 ) {
-    const int rc = qvi_new(&thgroup, group_size, 0);
+    int rc = QV_SUCCESS;
+    // A context pointer was not provided, so create a new one.
+    if (!ctx) {
+        rc = qvi_new(&m_context);
+        if (qvi_unlikely(rc != QV_SUCCESS)) throw qvi_runtime_error();
+    }
+    else {
+        // Else a context pointer was provided, so use it.
+        m_context = ctx;
+        m_context->retain();
+    }
+    //
+    rc = qvi_new(&thgroup, m_context, group_size);
     if (qvi_unlikely(rc != QV_SUCCESS)) throw qvi_runtime_error();
 }
 
+qvi_group_pthread::qvi_group_pthread(
+    qvi_pthread_group_context *ctx,
+    qvi_pthread_group *thread_group
+) {
+    assert(ctx && thread_group);
+    m_context = ctx;
+    m_context->retain();
+    //
+    thgroup = thread_group;
+}
+
 qvi_group_pthread::~qvi_group_pthread(void)
 {
     qvi_delete(&thgroup);
@@ -40,13 +64,18 @@ qvi_group_pthread::split(
     int key,
     qvi_group **child
 ) {
+    // NOTE: This is a collective call across
+    // ALL threads in the parent thread group.
     qvi_group_pthread *ichild = nullptr;
-    int rc  = qvi_new(&ichild);
+
+    qvi_pthread_group *ithgroup = nullptr;
+    int rc = thgroup->split(color, key, &ithgroup);
     if (qvi_unlikely(rc != QV_SUCCESS)) goto out;
 
-    rc = thgroup->split(color, key, &ichild->thgroup);
+    rc = qvi_new(&ichild, m_context, ithgroup);
 out:
     if (qvi_unlikely(rc != QV_SUCCESS)) {
+        qvi_delete(&ithgroup);
         qvi_delete(&ichild);
     }
     *child = ichild;

src/qvi-group-pthread.h

@@ -20,14 +20,32 @@
 #include "qvi-bbuff.h"
 
 struct qvi_group_pthread : public qvi_group {
+protected:
+    /**
+     * Points to per-process, per-thread_split()
+     * information that maintains Pthread group context.
+     */
+    qvi_pthread_group_context *m_context = nullptr;
+public:
     /** Underlying group instance. */
     qvi_pthread_group *thgroup = nullptr;
-    /** Constructor. */
-    qvi_group_pthread(void) = default;
-    /** Constructor. */
+    /** Default constructor. */
+    qvi_group_pthread(void) = delete;
+    /**
+     * Constructor that is called by the parent process to setup
+     * base infrastructure required during a thread_split().
+     */
     qvi_group_pthread(
+        qvi_pthread_group_context *ctx,
         int group_size
     );
+    /**
+     * Constructor that is collective across ALL threads in the parent group.
+     */
+    qvi_group_pthread(
+        qvi_pthread_group_context *ctx,
+        qvi_pthread_group *thread_group
+    );
     /** Destructor. */
     virtual ~qvi_group_pthread(void);
 
@@ -59,7 +77,10 @@ struct qvi_group_pthread : public qvi_group {
     make_intrinsic(
         qv_scope_intrinsic_t
     ) {
-        // Nothing to do.
+        // Nothing to do here because a Pthread group cannot be created outside
+        // of another group. For example, a thread_split can be called from a
+        // process context, which can be an intrinsic group, but not from a
+        // threaded context alone.
         return QV_SUCCESS;
     }
 
@@ -69,14 +90,15 @@ struct qvi_group_pthread : public qvi_group {
     );
 
     virtual int
-    thsplit(
+    thread_split(
         int,
         qvi_group **
     ) {
-        // TODO(skg)
         return QV_ERR_NOT_SUPPORTED;
     }
-
+    /**
+     * This is a collective call across all threads in the parent thread group.
+     */
     virtual int
     split(
         int color,

src/qvi-group.cc

@@ -1,6 +1,6 @@
 /* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2021-2024 Triad National Security, LLC
+ * Copyright (c) 2021-2025 Triad National Security, LLC
  *                         All rights reserved.
  *
  * This file is part of the quo-vadis project. See the LICENSE file at the
@@ -23,12 +23,16 @@ qvi_group::hwloc(void)
 }
 
 int
-qvi_group::thsplit(
+qvi_group::thread_split(
     int nthreads,
     qvi_group **child
 ) {
     qvi_group_pthread *ichild = nullptr;
-    const int rc = qvi_new(&ichild, nthreads);
+    // This is the entry point for creating a new thread group, so nullptr
+    // passed to signal that a new context must be created by the new
+    // qvi_group_pthread. Also note this is called by a single thread of
+    // execution (i.e., the parent process).
+    const int rc = qvi_new(&ichild, nullptr, nthreads);
     if (qvi_unlikely(rc != QV_SUCCESS)) {
         qvi_delete(&ichild);
     }
@@ -44,14 +48,29 @@ qvi_group::next_id(
     // infrastructure (e.g., QVI_MPI_GROUP_WORLD) will never equal or exceed
     // this value.
     static std::atomic<qvi_group_id_t> group_id(64);
-    if (group_id == UINT64_MAX) {
-        qvi_log_error("Group ID space exhausted");
+    if (qvi_unlikely(group_id == UINT64_MAX)) {
+        qvi_log_error("Group ID space exhausted.");
         return QV_ERR_OOR;
     }
     *gid = group_id++;
     return QV_SUCCESS;
 }
 
+int
+qvi_group::next_ids(
+    size_t n,
+    std::vector<qvi_group_id_t> &gids
+) {
+    gids.resize(n);
+    for (size_t i = 0; i < n; ++i) {
+        qvi_group_id_t gid = 0;
+        const int rc = next_id(&gid);
+        if (qvi_unlikely(rc != QV_SUCCESS)) return rc;
+        gids[i] = gid;
+    }
+    return QV_SUCCESS;
+}
+
 /*
  * vim: ft=cpp ts=4 sts=4 sw=4 expandtab
  */

src/qvi-group.h

@@ -61,10 +61,11 @@ struct qvi_group : qvi_refc {
         qvi_group **child
     ) = 0;
     /**
-     * Creates a new thread group by splitting off of the caller's group.
+     * Creates a new thread group by splitting off of the calling process'
+     * group.
      */
     virtual int
-    thsplit(
+    thread_split(
         int nthreads,
         qvi_group **child
     );
@@ -98,6 +99,12 @@ struct qvi_group : qvi_refc {
     next_id(
         qvi_group_id_t *gid
     );
+    /** Populates gids with n unique group IDs after each call. */
+    static int
+    next_ids(
+        size_t n,
+        std::vector<qvi_group_id_t> &gids
+    );
 };
 
 #endif