Reorganize MPI sources (#526)

* Reorganize MPI sources

* Update build documentation

Author: daboehme

doc/sphinx/build.rst

@@ -133,9 +133,6 @@ The CMake package defines the following variables and targets:
 +----------------------------+------------------------------------------+
 | caliper                    | The Caliper runtime library (target)     |
 +----------------------------+------------------------------------------+
-| caliper-serial             | Caliper runtime library without MPI      |
-|                            | dependencies (target)                    |
-+----------------------------+------------------------------------------+
 | caliper-tools-util         | Utilities for caliper tools (target)     |
 +----------------------------+------------------------------------------+
 

src/CMakeLists.txt

@@ -17,10 +17,6 @@ add_subdirectory(reader)
 add_subdirectory(services)
 add_subdirectory(caliper)
 
-if (CALIPER_HAVE_MPI)
-  add_subdirectory(mpi)
-endif()
-
 set(CALIPER_COMMON_OBJS
   $<TARGET_OBJECTS:caliper-common>
   $<TARGET_OBJECTS:caliper-interface>
@@ -30,38 +26,9 @@ set(CALIPER_COMMON_OBJS
   ${CALIPER_SERVICES_LIBS}
   ${CALIPER_EXTERNAL_SOURCES})
 
-list(APPEND CALIPER_SERIAL_OBJS
-  ${CALIPER_COMMON_OBJS})
 list(APPEND CALIPER_ALL_OBJS
   ${CALIPER_COMMON_OBJS})
 
-# Make (non-mpi) libcaliper-serial
-
-add_library(caliper-serial ${CALIPER_SERIAL_OBJS})
-
-set_target_properties(caliper-serial PROPERTIES SOVERSION ${CALIPER_MAJOR_VERSION})
-set_target_properties(caliper-serial PROPERTIES VERSION ${CALIPER_VERSION})
-
-target_compile_features(caliper-serial PUBLIC cxx_std_11)
-
-target_include_directories(
-  caliper-serial
-  INTERFACE
-  "$<INSTALL_INTERFACE:include>")
-
-list(APPEND CALIPER_EXTERNAL_LIBS
-  Threads::Threads)
-
-foreach (_extlib ${CALIPER_EXTERNAL_LIBS})
-  target_link_libraries(caliper-serial PRIVATE ${_extlib})
-endforeach()
-
-if (WIN32)
-  target_link_libraries(caliper-serial PUBLIC ws2_32)
-endif()
-
-# Make the combined libcaliper
-
 add_library(caliper ${CALIPER_ALL_OBJS})
 
 set_target_properties(caliper PROPERTIES SOVERSION ${CALIPER_MAJOR_VERSION})
@@ -89,11 +56,6 @@ if (WITH_TOOLS)
   add_subdirectory(tools)
 endif()
 
-install(TARGETS caliper-serial
-  EXPORT caliper-targets
-  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
-
 install(TARGETS caliper
   EXPORT caliper-targets
   ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}

src/caliper/CMakeLists.txt

@@ -16,31 +16,33 @@ set(CALIPER_RUNTIME_SOURCES
   cali_datatracker.cpp
   config_sanity_check.cpp)
 
+if (CALIPER_HAVE_MPI)
+  list(APPEND CALIPER_RUNTIME_SOURCES
+    aggregate_over_mpi.cpp
+    collective_flush.cpp
+    CollectiveOutputChannel.cpp
+    OutputCommMpi.cpp
+    MpiChannelManager.cpp
+    mpi_flush.cpp
+    machine_mpi.cpp
+    setup_mpi.cpp)
+else()
+  list(APPEND CALIPER_RUNTIME_SOURCES
+    machine_serial.cpp
+    setup_serial.cpp)
+endif()
+
 add_subdirectory(controllers)
 
 add_library(caliper-runtime OBJECT ${CALIPER_RUNTIME_SOURCES})
 
 target_compile_options(caliper-runtime PRIVATE ${Wall_flag})
 target_compile_features(caliper-runtime PUBLIC cxx_std_11)
 
-list(APPEND CALIPER_SERIAL_OBJS
-  caliper/machine_serial.cpp
-  caliper/setup_serial.cpp)
-
 if (CALIPER_HAVE_MPI)
   target_include_directories(caliper-runtime PRIVATE ${MPI_CXX_INCLUDE_PATH})
-  list(APPEND CALIPER_ALL_OBJS
-    caliper/machine_mpi.cpp
-    caliper/setup_mpi.cpp)
-else()
-  list(APPEND CALIPER_ALL_OBJS
-    caliper/machine_serial.cpp
-    caliper/setup_serial.cpp)
 endif()
 
-set(CALIPER_SERIAL_OBJS ${CALIPER_SERIAL_OBJS} PARENT_SCOPE)
-set(CALIPER_ALL_OBJS    ${CALIPER_ALL_OBJS}    PARENT_SCOPE)
-
 if (BUILD_TESTING)
   add_subdirectory(test)
 endif()

src/mpi/CollectiveOutputChannel.cpp src/caliper/CollectiveOutputChannel.cpp

@@ -7,9 +7,58 @@
 
 #include "caliper/common/Log.h"
 
-#ifdef CALIPER_HAVE_MPI
-#include "../mpi/mpi_machine.h"
-#endif
+#include <mpi.h>
+
+#include <unistd.h>
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace
+{
+
+int get_rank_for_hash(uint64_t hash, MPI_Comm comm)
+{
+    int myrank = 0;
+    int ccsize = 0;
+
+    PMPI_Comm_rank(comm, &myrank);
+    PMPI_Comm_size(comm, &ccsize);
+
+    std::vector<uint64_t> res(ccsize, 0);
+
+    int ret = PMPI_Allgather(&hash,      1, MPI_UINT64_T,
+                             res.data(), 1, MPI_UINT64_T, comm);
+
+    if (ret != MPI_SUCCESS)
+        return -1;
+
+    int hrank = 0;
+
+    for (int i = 0; i < myrank; ++i)
+        if (hash == res[i])
+            ++hrank;
+
+    return hrank;
+}
+
+int get_rank_for_node()
+{
+    constexpr std::size_t max_len = 1024;
+    char hostname[max_len];
+    if (gethostname(hostname, max_len) < 0)
+        return -1;
+
+    int initialized = 0;
+    PMPI_Initialized(&initialized);
+    if (!initialized)
+        return 0;
+
+    return ::get_rank_for_hash(std::hash<std::string>{}(std::string(hostname)), MPI_COMM_WORLD);
+}
+
+} // namespace [anonymous]
 
 namespace cali
 {
@@ -23,11 +72,7 @@ int get_rank_for(MachineLevel level)
     case MachineLevel::Process:
         return 0;
     case MachineLevel::Node:
-#ifdef CALIPER_HAVE_MPI
-        return mpi::get_rank_for_node();
-#else
-        return 0;
-#endif
+        return ::get_rank_for_node();
     default:
         Log(0).stream() << "machine::get_rank_for(MachineLevel): level "
                         << level << " is not supported"
@@ -37,6 +82,6 @@ int get_rank_for(MachineLevel level)
     return -1;
 }
 
-}
+} // namespace machine
 
 }

src/mpi/MpiChannelManager.cpp src/caliper/MpiChannelManager.cpp

@@ -3,35 +3,102 @@
 
 #include "caliper/caliper-config.h"
 
+#include "OutputCommMpi.h"
+
+#include "caliper/CaliperService.h"
+#include "caliper/CustomOutputController.h"
+
+#include "caliper/common/Log.h"
+#include "caliper/common/OutputStream.h"
+
 #include "../services/Services.h"
 
-namespace cali
+#include <mpi.h>
+
+#include <sstream>
+
+using namespace cali;
+
+using COC = cali::internal::CustomOutputController;
+
+namespace
 {
 
-#ifdef CALIPER_HAVE_MPI
-namespace mpi
+void
+setup_log_prefix()
 {
+    static bool done = false;
+
+    if (done)
+        return;
 
-extern void setup_mpi();
-extern void add_mpi_controllers_and_services();
+    int mpi_is_initialized = 0;
+    MPI_Initialized(&mpi_is_initialized);
 
+    if (mpi_is_initialized && Log::is_initialized()) {
+        int rank = 0;
+
+        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+        // Disable (most) logging on non-0 ranks by default
+        std::ostringstream logprefix;
+        logprefix << "(" << rank << "): ";
+
+        Log::add_prefix(logprefix.str());
+
+        if (rank > 0)
+            Log::set_verbosity(0);
+
+        done = true;
+    }
 }
-#endif
+
+// Implement flush over MPI for CustomOutputController objects
+void
+custom_output_controller_flush_mpi(COC* controller)
+{
+    Log(2).stream() << controller->name() << ": CustomOutputController::flush(): using MPI" << std::endl;
+
+    OutputCommMpi comm;
+    OutputStream  stream;
+
+    controller->collective_flush(stream, comm);
+}
+
+} // namespace [anonymous]
+
+namespace cali
+{
+
+extern CaliperService mpiflush_service;
 
 void add_submodule_controllers_and_services()
 {
+    static const CaliperService mpi_services[] = {
+        mpiflush_service, { nullptr, nullptr }
+    };
+
+    services::add_service_specs(mpi_services);
     services::add_default_service_specs();
-#ifdef CALIPER_HAVE_MPI
-    mpi::add_mpi_controllers_and_services();
-#endif
 }
 
 void init_submodules()
 {
-#ifdef CALIPER_HAVE_MPI
-    mpi::setup_mpi();
-#endif
+    ::setup_log_prefix();
+    COC::set_flush_fn(::custom_output_controller_flush_mpi);
+
     add_submodule_controllers_and_services();
 }
 
 } // namespace cali
+
+extern "C"
+{
+
+void
+cali_mpi_init()
+{
+    ::setup_log_prefix();
+}
+
+}

src/mpi/MpiEvents.h src/caliper/MpiEvents.h

@@ -13,7 +13,7 @@ set(CALIPER_TEST_SOURCES
 
 add_executable(test_caliper ${CALIPER_TEST_SOURCES})
 
-target_link_libraries(test_caliper caliper-serial gtest_main)
+target_link_libraries(test_caliper caliper gtest_main)
 
 set(CALIPER_TEST_JSON_FILES
   test_single_config.json