Features/ams new interface

.gitlab/custom-jobs-and-variables.yml

@@ -48,7 +48,7 @@ variables:
 
 # Lassen uses a different job scheduler (spectrum lsf) that does not allow
 # pre-allocation the same way slurm does. Arguments for job level allocation
-  LASSEN_JOB_ALLOC: "1 -W 30 -q pci"
+  LASSEN_JOB_ALLOC: "1 -W 45 -q pci"
 # Add variables that should apply to all the jobs on a machine:
 #  LASSEN_MY_VAR: "..."
 

CMakeLists.txt

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 cmake_minimum_required(VERSION 3.18)
-project(AMS VERSION 0.1.0 LANGUAGES CXX C)
+project(AMS VERSION 0.1.1 LANGUAGES CXX C)
 cmake_policy(SET CMP0074 NEW)
 
 # NOTE: This may break some of our integrations with the applications. But flux requires > C++20, RMQ requires C++17 and although AMS does not have 
@@ -33,213 +33,103 @@ set(AMS_EXAMPLE_INCLUDES "")
 
 # ------------------------------------------------------------------------------
 option(WITH_CUDA           "Option to enable CUDA" OFF)
-option(WITH_EXAMPLES       "Build examples" OFF)
 option(WITH_MPI            "Option to enable MPI" OFF)
 option(WITH_CALIPER        "Use Caliper for Profiling" OFF)
-option(WITH_FAISS          "Use C/C++ FAISS interface for HD Cache" OFF)
-option(WITH_TORCH          "Use C/C++ Torch interface for Surrogate Model Inference" OFF)
-option(WITH_TORCH_DEBUG    "Compute RMSE of Surrogate Model and Physics Module" OFF)
 option(WITH_TESTS          "Compile tests" OFF)
-option(WITH_REDIS          "Use REDIS as a database back end" OFF)
 option(WITH_HDF5           "Use HDF5 as a database back end" OFF)
 option(WITH_RMQ            "Use RabbitMQ as a database back end (require a reachable and running RabbitMQ server service)" OFF)
 option(WITH_AMS_DEBUG      "Enable verbose messages" OFF)
 option(WITH_PERFFLOWASPECT "Use PerfFlowAspect for Profiling" OFF)
 option(WITH_WORKFLOW       "Install python drivers used by the outer workflow" OFF)
 option(WITH_AMS_LIB        "Install C++ library to support scientific applications" ON)
-option(WITH_ADIAK          "Use Adiak for recording metadata" OFF)
-option(BUILD_SHARED_LIBS   "Build using shared libraries" ON)
 
-if (WITH_MPI)
-  #  SET(CMAKE_CXX_COMPILER "${MPI_CXX_COMPILER}" CACHE FILEPATH "CXX compiler overridden with MPI C++ wrapper")
-  #SET(CMAKE_C_COMPILER "${MPI_C_COMPILER}" CACHE FILEPATH "CXX compiler overridden with MPI C++ wrapper")
+# ------------------------------------------------------------------------------
+find_package(nlohmann_json REQUIRED)
+list(APPEND AMS_APP_LIBRARIES nlohmann_json::nlohmann_json)
 
+# ------------------------------------------------------------------------------
+find_package(Threads REQUIRED)
+# ------------------------------------------------------------------------------
+
+set(MPI_DIRECTORIES "")
+if (WITH_MPI)
   find_package(MPI REQUIRED)
   message(STATUS "MPICC:  ${MPI_C_COMPILER}")
   message(STATUS "MPICXX: ${MPI_CXX_COMPILER}")
-  list(APPEND AMS_APP_LIBRARIES MPI::MPI_CXX)
   message(STATUS "MPI Library used: " MPI::MPI_CXX)
-  list(APPEND AMS_APP_DEFINES "-D__ENABLE_MPI__")
+  foreach(LIBRARY ${MPI_CXX_LIBRARIES})
+    get_filename_component(LIB_DIR ${LIBRARY} DIRECTORY)
+    list(APPEND MPI_DIRECTORIES ${LIB_DIR})
+  endforeach()
+  list(REMOVE_DUPLICATES MPI_DIRECTORIES)
+  message(STATUS "MPI Liraries: ${MPI_DIRECTORIES}")
 endif()
 
 # ------------------------------------------------------------------------------
 if (WITH_CUDA)
-  if (NOT DEFINED AMS_CUDA_ARCH)
-    message(WARNING "CUDA ARCH IS NOT DEFINED, USING 70")
-    set(AMS_CUDA_ARCH 70)
-  endif()
-
-  if (BUILD_SHARED_LIBS)
-    set(CUDA_RUNTIME_LIBRARY "Shared")
-  else()
-    set(CUDA_RUNTIME_LIBRARY "Static")
-  endif()
-
-  # we need to enable nvcc
-  enable_language(CUDA)
-  set(CMAKE_CUDA_STANDARD 14)
-  # Turn off message of mfem
-  set(ENABLE_CUDA True)
-  list(APPEND AMS_APP_LIBRARIES ${CUDA_LIBRARIES} cuda)
-  list(APPEND AMS_APP_DEFINES "-D__ENABLE_CUDA__")
-  set(THRUST_IGNORE_CUB_VERSION_CHECK True)
+  find_package(CUDAToolkit)
 endif()
 
+
 # ------------------------------------------------------------------------------
 if (WITH_CALIPER)
   find_package(caliper REQUIRED)
-  list(APPEND AMS_APP_INCLUDES ${caliper_INCLUDE_DIR})
-  list(APPEND AMS_APP_LIBRARIES caliper)
-  list(APPEND AMS_APP_DEFINES "-D__ENABLE_CALIPER__")
+  message(STATUS "Caliper Directory is ${caliper_DIR}")
+  list(APPEND AMS_APP_DEFINES "__AMS_ENABLE_CALIPER__")
 endif()
 
 if (WITH_AMS_DEBUG)
-  list(APPEND AMS_APP_DEFINES "-DLIBAMS_VERBOSE")
+  list(APPEND AMS_APP_DEFINES "LIBAMS_VERBOSE")
 endif()
 
-# ------------------------------------------------------------------------------
-list(APPEND AMS_APP_DEFINES "-D__ENABLE_DB__")
-
-if (WITH_REDIS)
-  # Temprorary fix for:= the following error which happens when using Cuda 11.6 and Redis backend
-  # error: #error The version of CUB in your include path is not compatible with this release of Thrust.
-  if (WITH_CUDA)
-    add_compile_definitions(THRUST_IGNORE_CUB_VERSION_CHECK)
-  endif()
-  if (HIREDIS_DIR)
-    find_path(HIREDIS_HEADER NAMES hiredis HINTS ${HIREDIS_DIR} PATH_SUFFIXES include)
-    find_library(HIREDIS_LIB NAMES hiredis HINTS ${HIREDIS_DIR} PATH_SUFFIXES lib)
-  else()
-    find_package(hiredis REQUIRED)
-    find_path(HIREDIS_HEADER hiredis)
-    find_library(HIREDIS_LIB hiredis)
-  endif()
-  message(STATUS "Hiredis library is ${HIREDIS_LIB}")
-  message(STATUS "Hiredis headers are ${HIREDIS_HEADER}")
-  list(APPEND AMS_APP_INCLUDES ${HIREDIS_HEADER})
-  list(APPEND AMS_APP_LIBRARIES ${HIREDIS_LIB})
-  list(APPEND AMS_APP_DEFINES "-D__ENABLE_REDIS__")
-
-  if (REDIS_PLUS_PLUS_DIR)
-    find_path(REDIS_PLUS_PLUS_HEADER NAMES sw PATHS ${REDIS_PLUS_PLUS_DIR} PATH_SUFFIXES include)
-    find_library(REDIS_PLUS_PLUS_LIB NAMES redis++ PATHS ${REDIS_PLUS_PLUS_DIR} PATH_SUFFIXES lib)
-  else()
-    find_path(REDIS_PLUS_PLUS_HEADER sw)
-    find_library(REDIS_PLUS_PLUS_LIB redis++)
-  endif()
-  message(STATUS "Redis++ library is ${REDIS_PLUS_PLUS_LIB}")
-  list(APPEND AMS_APP_INCLUDES ${REDIS_PLUS_PLUS_HEADER})
-  list(APPEND AMS_APP_LIBRARIES ${REDIS_PLUS_PLUS_LIB})
-endif() # WITH_REDIS
-
-
 if (WITH_HDF5)
   if (HDF5_USE_STATIC_LIBRARIES)
-  find_package(HDF5 NAMES hdf5 COMPONENTS C static NO_DEFAULT_PATH PATHS ${AMS_HDF5_DIR} ${AMS_HDF5_DIR}/share/cmake)
-  list(APPEND AMS_APP_LIBRARIES ${HDF5_C_STATIC_LIBRARY})
-  message(STATUS "HDF5 Static Library : ${HDF5_C_STATIC_LIBRARY}")
-else()
-  find_package(HDF5 NAMES hdf5 COMPONENTS C shared NO_DEFAULT_PATH PATHS ${AMS_HDF5_DIR} ${AMS_HDF5_DIR}/share/cmake)
-  list(APPEND AMS_APP_LIBRARIES ${HDF5_C_SHARED_LIBRARY})
-  message(STATUS "HDF5 Shared Library : ${HDF5_C_SHARED_LIBRARY}")
-endif()
-  list(APPEND AMS_APP_INCLUDES ${HDF5_INCLUDE_DIR})
-  list(APPEND AMS_APP_DEFINES "-D__ENABLE_HDF5__")
-  message(STATUS "HDF5 Include directories: ${HDF5_INCLUDE_DIR}")
+    find_package(HDF5 COMPONENTS C static NO_DEFAULT_PATH PATHS ${AMS_HDF5_DIR} ${AMS_HDF5_DIR}/share/cmake)
+    set(AMS_HDF5_TARGET hdf5-static)
+    set(AMS_HDF5_LIB_TYPE "static")
+  else()
+      find_package(HDF5 COMPONENTS C shared NO_DEFAULT_PATH PATHS ${AMS_HDF5_DIR} ${AMS_HDF5_DIR}/share/cmake)
+      set(AMS_HDF5_LIB_TYPE "shared")
+      set(AMS_HDF5_TARGET hdf5-shared)
+  endif()
+  message(STATUS "HDF5 Dir is ${HDF5_DIR}")
+  list(APPEND AMS_APP_DEFINES "__ENABLE_HDF5__")
 endif() # WITH_HDF5
 
 if (WITH_RMQ)
   if (WITH_CUDA)
     add_compile_definitions(THRUST_IGNORE_CUB_VERSION_CHECK)
   endif()
-  list(APPEND AMS_APP_DEFINES "-D__ENABLE_RMQ__")
+  list(APPEND AMS_APP_DEFINES "__ENABLE_RMQ__")
 
   find_package(amqpcpp REQUIRED)
   get_target_property(amqpcpp_INCLUDE_DIR amqpcpp INTERFACE_INCLUDE_DIRECTORIES)
   list(APPEND AMS_APP_INCLUDES ${amqpcpp_INCLUDE_DIR})
 
   find_package(OpenSSL REQUIRED)
   if (OPENSSL_FOUND)
-    list(APPEND AMS_APP_INCLUDES ${OPENSSL_INCLUDE_DIR})
-    list(APPEND AMS_APP_LIBRARIES "${OPENSSL_LIBRARIES}")
-    list(APPEND AMS_APP_LIBRARIES ssl)
+    message(STATUS "OpenSSL libraries found: ${OPENSSL_LIBRARIES}")
     message(STATUS "OpenSSL includes found: " ${OPENSSL_INCLUDE_DIR})
-    message(STATUS "OpenSSL libraries found: " ${OPENSSL_LIBRARIES})
+    get_filename_component(AMS_OPENSSL_FOUND_ROOT "${OPENSSL_SSL_LIBRARY}" DIRECTORY)
+    get_filename_component(AMS_OPENSSL_FOUND_ROOT "${AMS_OPENSSL_FOUND_ROOT}" DIRECTORY)
+    message(STATUS "OPENSSL Root dir is ${AMS_OPENSSL_FOUND_ROOT}")
   else()
     message(STATUS "OpenSSL Not Found")
   endif()
-
+  set(AMS_LIBEVENT_HINTS ${MPI_DIRECTORIES})
+
   find_package(libevent REQUIRED) # event loop library
-  list(APPEND AMS_APP_INCLUDES ${LIBEVENT_INCLUDE_DIR})
-  list(APPEND AMS_APP_LIBRARIES "${LIBEVENT_LIBRARIES}")
-  list(APPEND AMS_APP_LIBRARIES amqpcpp event_pthreads event)
 endif() # WITH_RMQ
 
 # ------------------------------------------------------------------------------
-if(NOT DEFINED UMPIRE_DIR)
-  message(FATAL_ERROR "Missing required 'UMPIRE_DIR' variable pointing to an installed Umpire ${UMPIRE_DIR}")
-endif()
-
-find_package(UMPIRE REQUIRED
-             NO_DEFAULT_PATH
-             PATHS ${UMPIRE_DIR}/share/umpire/cmake ${UMPIRE_DIR}/lib/cmake/umpire)
-list(APPEND AMS_APP_LIBRARIES umpire)
-list(APPEND AMS_APP_INCLUDES ${UMPIRE_INCLUDE_DIR})
-
-find_package(nlohmann_json REQUIRED)
-list(APPEND AMS_APP_LIBRARIES nlohmann_json::nlohmann_json)
+find_package(Torch REQUIRED)
+# This is annoying, torch populates all my cuda flags
+# and resets them
+set(CMAKE_CUDA_FLAGS "")
+set(CMAKE_CUDA_ARCHITECTURES ON)
 
-# ------------------------------------------------------------------------------
-find_package(Threads REQUIRED)
 
 # ------------------------------------------------------------------------------
-if (WITH_TORCH)
-  find_package(Torch REQUIRED)
-  # This is annoying, torch populates all my cuda flags
-  # and resets them
-  set(CMAKE_CUDA_FLAGS "")
-  set(CMAKE_CUDA_ARCHITECTURES ON)
-
-  list(APPEND AMS_APP_INCLUDES "${TORCH_INCLUDE_DIRS}")
-  list(APPEND AMS_APP_LIBRARIES "${TORCH_LIBRARIES}")
-
-  list(APPEND AMS_APP_DEFINES "-D__ENABLE_TORCH__")
-  set(BLA_VENDER OpenBLAS)
-  find_package(BLAS REQUIRED)
-  list(APPEND AMS_APP_LIBRARIES "${BLAS_LIBRARIES}")
-endif()
-
-# ------------------------------------------------------------------------------
-if (WITH_FAISS)
-  ## TODO: still need to create FindFaiss.cmake
-  #find_package(FAISS REQUIRED HINTS ${FAISS_DIR})
-  #include(${PROJECT_SOURCE_DIR}/cmake/FindFAISS.cmake)
-
-  if (FAISS_DIR)
-    message(STATUS "FAISS_DIR = ${FAISS_DIR}")
-    set(FAISS_INCLUDE_DIRS "${FAISS_DIR}/include")
-    set(FAISS_LIB_DIR "${FAISS_DIR}/lib")
-    set(FAISS_LIB64_DIR "${FAISS_DIR}/lib64")
-    set(FAISS_LIBRARIES "faiss")
-  else()
-    message(FATAL_ERROR "Cannot find FAISS. FAISS_DIR is not defined.")
-  endif()
-
-  ## TODO: we need to make a FindFaiss.cmake
-  list(APPEND AMS_APP_INCLUDES "${FAISS_INCLUDE_DIRS}")
-  list(APPEND AMS_APP_LIB_DIRS "${FAISS_LIB_DIR}" "${FAISS_LIB64_DIR}")
-  list(APPEND AMS_APP_LIBRARIES "${FAISS_LIBRARIES}")
-  list(APPEND AMS_APP_DEFINES "-D__ENABLE_FAISS__")
-
-
-  find_package(OpenMP)
-  if (OPENMP_FOUND)
-     set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
-     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-     set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
-  endif()
-endif()
-
 if (WITH_RZ)
   find_package(MPI REQUIRED)
   add_subdirectory(rz)
@@ -252,7 +142,7 @@ endif()
 if (WITH_PERFFLOWASPECT)
   find_package(perfflowaspect CONFIG REQUIRED)
 
-  list(APPEND AMS_APP_DEFINES "-D__ENABLE_PERFFLOWASPECT__")
+  list(APPEND AMS_APP_DEFINES "__ENABLE_PERFFLOWASPECT__")
   list(APPEND AMS_APP_LIB_DIRS "${PERFFLOWASPECT_LIB_DIR}")
 
   list(APPEND AMS_APP_LIBRARIES "perfflow_runtime")
@@ -261,80 +151,7 @@ endif()
 
 add_subdirectory(src)
 
-# ------------------------------------------------------------------------------
-if (WITH_AMS_LIB)
-if (WITH_EXAMPLES)
-  #The AMS Runtime does not depend on MFEM. The
-  #examples we use do.
-  if (WITH_MPI)
-    list(APPEND AMS_EXAMPLE_LIBRARIES MPI::MPI_CXX)
-    list(APPEND AMS_EXAMPLE_DEFINES "-D__ENABLE_MPI__")
-  endif()
-
-  list(APPEND AMS_EXAMPLE_LIBRARIES umpire)
-  list(APPEND AMS_EXAMPLE_INCLUDES ${UMPIRE_INCLUDE_DIR})
-
-  if (MFEM_DIR)
-    include(${PROJECT_SOURCE_DIR}/cmake/FindMFEM.cmake)
-  else()
-    find_package(MFEM REQUIRED)
-  endif()
-  list(APPEND AMS_EXAMPLE_INCLUDES "${MFEM_INCLUDE_DIRS}")
-  list(APPEND AMS_EXAMPLE_LIBRARIES "${MFEM_LIBRARIES}")
-  list(APPEND AMS_EXAMPLE_LIB_DIRS "${MFEM_LIB_DIR}")
-
-  if (WITH_REDIS)
-    list(APPEND AMS_EXAMPLE_DEFINES "-D__ENABLE_REDIS__")
-  endif()
-
-  if (WITH_RMQ)
-    list(APPEND AMS_EXAMPLE_DEFINES "-D__ENABLE_RMQ__")
-    list(APPEND AMS_EXAMPLE_INCLUDES ${amqpcpp_INCLUDE_DIR})
-    list(APPEND AMS_EXAMPLE_INCLUDES ${OPENSSL_INCLUDE_DIR})
-    list(APPEND AMS_EXAMPLE_INCLUDES ${LIBEVENT_INCLUDE_DIR})
-    list(APPEND AMS_EXAMPLE_LIBRARIES OpenSSL::SSL amqpcpp pthread event_pthreads event)
-    if (WITH_MPI)
-      list(APPEND AMS_EXAMPLE_LIBRARIES MPI::MPI_CXX)
-    endif()
-  endif()
-
-  if (WITH_CUDA)
-    list(APPEND AMS_EXAMPLE_LIBRARIES ${CUDA_LIBRARIES} cuda)
-    list(APPEND AMS_EXAMPLE_DEFINES "-D__ENABLE_CUDA__")
-  endif()
-
-  if (WITH_CALIPER)
-    list(APPEND AMS_EXAMPLE_INCLUDES ${caliper_INCLUDE_DIR})
-    list(APPEND AMS_EXAMPLE_LIBRARIES caliper)
-    list(APPEND AMS_EXAMPLE_DEFINES "-D__ENABLE_CALIPER__")
-  endif()
-
-  if (WITH_FAISS)
-    list(APPEND AMS_EXAMPLE_DEFINES "-D__ENABLE_FAISS__")
-  endif()
-
-  if (WITH_TORCH)
-    list(APPEND AMS_EXAMPLE_DEFINES "-D__ENABLE_TORCH__")
-  endif()
-
-  if (WITH_PERFFLOWASPECT)
-    list(APPEND AMS_EXAMPLE_DEFINES "-D__ENABLE_PERFFLOWASPECT__")
-  endif()
-
-  if (WITH_ADIAK)
-    find_package(adiak REQUIRED)
-    list(APPEND AMS_EXAMPLE_DEFINES "-D__ENABLE_ADIAK__")
-    list(APPEND AMS_EXAMPLE_INCLUDES ${adiak_INCLUDE_DIR})
-    list(APPEND AMS_EXAMPLE_LIBRARIES adiak::adiak)
-  endif()
-
-  add_subdirectory(examples)
-endif()
-endif()
-
 if (WITH_TESTS)
   include(CTest)
   add_subdirectory(tests)
 endif()
-
-# ------------------------------------------------------------------------------