Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Workflow support #82

Merged
merged 36 commits into from
Sep 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
2780a46
Add working correct version of binomial options
koparasy Jun 25, 2024
2bb2c30
Add missing files, clean interface
koparasy Jun 25, 2024
9ab2c98
Add AMS support for binomial options
koparasy Jun 25, 2024
4744f76
Enable MPI for Binomial with weak scaling
koparasy Jun 26, 2024
af41a4d
Add CMake for examples
koparasy Jun 26, 2024
d1c41a7
srand seed based on rank id
koparasy Jun 26, 2024
a0837c8
Add missing files
koparasy Jun 26, 2024
d32d391
Implements sequential workflow including staging
koparasy Jun 26, 2024
d8afdb9
FS stager now can delete files from source directory
koparasy Jul 24, 2024
6d16b08
Seperates 'domain_name' from 'dbLabel'
koparasy Jul 25, 2024
bd123be
Fixes kosh store to group ensembles based on domain name
koparasy Jul 25, 2024
973fe94
Register data per domain into kosh
koparasy Jul 25, 2024
b703808
End to end sequential workflow
koparasy Jul 29, 2024
e23e2ce
Sub-select and train script
koparasy Jul 30, 2024
e799dd0
ams flux executor
koparasy Aug 2, 2024
a13a77f
Initial extensions for orchestrator
koparasy Aug 6, 2024
d9186c2
Working orchestrator
koparasy Aug 9, 2024
e3066dc
Adding orchestrator
koparasy Aug 9, 2024
4b48bd2
Added fake rmq publish option
lpottier Aug 9, 2024
6deecb0
typo
lpottier Aug 9, 2024
5dcbe64
Fix bug
lpottier Aug 9, 2024
1c9f28c
Pushes model updates to the stager
koparasy Aug 12, 2024
ef92644
Workflow sketch
koparasy Aug 13, 2024
b2c8342
Working stager with update models and initial workflow descr
koparasy Aug 19, 2024
a5c0c74
Workflow manager starts orchestrator
koparasy Aug 19, 2024
27d09b8
Supports graceful shutdown of AMSOrchestrator
koparasy Aug 20, 2024
257ea04
End to end workflow running
koparasy Aug 21, 2024
262be05
Add deployment tool to be moved under 'ams_wf'
koparasy Aug 21, 2024
ab060a0
Add comments for ams flux
koparasy Aug 22, 2024
046f63d
Add comments
koparasy Aug 23, 2024
c29a5b3
Fix execution errors
koparasy Aug 23, 2024
b2edfd0
Fix tests
koparasy Aug 23, 2024
2e0e93f
Fix tests
koparasy Aug 23, 2024
c3787b0
Guard inclusion of mpi.h
koparasy Aug 23, 2024
9990a49
Fix train/test split
koparasy Aug 27, 2024
000e7ec
Fix PR comments
koparasy Sep 18, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@ cmake_minimum_required(VERSION 3.18)
project(AMS VERSION 0.1.0 LANGUAGES CXX C)
cmake_policy(SET CMP0074 NEW)

set(CMAKE_CXX_STANDARD 14)
# NOTE: This may break some of our integrations with the applications. But flux requires > C++20, RMQ requires C++17 and although AMS does not have
# any restrictions on the CXX standard the application may impose those. The solution would be to compile RMQ with an older GCC version (8) and
# have flux to be an external of the system environment
set(CMAKE_CXX_STANDARD 17)

# Enable this to get compile_commands.json
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
Expand Down
55 changes: 4 additions & 51 deletions examples/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,55 +3,8 @@
#
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

file(GLOB_RECURSE AMS_CURRENT_EXAMPLE_INCLUDES "*.hpp")

set(AMS_EXAMPLE_SRC ${MINIAPP_INCLUDES} main.cpp app/eos_ams.cpp)


function(ADDExec binary_name definitions)
if (WITH_RZ)
list(APPEND AMS_EXAMPLE_SRC ${RZ_AMS_SOURCES} ${AMS_CURRENT_EXAMPLE_INCLUDES})
endif()

target_include_directories(${binary_name} PRIVATE ${AMS_EXAMPLE_INCLUDES}
${CMAKE_CURRENT_SOURCE_DIR} ${PROJECT_BINARY_DIR}/include)
target_compile_definitions(${binary_name} PRIVATE ${definitions})
target_link_directories(${binary_name} PRIVATE ${AMS_EXAMPLE_LIB_DIRS})
target_link_libraries(${binary_name} PUBLIC AMS ${AMS_EXAMPLE_LIBRARIES})

if (WITH_CUDA)
set_source_files_properties(main.cpp PROPERTIES LANGUAGE CUDA)
set_source_files_properties(main.cpp PROPERTIES COMPILE_FLAGS "--expt-extended-lambda")
set_source_files_properties(app/eos_ams.cpp PROPERTIES LANGUAGE CUDA)
set_source_files_properties(app/eos_ams.cpp PROPERTIES COMPILE_FLAGS "--expt-extended-lambda")
set_target_properties(${binary_name} PROPERTIES CUDA_ARCHITECTURES "${AMS_CUDA_ARCH}")

if (WITH_RZ)
set_source_files_properties(${RZ_AMS_SOURCES} PROPERTIES LANGUAGE CUDA)
set_source_files_properties(${RZ_AMS_SOURCES} PROPERTIES COMPILE_FLAGS "--expt-extended-lambda")
set_property(TARGET ${binary_name} PROPERTY CUDA_SEPARABLE_COMPILATION ON)
endif()

if (WITH_PERFFLOWASPECT)
set_property(SOURCE ${AMS_EXAMPLE_SRC} APPEND_STRING PROPERTY COMPILE_FLAGS " -Xcompiler=-Xclang -Xcompiler=-load -Xcompiler=-Xclang -Xcompiler=${PERFFLOWASPECT_LIB_DIR}/libWeavePass.so")
endif()
endif()
endfunction()

add_executable(no_ams_example ${AMS_EXAMPLE_SRC} ${MINIAPP_INCLUDES})
ADDExec(no_ams_example "${AMS_EXAMPLE_DEFINES}")

list(APPEND AMS_EXAMPLE_DEFINES "-DUSE_AMS")
add_executable(ams_example ${AMS_EXAMPLE_SRC} ${MINIAPP_INCLUDES})
ADDExec(ams_example "${AMS_EXAMPLE_DEFINES}")

if (WITH_WORKFLOW)
set(TRAIN_DEVICE "cpu")
if (WITH_CUDA)
set(TRAIN_DEVICE "gpu")
endif()
configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/prune.py" "${CMAKE_CURRENT_BINARY_DIR}/prune.py")
configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/train.py" "${CMAKE_CURRENT_BINARY_DIR}/train.py")
configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/sub_selection.py" "${CMAKE_CURRENT_BINARY_DIR}/sub_selection.py")
configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/example_run.sh" "${CMAKE_CURRENT_BINARY_DIR}/example_run.sh" @ONLY)
add_subdirectory(ideal_gas)
if (WITH_CUDA)
add_subdirectory(bnm_opt)
endif()

27 changes: 27 additions & 0 deletions examples/bnm_opt/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Copyright 2021-2023 Lawrence Livermore National Security, LLC and other
# AMSLib Project Developers
#
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

set(binomial_options_src binomial_options.cpp kernel.cpp)

function(ADDExec binary_name definitions)
target_include_directories(${binary_name} PUBLIC "${AMS_EXAMPLE_INCLUDES}")
target_compile_definitions(${binary_name} PRIVATE ${definitions})
target_compile_definitions(${binary_name} PRIVATE ${AMS_EXAMPLE_DEFINES})
target_link_directories(${binary_name} PRIVATE ${AMS_EXAMPLE_LIB_DIRS})
target_link_libraries(${binary_name} PUBLIC AMS ${AMS_EXAMPLE_LIBRARIES})

if (WITH_CUDA)
set_source_files_properties(kernel.cpp PROPERTIES LANGUAGE CUDA)
set_target_properties(${binary_name} PROPERTIES CUDA_ARCHITECTURES "${AMS_CUDA_ARCH}")
endif()
endfunction()

add_executable(no_ams_bo ${binomial_options_src} ${MINIAPP_INCLUDES})
ADDExec(no_ams_bo minibude "${AMS_EXAMPLE_DEFINES}")

list(APPEND AMS_EXAMPLE_DEFINES "-DUSE_AMS")
add_executable(ams_bo ${binomial_options_src} ${MINIAPP_INCLUDES})
ADDExec(ams_bo "${AMS_EXAMPLE_DEFINES}")

40 changes: 40 additions & 0 deletions examples/bnm_opt/binomialOptions.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Copyright 1993-2015 NVIDIA Corporation. All rights reserved.
*
* Please refer to the NVIDIA end user license agreement (EULA) associated
* with this source code for terms and conditions that govern your use of
* this software. Any use, reproduction, disclosure, or distribution of
* this software and related documentation outside the terms of the EULA
* is strictly prohibited.
*
*/


#ifndef BINOMIALOPTIONS_COMMON_H
#define BINOMIALOPTIONS_COMMON_H

#include "realtype.h"

////////////////////////////////////////////////////////////////////////////////
// Global types
////////////////////////////////////////////////////////////////////////////////
typedef struct {
real S;
real X;
real T;
real R;
real V;
} TOptionData;


////////////////////////////////////////////////////////////////////////////////
// Global parameters
////////////////////////////////////////////////////////////////////////////////
//Number of time steps
#define NUM_STEPS 2048
//Max option batch size
#define MAX_OPTIONS 1024

#define NUM_ITERATIONS 20

#endif
Loading
Loading