Skip to content

Commit 48f0bf7

Browse files
committed
Add CI container compilation tests for GPU code
1 parent 891b9d2 commit 48f0bf7

27 files changed

+1944
-0
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
FROM nvidia/cuda:11.6.1-devel-ubi8 AS base
2+
MAINTAINER Giorgis Georgakoudis <[email protected]>
3+
RUN \
4+
yum install -y dnf &&\
5+
dnf install -y git xz autoconf automake unzip patch gcc-gfortran bzip2 file &&\
6+
dnf upgrade -y &&\
7+
dnf clean all
8+
COPY repo repo
9+
RUN \
10+
mkdir -p ams-spack-env
11+
COPY spack.yaml ams-spack-env/spack.yaml
12+
13+
FROM base AS setup-spack-env
14+
RUN \
15+
git clone --depth 1 --branch releases/v0.20 https://github.com/spack/spack.git &&\
16+
source spack/share/spack/setup-env.sh &&\
17+
spack env activate -p ams-spack-env &&\
18+
spack external find --all --not-buildable --exclude openssl --exclude openblas --exclude bzip2
19+
20+
FROM setup-spack-env AS install-spack-env
21+
RUN \
22+
source spack/share/spack/setup-env.sh &&\
23+
spack env activate -p ams-spack-env &&\
24+
spack install --fail-fast
25+
26+
FROM install-spack-env AS clean-spack
27+
RUN \
28+
spack clean --all
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
2+
# AMSLib Project Developers. See the top-level COPYRIGHT file for details.
3+
#
4+
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5+
6+
from spack.package import *
7+
8+
9+
class Ams(CMakePackage, CudaPackage):
10+
11+
homepage = "https://lc.llnl.gov/gitlab/autonomous-multiscale-project/marbl-matprops-miniapp"
12+
git = "ssh://[email protected]:7999/autonomous-multiscale-project/marbl-matprops-miniapp.git"
13+
14+
maintainers = ["parasyris1", "koparasy"]
15+
16+
version("develop", branch="develop", submodules=False)
17+
version("release", branch="release", submodules=False)
18+
version("main", branch="main", submodules=False)
19+
20+
variant("faiss", default=False, description="Build with FAISS index as uncertainty quantification module")
21+
variant("caliper", default=False, description="Build with caliper for gather performance counters")
22+
variant("torch", default=False, description="Use torch for surrogate models")
23+
variant("mpi", default=False, description="Enable MPI support")
24+
variant("examples", default=False, description="Enable MPI support")
25+
variant("redis", default=False, description="Enable redis database")
26+
variant("hdf5", default=False, description="Enable HDF5 data storage")
27+
variant("rabbitmq", default=False, description="Enable RabbitMQ as data broker")
28+
variant("verbose", default=False, description="Enable AMSLib verbose output (controlled by environment variable)")
29+
30+
depends_on("umpire")
31+
depends_on("mpi", when="+mpi")
32+
33+
depends_on("caliper+cuda", when="+caliper +cuda")
34+
depends_on("faiss+cuda", when="+faiss +cuda")
35+
depends_on("mfem+cuda", when="+examples +cuda")
36+
depends_on("py-torch+cuda", when="+torch +cuda")
37+
38+
depends_on("py-torch~cuda", when="+torch ~cuda")
39+
depends_on("caliper ~cuda", when="+caliper ~cuda")
40+
depends_on("faiss ~cuda", when="+faiss ~cuda")
41+
depends_on("mfem ~cuda", when="+examples ~cuda")
42+
43+
depends_on("redis-plus-plus", when="+redis")
44+
depends_on("hdf5", when="+hdf5")
45+
depends_on("amqp-cpp +tcp", when="+rabbitmq")
46+
47+
with when("+cuda"):
48+
cuda_archs=CudaPackage.cuda_arch_values
49+
with when("+examples"):
50+
depends_on("mfem+cuda")
51+
for sm_ in cuda_archs:
52+
depends_on("mfem +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_))
53+
54+
with when("+torch"):
55+
depends_on("py-torch+cuda")
56+
for sm_ in cuda_archs:
57+
depends_on("py-torch +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_))
58+
59+
with when("+caliper"):
60+
depends_on("caliper+cuda", when="+caliper")
61+
for sm_ in cuda_archs:
62+
depends_on("caliper +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_))
63+
64+
depends_on("umpire+cuda")
65+
for sm_ in cuda_archs:
66+
depends_on("umpire +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_))
67+
68+
with when("+faiss"):
69+
depends_on("faiss+cuda", when="+faiss")
70+
for sm_ in cuda_archs:
71+
depends_on("umpire +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_))
72+
73+
def cmake_args(self):
74+
spec = self.spec
75+
args = []
76+
args.append("-DUMPIRE_DIR={0}".format(spec["umpire"].prefix))
77+
args.append("-DWITH_MPI={0}".format("On" if "+mpi" in spec else "Off"))
78+
79+
args.append("-DWITH_DB={0}".format("On" if ("+redis" in spec or "hdf5" in spec or "+rabbitmq" in spec) else "Off"))
80+
81+
if "+verbose" in spec:
82+
args.append("-DWITH_AMS_DEBUG=On")
83+
84+
if "+hdf5" in spec:
85+
args.append("-DWITH_HDF5=On")
86+
args.append("-DHDF5_Dir={0}".format(spec["hdf5"].prefix))
87+
88+
if "+cuda" in spec:
89+
args.append("-DWITH_CUDA=On")
90+
cuda_arch = spec.variants["cuda_arch"].value[0]
91+
args.append("-DAMS_CUDA_ARCH={0}".format(cuda_arch))
92+
93+
if "+caliper" in spec:
94+
args.append("-DWITH_CALIPER=On")
95+
args.append("-DCALIPER_DIR={0}/share/cmake/caliper".format(spec["caliper"].prefix))
96+
else:
97+
args.append("-DWITH_CALIPER=Off")
98+
99+
if "+faiss" in spec:
100+
args.append("-DWITH_FAISS=On")
101+
args.append("-DFAISS_DIR={0}".format(spec['faiss'].prefix))
102+
else:
103+
args.append("-DWITH_FAISS=Off")
104+
105+
if "+torch" in spec:
106+
args.append("-DWITH_TORCH=On")
107+
args.append("-DTorch_DIR={0}/lib/python{1}/site-packages/torch/share/cmake/Torch".format(spec['py-torch'].prefix, spec['python'].version.up_to(2)))
108+
109+
if "+redis" in spec:
110+
args.append("-DWITH_REDIS=On")
111+
args.append("-DREDIS_PLUS_PLUS_DIR={0}".format(spec["redis-plus-plus"].prefix))
112+
113+
if "+rabbitmq" in spec:
114+
args.append("-DWITH_RMQ=On")
115+
args.append("-Damqpcpp_DIR={0}/cmake".format(spec["amqp-cpp"].prefix))
116+
117+
if "+examples" in spec:
118+
args.append("-DWITH_EXAMPLES=On")
119+
args.append("-DMFEM_DIR={0}".format(spec['mfem'].prefix))
120+
121+
return args
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
From ef513fe3d1d864d865d7143699834228988a7cd7 Mon Sep 17 00:00:00 2001
2+
From: Brad King <[email protected]>
3+
Date: Fri, 5 Mar 2021 08:08:16 -0500
4+
Subject: [PATCH] Cray: Enable explicit Fortran preprocessing for Ninja
5+
generator
6+
7+
Cray 11.0 adds support for preprocessing with output written to a
8+
specified file (instead of always next to the source). Use it to
9+
enable Cray Fortran with the Ninja generator.
10+
11+
Patch-by: James Elliott
12+
Fixes: #20731
13+
---
14+
Modules/Compiler/Cray-Fortran.cmake | 4 ++++
15+
1 file changed, 4 insertions(+)
16+
17+
diff --git a/Modules/Compiler/Cray-Fortran.cmake b/Modules/Compiler/Cray-Fortran.cmake
18+
index 696ae76074..0d5e1c7679 100644
19+
--- a/Modules/Compiler/Cray-Fortran.cmake
20+
+++ b/Modules/Compiler/Cray-Fortran.cmake
21+
@@ -19,3 +19,7 @@ else()
22+
set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-eZ")
23+
set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-dZ")
24+
endif()
25+
+
26+
+if (NOT CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 11.0)
27+
+ set(CMAKE_Fortran_PREPROCESS_SOURCE "<CMAKE_Fortran_COMPILER> -o <PREPROCESSED_SOURCE> <DEFINES> <INCLUDES> <FLAGS> -eP <SOURCE>")
28+
+endif()
29+
--
30+
GitLab
31+
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
commit 475e78d9071b34690617a85853433a9fc15da057
2+
Author: Chuck Atkins <[email protected]>
3+
Date: Mon Jan 28 16:28:28 2019 -0500
4+
5+
macOS: Add missing explicit dependency on CoreServices framework
6+
7+
On Apple, the implementation of cmGlobalXCodeGenerator::Open uses
8+
LSOpenCFURLRef from CoreServices. This get's transitively pulled in
9+
from CMake's libuv build but ends up generating a linker error when
10+
using an external libuv. This explicitly adds the appropriate
11+
dependency.
12+
13+
diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt
14+
index 311f3f4e56..8aff8f6b2f 100644
15+
--- a/Source/CMakeLists.txt
16+
+++ b/Source/CMakeLists.txt
17+
@@ -791,9 +791,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "sparc"
18+
endif()
19+
endif()
20+
21+
-# On Apple we need CoreFoundation
22+
+# On Apple we need CoreFoundation and CoreServices
23+
if(APPLE)
24+
target_link_libraries(CMakeLib "-framework CoreFoundation")
25+
+ target_link_libraries(CMakeLib "-framework CoreServices")
26+
endif()
27+
28+
if(WIN32 AND NOT UNIX)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
From 89fc3b1fd22f97f9380990b521dd79f306ac18fd Mon Sep 17 00:00:00 2001
2+
From: Chuck Atkins <[email protected]>
3+
Date: Thu, 25 Jul 2019 09:37:20 -0400
4+
Subject: [PATCH] Revert "FindMPI: Store imported target link flags as a list
5+
instead of a string"
6+
7+
This reverts commit f7eaa342de316707d99e6ae29c693a480861560d.
8+
---
9+
Modules/FindMPI.cmake | 4 +---
10+
1 file changed, 1 insertion(+), 3 deletions(-)
11+
12+
diff --git a/Modules/FindMPI.cmake b/Modules/FindMPI.cmake
13+
index a80f799..fe09764 100644
14+
--- a/Modules/FindMPI.cmake
15+
+++ b/Modules/FindMPI.cmake
16+
@@ -1144,9 +1144,7 @@ macro(_MPI_create_imported_target LANG)
17+
18+
set_property(TARGET MPI::MPI_${LANG} PROPERTY INTERFACE_LINK_LIBRARIES "")
19+
if(MPI_${LANG}_LINK_FLAGS)
20+
- separate_arguments(_MPI_${LANG}_LINK_FLAGS NATIVE_COMMAND "${MPI_${LANG}_LINK_FLAGS}")
21+
- set_property(TARGET MPI::MPI_${LANG} APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${_MPI_${LANG}_LINK_FLAGS}")
22+
- unset(_MPI_${LANG}_LINK_FLAGS)
23+
+ set_property(TARGET MPI::MPI_${LANG} APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${MPI_${LANG}_LINK_FLAGS}")
24+
endif()
25+
# If the compiler links MPI implicitly, no libraries will be found as they're contained within
26+
# CMAKE_<LANG>_IMPLICIT_LINK_LIBRARIES already.
27+
--
28+
2.5.5
29+
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
From 19f267c75e84b72c4de42570be0c4222bb93aaff Mon Sep 17 00:00:00 2001
2+
From: Brad King <[email protected]>
3+
Date: Thu, 21 Nov 2019 14:38:35 -0500
4+
Subject: [PATCH] XL: Add support for Ninja and XL Fortran
5+
6+
The Ninja generator's support for Fortran requires that source files
7+
be preprocessed explicitly first. However, the `xlf` compiler does
8+
not have a simple `-E` option or equivalent to do preprocessing.
9+
The only documented way to get preprocessed output is to use `-d`
10+
to leave it behind, but only at an inflexible location.
11+
12+
Instead, create our own `cpp` wrapper script and substitute it for the
13+
real preprocessor using `-tF -B ...`. Teach the wrapper to map the
14+
`cpp` output to the location we need and then invoke the real `cpp`
15+
underneath.
16+
17+
Fixes: #19450
18+
---
19+
Help/release/dev/xlf-ninja.rst | 5 ++++
20+
Modules/CMakeDetermineCompilerId.cmake | 10 +++++++
21+
Modules/CMakeDetermineFortranCompiler.cmake | 5 ++++
22+
Modules/CMakeFortranCompiler.cmake.in | 1 +
23+
Modules/Compiler/XL-Fortran.cmake | 4 +++
24+
Modules/Compiler/XL-Fortran/cpp | 29 +++++++++++++++++++++
25+
6 files changed, 54 insertions(+)
26+
create mode 100644 Help/release/dev/xlf-ninja.rst
27+
create mode 100755 Modules/Compiler/XL-Fortran/cpp
28+
29+
diff --git a/Help/release/dev/xlf-ninja.rst b/Help/release/dev/xlf-ninja.rst
30+
new file mode 100644
31+
index 0000000000..916e713008
32+
--- /dev/null
33+
+++ b/Help/release/dev/xlf-ninja.rst
34+
@@ -0,0 +1,5 @@
35+
+xlf-ninja
36+
+---------
37+
+
38+
+* The IBM XL Fortran compiler is now supported by the :generator:`Ninja`
39+
+ generator.
40+
diff --git a/Modules/CMakeDetermineCompilerId.cmake b/Modules/CMakeDetermineCompilerId.cmake
41+
index f7ef755aeb..0b3664c5de 100644
42+
--- a/Modules/CMakeDetermineCompilerId.cmake
43+
+++ b/Modules/CMakeDetermineCompilerId.cmake
44+
@@ -182,6 +182,10 @@ function(CMAKE_DETERMINE_COMPILER_ID lang flagvar src)
45+
message(STATUS "The ${lang} compiler identification is unknown")
46+
endif()
47+
48+
+ if(lang STREQUAL "Fortran" AND CMAKE_${lang}_COMPILER_ID STREQUAL "XL")
49+
+ set(CMAKE_${lang}_XL_CPP "${CMAKE_${lang}_COMPILER_ID_CPP}" PARENT_SCOPE)
50+
+ endif()
51+
+
52+
set(CMAKE_${lang}_COMPILER_ID "${CMAKE_${lang}_COMPILER_ID}" PARENT_SCOPE)
53+
set(CMAKE_${lang}_PLATFORM_ID "${CMAKE_${lang}_PLATFORM_ID}" PARENT_SCOPE)
54+
set(CMAKE_${lang}_COMPILER_ARCHITECTURE_ID "${CMAKE_${lang}_COMPILER_ARCHITECTURE_ID}" PARENT_SCOPE)
55+
@@ -542,6 +546,12 @@ Id flags: ${testflags} ${CMAKE_${lang}_COMPILER_ID_FLAGS_ALWAYS}
56+
ERROR_VARIABLE CMAKE_${lang}_COMPILER_ID_OUTPUT
57+
RESULT_VARIABLE CMAKE_${lang}_COMPILER_ID_RESULT
58+
)
59+
+ if("${CMAKE_${lang}_COMPILER_ID_OUTPUT}" MATCHES "exec: [^\n]*\\((/[^,\n]*/cpp),CMakeFortranCompilerId.F")
60+
+ set(_cpp "${CMAKE_MATCH_1}")
61+
+ if(EXISTS "${_cpp}")
62+
+ set(CMAKE_${lang}_COMPILER_ID_CPP "${_cpp}" PARENT_SCOPE)
63+
+ endif()
64+
+ endif()
65+
endif()
66+
67+
# Check the result of compilation.
68+
diff --git a/Modules/CMakeDetermineFortranCompiler.cmake b/Modules/CMakeDetermineFortranCompiler.cmake
69+
index 5ddd64fae8..e8505417d6 100644
70+
--- a/Modules/CMakeDetermineFortranCompiler.cmake
71+
+++ b/Modules/CMakeDetermineFortranCompiler.cmake
72+
@@ -271,6 +271,11 @@ include(CMakeFindBinUtils)
73+
include(Compiler/${CMAKE_Fortran_COMPILER_ID}-FindBinUtils OPTIONAL)
74+
unset(_CMAKE_PROCESSING_LANGUAGE)
75+
76+
+if(CMAKE_Fortran_XL_CPP)
77+
+ set(_SET_CMAKE_Fortran_XL_CPP
78+
+ "set(CMAKE_Fortran_XL_CPP \"${CMAKE_Fortran_XL_CPP}\")")
79+
+endif()
80+
+
81+
if(CMAKE_Fortran_COMPILER_ARCHITECTURE_ID)
82+
set(_SET_CMAKE_Fortran_COMPILER_ARCHITECTURE_ID
83+
"set(CMAKE_Fortran_COMPILER_ARCHITECTURE_ID ${CMAKE_Fortran_COMPILER_ARCHITECTURE_ID})")
84+
diff --git a/Modules/CMakeFortranCompiler.cmake.in b/Modules/CMakeFortranCompiler.cmake.in
85+
index ae7b73ac4a..34f44aa542 100644
86+
--- a/Modules/CMakeFortranCompiler.cmake.in
87+
+++ b/Modules/CMakeFortranCompiler.cmake.in
88+
@@ -6,6 +6,7 @@ set(CMAKE_Fortran_COMPILER_WRAPPER "@CMAKE_Fortran_COMPILER_WRAPPER@")
89+
set(CMAKE_Fortran_PLATFORM_ID "@CMAKE_Fortran_PLATFORM_ID@")
90+
set(CMAKE_Fortran_SIMULATE_ID "@CMAKE_Fortran_SIMULATE_ID@")
91+
set(CMAKE_Fortran_SIMULATE_VERSION "@CMAKE_Fortran_SIMULATE_VERSION@")
92+
+@_SET_CMAKE_Fortran_XL_CPP@
93+
@_SET_CMAKE_Fortran_COMPILER_ARCHITECTURE_ID@
94+
@SET_MSVC_Fortran_ARCHITECTURE_ID@
95+
set(CMAKE_AR "@CMAKE_AR@")
96+
diff --git a/Modules/Compiler/XL-Fortran.cmake b/Modules/Compiler/XL-Fortran.cmake
97+
index c4fb09712a..1683dff4f0 100644
98+
--- a/Modules/Compiler/XL-Fortran.cmake
99+
+++ b/Modules/Compiler/XL-Fortran.cmake
100+
@@ -18,3 +18,7 @@ string(APPEND CMAKE_Fortran_FLAGS_INIT " -qthreaded -qhalt=e")
101+
# xlf: 1501-214 (W) command option E reserved for future use - ignored
102+
set(CMAKE_Fortran_CREATE_PREPROCESSED_SOURCE)
103+
set(CMAKE_Fortran_CREATE_ASSEMBLY_SOURCE)
104+
+
105+
+set(CMAKE_Fortran_PREPROCESS_SOURCE
106+
+ "<CMAKE_Fortran_COMPILER> <DEFINES> <INCLUDES> <FLAGS> -qpreprocess -qnoobject -qsuppress=1517-020 -tF -B \"${CMAKE_CURRENT_LIST_DIR}/XL-Fortran/\" -WF,--cpp,\"${CMAKE_Fortran_XL_CPP}\",--out,<PREPROCESSED_SOURCE> <SOURCE>"
107+
+ )
108+
diff --git a/Modules/Compiler/XL-Fortran/cpp b/Modules/Compiler/XL-Fortran/cpp
109+
new file mode 100755
110+
index 0000000000..1fd62c26a0
111+
--- /dev/null
112+
+++ b/Modules/Compiler/XL-Fortran/cpp
113+
@@ -0,0 +1,29 @@
114+
+#!/usr/bin/env bash
115+
+
116+
+# Source file.
117+
+src="$(printf %q "$1")"
118+
+shift
119+
+
120+
+# Output file the compiler expects.
121+
+out="$(printf %q "$1")"
122+
+shift
123+
+
124+
+# Create the file the compiler expects. It will check syntax.
125+
+>"$out"
126+
+
127+
+cpp='cpp'
128+
+opts=''
129+
+while test "$#" != 0; do
130+
+ case "$1" in
131+
+ # Extract the option for the path to cpp.
132+
+ --cpp) shift; cpp="$(printf %q "$1")" ;;
133+
+ # Extract the option for our own output file.
134+
+ --out) shift; out="$(printf %q "$1")" ;;
135+
+ # Collect the rest of the command line.
136+
+ *) opts="$opts $(printf %q "$1")" ;;
137+
+ esac
138+
+ shift
139+
+done
140+
+
141+
+# Execute the real preprocessor tool.
142+
+eval "exec $cpp $src $out $opts"
143+
--
144+
2.24.1
145+
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
--- spack-src/Modules/FortranCInterface/Verify/CMakeLists.txt.org 2020-06-05 15:54:59.559043595 +0900
2+
+++ spack-src/Modules/FortranCInterface/Verify/CMakeLists.txt 2020-06-05 15:58:28.150062948 +0900
3+
@@ -4,6 +4,7 @@
4+
cmake_minimum_required(VERSION ${CMAKE_VERSION})
5+
project(VerifyFortranC C Fortran)
6+
7+
+set (CMAKE_EXE_LINKER_FLAGS "--linkfortran")
8+
option(VERIFY_CXX "Whether to verify C++ and Fortran" OFF)
9+
if(VERIFY_CXX)
10+
enable_language(CXX)

0 commit comments

Comments
 (0)