Skip to content

Commit 9a039f7

Browse files
ldowenldowen
authored
Merge pull request #319 from LLNL/feature/perf_assess
Performance Testing and Assessment
2 parents 721c079 + e4edecd commit 9a039f7

39 files changed

+835
-494
lines changed

‎.gitlab-ci.yml

+2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ variables:
66

77
SCRIPT_DIR: "scripts"
88
ATS_FILE: "tests/integration.ats"
9+
PERF_ATS_FILE: "tests/performance.py"
910
SPHERAL_MODULE: "Spheral"
1011
SPACK_NAME: spheral
1112
SPACK_PKG_NAME: spheral
@@ -20,6 +21,7 @@ stages:
2021
- build_and_install
2122
- run_ats
2223
- update_tpls
24+
- run_perf
2325
- generate_buildcache
2426
- install_production
2527
- update_permissions

‎.gitlab/jobs-mpi.yml

+15-2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@ toss_gcc_mvapich2_cleanup:
2929
extends: [.toss_resource2, .gcc_mvapich2, .cleanup_dir]
3030
needs: [toss_gcc_mvapich2_test]
3131

32+
toss_gcc_mvapich2_perf:
33+
extends: [.toss_resource2, .gcc_mvapich2, .run_perf, .merge_pr_rule]
34+
needs: [toss_gcc_mvapich2_build]
35+
3236

3337

3438
toss_clang_mvapich2_tpls:
@@ -46,6 +50,10 @@ toss_clang_mvapich2_cleanup:
4650
extends: [.toss_resource2, .clang_mvapich2, .cleanup_dir]
4751
needs: [toss_clang_mvapich2_test]
4852

53+
toss_clang_mvapich2_perf:
54+
extends: [.toss_resource2, .clang_mvapich2, .run_perf, .merge_pr_rule]
55+
needs: [toss_clang_mvapich2_build]
56+
4957

5058

5159
#toss_oneapi_2022_1_mvapich2:
@@ -68,6 +76,10 @@ blueos_gcc_spectrum_cleanup:
6876
extends: [.blueos_resource1, .gcc_spectrum, .cleanup_dir]
6977
needs: [blueos_gcc_spectrum_test]
7078

79+
blueos_gcc_spectrum_perf:
80+
extends: [.blueos_resource1, .gcc_spectrum, .run_perf, .merge_pr_rule]
81+
needs: [blueos_gcc_spectrum_build]
82+
7183

7284

7385
blueos_cuda_11_gcc_spectrum_tpls:
@@ -85,5 +97,6 @@ blueos_cuda_11_gcc_spectrum_cleanup:
8597
extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .cleanup_dir]
8698
needs: [blueos_cuda_11_gcc_spectrum_test]
8799

88-
89-
100+
blueos_cuda_11_gcc_spectrum_perf:
101+
extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .run_perf, .merge_pr_rule]
102+
needs: [blueos_cuda_11_gcc_spectrum_build]

‎.gitlab/scripts.yml

+17-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
script:
2828
- CI_BUILD_DIR=$(cat ci-dir.txt)
2929
- cd $CI_BUILD_DIR && cat job-name.txt
30-
- ./$SCRIPT_DIR/devtools/host-config-build.py --no-clean --build --nprocs $NPROC --host-config gitlab.cmake $EXTRA_CMAKE_ARGS
30+
- ./$SCRIPT_DIR/devtools/host-config-build.py --no-clean --build --nprocs $NPROC --host-config gitlab.cmake -DENABLE_TIMER=ON $EXTRA_CMAKE_ARGS
3131
artifacts:
3232
paths:
3333
- ci-dir.txt
@@ -67,6 +67,22 @@
6767
exit_codes:
6868
- 80
6969

70+
# Run the performance suite and move the resulting Caliper files into the benchmark location
71+
.run_perf:
72+
stage: run_perf
73+
variables:
74+
GIT_STRATEGY: none
75+
script:
76+
- CI_BUILD_DIR=$(cat ci-dir.txt)
77+
- cd $CI_BUILD_DIR && cat job-name.txt
78+
79+
- ./build_gitlab/install/spheral-ats --ciRun --numNodes 2 --logs perf_logs ./build_gitlab/install/$PERF_ATS_FILE
80+
artifacts:
81+
when: always
82+
paths:
83+
- ci-dir.txt
84+
- perf_logs/
85+
7086
.cleanup_dir:
7187
stage: cleanup
7288
variables:

‎.gitlab/specs.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
.cuda_11_gcc_spectrum:
3838
variables:
3939
SPEC: 'gcc@$GCC_VERSION+cuda cuda_arch=70'
40-
EXTRA_CMAKE_ARGS: '-DENABLE_TIMER=On'
40+
EXTRA_CMAKE_ARGS: ''
4141

4242
.oneapi_2022_1_mvapich2:
4343
variables:

‎RELEASE_NOTES.md

+1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ Notable changes include:
3232
* Cleaned up use of std::any in State objects using a visitor pattern to be rigorous ensuring all state entries are handled properly
3333
during assignement, equality, and cloning operations. This is intended to help ensure our Physics advance during time integration
3434
is correct.
35+
* Performance regression testing is now available. All developers are encouraged to run the performance testing suite for any code changes that might impact performance. See documentation for more details.
3536
3637
* Build changes / improvements:
3738
* Distributed source directory must always be built now.

‎cmake/SetupSpheral.cmake

+4-9
Original file line numberDiff line numberDiff line change
@@ -153,15 +153,10 @@ endif()
153153
# Build C++ tests and install tests to install directory
154154
#-------------------------------------------------------------------------------
155155
if (ENABLE_TESTS)
156-
install(DIRECTORY ${SPHERAL_ROOT_DIR}/tests/
157-
USE_SOURCE_PERMISSIONS
158-
DESTINATION "${SPHERAL_TEST_INSTALL_PREFIX}"
159-
PATTERN "*CMakeLists.txt*" EXCLUDE
160-
PATTERN "*.cmake" EXCLUDE
161-
PATTERN "*.in" EXCLUDE
162-
PATTERN "*.pyc" EXCLUDE
163-
PATTERN "*~" EXCLUDE)
164-
add_subdirectory(${SPHERAL_ROOT_DIR}/tests/unit)
156+
spheral_install_python_tests(${SPHERAL_ROOT_DIR}/tests/ ${SPHERAL_TEST_INSTALL_PREFIX})
157+
# Always install performance.py in the top of the testing script
158+
install(FILES ${SPHERAL_ROOT_DIR}/tests/performance.py
159+
DESTINATION ${CMAKE_INSTALL_PREFIX}/tests)
165160
endif()
166161

167162
include(${SPHERAL_ROOT_DIR}/cmake/SpheralConfig.cmake)

‎cmake/spheral/SpheralInstallPythonFiles.cmake

+21
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,24 @@ function(spheral_install_python_files)
1919
endif()
2020

2121
endfunction()
22+
23+
#----------------------------------------------------------------------------------------
24+
# spheral_instalL_python_tests
25+
#----------------------------------------------------------------------------------------
26+
# ----------------------
27+
# INPUT VARIABLES
28+
# ----------------------
29+
# test_dir : REQUIRED : Source directory of tests to install
30+
# test_dest : REQUIRED : Destination for tests
31+
function(spheral_install_python_tests test_dir test_dest)
32+
install(DIRECTORY ${test_dir}
33+
USE_SOURCE_PERMISSIONS
34+
DESTINATION "${test_dest}"
35+
PATTERN "*CMakeLists.txt*" EXCLUDE
36+
PATTERN "*.cmake" EXCLUDE
37+
PATTERN "*.in" EXCLUDE
38+
PATTERN "*.pyc" EXCLUDE
39+
PATTERN "performance.py" EXCLUDE
40+
PATTERN "*~" EXCLUDE)
41+
# performance.py must be installed in the top test directory
42+
endfunction()

‎docs/build_guide/external/quickstart.rst

+2-2
Original file line numberDiff line numberDiff line change
@@ -24,5 +24,5 @@ Quickstart
2424
:end-before: [ex_install-section-end]
2525

2626
.. include:: ../include/quickstart.rst.inc
27-
:start-after: [ex_test-section-start]
28-
:end-before: [ex_test-section-end]
27+
:start-after: [test-section-start]
28+
:end-before: [test-section-end]

‎docs/build_guide/external/tests.rst

+2-10
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,5 @@ Running Tests
66
:end-before: [smoke_test-end]
77

88
.. include:: ../include/tests.rst.inc
9-
:start-after: [ex_ats_test-start]
10-
:end-before: [ex_ats_test-end]
11-
12-
.. include:: ../include/tests.rst.inc
13-
:start-after: [filter_options-start]
14-
:end-before: [filter_options-end]
15-
16-
.. include:: ../include/tests.rst.inc
17-
:start-after: [ex_filter_options_cmd-start]
18-
:end-before: [ex_filter_options_cmd-end]
9+
:start-after: [ats_test-start]
10+
:end-before: [ats_test-end]

‎docs/build_guide/include/quickstart.rst.inc

+3-18
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ Build and install Spheral.
120120
TESTING
121121
----------------------------------------
122122

123-
[ex_test-section-start]
123+
[test-section-start]
124124
Run a basic smoke test for Spheral
125125
::
126126

@@ -130,22 +130,7 @@ Run a basic smoke test for Spheral
130130
Run our full test suite.
131131
::
132132

133-
./.venv/bin/ats -e spheral test/integration.ats
133+
./spheral-ats test/integration.ats
134134

135135
These commands are explained in further sections.
136-
[ex_test-section-end]
137-
138-
[lc_test-section-start]
139-
Run a basic smoke test for Spheral
140-
::
141-
142-
cd ../install
143-
./spheral -c "import Spheral"
144-
145-
Run our full test suite.
146-
::
147-
148-
./spheral-atstest test/integration.ats
149-
150-
These commands are explained in further sections.
151-
[lc_test-section-end]
136+
[test-section-end]

‎docs/build_guide/include/tests.rst.inc

+8-82
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,14 @@
88
Basic Smoke Test
99
================
1010

11-
After a build and install it's recommended to perform a quick smoke test with Spheral to see if the Spheral environment was installed and all of the libraries were built and linked together correctly.
11+
After a build and install, we recommend you perform a smoke test with Spheral to ensure the Spheral environment is installed and linked correctly.
1212

1313
From your install directory run:
1414
::
1515

1616
./spheral -c "import Spheral"
1717

18-
This will run the Spheral Python process and try importing the Spheral modules. If successful you should see a banner printed with information about the version of Spheral, after which Python exits.
18+
This will run the Spheral python process and import the Spheral modules. If successful, you should see a banner printed with information about the version of Spheral, after which Python exits.
1919

2020
[smoke_test-end]
2121

@@ -24,94 +24,20 @@ This will run the Spheral Python process and try importing the Spheral modules.
2424
..
2525
RUNNING ATS
2626
----------------------------------------
27-
28-
[ex_ats_test-start]
29-
ATS Testing Suite
30-
=================
31-
32-
Spheral uses ATS to execute a suite of parallel tests. To run this on an external system we need to use Spheral's virtual-env installation of ATS, as external users will not have access to some LC available scripts.
33-
34-
From the install directory run:
35-
::
36-
37-
./.venv/bin/ats -e spheral tests/integration.ats
38-
39-
[ex_ats_test-end]
40-
[lc_ats_test-start]
27+
[ats_test-start]
4128
ATS Testing Suite
4229
=================
4330

44-
Spheral uses ATS (the Python Automated Testing System) to execute a suite of parallel tests. During install a script is generated ``spheral-atstest`` which handles calling the ats script in ``/usr/gapps/Spheral/bin`` and passing the generated ``spheral`` executor script to it.
31+
Spheral uses ATS (Automated Testing System) to execute a suite of parallel tests. During install, a script ``spheral-ats`` is created. This script handles launching allocations and setting appropriate ATS and hardware flags for running the testing suite.
4532

4633
From the install directory run:
4734
::
4835

49-
./spheral-atstest tests/integration.ats
50-
51-
[lc_ats_test-end]
52-
53-
54-
55-
..
56-
ATS FILTER OPTIONS
57-
----------------------------------------
58-
59-
[filter_options-start]
60-
ATS Filters and Options
61-
-----------------------
62-
63-
Appropriate filters are injected into the spheral-atstest script based on build configurations. However if you are using ats directly or wish to override some of these filters you can pass them as defined below.
64-
65-
Non MPI Filter
66-
..............
67-
68-
If Spheral was built without ``MPI`` support we will need to pass a filter to our ``ats`` command to prevent parallel tests from being performed:
69-
::
70-
71-
--filter='"np<2"'
72-
73-
This stops the ATS suite from attempting to run any tests that rely on more than one process/rank.
74-
75-
Debug Build Filter
76-
..................
77-
78-
If Spheral was built in Debug mode it is recommended to pass the below filter if you value your time.
79-
::
80-
81-
--filter='"level<100"'
82-
83-
These filters stack when invoked. So if you are running the test suite on a non-mpi debug build the command would be:
84-
85-
[filter_options-end]
86-
87-
[ex_filter_options_cmd-start]
88-
::
89-
90-
./.venv/bin/ats -e spheral tests/integration.ats --filter='"np<2"' --filter='"level<100"'
91-
92-
[ex_filter_options_cmd-end]
93-
[lc_filter_options_cmd-start]
94-
::
95-
96-
./spheral-atstest tests/integration.ats --filter='"np<2"' --filter='"level<100"'
97-
98-
[lc_filter_options_cmd-end]
99-
100-
[lc_cuda_filter-start]
101-
CUDA Filter
102-
...........
103-
104-
When building Spheral with CUDA support and without MPI support on Blue-OS systems we need to pass --smpi_off to ATS. When using spheral-atstest this flag is injected for you. However, for veboseness the flag is:
105-
::
106-
107-
--addOp "--smpi_off"
108-
109-
.. note::
110-
We need to pass ``--addOp`` as a way to forward the ``-smpi_off`` option to ATS. This is because spheral-atstest uses an intermediate layer to control ATS execution.
36+
./spheral-ats tests/integration.ats
11137

112-
If you are using ATS directly you only need to pass:
38+
For more information about using ``spheral-ats``, run the following from the install directory:
11339
::
11440

115-
--smpi_off
41+
./spheral-ats --help
11642

117-
[lc_cuda_filter-end]
43+
[ats_test-end]

‎docs/build_guide/lc/quickstart.rst

+2-2
Original file line numberDiff line numberDiff line change
@@ -20,5 +20,5 @@ Quickstart
2020
:end-before: [lc_install-section-end]
2121

2222
.. include:: ../include/quickstart.rst.inc
23-
:start-after: [lc_test-section-start]
24-
:end-before: [lc_test-section-end]
23+
:start-after: [test-section-start]
24+
:end-before: [test-section-end]

‎docs/build_guide/lc/tests.rst

+2-14
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,5 @@ Running Tests
66
:end-before: [smoke_test-end]
77

88
.. include:: ../include/tests.rst.inc
9-
:start-after: [lc_ats_test-start]
10-
:end-before: [lc_ats_test-end]
11-
12-
.. include:: ../include/tests.rst.inc
13-
:start-after: [filter_options-start]
14-
:end-before: [filter_options-end]
15-
16-
.. include:: ../include/tests.rst.inc
17-
:start-after: [lc_filter_options_cmd-start]
18-
:end-before: [lc_filter_options_cmd-end]
19-
20-
.. include:: ../include/tests.rst.inc
21-
:start-after: [lc_cuda_filter-start]
22-
:end-before: [lc_cuda_filter-end]
9+
:start-after: [ats_test-start]
10+
:end-before: [ats_test-end]

0 commit comments

Comments
 (0)