Skip to content

Commit 92233a9

Browse files
authored
Merge pull request #187 from hmeiland/feature-archdetect
add support in init script for using 'archdetect' alternative to archspec (only if $EESSI_USE_ARCHDETECT is set to '1')
2 parents f4f2fb4 + 2d77d74 commit 92233a9

22 files changed

+363
-3
lines changed
+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
2+
name: Tests for eessi_archdetect.sh
3+
on: [push, pull_request]
4+
jobs:
5+
build:
6+
runs-on: ubuntu-20.04
7+
strategy:
8+
matrix:
9+
proc_cpuinfo:
10+
- x86_64/intel/haswell/archspec-linux-E5-2680-v3
11+
- x86_64/intel/skylake_avx512/archspec-linux-6132
12+
- x86_64/amd/zen2/Azure-CentOS7-7V12
13+
- x86_64/amd/zen3/Azure-CentOS7-7V73X
14+
- ppc64le/power9le/unknown-power9le
15+
- aarch64/arm/neoverse-n1/Azure-Ubuntu20-Altra
16+
- aarch64/arm/neoverse-n1/AWS-awslinux-graviton2
17+
- aarch64/arm/neoverse-v1/AWS-awslinux-graviton3
18+
fail-fast: false
19+
steps:
20+
- uses: actions/checkout@v2
21+
22+
- name: test eessi_archdetect.sh
23+
run: |
24+
export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}}
25+
export EESSI_MACHINE_TYPE=${EESSI_MACHINE_TYPE%%/*}
26+
export EESSI_PROC_CPUINFO=./tests/archdetect/${{matrix.proc_cpuinfo}}.cpuinfo
27+
CPU_ARCH=$(./init/eessi_archdetect.sh cpupath)
28+
if [[ $CPU_ARCH == "$( cat ./tests/archdetect/${{matrix.proc_cpuinfo}}.output )" ]]; then
29+
echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCH" >&2
30+
else
31+
echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCH" >&2
32+
exit 1
33+
fi

init/arch_specs/eessi_arch_arm.spec

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# ARM CPU architecture specifications
2+
# Software path in EESSI | Vendor ID | List of defining CPU features
3+
"aarch64/arm/neoverse-n1" "ARM" "asimd" # Ampere Altra
4+
"aarch64/arm/neoverse-n1" "" "asimd" # AWS Graviton2
5+
"aarch64/arm/neoverse-v1" "ARM" "asimd svei8mm"
6+
"aarch64/arm/neoverse-v1" "" "asimd svei8mm" # AWS Graviton3

init/arch_specs/eessi_arch_ppc.spec

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# POWER CPU architecture specifications
2+
# Software path in EESSI | Vendor ID | List of defining CPU features
3+
"ppc64le/power9le" "" "POWER9" # IBM Power9

init/arch_specs/eessi_arch_x86.spec

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# x86_64 CPU architecture specifications
2+
# Software path in EESSI | Vendor ID | List of defining CPU features
3+
"x86_64/intel/haswell" "GenuineIntel" "avx2 fma" # Intel Haswell, Broadwell
4+
"x86_64/intel/skylake_avx512" "GenuineIntel" "avx2 fma avx512f avx512bw avx512cd avx512dq avx512vl" # Intel Skylake, Cascade Lake
5+
"x86_64/amd/zen2" "AuthenticAMD" "avx2 fma" # AMD Rome
6+
"x86_64/amd/zen3" "AuthenticAMD" "avx2 fma vaes" # AMD Milan, Milan-X

init/eessi_archdetect.sh

+143
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
#!/usr/bin/env bash
2+
VERSION="1.0.0"
3+
4+
# Logging
5+
LOG_LEVEL="INFO"
6+
7+
timestamp () {
8+
date "+%Y-%m-%d %H:%M:%S"
9+
}
10+
11+
log () {
12+
# Simple logger function
13+
declare -A levels=([DEBUG]=0 [INFO]=1 [WARN]=2 [ERROR]=3)
14+
msg_type="${1:-INFO}"
15+
msg_body="${2:-'null'}"
16+
17+
[ ${levels[$msg_type]} ] || log "ERROR" "Unknown log level $msg_type"
18+
19+
# ignore messages below log level
20+
[ ${levels[$msg_type]} -lt ${levels[$LOG_LEVEL]} ] && return 0
21+
# print log message to standard error
22+
echo "$(timestamp) [$msg_type] $msg_body" >&2
23+
# exit after any error message
24+
[ $msg_type == "ERROR" ] && exit 1
25+
}
26+
27+
# Supported CPU specifications
28+
update_arch_specs(){
29+
# Add contents of given spec file into an array
30+
# 1: name of array with CPU arch specs
31+
# 2: spec file with the additional specs
32+
33+
[ -z "$1" ] && echo "[ERROR] update_arch_specs: missing array in argument list" >&2 && exit 1
34+
local -n arch_specs=$1
35+
36+
[ ! -f "$2" ] && echo "[ERROR] update_arch_specs: spec file not found: $2" >&2 && exit 1
37+
local spec_file="$2"
38+
while read spec_line; do
39+
# format spec line as an array and append it to array with all CPU arch specs
40+
arch_specs+=("(${spec_line})")
41+
# remove comments from spec file
42+
done < <(sed -E 's/(^|[\s\t])#.*$//g;/^\s*$/d' "$spec_file")
43+
}
44+
45+
# CPU specification of host system
46+
get_cpuinfo(){
47+
# Return the value from cpuinfo for the matching key
48+
# 1: string with key pattern
49+
50+
[ -z "$1" ] && log "ERROR" "get_cpuinfo: missing key pattern in argument list"
51+
cpuinfo_pattern="^${1}\s*:\s*"
52+
53+
# case insensitive match of key pattern and delete key pattern from result
54+
grep -i "$cpuinfo_pattern" ${EESSI_PROC_CPUINFO:-/proc/cpuinfo} | tail -n 1 | sed "s/$cpuinfo_pattern//i"
55+
}
56+
57+
check_allinfirst(){
58+
# Return true if all given arguments after the first are found in the first one
59+
# 1: reference string of space separated values
60+
# 2,3..: each additional argument is a single value to be found in the reference string
61+
62+
[ -z "$1" ] && log "ERROR" "check_allinfirst: missing argument with reference string"
63+
reference="$1"
64+
shift
65+
66+
for candidate in "$@"; do
67+
[[ " $reference " == *" $candidate "* ]] || return 1
68+
done
69+
return 0
70+
}
71+
72+
cpupath(){
73+
# Identify the best matching CPU architecture from a list of supported specifications for the host CPU
74+
# Return the path to the installation files in EESSI of the best matching architecture
75+
local cpu_arch_spec=()
76+
77+
# Identify the host CPU architecture
78+
local machine_type=${EESSI_MACHINE_TYPE:-$(uname -m)}
79+
log "DEBUG" "cpupath: Host CPU architecture identified as '$machine_type'"
80+
81+
# Populate list of supported specs for this architecture
82+
case $machine_type in
83+
"x86_64") local spec_file="eessi_arch_x86.spec";;
84+
"aarch64") local spec_file="eessi_arch_arm.spec";;
85+
"ppc64le") local spec_file="eessi_arch_ppc.spec";;
86+
*) log "ERROR" "cpupath: Unsupported CPU architecture $machine_type"
87+
esac
88+
# spec files are located in a subfolder with this script
89+
local base_dir=$(dirname $(realpath $0))
90+
update_arch_specs cpu_arch_spec "$base_dir/arch_specs/${spec_file}"
91+
92+
# Identify the host CPU vendor
93+
local cpu_vendor_tag="vendor[ _]id"
94+
local cpu_vendor=$(get_cpuinfo "$cpu_vendor_tag")
95+
log "DEBUG" "cpupath: CPU vendor of host system: '$cpu_vendor'"
96+
97+
# Identify the host CPU flags or features
98+
local cpu_flag_tag='flags'
99+
# cpuinfo systems print different line identifiers, eg features, instead of flags
100+
[ "${cpu_vendor}" == "ARM" ] && cpu_flag_tag='flags'
101+
[ "${machine_type}" == "aarch64" ] && [ "${cpu_vendor}x" == "x" ] && cpu_flag_tag='features'
102+
[ "${machine_type}" == "ppc64le" ] && cpu_flag_tag='cpu'
103+
104+
local cpu_flags=$(get_cpuinfo "$cpu_flag_tag")
105+
log "DEBUG" "cpupath: CPU flags of host system: '$cpu_flags'"
106+
107+
# Default to generic CPU
108+
local best_arch_match="generic"
109+
110+
# Iterate over the supported CPU specifications to find the best match for host CPU
111+
# Order of the specifications matters, the last one to match will be selected
112+
for arch in "${cpu_arch_spec[@]}"; do
113+
eval "arch_spec=$arch"
114+
if [ "${cpu_vendor}x" == "${arch_spec[1]}x" ]; then
115+
# each flag in this CPU specification must be found in the list of flags of the host
116+
check_allinfirst "${cpu_flags[*]}" ${arch_spec[2]} && best_arch_match=${arch_spec[0]} && \
117+
log "DEBUG" "cpupath: host CPU best match updated to $best_arch_match"
118+
fi
119+
done
120+
121+
log "INFO" "cpupath: best match for host CPU: $best_arch_match"
122+
echo "$best_arch_match"
123+
}
124+
125+
# Parse command line arguments
126+
USAGE="Usage: eessi_archdetect.sh [-h][-d] <action>"
127+
128+
while getopts 'hdv' OPTION; do
129+
case "$OPTION" in
130+
h) echo "$USAGE"; exit 0;;
131+
d) LOG_LEVEL="DEBUG";;
132+
v) echo "eessi_archdetect.sh v$VERSION"; exit 0;;
133+
?) echo "$USAGE"; exit 1;;
134+
esac
135+
done
136+
shift "$(($OPTIND -1))"
137+
138+
ARGUMENT=${1:-none}
139+
140+
case "$ARGUMENT" in
141+
"cpupath") cpupath; exit;;
142+
*) echo "$USAGE"; log "ERROR" "Missing <action> argument";;
143+
esac

init/eessi_environment_variables

+10-3
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,16 @@ if [ -d $EESSI_PREFIX ]; then
2525
if [ -d $EESSI_EPREFIX ]; then
2626

2727
# determine subdirectory in software layer
28-
# note: eessi_software_subdir_for_host.py will pick up value from $EESSI_SOFTWARE_SUBDIR_OVERRIDE if it's defined!
29-
export EESSI_EPREFIX_PYTHON=$EESSI_EPREFIX/usr/bin/python3
30-
export EESSI_SOFTWARE_SUBDIR=$($EESSI_EPREFIX_PYTHON ${EESSI_INIT_DIR_PATH}/eessi_software_subdir_for_host.py $EESSI_PREFIX)
28+
if [ "$EESSI_USE_ARCHDETECT" == "1" ]; then
29+
# if archdetect is enabled, use internal code
30+
export EESSI_SOFTWARE_SUBDIR=$(${EESSI_INIT_DIR_PATH}/eessi_archdetect.sh cpupath)
31+
echo "archdetect says ${EESSI_SOFTWARE_SUBDIR}" >> $output
32+
else
33+
# note: eessi_software_subdir_for_host.py will pick up value from $EESSI_SOFTWARE_SUBDIR_OVERRIDE if it's defined!
34+
export EESSI_EPREFIX_PYTHON=$EESSI_EPREFIX/usr/bin/python3
35+
export EESSI_SOFTWARE_SUBDIR=$($EESSI_EPREFIX_PYTHON ${EESSI_INIT_DIR_PATH}/eessi_software_subdir_for_host.py $EESSI_PREFIX)
36+
echo "archspec says ${EESSI_SOFTWARE_SUBDIR}" >> $output
37+
fi
3138
if [ ! -z $EESSI_SOFTWARE_SUBDIR ]; then
3239

3340
echo "Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory." >> $output
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
processor : 0
2+
BogoMIPS : 243.75
3+
Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm lrcpc dcpop asimddp ssbs
4+
CPU implementer : 0x41
5+
CPU architecture: 8
6+
CPU variant : 0x3
7+
CPU part : 0xd0c
8+
CPU revision : 1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
aarch64/arm/neoverse-n1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
Architecture: aarch64
2+
CPU op-mode(s): 32-bit, 64-bit
3+
Byte Order: Little Endian
4+
CPU(s): 64
5+
On-line CPU(s) list: 0-63
6+
Thread(s) per core: 1
7+
Core(s) per socket: 64
8+
Socket(s): 1
9+
NUMA node(s): 1
10+
Vendor ID: ARM
11+
Model: 1
12+
Model name: Neoverse-N1
13+
Stepping: r3p1
14+
BogoMIPS: 50.00
15+
L1d cache: 4 MiB
16+
L1i cache: 4 MiB
17+
L2 cache: 64 MiB
18+
L3 cache: 32 MiB
19+
NUMA node0 CPU(s): 0-63
20+
Vulnerability Itlb multihit: Not affected
21+
Vulnerability L1tf: Not affected
22+
Vulnerability Mds: Not affected
23+
Vulnerability Meltdown: Mitigation; PTI
24+
Vulnerability Mmio stale data: Not affected
25+
Vulnerability Spec store bypass: Not affected
26+
Vulnerability Spectre v1: Mitigation; __user pointer sanitization
27+
Vulnerability Spectre v2: Mitigation; CSV2, BHB
28+
Vulnerability Srbds: Not affected
29+
Vulnerability Tsx async abort: Not affected
30+
Flags: fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm lrcpc dcpop asimddp
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
aarch64/arm/neoverse-n1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
processor : 0
2+
BogoMIPS : 2100.00
3+
Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm jscvt fcma lrcpc dcpop sha3 sm3 sm4 asimddp sha512 sve asimdfhm dit uscat ilrcpc flagm ssbs paca pacg dcpodp svei8mm svebf16 i8mm bf16 dgh rng
4+
CPU implementer : 0x41
5+
CPU architecture: 8
6+
CPU variant : 0x1
7+
CPU part : 0xd40
8+
CPU revision : 1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
aarch64/arm/neoverse-v1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
processor : 0
2+
cpu : POWER9 (architected), altivec supported
3+
clock : 2200.000000MHz
4+
revision : 2.2 (pvr 004e 1202)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ppc64le/power9le
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
Architecture: x86_64
2+
CPU op-mode(s): 32-bit, 64-bit
3+
Byte Order: Little Endian
4+
CPU(s): 120
5+
On-line CPU(s) list: 0-119
6+
Thread(s) per core: 1
7+
Core(s) per socket: 60
8+
Socket(s): 2
9+
NUMA node(s): 4
10+
Vendor ID: AuthenticAMD
11+
CPU family: 23
12+
Model: 49
13+
Model name: AMD EPYC 7V12 64-Core Processor
14+
Stepping: 0
15+
CPU MHz: 2445.424
16+
BogoMIPS: 4890.84
17+
Hypervisor vendor: Microsoft
18+
Virtualization type: full
19+
L1d cache: 32K
20+
L1i cache: 32K
21+
L2 cache: 512K
22+
L3 cache: 16384K
23+
NUMA node0 CPU(s): 0-29
24+
NUMA node1 CPU(s): 30-59
25+
NUMA node2 CPU(s): 60-89
26+
NUMA node3 CPU(s): 90-119
27+
Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm art rep_good nopl extd_apicid aperfmperf eagerfpu pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext retpoline_amd ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr arat umip
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
x86_64/amd/zen2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
Architecture: x86_64
2+
CPU op-mode(s): 32-bit, 64-bit
3+
Byte Order: Little Endian
4+
CPU(s): 120
5+
On-line CPU(s) list: 0-119
6+
Thread(s) per core: 1
7+
Core(s) per socket: 60
8+
Socket(s): 2
9+
NUMA node(s): 4
10+
Vendor ID: AuthenticAMD
11+
CPU family: 25
12+
Model: 1
13+
Model name: AMD EPYC 7V73X 64-Core Processor
14+
Stepping: 2
15+
CPU MHz: 1846.550
16+
BogoMIPS: 3693.10
17+
Hypervisor vendor: Microsoft
18+
Virtualization type: full
19+
L1d cache: 32K
20+
L1i cache: 32K
21+
L2 cache: 512K
22+
L3 cache: 98304K
23+
NUMA node0 CPU(s): 0-29
24+
NUMA node1 CPU(s): 30-59
25+
NUMA node2 CPU(s): 60-89
26+
NUMA node3 CPU(s): 90-119
27+
Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm art rep_good nopl extd_apicid aperfmperf eagerfpu pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single retpoline_amd vmmcall fsgsbase bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr arat umip vaes vpclmulqdq
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
x86_64/amd/zen3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
processor : 0
2+
vendor_id : GenuineIntel
3+
cpu family : 6
4+
model : 63
5+
model name : Intel(R) Xeon(R) CPU E5-2680 v3 @ 2.50GHz
6+
stepping : 2
7+
microcode : 0x3c
8+
cpu MHz : 1757.910
9+
cache size : 30720 KB
10+
physical id : 0
11+
siblings : 12
12+
core id : 0
13+
cpu cores : 12
14+
apicid : 0
15+
initial apicid : 0
16+
fpu : yes
17+
fpu_exception : yes
18+
cpuid level : 15
19+
wp : yes
20+
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf eagerfpu pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm epb invpcid_single tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm xsaveopt cqm_llc cqm_occup_llc ibpb ibrs stibp dtherm arat pln pts spec_ctrl intel_stibp
21+
bogomips : 4987.97
22+
clflush size : 64
23+
cache_alignment : 64
24+
address sizes : 46 bits physical, 48 bits virtual
25+
power management:
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
x86_64/intel/haswell
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
processor : 0
2+
vendor_id : GenuineIntel
3+
cpu family : 6
4+
model : 85
5+
model name : Intel(R) Xeon(R) Gold 6132 CPU @ 2.60GHz
6+
stepping : 4
7+
microcode : 0x200004d
8+
cpu MHz : 2600.000
9+
cache size : 19712 KB
10+
physical id : 0
11+
siblings : 14
12+
core id : 0
13+
cpu cores : 14
14+
apicid : 0
15+
initial apicid : 0
16+
fpu : yes
17+
fpu_exception : yes
18+
cpuid level : 22
19+
wp : yes
20+
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf eagerfpu pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch epb cat_l3 cdp_l3 invpcid_single intel_pt tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local ibpb ibrs stibp dtherm ida arat pln pts spec_ctrl intel_stibp ssbd
21+
bogomips : 5200.00
22+
clflush size : 64
23+
cache_alignment : 64
24+
address sizes : 46 bits physical, 48 bits virtual
25+
power management:
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
x86_64/intel/skylake_avx512

0 commit comments

Comments
 (0)