Skip to content

Commit a6142b1

Browse files
ahmadelyousseffacebook-github-bot
ahmadelyoussef
authored and
facebook-github-bot
committed
Implmenting CI workflow for wdl, syscall and health_check (#122)
Summary: Pull Request resolved: #122 Implmenting CI workflow for wdl, syscall and health_check Reviewed By: excelle08 Differential Revision: D75171333 fbshipit-source-id: 00e536584c04ca99c1ae0dc2587fbafcfbe9b6ac
1 parent 530fe73 commit a6142b1

File tree

10 files changed

+428
-10
lines changed

10 files changed

+428
-10
lines changed

‎.github/scripts/dcperf_install_health_check.bash‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Install Health Check
15+
################################################################################
16+
17+
install_health_check () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Install Health Check"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
36+
echo "[INSTALL] Installing Health Check ..."
37+
(print_exec conda run --no-capture-output ${env_prefix} \
38+
python ./benchpress_cli.py install health_check) || return 1
39+
}

‎.github/scripts/dcperf_install_syscall.bash‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Install wdl
15+
################################################################################
16+
17+
install_syscall () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Install Syscall"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
36+
echo "[INSTALL] Installing Syscall ..."
37+
(print_exec conda run --no-capture-output ${env_prefix} \
38+
python ./benchpress_cli.py -b system install syscall_autoscale) || return 1
39+
}

‎.github/scripts/dcperf_install_wdl.bash‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Install wdl
15+
################################################################################
16+
17+
install_wdl () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Install wdl"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
36+
echo "[INSTALL] Installing wdl ..."
37+
(print_exec conda run --no-capture-output ${env_prefix} \
38+
python ./benchpress_cli.py -b wdl install folly_single_core) || return 1
39+
}

‎.github/scripts/setup_env.bash‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,9 @@
2525
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_video_transcode_bench.bash"
2626
# shellcheck disable=SC1091,SC2128
2727
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_mediawiki.bash"
28+
# shellcheck disable=SC1091,SC2128
29+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_wdl.bash"
30+
# shellcheck disable=SC1091,SC2128
31+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_health_check.bash"
32+
# shellcheck disable=SC1091,SC2128
33+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_syscall.bash"

‎.github/workflows/dcperf_ci_health_check.yml‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
# This workflow is used for DCPerf CI.
7+
name: DCPerf Health_check CI
8+
9+
on:
10+
# PR Trigger
11+
#
12+
pull_request:
13+
14+
# Push Trigger (enable to catch errors coming out of multiple merges)
15+
#
16+
push:
17+
branches:
18+
- main
19+
20+
# Manual Trigger
21+
#
22+
workflow_dispatch:
23+
24+
concurrency:
25+
# Cancel previous runs in the PR if a new commit is pushed
26+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
27+
cancel-in-progress: true
28+
29+
jobs:
30+
# Build on CPU hosts (generic GitHub runners)
31+
build_artifact:
32+
runs-on: ${{ matrix.host-machine.instance }}
33+
container:
34+
image: ${{ matrix.host-machine.container }}
35+
options: --user root --privileged --pid=host
36+
volumes:
37+
- /var/run/docker.sock:/var/run/docker.sock
38+
defaults:
39+
run:
40+
shell: bash
41+
env:
42+
PRELUDE: .github/scripts/setup_env.bash
43+
BUILD_ENV: build_env
44+
continue-on-error: false
45+
strategy:
46+
fail-fast: false
47+
matrix:
48+
host-machine: [
49+
{ arch: x86, instance: "ubuntu-latest", container: "ubuntu:22.04", type: "ubuntu" },
50+
{ arch: aarch64, instance: "ubuntu-24.04-arm", container: "ubuntu:22.04", type: "ubuntu" },
51+
]
52+
python-version: [ "3.12" ]
53+
steps:
54+
- name: Setup Build Container (ubuntu-based)
55+
if: ${{ matrix.host-machine.type == 'ubuntu' }}
56+
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode
57+
58+
- name: Setup Build Container (centos-based)
59+
if: ${{ matrix.host-machine.type == 'centos' }}
60+
run: |
61+
dnf update -y
62+
dnf install -y git pciutils which dmidecode sudo shadow-utils passwd
63+
64+
- name: Checkout the Repository
65+
uses: actions/checkout@v4
66+
with:
67+
submodules: true
68+
69+
- name: Fix PAM configuration for sudo
70+
if: ${{ matrix.host-machine.type == 'centos' }}
71+
run: |
72+
echo "auth sufficient pam_rootok.so" > /etc/pam.d/sudo
73+
74+
- name: Enable Sudo without password
75+
if: ${{ matrix.host-machine.type == 'centos' }}
76+
run: |
77+
echo "root ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers
78+
79+
- name: Unlock root user account
80+
if: ${{ matrix.host-machine.type == 'centos' }}
81+
run: |
82+
passwd -u root
83+
84+
- name: Free Disk Space on Host
85+
run: . $PRELUDE; free_disk_space_on_host
86+
87+
- name: Display System Info
88+
run: . $PRELUDE; print_system_info
89+
90+
- name: Install Build Tools
91+
run: . $PRELUDE; install_build_tools
92+
93+
- name: Setup Miniconda
94+
run: . $PRELUDE; setup_miniconda $HOME/miniconda
95+
96+
- name: Create Conda Environment
97+
run: . $PRELUDE; create_conda_environment $BUILD_ENV ${{ matrix.python-version }}
98+
99+
- name: Install Python Tools
100+
run: . $PRELUDE; install_python_tools $BUILD_ENV
101+
102+
- name: Install Health_check
103+
run: . $PRELUDE; install_health_check $BUILD_ENV

‎.github/workflows/dcperf_ci_syscall.yml‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
# This workflow is used for DCPerf CI.
7+
name: DCPerf Syscall CI
8+
9+
on:
10+
# PR Trigger
11+
#
12+
pull_request:
13+
14+
# Push Trigger (enable to catch errors coming out of multiple merges)
15+
#
16+
push:
17+
branches:
18+
- main
19+
20+
# Manual Trigger
21+
#
22+
workflow_dispatch:
23+
24+
concurrency:
25+
# Cancel previous runs in the PR if a new commit is pushed
26+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
27+
cancel-in-progress: true
28+
29+
jobs:
30+
# Build on CPU hosts (generic GitHub runners)
31+
build_artifact:
32+
runs-on: ${{ matrix.host-machine.instance }}
33+
container:
34+
image: ${{ matrix.host-machine.container }}
35+
options: --user root --privileged --pid=host
36+
volumes:
37+
- /var/run/docker.sock:/var/run/docker.sock
38+
defaults:
39+
run:
40+
shell: bash
41+
env:
42+
PRELUDE: .github/scripts/setup_env.bash
43+
BUILD_ENV: build_env
44+
continue-on-error: false
45+
strategy:
46+
fail-fast: false
47+
matrix:
48+
host-machine: [
49+
{ arch: x86, instance: "ubuntu-latest", container: "ubuntu:22.04", type: "ubuntu" },
50+
{ arch: aarch64, instance: "ubuntu-24.04-arm", container: "ubuntu:22.04", type: "ubuntu" },
51+
{ arch: x86, instance: "ubuntu-latest", container: "quay.io/centos/centos:stream9", type: "centos" },
52+
{ arch: aarch64, instance: "ubuntu-24.04-arm", container: "quay.io/centos/centos:stream9", type: "centos" },
53+
]
54+
python-version: [ "3.12" ]
55+
steps:
56+
- name: Setup Build Container (ubuntu-based)
57+
if: ${{ matrix.host-machine.type == 'ubuntu' }}
58+
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode pkgconf
59+
60+
- name: Setup Build Container (centos-based)
61+
if: ${{ matrix.host-machine.type == 'centos' }}
62+
run: dnf update -y; dnf install -y git pciutils which dmidecode
63+
64+
- name: Checkout the Repository
65+
uses: actions/checkout@v4
66+
with:
67+
submodules: true
68+
69+
- name: Free Disk Space on Host
70+
run: . $PRELUDE; free_disk_space_on_host
71+
72+
- name: Display System Info
73+
run: . $PRELUDE; print_system_info
74+
75+
- name: Install Build Tools
76+
run: . $PRELUDE; install_build_tools
77+
78+
- name: Setup Miniconda
79+
run: . $PRELUDE; setup_miniconda $HOME/miniconda
80+
81+
- name: Create Conda Environment
82+
run: . $PRELUDE; create_conda_environment $BUILD_ENV ${{ matrix.python-version }}
83+
84+
- name: Install Python Tools
85+
run: . $PRELUDE; install_python_tools $BUILD_ENV
86+
87+
- name: Install Syscall
88+
run: . $PRELUDE; install_syscall $BUILD_ENV
89+
continue-on-error: false

0 commit comments

Comments
 (0)