Skip to content

Commit 062b450

Browse files
NickeZNickeZ
authored
Merge pull request #1278 from NickeZ/nickez/build-container-in-ci
Nickez/build container in ci
2 parents 8b0d8a0 + 51bc17b commit 062b450

14 files changed

+171
-62
lines changed

‎.ci/README.md

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
CI Design guidelines
2+
3+
* It is more maintainable to create scripts in `.ci` and then call them from the workflows than to
4+
have scripts inline in the workflows. However, it is also good to split up scripts in multiple
5+
steps and jobs depending on what is being done.
6+
7+
* The docker image is rebuilt if the `Dockerfile` or `.containerversion` file is modified. (In case
8+
of a push event it is also automatically published to docker hub).
9+
10+
* If there are changes in the `Dockerfile`, then `.containerversion` must be updated with an
11+
unpublished version number.
12+
13+
* We listen to two kinds of events, `pull_request` and `push` using two different workflows,
14+
`pr-ci.yml` and `ci.yml`.
15+
* On pull request events, github will checkout a version of the tree that is the PR branch merged
16+
into the base branch. When we look for what is modifed we can diff HEAD^1 to HEAD. If github
17+
didn't do this, it would've missed commits added to the base branch since the PR branch was
18+
forked.
19+
20+
o--o--o--o <-- (base branch, typically 'master', parent 1)
21+
\ \
22+
\ o <-- (HEAD)
23+
\ /
24+
o----o <-- Pull requst branch (parent 2)
25+
26+
* On push events we get hashes of last commit before and after the push. When we look for what
27+
changed we can diff github.event.before with HEAD.
28+
29+
o--o--o--o--o--o <-- github.event.after (HEAD)
30+
\
31+
github.event.before

‎.ci/build-container

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
CONTAINER_REPO=shiftcrypto/firmware_v2
6+
CONTAINER_VERSION=$(cat .containerversion)
7+
8+
docker build --pull --no-cache -t $CONTAINER_REPO:latest -t $CONTAINER_REPO:$CONTAINER_VERSION .

‎.ci/check-container-sources-modified

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
#
3+
# This script works on merge commits. <rev>^1 means the first parent of <rev>.
4+
#
5+
# When the github action creates a temporary merge commit for a pull request, the first parent will
6+
# be the base (the branch being merged into).
7+
8+
set -e
9+
10+
if git diff --name-only HEAD^1 HEAD | grep -E '^(\.containerversion|Dockerfile)' >/dev/null; then
11+
echo "true"
12+
exit
13+
fi
14+
echo "false"

‎.ci/check-container-version-published

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
CONTAINER_REPO=shiftcrypto/firmware_v2
6+
CONTAINER_VERSION=$(cat .containerversion)
7+
8+
# docker manifest returns 1 (error) if the container doesn't exist and 0 (success) if it does.
9+
if docker manifest inspect $CONTAINER_REPO:$CONTAINER_VERSION > /dev/null; then
10+
>&2 echo Container version \'$CONTAINER_VERSION\' exists.
11+
echo true
12+
exit
13+
fi
14+
echo false

‎.ci/check-pep8

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ set -o pipefail
1111
command -v git >/dev/null 2>&1 || { echo >&2 "git is missing"; exit 1; }
1212

1313
# grep will exit with 1 if no lines are found
14-
FILES=$(git --no-pager diff --diff-filter=d --name-only ${TARGET_BRANCH} | grep -v -e "old/" -e "generated/" -e "rust/vendor/" | grep -E ".py\$" || exit 0)
14+
FILES=$(git --no-pager diff --diff-filter=d --name-only ${TARGET_BRANCH} HEAD | grep -v -e "old/" -e "generated/" -e "rust/vendor/" | grep -E ".py\$" || exit 0)
1515
if [ -z "${FILES}" ] ; then
1616
exit 0
1717
fi

‎.ci/check-style

+2-2
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,10 @@ if test -t 1; then
2525
fi
2626
fi
2727

28-
if git --no-pager diff --diff-filter=d --name-only ${TARGET_BRANCH} | grep -v -E "(^src/(rust|ui/fonts)|.*ugui.*|.*base32.*)" | grep -E "^(src|test)" | grep -E "\.(c|h)\$" | xargs -n1 "$CLANGFORMAT" -output-replacements-xml | grep -c "<replacement " >/dev/null; then
28+
if git --no-pager diff --diff-filter=d --name-only ${TARGET_BRANCH} HEAD | grep -v -E "(^src/(rust|ui/fonts)|.*ugui.*|.*base32.*)" | grep -E "^(src|test)" | grep -E "\.(c|h)\$" | xargs -n1 "$CLANGFORMAT" -output-replacements-xml | grep -c "<replacement " >/dev/null; then
2929
echo -e "${red}Not $CLANGFORMAT clean${normal}"
3030
# Apply CF to the files
31-
git --no-pager diff --diff-filter=d --name-only ${TARGET_BRANCH} | grep -v -E "(^src/(rust|ui/fonts)|.*ugui.*|.*base32.*)" | grep -E "^(src|test)" | grep -E "\.(c|h)\$" | xargs -n1 "$CLANGFORMAT" -i
31+
git --no-pager diff --diff-filter=d --name-only ${TARGET_BRANCH} HEAD | grep -v -E "(^src/(rust|ui/fonts)|.*ugui.*|.*base32.*)" | grep -E "^(src|test)" | grep -E "\.(c|h)\$" | xargs -n1 "$CLANGFORMAT" -i
3232
# Print list of files that weren't formatted correctly
3333
echo -e "Incorrectly formatted files:"
3434
git --no-pager diff --name-only

‎.ci/check-tidy

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ for dir in build build-build; do
3636
s/-Wno-cast-function-type//g; s/-mfpu=fpv4-sp-d16//g; s/-Wformat-signedness//g' ${dir}/compile_commands.json
3737

3838
# Only check our files
39-
SOURCES1=$(git --no-pager diff --diff-filter=d --name-only ${TARGET_BRANCH} |\
39+
SOURCES1=$(git --no-pager diff --diff-filter=d --name-only ${TARGET_BRANCH} HEAD |\
4040
grep -v -E "(^src/(drivers|ui/fonts)|.*ugui.*|.*base32.*)" |\
4141
grep -E "^(src)" |\
4242
grep -v "^test/unit-test/u2f/" |\

‎.ci/publish-container

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
CONTAINER_REPO=shiftcrypto/firmware_v2
6+
CONTAINER_VERSION=$(cat .containerversion)
7+
8+
docker push $CONTAINER_REPO:latest
9+
docker push $CONTAINER_REPO:$CONTAINER_VERSION

‎.ci/pull-container

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
CONTAINER_REPO=shiftcrypto/firmware_v2
6+
CONTAINER_VERSION=$(cat .containerversion)
7+
8+
docker pull $CONTAINER_REPO:$CONTAINER_VERSION

‎.ci/run-container-ci

+4-21
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,9 @@
1616

1717
# The script runs all CI builds and checks in a Docker container.
1818
# It accepts two positional arguments:
19-
# 1. A workspace dir, the root of the git repo clone, or "pull" literal.
20-
# In the latter case, CI container image is pulled from a registry.
21-
# 2. An optional target branch for code style diffs. Defaults to "master" for
22-
# push commits and overwritten with TRAVIS_BRANCH env var for pull requests
23-
# when run on Travis CI.
19+
# 1. A workspace dir, the root of the git repo clone, to be mounted in the container.
20+
# 2. A git revision (see man gitrevisions) to compare against HEAD to filter out modified and new
21+
# files. Some scripts only run on that subset.
2422

2523
set -e
2624
set -x
@@ -29,28 +27,13 @@ CONTAINER_REPO=shiftcrypto/firmware_v2
2927
CONTAINER_VERSION=$(cat .containerversion)
3028
CONTAINER=$CONTAINER_REPO:${CONTAINER_VERSION}
3129

32-
if [ "$1" == "pull" ] ; then
33-
docker pull "$CONTAINER"
34-
exit 0
35-
fi
36-
3730
WORKSPACE_DIR="$1"
3831
if [ -z "${WORKSPACE_DIR}" ]; then
3932
echo "Workspace dir path is empty."
4033
exit 1
4134
fi
4235

43-
TARGET_BRANCH="${2:-master}"
44-
if [ "${TRAVIS}" == "true" ] && [ "${TRAVIS_PULL_REQUEST}" != "false" ] ; then
45-
TARGET_BRANCH=${TRAVIS_BRANCH}
46-
fi
47-
48-
# Fetch origin/master so that we can diff when checking coding style.
49-
git remote set-branches --add origin ${TARGET_BRANCH}
50-
git fetch origin
51-
52-
TARGET_BRANCH=origin/${TARGET_BRANCH}
53-
36+
TARGET_BRANCH="$2"
5437
# The safe.directory config is so that git commands work. even though the repo folder mounted in
5538
# Docker is owned by root, which can be different from the owner on the host.
5639
docker run -e TARGET_BRANCH="${TARGET_BRANCH}" \

‎.github/actions/pr-ci-common/action.yml

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: Pull request CI common
2+
3+
inputs:
4+
base-sha:
5+
required: true
6+
runs:
7+
using: "composite"
8+
steps:
9+
- name: Check if container files was modified and if container version already exists
10+
id: checks
11+
shell: bash
12+
run: |
13+
echo modified=$(./.ci/check-container-sources-modified) >> "$GITHUB_OUTPUT"
14+
echo container-published=$(./.ci/check-container-version-published) >> "$GITHUB_OUTPUT"
15+
16+
- name: Build container image
17+
if: steps.checks.outputs.modified == 'true'
18+
shell: bash
19+
run: |
20+
if "${{ steps.checks.outputs.container-published }}" == "true"; then
21+
echo "::error::Container modified but version $(cat .containerversion) already published"
22+
exit 1
23+
fi
24+
./.ci/build-container
25+
26+
- name: Pull container image
27+
if: steps.checks.outputs.modified == 'false'
28+
shell: bash
29+
run: ./.ci/pull-container
30+
31+
- name: Run CI in container
32+
shell: bash
33+
run: ./.ci/run-container-ci ${{github.workspace}} ${{ inputs.base-sha }}

‎.github/workflows/ci.yml

+26-3
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,37 @@ on:
88
- master
99

1010
jobs:
11-
linux-docker:
11+
ci:
1212
runs-on: ubuntu-22.04
1313
steps:
1414
- name: Clone the repo
1515
uses: actions/checkout@v4
1616
with:
17+
fetch-depth: 0
18+
fetch-tags: true
1719
submodules: recursive
20+
21+
- name: Check if container should be published
22+
id: checks
23+
run: echo container-published=$(./.ci/check-container-version-published) >> $GITHUB_OUTPUT
24+
25+
- name: Build container
26+
if: steps.checks.outputs.container-published == 'false'
27+
run: ./.ci/build-container
28+
29+
- name: Login to Docker Hub
30+
uses: docker/login-action@v3
31+
with:
32+
username: ${{ secrets.DOCKER_USERNAME }}
33+
password: ${{ secrets.DOCKER_TOKEN }}
34+
35+
- name: Publish container
36+
if: steps.checks.outputs.container-published == 'false'
37+
run: ./.ci/publish-container
38+
1839
- name: Pull CI container image
19-
run: ./.ci/run-container-ci pull
40+
if: steps.checks.outputs.container-published == 'true'
41+
run: ./.ci/pull-container
42+
2043
- name: Run CI in container
21-
run: ./.ci/run-container-ci ${{github.workspace}}
44+
run: ./.ci/run-container-ci ${{github.workspace}} ${{ github.event.before }}

‎.github/workflows/pr-ci.yml

+20-13
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,12 @@ jobs:
1212
uses: actions/checkout@v4
1313
with:
1414
submodules: recursive
15+
fetch-depth: 0
1516

16-
- name: Pull container image
17-
run: ./.ci/run-container-ci pull
18-
19-
- name: Run CI in container
20-
run: ./.ci/run-container-ci ${{github.workspace}} ${{ github.base_ref }}
17+
- name: CI
18+
uses: ./.github/actions/pr-ci-common
19+
with:
20+
base-sha: ${{ github.event.pull_request.base.sha }}
2121

2222
# Generate a list of commits to run CI on
2323
generate-matrix:
@@ -34,9 +34,15 @@ jobs:
3434
- name: Create jobs for commits in PR history
3535
id: set-matrix
3636
run: |
37-
echo matrix=$(.ci/matrix-from-commit-log origin/${{github.base_ref}}..${{ github.event.pull_request.head.sha}}~) >> $GITHUB_OUTPUT
37+
echo matrix=$(.ci/matrix-from-commit-log ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}~) >> $GITHUB_OUTPUT
3838
3939
# Run this job for every commit in the PR except HEAD.
40+
# This job simulates what github does for the PR HEAD commit but for every other commit in the
41+
# PR. So for every commit, it creates a merge commit between that commit and the base branch.
42+
# Then it runs the CI on that merge commit.
43+
# The only caveat is that this file (pr-ci.yml) is already loaded from the PR HEAD merge commit,
44+
# and therefore we need to load the `.ci` scripts from the PR HEAD merge commit. The outcome of
45+
# that is that changes to the CI is not tested per commit. All commits use the final version.
4046
pr-commit-ci:
4147
runs-on: ubuntu-22.04
4248
needs: [ generate-matrix ]
@@ -58,13 +64,14 @@ jobs:
5864
GIT_COMMITTER_NAME: Bot
5965
GIT_COMMITTER_EMAIL: [email protected]
6066
run: |
61-
git fetch origin ${{ matrix.commit }}
67+
git fetch origin ${{ matrix.commit }} ${{ github.event.pull_request.merge_commit_sha }}
6268
git merge --no-ff --no-edit ${{ matrix.commit }}
63-
echo "merge commit parents:"
6469
git log -1 --format="Head %H, Parents %P"
70+
# Since the workflow definition is taken from the pull request merge commit, we need to
71+
# get the .ci scripts from there as well.
72+
git checkout -f ${{ github.event.pull_request.merge_commit_sha }} -- .ci .github
6573
66-
- name: Pull container image
67-
run: ./.ci/run-container-ci pull
68-
69-
- name: Run CI in container
70-
run: ./.ci/run-container-ci ${{github.workspace}} ${{ github.base_ref }}
74+
- name: CI
75+
uses: ./.github/actions/pr-ci-common
76+
with:
77+
base-sha: ${{ github.event.pull_request.base.sha }}

‎.travis.yml

-21
This file was deleted.

0 commit comments

Comments
 (0)