Skip to content

Commit c967beb

Browse files
NickeZNickeZ
committed
Make docker image easier to reproduce
1 parent d29e5a6 commit c967beb

9 files changed

+175
-103
lines changed

‎.ci/build-base-container

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
3+
docker build --no-cache -f Dockerfile.ubuntu .

‎.ci/build-dev-container

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
docker build --pull --no-cache -t shiftcrypto/firmware_v2 .
4+
docker pull shiftcrypto/firmware_v2:41
5+
6+
# Print out the differences but ignore them. We don't require reproducible builds right now.
7+
ocidiff shiftcrypto/firmware_v2:latest shiftcrypto/firmware_v2:41 || true

‎.ci/publish-container

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
CONTAINER_REPO=shiftcrypto/firmware_v2
6+
CONTAINER_VERSION=$(grep ^CONTAINER_VERSION Makefile | cut -d ' ' -f3)
7+
8+
if docker manifest inspect $CONTAINER_REPO:$CONTAINER_VERSION; then
9+
echo $CONTAINER_VERSION already exists, will not publish
10+
exit 1
11+
fi
12+
13+
docker build --no-cache -t $CONTAINER_REPO:latest -t $CONTAINER_REPO:$CONTAINER_VERSION .
14+
docker push $CONTAINER_REPO:latest $CONTAINER_REPO:$CONTAINER_VERSION

‎.ci/run-container-ci

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
set -e
2626
set -x
2727

28-
CONTAINER=shiftcrypto/firmware_v2:41
28+
CONTAINER=shiftcrypto/firmware_v2:42
2929

3030
if [ "$1" == "pull" ] ; then
3131
docker pull "$CONTAINER"

‎.github/workflows/container-images.yml

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: Container nightly check
2+
on:
3+
workflow_dispatch:
4+
schedule:
5+
- cron: '0 2 * * *'
6+
env:
7+
DIFFOCI_URL: https://github.com/reproducible-containers/diffoci/releases/download/v0.1.4/diffoci-v0.1.4.linux-amd64
8+
9+
jobs:
10+
check-base-image:
11+
runs-on: ubuntu-22.04
12+
steps:
13+
- uses: actions/checkout@v4
14+
- name: Fetch diffoci
15+
run: curl -L $DIFFOCI_URL -o /usr/local/bin/diffoci
16+
- name: Check that base container builds
17+
run: ./.ci/build-base-container
18+
19+
check-dev-image:
20+
runs-on: ubuntu-22.04
21+
steps:
22+
- uses: actions/checkout@v4
23+
- name: Fetch diffoci
24+
run: curl -L $DIFFOCI_URL -o /usr/local/bin/diffoci
25+
- name: Check that dev container builds
26+
run: ./.ci/build-dev-container

‎.github/workflows/publish-container-image.yml

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
on:
2+
workflow_dispatch:
3+
push:
4+
branches:
5+
- master
6+
paths:
7+
- Dockerfile
8+
jobs:
9+
check-base-image:
10+
runs-on: ubuntu-22.04
11+
steps:
12+
- uses: actions/checkout@v4
13+
- name: Publish new container version
14+
run: ./.ci/publish-container

‎Dockerfile

+40-98
Original file line numberDiff line numberDiff line change
@@ -13,136 +13,78 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515

16-
# Latest Ubuntu LTS
17-
FROM ubuntu:22.04
18-
ENV DEBIAN_FRONTEND noninteractive
19-
20-
RUN apt-get update && apt-get upgrade -y && apt-get install -y wget nano rsync curl gnupg2 jq unzip bzip2
21-
22-
# for clang-*-15, see https://apt.llvm.org/
23-
RUN echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" >> /etc/apt/sources.list && \
24-
echo "deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" >> /etc/apt/sources.list && \
25-
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
16+
FROM shiftcrypto/ubuntu:1
2617

2718
# Install gcc8-arm-none-eabi
28-
RUN mkdir ~/Downloads &&\
29-
cd ~/Downloads &&\
30-
wget -O gcc.tar.bz2 https://developer.arm.com/-/media/Files/downloads/gnu-rm/8-2018q4/gcc-arm-none-eabi-8-2018-q4-major-linux.tar.bz2?revision=d830f9dd-cd4f-406d-8672-cca9210dd220?product=GNU%20Arm%20Embedded%20Toolchain,64-bit,,Linux,8-2018-q4-major &&\
19+
RUN wget -O gcc.tar.bz2 https://developer.arm.com/-/media/Files/downloads/gnu-rm/8-2018q4/gcc-arm-none-eabi-8-2018-q4-major-linux.tar.bz2?revision=d830f9dd-cd4f-406d-8672-cca9210dd220?product=GNU%20Arm%20Embedded%20Toolchain,64-bit,,Linux,8-2018-q4-major &&\
3120
echo "fb31fbdfe08406ece43eef5df623c0b2deb8b53e405e2c878300f7a1f303ee52 gcc.tar.bz2" | sha256sum -c &&\
32-
cd ~/Downloads &&\
33-
tar -xjvf gcc.tar.bz2 &&\
34-
rm -f gcc.tar.bz2 &&\
35-
cd ~/Downloads && rsync -a gcc-arm-none-eabi-8-2018-q4-major/ /usr/local/
36-
37-
# Tools for building
38-
RUN apt-get update && apt-get install -y \
39-
build-essential \
40-
llvm-18 \
41-
gcc-10 \
42-
binutils \
43-
valgrind \
44-
cmake \
45-
git \
46-
autotools-dev \
47-
automake \
48-
autoconf \
49-
libtool \
50-
pkg-config \
51-
libcmocka-dev \
52-
libc6-i386 \
53-
lib32stdc++6 \
54-
lib32z1 \
55-
libusb-1.0-0-dev \
56-
libudev-dev \
57-
libhidapi-dev
58-
59-
RUN apt-get update && apt-get install -y \
60-
doxygen \
61-
graphviz
21+
tar -xjf gcc.tar.bz2 -C /usr/local --strip-components=1 &&\
22+
rm gcc.tar.bz2 /root/.wget-hsts
6223

6324
# Set gcc-10 as the default gcc
64-
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100
65-
RUN update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-10 100
66-
67-
# Tools for CI
68-
RUN apt-get update && apt-get install -y \
69-
python3 \
70-
python3-pip \
71-
clang-format-18 \
72-
clang-tidy-18
25+
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && rm /var/log/alternatives.log
26+
RUN update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-10 100 && rm /var/log/alternatives.log
7327

74-
RUN python3 -m pip install --upgrade pip
28+
# Make Python3 the default
29+
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && rm /var/log/alternatives.log
7530

7631
# Python modules
77-
COPY py/bitbox02 /tmp/bitbox02
78-
RUN python3 -m pip install /tmp/bitbox02
79-
RUN rm -r /tmp/bitbox02
80-
COPY py/requirements.txt /tmp
81-
RUN python3 -m pip install --upgrade --requirement /tmp/requirements.txt
82-
RUN rm /tmp/requirements.txt
32+
RUN --mount=source=py/bitbox02,target=/mnt/bitbox02,rw \
33+
pip install --no-compile --no-cache-dir /mnt/bitbox02
34+
RUN --mount=source=py/requirements.txt,target=/mnt/requirements.txt \
35+
pip install --no-compile --no-cache-dir --upgrade --requirement /mnt/requirements.txt
8336

8437
# Python modules for CI
85-
RUN python3 -m pip install --upgrade \
38+
RUN pip install --no-compile --no-cache-dir --upgrade \
8639
pylint==2.13.9 \
8740
pylint-protobuf==0.20.2 \
8841
black==22.3.0 \
8942
mypy==0.960 \
90-
mypy-protobuf==3.2.0
91-
92-
# Python modules for packaging
93-
RUN python3 -m pip install --upgrade \
43+
mypy-protobuf==3.2.0 \
9444
setuptools==41.2.0 \
9545
wheel==0.33.6 \
96-
twine==1.15.0
46+
twine==1.15.0 \
47+
gcovr==7.2
9748

9849
#Install protoc from release, because the version available on the repo is too old
9950
RUN mkdir -p /opt/protoc && \
10051
curl -L0 https://github.com/protocolbuffers/protobuf/releases/download/v21.2/protoc-21.2-linux-x86_64.zip -o /tmp/protoc-21.2-linux-x86_64.zip && \
101-
unzip /tmp/protoc-21.2-linux-x86_64.zip -d /opt/protoc
102-
ENV PATH /opt/protoc/bin:$PATH
103-
104-
# Make Python3 the default
105-
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
106-
107-
# Developer tools
108-
RUN apt-get update && apt-get install -y \
109-
bash-completion
110-
# Install gcovr from PIP to get a newer version than in apt repositories
111-
RUN python3 -m pip install gcovr
52+
unzip /tmp/protoc-21.2-linux-x86_64.zip -d /opt/protoc &&\
53+
rm /tmp/protoc-21.2-linux-x86_64.zip
54+
ENV PATH=/opt/protoc/bin:$PATH
11255

11356
# Install Go, used for the tools in tools/go and for test/gounittest
114-
ENV GOPATH /opt/go
115-
ENV GOROOT /opt/go_dist/go
116-
ENV PATH $GOROOT/bin:$GOPATH/bin:$PATH
57+
ENV GOPATH=/opt/go
58+
ENV GOROOT=/opt/go_dist/go
59+
ENV PATH=$GOROOT/bin:$GOPATH/bin:$PATH
11760
RUN mkdir -p /opt/go_dist && \
11861
curl https://dl.google.com/go/go1.19.3.linux-amd64.tar.gz | tar -xz -C /opt/go_dist
11962

12063
# Install lcov from release (the one from the repos is too old).
121-
RUN cd /opt && wget https://github.com/linux-test-project/lcov/releases/download/v1.14/lcov-1.14.tar.gz && tar -xf lcov-1.14.tar.gz
122-
ENV PATH /opt/lcov-1.14/bin:$PATH
64+
RUN curl -L https://github.com/linux-test-project/lcov/releases/download/v1.14/lcov-1.14.tar.gz | tar -xz -C /opt
65+
ENV PATH=/opt/lcov-1.14/bin:$PATH
12366

12467
# Install rust compiler
125-
ENV PATH /opt/cargo/bin:$PATH
68+
# Since bindgen embeds information about its target directory, use a deterministic path for it.
69+
ENV PATH=/opt/cargo/bin:$PATH
12670
ENV RUSTUP_HOME=/opt/rustup
127-
COPY src/rust/rust-toolchain.toml /tmp/rust-toolchain.toml
128-
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | CARGO_HOME=/opt/cargo sh -s -- --default-toolchain $(grep -oP '(?<=channel = ")[^"]+' /tmp/rust-toolchain.toml) -y
129-
RUN rustup target add thumbv7em-none-eabi
130-
RUN rustup component add rustfmt
131-
RUN rustup component add clippy
132-
RUN rustup component add rust-src
133-
RUN CARGO_HOME=/opt/cargo cargo install cbindgen --version 0.26.0 --locked
134-
RUN CARGO_HOME=/opt/cargo cargo install bindgen-cli --version 0.69.4 --locked
71+
RUN --mount=source=tools/prost-build-proto,target=/mnt/prost-build-proto,rw \
72+
--mount=source=src/rust/rust-toolchain.toml,target=/mnt/rust-toolchain.toml \
73+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs |\
74+
CARGO_HOME=/opt/cargo sh -s -- --default-toolchain $(grep -oP '(?<=channel = ")[^"]+' /mnt/rust-toolchain.toml) -y &&\
75+
rustup target add thumbv7em-none-eabi &&\
76+
rustup component add rustfmt &&\
77+
rustup component add clippy &&\
78+
rustup component add rust-src &&\
79+
CARGO_HOME=/opt/cargo cargo install cbindgen --version 0.26.0 --locked &&\
80+
CARGO_HOME=/opt/cargo cargo install bindgen-cli --version 0.69.4 --locked --target-dir=/tmp/bindgen-target &&\
81+
CARGO_HOME=/opt/cargo cargo install --path /mnt/prost-build-proto --locked &&\
82+
rm -r /tmp/bindgen-target /opt/cargo/registry/index /opt/cargo/.global-cache
13583

13684
# Until cargo vendor supports vendoring dependencies of the rust std libs we
13785
# need a copy of this file next to the toml file. It also has to be world
13886
# writable so that invocations of `cargo vendor` can update it. Below is the
13987
# tracking issue for `cargo vendor` to support rust std libs.
14088
# https://github.com/rust-lang/wg-cargo-std-aware/issues/23
141-
RUN cp "$(rustc --print=sysroot)/lib/rustlib/src/rust/Cargo.lock" "$(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/"
142-
RUN chmod 777 $(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/Cargo.lock
143-
144-
COPY tools/prost-build-proto prost-build-proto
145-
RUN CARGO_HOME=/opt/cargo cargo install --path prost-build-proto --locked
146-
147-
# Clean temporary files to reduce image size
148-
RUN rm -rf /var/lib/apt/lists/*
89+
RUN cp "$(rustc --print=sysroot)/lib/rustlib/src/rust/Cargo.lock" "$(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/" &&\
90+
chmod 777 $(rustc --print=sysroot)/lib/rustlib/src/rust/library/test/Cargo.lock

‎Dockerfile.ubuntu

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# Copyright 2019 Shift Cryptosecurity AG
2+
# Copyright 2020 Shift Crypto AG
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# Latest Ubuntu LTS
17+
FROM ubuntu:22.04@sha256:0eb0f877e1c869a300c442c41120e778db7161419244ee5cbc6fa5f134e74736
18+
19+
RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y \
20+
wget \
21+
curl \
22+
gnupg2 \
23+
&& rm -r /var/lib/apt/lists/*
24+
25+
# for clang-*-18, see https://apt.llvm.org/
26+
RUN echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" >> /etc/apt/sources.list && \
27+
echo "deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" >> /etc/apt/sources.list && \
28+
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
29+
rm /root/.wget-hsts
30+
31+
RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y \
32+
make \
33+
git \
34+
pkg-config \
35+
nano \
36+
jq \
37+
unzip \
38+
python3 \
39+
python3-pip \
40+
binutils \
41+
valgrind \
42+
cmake \
43+
autotools-dev \
44+
automake \
45+
autoconf \
46+
libtool \
47+
libcmocka-dev \
48+
bzip2 \
49+
llvm-18 \
50+
gcc-10 \
51+
libc6-i386 \
52+
lib32stdc++6 \
53+
lib32z1 \
54+
libusb-1.0-0-dev \
55+
libudev-dev \
56+
libhidapi-dev \
57+
doxygen \
58+
graphviz \
59+
clang-format-18 \
60+
clang-tidy-18 \
61+
bash-completion \
62+
&& rm -r /var/lib/apt/lists/*

‎Makefile

+8-4
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@
1414

1515
# This makefile is used as a command runner and not for tracking dependencies between recipies
1616

17+
# This is the current version of the development container. If you make changes to the Dockerfile,
18+
# bump this number to release a new version.
19+
CONTAINER_VERSION := 41
20+
1721
.DEFAULT_GOAL := firmware
1822
SANITIZE ?= ON
1923
simulator: SANITIZE = OFF
@@ -129,15 +133,15 @@ jlink-flash-factory-setup: | build
129133
jlink-flash-firmware-semihosting: | build-semihosting
130134
JLinkExe -if SWD -device ATSAMD51J20 -speed 4000 -autoconnect 1 -CommanderScript ./build-semihosting/scripts/firmware.jlink
131135
dockerinit:
132-
./scripts/container.sh build --pull --platform linux/amd64 --force-rm --no-cache -t shiftcrypto/firmware_v2 .
136+
./scripts/container.sh pull shiftcrypto/firmware_v2:${CONTAINER_VERSION}
133137
dockerdev:
134-
./scripts/dockerenv.sh
138+
./scripts/dockerenv.sh ${CONTAINER_VERSION}
135139
dockerrel:
136-
./scripts/dockerenv.sh release
140+
./scripts/dockerenv.sh release ${CONTAINER_VERSION}
137141
generate-atecc608-config:
138142
cd tools/atecc608 && go run main.go
139143
ci:
140-
./.ci/ci
144+
./.ci/ci ${CONTAINER_VERSION}
141145
prepare-tidy: | build build-build
142146
make -C build rust-cbindgen
143147
make -C build-build rust-cbindgen

0 commit comments

Comments
 (0)