Skip to content

Commit 904775c

Browse files
authored
chore: new docker build pipeline (#4503)
Our previous weekly pipeline used qemu, was very slow and over-complicated. This one uses matrix with proper parallelization and the latest arm64 github runners. now it takes less than 30 minutes to build everything. lets make it daily.
1 parent 23af41c commit 904775c

File tree

4 files changed

+135
-3
lines changed

4 files changed

+135
-3
lines changed

.dockerignore

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
_deps/*
22
build-*
33
tools/packaging/*
4+
.github/*
+131
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
name: Development Docker Build
2+
3+
on:
4+
schedule:
5+
- cron: '15 0 * * *'
6+
workflow_dispatch:
7+
8+
9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
11+
cancel-in-progress: true
12+
13+
env:
14+
image: ghcr.io/dragonflydb/dragonfly-dev
15+
16+
jobs:
17+
build_and_tag:
18+
name: Build and Push ${{matrix.flavor}} ${{ matrix.os.arch }} image
19+
strategy:
20+
matrix:
21+
flavor: [alpine,ubuntu]
22+
os:
23+
- image: ubuntu-24.04
24+
arch: amd64
25+
- image: ubuntu-24.04-arm
26+
arch: arm64
27+
28+
runs-on: ${{ matrix.os.image }}
29+
permissions:
30+
contents: read
31+
packages: write
32+
id-token: write
33+
steps:
34+
- name: checkout
35+
uses: actions/checkout@v4
36+
with:
37+
fetch-depth: 1
38+
submodules: true
39+
- name: Set up Docker Buildx
40+
uses: docker/setup-buildx-action@v3
41+
- name: Login to GitHub Container Registry
42+
uses: docker/login-action@v3
43+
with:
44+
registry: ghcr.io
45+
username: ${{ github.repository_owner }}
46+
password: ${{ secrets.GITHUB_TOKEN }}
47+
- name: Get Build Information
48+
id: build_info
49+
run: |
50+
echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
51+
52+
- name: Docker meta
53+
id: metadata
54+
uses: docker/metadata-action@v5
55+
with:
56+
images: |
57+
${{ env.image }}
58+
tags: |
59+
type=sha,enable=true,prefix=${{ matrix.flavor}}-,suffix=-${{ matrix.os.arch }},format=short
60+
labels: |
61+
org.opencontainers.image.vendor=DragonflyDB LTD
62+
org.opencontainers.image.title=Dragonfly Development Image
63+
org.opencontainers.image.description=The fastest in-memory store
64+
- name: Build image
65+
id: build
66+
uses: docker/build-push-action@v6
67+
with:
68+
context: .
69+
push: true
70+
provenance: false # Prevent pushing a docker manifest
71+
tags: |
72+
${{ steps.metadata.outputs.tags }}
73+
labels: ${{ steps.metadata.outputs.labels }}
74+
file: tools/packaging/Dockerfile.${{ matrix.flavor }}-dev
75+
cache-from: type=gha,scope=tagged${{ matrix.flavor }}
76+
cache-to: type=gha,scope=tagged${{ matrix.flavor }},mode=max
77+
load: true # Load the build images into the local docker.
78+
- name: Test Image
79+
run: |
80+
image_tag=${{ steps.metadata.outputs.tags }}
81+
echo ${{ steps.build.outputs.digest }}
82+
image_digest=${{ env.image }}@${{ steps.build.outputs.digest }}
83+
84+
# install redis-tools
85+
sudo apt-get install redis-tools -y
86+
87+
docker image inspect ${image_tag}
88+
echo "Testing ${{ matrix.flavor }} image"
89+
90+
# docker run with port-forwarding
91+
docker run -d -p 6379:6379 ${image_tag}
92+
sleep 5
93+
if [[ $(redis-cli -h localhost ping) != "PONG" ]]; then
94+
echo "Redis ping failed"
95+
exit 1
96+
fi
97+
echo "Redis ping succeeded"
98+
exit 0
99+
100+
outputs:
101+
# matrix jobs outputs override each other, but we use the same sha
102+
# for all images, so we can use the same output name.
103+
sha: ${{ steps.build_info.outputs.short_sha }}
104+
105+
merge_manifest:
106+
needs: [build_and_tag]
107+
runs-on: ubuntu-latest
108+
strategy:
109+
matrix:
110+
flavor: [alpine,ubuntu]
111+
steps:
112+
- name: Login to GitHub Container Registry
113+
uses: docker/login-action@v3
114+
with:
115+
registry: ghcr.io
116+
username: ${{ github.repository_owner }}
117+
password: ${{ secrets.GITHUB_TOKEN }}
118+
119+
- name: Merge and Push
120+
run: |
121+
flavor_tag=${{ env.image }}:${{matrix.flavor}}
122+
sha_tag=${flavor_tag}-${{ needs.build_and_tag.outputs.sha }}
123+
124+
# Create and push the manifest like dragonfly-dev:alpine-<sha>
125+
docker manifest create ${sha_tag} --amend ${sha_tag}-amd64 --amend ${sha_tag}-arm64
126+
docker manifest push ${sha_tag}
127+
128+
# Create and push the manifest like dragonfly-dev:alpine
129+
docker manifest create ${flavor_tag} --amend ${sha_tag}-amd64 --amend ${sha_tag}-arm64
130+
docker manifest push ${flavor_tag}
131+

tools/packaging/Dockerfile.alpine-dev

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# syntax=docker/dockerfile:1
2-
FROM gcr.io/cadvisor/cadvisor:v0.46.0 as libpfm_donor
2+
FROM gcr.io/cadvisor/cadvisor:v0.46.0 AS libpfm_donor
33

4-
FROM alpine:3 as builder
4+
FROM alpine:3 AS builder
55

66
# "openssl-libs-static" fixes "Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the"
77
RUN apk add autoconf-archive automake bash bison boost-dev cmake coreutils net-tools \

tools/packaging/Dockerfile.ubuntu-dev

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# syntax=docker/dockerfile:1
2-
FROM ghcr.io/romange/ubuntu-dev:20 as builder
2+
FROM ghcr.io/romange/ubuntu-dev:20 AS builder
33

44
WORKDIR /build
55

0 commit comments

Comments
 (0)