Skip to content

Commit f5adadd

Browse files
luizirberluizirber
authored
build: add docker images (#79)
Add docker images, following what was done for branchwater Prefer sralite format, we get same data but with smaller downloads: https://github.com/ncbi/workshop-asm-ngs-2024/wiki/3.3-Using-SRAToolkit-to-retrieve-Data#approach-3-force-toolkit-to-retrieve-sra-lite-format
1 parent 70bd1eb commit f5adadd

File tree

7 files changed

+3314
-2139
lines changed

7 files changed

+3314
-2139
lines changed

‎.dockerignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,6 @@
55
!wort/
66
!pixi.lock
77
!config/
8+
!migrations/
89
!wortapp.py
910
!pyproject.toml

‎.github/workflows/docker-images.yml

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
# Based on
2+
# - https://docs.github.com/en/actions/use-cases-and-examples/publishing-packages/publishing-docker-images
3+
# - https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners
4+
# and the branchwater config
5+
# - https://github.com/sourmash-bio/branchwater/blob/main/.github/workflows/docker-images.yml
6+
7+
name: Create and publish Docker images
8+
9+
on:
10+
push:
11+
branches:
12+
- 'main'
13+
tags:
14+
- 'v*'
15+
pull_request:
16+
paths:
17+
- .github/workflows/docker-images.yml
18+
- Dockerfile
19+
## disabling for most PRs, too many builds
20+
# branches:
21+
# - 'main'
22+
23+
env:
24+
REGISTRY: ghcr.io
25+
IMAGE_NAME: ${{ github.repository }}
26+
27+
jobs:
28+
build:
29+
runs-on: ${{ matrix.os }}
30+
strategy:
31+
matrix:
32+
target: [web, worker]
33+
platform:
34+
- linux/amd64
35+
- linux/arm64
36+
include:
37+
- platform: linux/amd64
38+
os: ubuntu-latest
39+
- platform: linux/arm64
40+
os: ubuntu-24.04-arm
41+
42+
permissions:
43+
contents: read
44+
packages: write
45+
id-token: write
46+
47+
steps:
48+
- name: Delete huge unnecessary tools folder
49+
run: |
50+
rm -rf /opt/hostedtoolcache
51+
cd /opt
52+
find . -maxdepth 1 -mindepth 1 '!' -path ./containerd '!' -path ./actionarchivecache '!' -path ./runner '!' -path ./runner-cache -exec rm -rf '{}' ';'
53+
54+
- name: Checkout repository
55+
uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2
56+
57+
- name: Log in to the Container registry
58+
uses: docker/login-action@327cd5a69de6c009b9ce71bce8395f28e651bf99
59+
with:
60+
registry: ${{ env.REGISTRY }}
61+
username: ${{ github.actor }}
62+
password: ${{ secrets.GITHUB_TOKEN }}
63+
64+
- name: Set up Docker Buildx
65+
uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5
66+
67+
- name: Prepare
68+
run: |
69+
platform=${{ matrix.platform }}
70+
echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
71+
72+
- name: Extract metadata (tags, labels) for Docker
73+
id: meta
74+
uses: docker/metadata-action@8e1d5461f02b7886d3c1a774bfbd873650445aa2
75+
with:
76+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
77+
78+
- name: Build and push by digest
79+
id: build
80+
uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991
81+
with:
82+
context: .
83+
platforms: ${{ matrix.platform }}
84+
labels: ${{ steps.meta.outputs.labels }}
85+
target: ${{ matrix.target }}
86+
outputs: type=image,"name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}",push-by-digest=true,name-canonical=true,push=true
87+
cache-from: type=gha
88+
cache-to: type=gha,mode=max
89+
# don't push on PRs
90+
push: ${{ ! startsWith(github.ref, 'refs/pull/') }}
91+
92+
- name: Export digest
93+
run: |
94+
mkdir -p ${{ runner.temp }}/digests
95+
digest="${{ steps.build.outputs.digest }}"
96+
touch "${{ runner.temp }}/digests/${digest#sha256:}"
97+
98+
- name: Upload digest
99+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08
100+
with:
101+
name: digests-${{ matrix.target }}-${{ env.PLATFORM_PAIR }}
102+
path: ${{ runner.temp }}/digests/*
103+
if-no-files-found: error
104+
retention-days: 1
105+
106+
merge:
107+
runs-on: ubuntu-latest
108+
# avoid tagging on PRs
109+
if: ${{ ! startsWith(github.ref, 'refs/pull/') }}
110+
needs:
111+
- build
112+
113+
strategy:
114+
matrix:
115+
target: [web, worker]
116+
117+
permissions:
118+
contents: read
119+
packages: write
120+
attestations: write
121+
id-token: write
122+
123+
steps:
124+
- name: Download digests
125+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16
126+
with:
127+
path: ${{ runner.temp }}/digests
128+
pattern: digests-${{ matrix.target }}-*
129+
merge-multiple: true
130+
131+
- name: Log in to the Container registry
132+
uses: docker/login-action@327cd5a69de6c009b9ce71bce8395f28e651bf99
133+
with:
134+
registry: ${{ env.REGISTRY }}
135+
username: ${{ github.actor }}
136+
password: ${{ secrets.GITHUB_TOKEN }}
137+
138+
- name: Set up Docker Buildx
139+
uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5
140+
141+
- name: Extract metadata (tags, labels) for Docker
142+
id: meta
143+
uses: docker/metadata-action@8e1d5461f02b7886d3c1a774bfbd873650445aa2
144+
with:
145+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
146+
tags: |
147+
type=ref,event=branch,prefix=${{ matrix.target }}-
148+
type=ref,event=pr,prefix=${{ matrix.target }}-pr-
149+
type=semver,prefix=${{ matrix.target }}-,pattern={{version}}
150+
151+
- name: Create manifest list and push
152+
working-directory: ${{ runner.temp }}/digests
153+
run: |
154+
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
155+
$(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
156+
157+
- name: Inspect image
158+
run: |
159+
docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
160+
161+
# - name: Generate artifact attestation
162+
# uses: actions/attest-build-provenance@v2
163+
# with:
164+
# subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
165+
# subject-digest: ${{ steps.build.outputs.digest }}
166+
# push-to-registry: true

‎Dockerfile

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,24 @@
11
FROM ghcr.io/prefix-dev/pixi:0.40.3 AS build
22

3-
COPY . /app
43
WORKDIR /app
4+
5+
COPY . .
6+
7+
RUN --mount=type=cache,target=/root/.cache/rattler/cache,sharing=private pixi install
8+
59
RUN pixi run build-wheel
10+
611
RUN pixi run -e web postinstall-prod
712
RUN pixi shell-hook -e web > /shell-hook-web
813
RUN echo 'exec "$@"' >> /shell-hook-web
14+
915
RUN pixi run -e worker postinstall-prod
1016
RUN pixi shell-hook -e worker > /shell-hook-worker
1117
RUN echo 'exec "$@"' >> /shell-hook-worker
1218

1319
#--------------------
1420

15-
FROM ubuntu:24.04 AS web
21+
FROM ubuntu:24.04 AS web
1622

1723
# only copy the production environment into prod container
1824
COPY --from=build /app/.pixi/envs/web /app/.pixi/envs/web
@@ -21,13 +27,14 @@ COPY --from=build /shell-hook-web /shell-hook
2127
RUN groupadd user && \
2228
useradd --create-home --home-dir /home/user -g user -s /bin/bash user
2329

30+
USER user
31+
2432
COPY wortapp.py /app
2533
COPY config/ /app/config
34+
COPY migrations/ /app/migrations
2635

2736
WORKDIR /app
2837

29-
#USER user
30-
3138
ENTRYPOINT ["/bin/bash", "/shell-hook"]
3239
CMD ["gunicorn", "-b", "0.0.0.0:5000", "--access-logfile", "-", "'wortapp:create_app()'"]
3340

@@ -47,7 +54,7 @@ COPY config/ /app/config
4754

4855
WORKDIR /app
4956

50-
USER user
57+
USER user
5158

5259
# Configure sra-toolkit to disable cache
5360
RUN mkdir ~/.ncbi
@@ -56,16 +63,19 @@ RUN echo '## auto-generated configuration file - DO NOT EDIT ##''' >> ~/.ncbi/us
5663
RUN echo '' >> ~/.ncbi/user-settings.mkfg
5764
RUN echo '/LIBS/GUID = "7737545d-30d4-4d05-875a-2c562df521d5"' >> ~/.ncbi/user-settings.mkfg
5865
RUN echo '/config/default = "false"' >> ~/.ncbi/user-settings.mkfg
66+
RUN echo '/libs/vdb/quality = "ZR"' >> ~/.ncbi/user-settings.mkfg
5967
RUN echo '/libs/cloud/accept_aws_charges = "false"' >> ~/.ncbi/user-settings.mkfg
60-
RUN echo '/libs/cloud/report_instance_identity = "true"' >> ~/.ncbi/user-settings.mkfg
68+
RUN echo '/libs/cloud/report_instance_identity = "false"' >> ~/.ncbi/user-settings.mkfg
6169
RUN echo '/repository/user/ad/public/apps/file/volumes/flatAd = "."' >> ~/.ncbi/user-settings.mkfg
6270
RUN echo '/repository/user/ad/public/apps/refseq/volumes/refseqAd = "."' >> ~/.ncbi/user-settings.mkfg
6371
RUN echo '/repository/user/ad/public/apps/sra/volumes/sraAd = "."' >> ~/.ncbi/user-settings.mkfg
6472
RUN echo '/repository/user/ad/public/apps/sraPileup/volumes/ad = "."' >> ~/.ncbi/user-settings.mkfg
6573
RUN echo '/repository/user/ad/public/apps/sraRealign/volumes/ad = "."' >> ~/.ncbi/user-settings.mkfg
74+
RUN echo '/repository/user/ad/public/apps/wgs/volumes/wgsAd = "."' >> ~/.ncbi/user-settings.mkfg
6675
RUN echo '/repository/user/ad/public/root = "."' >> ~/.ncbi/user-settings.mkfg
67-
RUN echo '/repository/user/default-path = "/root/ncbi"' >> ~/.ncbi/user-settings.mkfg
76+
RUN echo '/repository/user/default-path = "/home/user/ncbi"' >> ~/.ncbi/user-settings.mkfg
6877
RUN echo '/repository/user/main/public/cache-disabled = "true"' >> ~/.ncbi/user-settings.mkfg
78+
RUN echo '/tools/prefetch/download_to_cache = "false"' >> ~/.ncbi/user-settings.mkfg
6979

7080
ENV RAYON_NUM_THREADS 3
7181
ENTRYPOINT ["/bin/bash", "/shell-hook"]

‎docker-compose.yml

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,19 @@
1-
version: '3.4'
2-
31
services:
42

53
db:
6-
restart: always
74
image: postgres:9.6-alpine
5+
restart: always
86
volumes:
97
- ./data/postgres-data:/var/lib/postgresql/data
108
env_file:
119
- env.production
1210

1311
worker:
12+
image: ghcr.io/sourmash-bio/wort:worker-main
1413
build:
1514
context: .
1615
dockerfile: Dockerfile
1716
target: worker
18-
image: wort-worker
1917
env_file:
2018
- iam/wort_s3.env
2119
command: >
@@ -26,11 +24,12 @@ services:
2624
-l INFO -c 1
2725
2826
web:
29-
restart: always
27+
image: ghcr.io/sourmash-bio/wort:web-main
3028
build:
3129
context: .
3230
dockerfile: Dockerfile
3331
target: web
32+
restart: always
3433
command: >
3534
gunicorn -b 0.0.0.0:5000
3635
--access-logfile -
@@ -40,8 +39,6 @@ services:
4039
FLASK_APP: 'wortapp.py'
4140
ports:
4241
- "8082:5000"
43-
links:
44-
- db
4542
depends_on:
4643
- db
4744
- redis

0 commit comments

Comments
 (0)