Skip to content

Commit b2cec4c

Browse files
razvanNickLarsenNZ
razvan
and
NickLarsenNZ
authored
feat: add spark connect client to image build flows (#1051)
* feat: add spark connect client to image build flows * Update .github/workflows/build_spark-connect-client.yaml Co-authored-by: Nick <[email protected]> --------- Co-authored-by: Nick <[email protected]>
1 parent a317716 commit b2cec4c

File tree

3 files changed

+181
-3
lines changed

3 files changed

+181
-3
lines changed

‎.github/workflows/build_spark-connect-client.yaml

+171
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
---
2+
name: Build Spark Connect Client
3+
run-name: |
4+
Build Spark Connect Client (attempt #${{ github.run_attempt }})
5+
6+
env:
7+
PRODUCT_NAME: spark-connect-client
8+
SDP_VERSION: ${{ github.ref_type == 'tag' && github.ref_name || '0.0.0-dev' }}
9+
10+
on:
11+
workflow_dispatch:
12+
schedule:
13+
- cron: '0 0 2/2 * *' # https://crontab.guru/#0_0_2/2_*_*
14+
push:
15+
branches: [main]
16+
tags: ['*']
17+
paths:
18+
# To check dependencies, run this ( you will need to consider transitive dependencies)
19+
# bake --product PRODUCT -d | grep -v 'docker buildx bake' | jq '.target | keys[]'
20+
- spark-k8s/**
21+
- spark-connect-client/**
22+
- stackable-base/**
23+
- java-base/**
24+
- .github/actions/**
25+
- .github/workflows/build_spark-connect-client.yaml
26+
27+
jobs:
28+
generate_matrix:
29+
name: Generate Version List
30+
runs-on: ubuntu-latest
31+
steps:
32+
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
33+
with:
34+
persist-credentials: false
35+
- id: shard
36+
uses: stackabletech/actions/shard@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
37+
with:
38+
product-name: ${{ env.PRODUCT_NAME }}
39+
outputs:
40+
versions: ${{ steps.shard.outputs.versions }}
41+
42+
build:
43+
name: Build/Publish ${{ matrix.versions }}-${{ matrix.runner.arch }} Image
44+
needs: [generate_matrix]
45+
permissions:
46+
id-token: write
47+
runs-on: ${{ matrix.runner.name }}
48+
strategy:
49+
fail-fast: false
50+
matrix:
51+
runner:
52+
- {name: "ubuntu-latest", arch: "amd64"}
53+
- {name: "ubicloud-standard-8-arm", arch: "arm64"}
54+
versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }}
55+
steps:
56+
- name: Checkout Repository
57+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
58+
with:
59+
persist-credentials: false
60+
61+
- name: Free Disk Space
62+
uses: stackabletech/actions/free-disk-space@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
63+
64+
- name: Build Product Image
65+
id: build
66+
uses: stackabletech/actions/build-product-image@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
67+
with:
68+
product-name: ${{ env.PRODUCT_NAME }}
69+
product-version: ${{ matrix.versions }}
70+
build-cache-password: ${{ secrets.BUILD_CACHE_NEXUS_PASSWORD }}
71+
sdp-version: ${{ env.SDP_VERSION }}
72+
73+
- name: Publish Container Image on docker.stackable.tech
74+
uses: stackabletech/actions/publish-image@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
75+
with:
76+
image-registry-uri: docker.stackable.tech
77+
image-registry-username: github
78+
image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
79+
image-repository: stackable/${{ env.PRODUCT_NAME }}
80+
image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
81+
source-image-uri: localhost/${{ env.PRODUCT_NAME }}:${{ steps.build.outputs.image-manifest-tag }}
82+
83+
- name: Publish Container Image on oci.stackable.tech
84+
uses: stackabletech/actions/publish-image@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
85+
with:
86+
image-registry-uri: oci.stackable.tech
87+
image-registry-username: robot$sdp+github-action-build
88+
image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
89+
image-repository: sdp/${{ env.PRODUCT_NAME }}
90+
image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
91+
source-image-uri: localhost/${{ env.PRODUCT_NAME }}:${{ steps.build.outputs.image-manifest-tag }}
92+
93+
publish_manifests:
94+
name: Build/Publish ${{ matrix.versions }} Manifests
95+
needs: [generate_matrix, build]
96+
permissions:
97+
id-token: write
98+
runs-on: ubuntu-latest
99+
strategy:
100+
fail-fast: false
101+
matrix:
102+
versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }}
103+
steps:
104+
- name: Checkout Repository
105+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
106+
with:
107+
persist-credentials: false
108+
109+
- name: Publish and Sign Image Index Manifest to docker.stackable.tech
110+
uses: stackabletech/actions/publish-index-manifest@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
111+
with:
112+
image-registry-uri: docker.stackable.tech
113+
image-registry-username: github
114+
image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
115+
image-repository: stackable/${{ env.PRODUCT_NAME }}
116+
image-index-manifest-tag: ${{ matrix.versions }}-stackable${{ env.SDP_VERSION }}
117+
118+
- name: Publish and Sign Image Index Manifest to oci.stackable.tech
119+
uses: stackabletech/actions/publish-index-manifest@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
120+
with:
121+
image-registry-uri: oci.stackable.tech
122+
image-registry-username: robot$sdp+github-action-build
123+
image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
124+
image-repository: sdp/${{ env.PRODUCT_NAME }}
125+
image-index-manifest-tag: ${{ matrix.versions }}-stackable${{ env.SDP_VERSION }}
126+
127+
notify:
128+
name: Failure Notification
129+
needs: [generate_matrix, build, publish_manifests]
130+
runs-on: ubuntu-latest
131+
if: failure()
132+
steps:
133+
- uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
134+
with:
135+
channel-id: "C07UG6JH44F" # notifications-container-images
136+
payload: |
137+
{
138+
"text": "*${{ github.workflow }}* failed (attempt ${{ github.run_attempt }})",
139+
"attachments": [
140+
{
141+
"pretext": "See the details below for a summary of which job(s) failed.",
142+
"color": "#aa0000",
143+
"fields": [
144+
{
145+
"title": "Generate Version List",
146+
"short": true,
147+
"value": "${{ needs.generate_matrix.result }}"
148+
},
149+
{
150+
"title": "Build/Publish Image",
151+
"short": true,
152+
"value": "${{ needs.build.result }}"
153+
},
154+
{
155+
"title": "Build/Publish Manifests",
156+
"short": true,
157+
"value": "${{ needs.publish_manifests.result }}"
158+
}
159+
],
160+
"actions": [
161+
{
162+
"type": "button",
163+
"text": "Go to workflow run",
164+
"url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
165+
}
166+
]
167+
}
168+
]
169+
}
170+
env:
171+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_CONTAINER_IMAGE_TOKEN }}

‎.scripts/update_readme_badges.sh

+4
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@ for BUILD_WORKFLOW_FILE in .github/workflows/build_*.yaml; do
5454
echo >> "$BADGES_TMP"
5555
fi
5656
done
57+
# This needs to add the remaning empty columns of the last row in the table
58+
# This is a hack to fix the status quo and make markdownlint happy.
59+
echo -n "| | | |" >> "$BADGES_TMP"
60+
echo >> "$BADGES_TMP"
5761
echo -n "<!-- end:badges -->" >> "$BADGES_TMP"
5862

5963
# Print the image and link shortcuts. Eg:

‎README.md

+6-3
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,10 @@ This repository contains Dockerfiles and scripts to build base images for use wi
88
| [![Build Airflow]][build_airflow.yaml] | [![Build Druid]][build_druid.yaml] | [![Build Hadoop]][build_hadoop.yaml] | [![Build HBase]][build_hbase.yaml] |
99
| [![Build Hello-World]][build_hello-world.yaml] | [![Build Hive]][build_hive.yaml] | [![Build Java Base]][build_java-base.yaml] | [![Build Java Development]][build_java-devel.yaml] |
1010
| [![Build Kafka Testing Tools]][build_kafka-testing-tools.yaml] | [![Build Kafka]][build_kafka.yaml] | [![Build kcat]][build_kcat.yaml] | [![Build Krb5]][build_krb5.yaml] |
11-
| [![Build NiFi]][build_nifi.yaml] | [![Build Omid]][build_omid.yaml] | [![Build OPA]][build_opa.yaml] | [![Build Spark K8s]][build_spark-k8s.yaml] |
12-
| [![Build Stackable Base]][build_stackable-base.yaml] | [![Build Superset]][build_superset.yaml] | [![Build Testing Tools]][build_testing-tools.yaml] | [![Build Tools]][build_tools.yaml] |
13-
| [![Build Trino CLI]][build_trino-cli.yaml] | [![Build Trino]][build_trino.yaml] | [![Build Vector]][build_vector.yaml] | [![Build ZooKeeper]][build_zookeeper.yaml] |
11+
| [![Build NiFi]][build_nifi.yaml] | [![Build Omid]][build_omid.yaml] | [![Build OPA]][build_opa.yaml] | [![Build Spark Connect Client]][build_spark-connect-client.yaml] |
12+
| [![Build Spark K8s]][build_spark-k8s.yaml] | [![Build Stackable Base]][build_stackable-base.yaml] | [![Build Superset]][build_superset.yaml] | [![Build Testing Tools]][build_testing-tools.yaml] |
13+
| [![Build Tools]][build_tools.yaml] | [![Build Trino CLI]][build_trino-cli.yaml] | [![Build Trino]][build_trino.yaml] | [![Build Vector]][build_vector.yaml] |
14+
| [![Build ZooKeeper]][build_zookeeper.yaml] | | | |
1415
<!-- end:badges -->
1516

1617
## Prerequisites
@@ -243,6 +244,8 @@ ENTRYPOINT ["/stackable-zookeeper-operator"]
243244
[build_omid.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_omid.yaml
244245
[Build OPA]: https://github.com/stackabletech/docker-images/actions/workflows/build_opa.yaml/badge.svg
245246
[build_opa.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_opa.yaml
247+
[Build Spark Connect Client]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-connect-client.yaml/badge.svg
248+
[build_spark-connect-client.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-connect-client.yaml
246249
[Build Spark K8s]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-k8s.yaml/badge.svg
247250
[build_spark-k8s.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-k8s.yaml
248251
[Build Stackable Base]: https://github.com/stackabletech/docker-images/actions/workflows/build_stackable-base.yaml/badge.svg

0 commit comments

Comments
 (0)