Skip to content

Commit 9ec4cc4

Browse files
MyroTkMyroTk
authored
Merge pull request #1837 from Altinity/gh-artifacts-26.3
Antalya 26.3 - Use GH artifacts in CICD
2 parents 0ef53dd + 154a75b commit 9ec4cc4

16 files changed

Lines changed: 1206 additions & 139 deletions

‎.github/workflows/master.yml‎

Lines changed: 362 additions & 0 deletions
Large diffs are not rendered by default.

‎.github/workflows/pull_request.yml‎

Lines changed: 395 additions & 0 deletions
Large diffs are not rendered by default.

‎.github/workflows/pull_request_community.yml‎

Lines changed: 99 additions & 99 deletions
Large diffs are not rendered by default.

‎.github/workflows/regression-reusable-suite.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ jobs:
9393
build_sha: ${{ inputs.build_sha }}
9494
pr_number: ${{ github.event.number }}
9595
artifacts: builds
96+
GH_BINARY_ARTIFACT_NAME: ${{ inputs.runner_arch == 'aarch64' && 'CH_ARM_BIN_GH' || 'CH_AMD_BINARY_GH' }}
9697
# Args
9798
args: --test-to-end
9899
--no-colors
@@ -127,7 +128,30 @@ jobs:
127128
- name: 🛠️ Setup
128129
run: .github/setup.sh
129130

131+
- name: 📥 Try get binary from GH artifact
132+
id: get_binary_gh
133+
continue-on-error: true
134+
uses: actions/download-artifact@v4
135+
with:
136+
name: ${{ env.GH_BINARY_ARTIFACT_NAME }}
137+
path: ${{ runner.temp }}/gh_binary
138+
139+
- name: 🔎 Resolve binary source
140+
id: resolve_binary_source
141+
run: |
142+
GH_BINARY="${{ runner.temp }}/gh_binary/clickhouse"
143+
if [ -f "$GH_BINARY" ]; then
144+
chmod +x "$GH_BINARY"
145+
echo "clickhouse_path=$GH_BINARY" >> "$GITHUB_ENV"
146+
echo "has_gh_binary=true" >> "$GITHUB_OUTPUT"
147+
echo "Using GH artifact binary: $GH_BINARY"
148+
else
149+
echo "has_gh_binary=false" >> "$GITHUB_OUTPUT"
150+
echo "GH artifact binary not found, fallback to S3 URL resolution"
151+
fi
152+
130153
- name: 📦 Get deb url
154+
if: ${{ steps.resolve_binary_source.outputs.has_gh_binary != 'true' }}
131155
env:
132156
S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/
133157
PR_NUMBER: ${{ github.event.pull_request.number || 0 }}

‎.github/workflows/release_builds.yml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,18 @@ jobs:
275275
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
276276
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log
277277
278+
- name: Upload artifact CH_AMD_DEBUG_GH
279+
id: upload_CH_AMD_DEBUG_GH
280+
uses: actions/upload-artifact@v7
281+
continue-on-error: true
282+
with:
283+
name: CH_AMD_DEBUG_GH
284+
path: ci/tmp/build/programs/self-extracting/clickhouse
285+
retention-days: 1
286+
- name: Warn on failed upload of CH_AMD_DEBUG_GH
287+
if: steps.upload_CH_AMD_DEBUG_GH.outcome == 'failure'
288+
run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_DEBUG_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."
289+
278290
build_amd_asan:
279291
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
280292
needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
@@ -322,6 +334,18 @@ jobs:
322334
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
323335
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log
324336
337+
- name: Upload artifact CH_AMD_ASAN_GH
338+
id: upload_CH_AMD_ASAN_GH
339+
uses: actions/upload-artifact@v7
340+
continue-on-error: true
341+
with:
342+
name: CH_AMD_ASAN_GH
343+
path: ci/tmp/build/programs/self-extracting/clickhouse
344+
retention-days: 1
345+
- name: Warn on failed upload of CH_AMD_ASAN_GH
346+
if: steps.upload_CH_AMD_ASAN_GH.outcome == 'failure'
347+
run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_ASAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."
348+
325349
build_amd_tsan:
326350
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
327351
needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
@@ -369,6 +393,18 @@ jobs:
369393
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
370394
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log
371395
396+
- name: Upload artifact CH_AMD_TSAN_GH
397+
id: upload_CH_AMD_TSAN_GH
398+
uses: actions/upload-artifact@v7
399+
continue-on-error: true
400+
with:
401+
name: CH_AMD_TSAN_GH
402+
path: ci/tmp/build/programs/self-extracting/clickhouse
403+
retention-days: 1
404+
- name: Warn on failed upload of CH_AMD_TSAN_GH
405+
if: steps.upload_CH_AMD_TSAN_GH.outcome == 'failure'
406+
run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_TSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."
407+
372408
build_amd_msan:
373409
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
374410
needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
@@ -416,6 +452,18 @@ jobs:
416452
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
417453
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log
418454
455+
- name: Upload artifact CH_AMD_MSAN_GH
456+
id: upload_CH_AMD_MSAN_GH
457+
uses: actions/upload-artifact@v7
458+
continue-on-error: true
459+
with:
460+
name: CH_AMD_MSAN_GH
461+
path: ci/tmp/build/programs/self-extracting/clickhouse
462+
retention-days: 1
463+
- name: Warn on failed upload of CH_AMD_MSAN_GH
464+
if: steps.upload_CH_AMD_MSAN_GH.outcome == 'failure'
465+
run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_MSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."
466+
419467
build_amd_ubsan:
420468
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
421469
needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
@@ -463,6 +511,18 @@ jobs:
463511
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
464512
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log
465513
514+
- name: Upload artifact CH_AMD_UBSAN_GH
515+
id: upload_CH_AMD_UBSAN_GH
516+
uses: actions/upload-artifact@v7
517+
continue-on-error: true
518+
with:
519+
name: CH_AMD_UBSAN_GH
520+
path: ci/tmp/build/programs/self-extracting/clickhouse
521+
retention-days: 1
522+
- name: Warn on failed upload of CH_AMD_UBSAN_GH
523+
if: steps.upload_CH_AMD_UBSAN_GH.outcome == 'failure'
524+
run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_UBSAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."
525+
466526
build_amd_binary:
467527
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
468528
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
@@ -510,6 +570,18 @@ jobs:
510570
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
511571
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log
512572
573+
- name: Upload artifact CH_AMD_BINARY_GH
574+
id: upload_CH_AMD_BINARY_GH
575+
uses: actions/upload-artifact@v7
576+
continue-on-error: true
577+
with:
578+
name: CH_AMD_BINARY_GH
579+
path: ci/tmp/build/programs/self-extracting/clickhouse
580+
retention-days: 1
581+
- name: Warn on failed upload of CH_AMD_BINARY_GH
582+
if: steps.upload_CH_AMD_BINARY_GH.outcome == 'failure'
583+
run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_AMD_BINARY_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."
584+
513585
build_arm_asan:
514586
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
515587
needs: [build_amd_binary, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
@@ -557,6 +629,18 @@ jobs:
557629
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
558630
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log
559631
632+
- name: Upload artifact CH_ARM_ASAN_GH
633+
id: upload_CH_ARM_ASAN_GH
634+
uses: actions/upload-artifact@v7
635+
continue-on-error: true
636+
with:
637+
name: CH_ARM_ASAN_GH
638+
path: ci/tmp/build/programs/self-extracting/clickhouse
639+
retention-days: 1
640+
- name: Warn on failed upload of CH_ARM_ASAN_GH
641+
if: steps.upload_CH_ARM_ASAN_GH.outcome == 'failure'
642+
run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_ASAN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."
643+
560644
build_arm_binary:
561645
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
562646
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
@@ -604,6 +688,18 @@ jobs:
604688
prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
605689
for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log
606690
691+
- name: Upload artifact CH_ARM_BIN_GH
692+
id: upload_CH_ARM_BIN_GH
693+
uses: actions/upload-artifact@v7
694+
continue-on-error: true
695+
with:
696+
name: CH_ARM_BIN_GH
697+
path: ci/tmp/build/programs/self-extracting/clickhouse
698+
retention-days: 1
699+
- name: Warn on failed upload of CH_ARM_BIN_GH
700+
if: steps.upload_CH_ARM_BIN_GH.outcome == 'failure'
701+
run: echo "::warning title=GH artifact upload failed::Failed to upload [CH_ARM_BIN_GH] to GitHub artifacts (e.g. quota/rate limit). Downstream consumers will fall back to S3."
702+
607703
build_amd_release:
608704
runs-on: [self-hosted, altinity-on-demand, altinity-builder]
609705
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
@@ -924,6 +1020,13 @@ jobs:
9241020
EOF
9251021
ENV_SETUP_SCRIPT_EOF
9261022
1023+
- name: Download artifact CH_ARM_BIN_GH
1024+
uses: actions/download-artifact@v8
1025+
continue-on-error: true
1026+
with:
1027+
name: CH_ARM_BIN_GH
1028+
path: ./ci/tmp
1029+
9271030
- name: Run
9281031
id: run
9291032
run: |
@@ -971,6 +1074,13 @@ jobs:
9711074
EOF
9721075
ENV_SETUP_SCRIPT_EOF
9731076
1077+
- name: Download artifact CH_ARM_BIN_GH
1078+
uses: actions/download-artifact@v8
1079+
continue-on-error: true
1080+
with:
1081+
name: CH_ARM_BIN_GH
1082+
path: ./ci/tmp
1083+
9741084
- name: Run
9751085
id: run
9761086
run: |

‎ci/defs/defs.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -426,6 +426,15 @@ class ArtifactNames:
426426
CH_RISCV64 = "CH_RISCV64_BIN"
427427
CH_S390X = "CH_S390X_BIN"
428428
CH_LOONGARCH64 = "CH_LOONGARCH64_BIN"
429+
CH_AMD_DEBUG_GH = "CH_AMD_DEBUG_GH"
430+
CH_AMD_BINARY_GH = "CH_AMD_BINARY_GH"
431+
CH_ARM_BINARY_GH = "CH_ARM_BIN_GH"
432+
CH_AMD_ASAN_GH = "CH_AMD_ASAN_GH"
433+
CH_AMD_TSAN_GH = "CH_AMD_TSAN_GH"
434+
CH_AMD_MSAN_GH = "CH_AMD_MSAN_GH"
435+
CH_AMD_UBSAN_GH = "CH_AMD_UBSAN_GH"
436+
CH_ARM_ASAN_GH = "CH_ARM_ASAN_GH"
437+
CH_ARM_TSAN_GH = "CH_ARM_TSAN_GH"
429438

430439
FAST_TEST = "FAST_TEST"
431440
UNITTEST_AMD_ASAN = "UNITTEST_AMD_ASAN"
@@ -536,6 +545,23 @@ class ArtifactConfigs:
536545
ArtifactNames.CH_LOONGARCH64,
537546
]
538547
)
548+
clickhouse_binaries_gh = Artifact.Config(
549+
name="...",
550+
type=Artifact.Type.GH,
551+
path=f"{TEMP_DIR}/build/programs/self-extracting/clickhouse",
552+
).parametrize(
553+
names=[
554+
ArtifactNames.CH_AMD_DEBUG_GH,
555+
ArtifactNames.CH_AMD_BINARY_GH,
556+
ArtifactNames.CH_ARM_BINARY_GH,
557+
ArtifactNames.CH_AMD_ASAN_GH,
558+
ArtifactNames.CH_AMD_TSAN_GH,
559+
ArtifactNames.CH_AMD_MSAN_GH,
560+
ArtifactNames.CH_AMD_UBSAN_GH,
561+
ArtifactNames.CH_ARM_ASAN_GH,
562+
ArtifactNames.CH_ARM_TSAN_GH,
563+
]
564+
)
539565
clickhouse_stripped_binaries = Artifact.Config(
540566
name="...",
541567
type=Artifact.Type.S3,

0 commit comments

Comments
 (0)