From 9425d790cd1c8da7f8af473e7cc5c6171725156a Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Thu, 27 Feb 2025 12:49:15 -0500 Subject: [PATCH 01/38] workflow for grype scanning --- .github/grype/parse_vulnerabilities_grype.py | 38 ++++++++++++ .github/grype/run_grype_scan.sh | 18 ++++++ .../grype/transform_and_upload_reports_s3.sh | 8 +++ .github/workflows/grype_scan.yml | 62 +++++++++++++++++++ .github/workflows/release_branches.yml | 12 ++++ 5 files changed, 138 insertions(+) create mode 100644 .github/grype/parse_vulnerabilities_grype.py create mode 100755 .github/grype/run_grype_scan.sh create mode 100755 .github/grype/transform_and_upload_reports_s3.sh create mode 100644 .github/workflows/grype_scan.yml diff --git a/.github/grype/parse_vulnerabilities_grype.py b/.github/grype/parse_vulnerabilities_grype.py new file mode 100644 index 000000000000..1e822935a4f1 --- /dev/null +++ b/.github/grype/parse_vulnerabilities_grype.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# Copyright 2022, Altinity Inc. All Rights Reserved. +# +# All information contained herein is, and remains the property +# of Altinity Inc. Any dissemination of this information or +# reproduction of this material is strictly forbidden unless +# prior written permission is obtained from Altinity Inc. + +from testflows.core import * +import json + +xfails = {} + + +@Name("docker vulnerabilities") +@XFails(xfails) +@TestModule +def docker_vulnerabilities(self): + with Given("I gather grype scan results"): + with open("./result.json", "r") as f: + results = json.load(f) + + for vulnerability in results["matches"]: + with Test( + f"{vulnerability['vulnerability']['id']}@{vulnerability['vulnerability']['namespace']},{vulnerability['vulnerability']['severity']}", + flags=TE, + ): + note(vulnerability) + critical_levels = set(["HIGH", "CRITICAL"]) + if vulnerability['vulnerability']["severity"] in critical_levels: + with Then( + f"Found vulnerability of {vulnerability['vulnerability']['severity']} severity" + ): + result(Fail) + + +if main(): + docker_vulnerabilities() diff --git a/.github/grype/run_grype_scan.sh b/.github/grype/run_grype_scan.sh new file mode 100755 index 000000000000..c5ce0b1b10d3 --- /dev/null +++ b/.github/grype/run_grype_scan.sh @@ -0,0 +1,18 @@ +set -x +set -e + +IMAGE=$1 + +GRYPE_VERSION="v0.80.1" + +docker pull $IMAGE +docker pull anchore/grype:${GRYPE_VERSION} + +docker run \ + --rm --volume /var/run/docker.sock:/var/run/docker.sock \ + --name Grype anchore/grype:${GRYPE_VERSION} \ + --scope all-layers \ + -o json \ + $IMAGE > result.json + +ls -sh diff --git a/.github/grype/transform_and_upload_reports_s3.sh b/.github/grype/transform_and_upload_reports_s3.sh new file mode 100755 index 000000000000..fcab607935a2 --- /dev/null +++ b/.github/grype/transform_and_upload_reports_s3.sh @@ -0,0 +1,8 @@ +tfs --no-colors transform nice raw.log nice.log.txt +tfs --no-colors report results -a "https://s3.amazonaws.com/$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" raw.log - --copyright "Altinity LTD" --logo | tfs --no-colors document convert > results.html + +S3_PATH="s3://$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" +aws s3 cp --no-progress nice.log.txt $S3_PATH/nice.log.txt --content-type "text/plain; charset=utf-8" || echo "nice log file not found". +aws s3 cp --no-progress results.html $S3_PATH/results.html || echo "results file not found". +aws s3 cp --no-progress raw.log $S3_PATH/raw.log || echo "raw.log file not found". +aws s3 cp --no-progress result.json $S3_PATH/result.json --content-type "text/plain; charset=utf-8" || echo "result.json not found". \ No newline at end of file diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml new file mode 100644 index 000000000000..4cd3b95449ab --- /dev/null +++ b/.github/workflows/grype_scan.yml @@ -0,0 +1,62 @@ +name: Grype Scan +run-name: Grype Scan ${{ inputs.docker_image }} + +on: + workflow_dispatch: + inputs: + docker_image: + description: 'Docker image with tag to scan' + required: true + workflow_call: + inputs: + docker_image: + description: 'Docker image without tag to scan' + required: true + type: string + +jobs: + grype_scan: + runs-on: [self-hosted, x86, type-cx42, image-x86-app-docker-ce] + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Docker + uses: docker/setup-buildx-action@v3 + + - name: Set up Python + run: | + export TESTFLOWS_VERSION="2.4.19" + sudo apt-get install -y python3-pip python3-venv + python3 -m venv venv + source venv/bin/activate + pip install --upgrade requests chardet urllib3 + pip install testflows==$TESTFLOWS_VERSION + echo PATH=$PATH >>$GITHUB_ENV + + - name: Set image tag if called by another workflow + if: ${{ github.event_name == 'workflow_call' }} + id: set_version + run: | + python3 ./tests/ci/version_helper.py | tee /tmp/version_info + source /tmp/version_info + echo "::set-output name=docker_image::${{ github.event.inputs.docker_image }}:${{ github.event.pull_request.number }}-$CLICKHOUSE_VERSION_STRING" + echo "::set-output name=commit_sha::$CLICKHOUSE_VERSION_GITHASH + + - name: Run Grype Scan + run: | + DOCKER_IMAGE=${{ steps.set_version.outputs.docker_image || inputs.docker_image }} + ./.github/grype/run_grype_scan.sh $DOCKER_IMAGE + + - name: Parse grype results + run: | + python3 -u ./.github/grype/parse_vulnerabilities_grype.py -o nice --no-colors --log raw.log --test-to-end + + - name: Transform and Upload Grype Results + env: + S3_BUCKET: "altinity-build-artifacts" + COMMIT_SHA: ${{ steps.set_version.outputs.commit_sha || github.sha }} + PR_NUMBER: ${{ github.event.number }} + DOCKER_IMAGE: ${{ steps.set_version.outputs.docker_image || inputs.docker_image }} + run: | + ./.github/grype/transform_and_upload_results_s3.sh diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 22e105df2cd1..809e0caddfc3 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -175,6 +175,18 @@ jobs: test_name: Docker keeper image runner_type: altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} + GrypeScanServerImage: + needs: [DockerServerImage] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/grype_scan.yml + with: + docker_image: "altinityinfra/clickhouse-server" + GrypeScanKeeperImage: + needs: [DockerKeeperImage] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/grype_scan.yml + with: + docker_image: "altinityinfra/clickhouse-keeper" ############################################################################################ ##################################### BUILD REPORTER ####################################### ############################################################################################ From 408da68df74f2bd2ebff697fe90a058425e8ca45 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Thu, 27 Feb 2025 15:26:31 -0500 Subject: [PATCH 02/38] Cleanup and fix default pr number and upload artifacts --- .github/grype/parse_vulnerabilities_grype.py | 8 +------- .github/workflows/grype_scan.yml | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/grype/parse_vulnerabilities_grype.py b/.github/grype/parse_vulnerabilities_grype.py index 1e822935a4f1..455d9ea2d39d 100644 --- a/.github/grype/parse_vulnerabilities_grype.py +++ b/.github/grype/parse_vulnerabilities_grype.py @@ -1,13 +1,7 @@ #!/usr/bin/env python3 -# Copyright 2022, Altinity Inc. All Rights Reserved. -# -# All information contained herein is, and remains the property -# of Altinity Inc. Any dissemination of this information or -# reproduction of this material is strictly forbidden unless -# prior written permission is obtained from Altinity Inc. +import json from testflows.core import * -import json xfails = {} diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 4cd3b95449ab..5b754e777237 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -19,7 +19,7 @@ jobs: runs-on: [self-hosted, x86, type-cx42, image-x86-app-docker-ce] steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Docker uses: docker/setup-buildx-action@v3 @@ -40,8 +40,8 @@ jobs: run: | python3 ./tests/ci/version_helper.py | tee /tmp/version_info source /tmp/version_info - echo "::set-output name=docker_image::${{ github.event.inputs.docker_image }}:${{ github.event.pull_request.number }}-$CLICKHOUSE_VERSION_STRING" - echo "::set-output name=commit_sha::$CLICKHOUSE_VERSION_GITHASH + echo "::set-output name=docker_image::${{ github.event.inputs.docker_image }}:${{ github.event.pull_request.number || 0 }}-$CLICKHOUSE_VERSION_STRING" + echo "::set-output name=commit_sha::$CLICKHOUSE_VERSION_GITHASH" - name: Run Grype Scan run: | @@ -56,7 +56,16 @@ jobs: env: S3_BUCKET: "altinity-build-artifacts" COMMIT_SHA: ${{ steps.set_version.outputs.commit_sha || github.sha }} - PR_NUMBER: ${{ github.event.number }} + PR_NUMBER: ${{ github.event.pull_request.number || 0 }} DOCKER_IMAGE: ${{ steps.set_version.outputs.docker_image || inputs.docker_image }} run: | ./.github/grype/transform_and_upload_results_s3.sh + + - name: Upload artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: grype-results + path: | + result.json + nice.log.txt From 9bf61cc4dbe56825439dca75060fa8f0efc966cb Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Thu, 6 Mar 2025 09:32:02 -0500 Subject: [PATCH 03/38] fix grype scan tags and add comments try fixing tags again --- .github/workflows/grype_scan.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 5b754e777237..8260bf280cc1 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -3,20 +3,23 @@ run-name: Grype Scan ${{ inputs.docker_image }} on: workflow_dispatch: + # Inputs for manual run inputs: docker_image: description: 'Docker image with tag to scan' required: true workflow_call: + # Inputs for workflow call inputs: docker_image: - description: 'Docker image without tag to scan' + description: 'Docker image without tag to scan. Tag will be determined from the repo state.' required: true type: string jobs: grype_scan: - runs-on: [self-hosted, x86, type-cx42, image-x86-app-docker-ce] + name: Grype Scan + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] steps: - name: Checkout repository uses: actions/checkout@v4 From aa89c70d9a2343ae98d12cc31660fcbe38bfb144 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Thu, 6 Mar 2025 11:21:36 -0500 Subject: [PATCH 04/38] add skipping support to regression --- .github/workflows/release_branches.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 809e0caddfc3..e633cc0868b5 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -487,8 +487,8 @@ jobs: ##################################### REGRESSION TESTS ###################################### ############################################################################################# RegressionTestsRelease: - needs: [BuilderDebRelease] - if: ${{ !failure() && !cancelled() }} + needs: [RunConfig, BuilderDebRelease] + if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.RunConfig.outputs.data).ci_settings.exclude_keywords, 'regression')}} uses: ./.github/workflows/regression.yml secrets: inherit with: @@ -498,8 +498,8 @@ jobs: build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout_minutes: 300 RegressionTestsAarch64: - needs: [BuilderDebAarch64] - if: ${{ !failure() && !cancelled() }} + needs: [RunConfig, BuilderDebAarch64] + if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.RunConfig.outputs.data).ci_settings.exclude_keywords, 'regression') && !contains(fromJson(needs.RunConfig.outputs.data).ci_settings.exclude_keywords, 'aarch64')}} uses: ./.github/workflows/regression.yml secrets: inherit with: From ef05cfdcef590ec8c0d91d12d01d418a86ceb054 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Thu, 6 Mar 2025 13:13:10 -0500 Subject: [PATCH 05/38] add missing apt update fix path to grype upload script Fix report command and workflow issues --- ...load_reports_s3.sh => transform_and_upload_results_s3.sh} | 2 +- .github/workflows/grype_scan.yml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) rename .github/grype/{transform_and_upload_reports_s3.sh => transform_and_upload_results_s3.sh} (89%) diff --git a/.github/grype/transform_and_upload_reports_s3.sh b/.github/grype/transform_and_upload_results_s3.sh similarity index 89% rename from .github/grype/transform_and_upload_reports_s3.sh rename to .github/grype/transform_and_upload_results_s3.sh index fcab607935a2..0690b0f81340 100755 --- a/.github/grype/transform_and_upload_reports_s3.sh +++ b/.github/grype/transform_and_upload_results_s3.sh @@ -1,5 +1,5 @@ tfs --no-colors transform nice raw.log nice.log.txt -tfs --no-colors report results -a "https://s3.amazonaws.com/$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" raw.log - --copyright "Altinity LTD" --logo | tfs --no-colors document convert > results.html +tfs --no-colors report results -a "https://s3.amazonaws.com/$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" raw.log - --copyright "Altinity LTD" | tfs --no-colors document convert > results.html S3_PATH="s3://$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" aws s3 cp --no-progress nice.log.txt $S3_PATH/nice.log.txt --content-type "text/plain; charset=utf-8" || echo "nice log file not found". diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 8260bf280cc1..5e69ab554828 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -30,6 +30,7 @@ jobs: - name: Set up Python run: | export TESTFLOWS_VERSION="2.4.19" + sudo apt-get update sudo apt-get install -y python3-pip python3-venv python3 -m venv venv source venv/bin/activate @@ -38,7 +39,7 @@ jobs: echo PATH=$PATH >>$GITHUB_ENV - name: Set image tag if called by another workflow - if: ${{ github.event_name == 'workflow_call' }} + if: ${{ github.event.inputs.docker_image }} id: set_version run: | python3 ./tests/ci/version_helper.py | tee /tmp/version_info @@ -68,7 +69,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: grype-results + name: grype-results-${{ github.run_id }}-${{ github.job }} path: | result.json nice.log.txt From 37c9517b06ce2a94dabad7d6193159720ba6c1a6 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Fri, 7 Mar 2025 10:35:16 -0500 Subject: [PATCH 06/38] fix copilot's follies fix version getter for grype scan fix another copilot mistake fix pr num in version add tag for image to access awscli smaller instance for grype too --- .github/workflows/grype_scan.yml | 16 ++++++++-------- .github/workflows/release_branches.yml | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 5e69ab554828..0238a51a8475 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -6,20 +6,20 @@ on: # Inputs for manual run inputs: docker_image: - description: 'Docker image with tag to scan' + description: 'Docker image. If no tag, it will be determined by version_helper.py' required: true workflow_call: # Inputs for workflow call inputs: docker_image: - description: 'Docker image without tag to scan. Tag will be determined from the repo state.' + description: 'Docker image. If no tag, it will be determined by version_helper.py' required: true type: string jobs: grype_scan: name: Grype Scan - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cax11, altinity-image-x86-app-docker-ce] steps: - name: Checkout repository uses: actions/checkout@v4 @@ -38,13 +38,13 @@ jobs: pip install testflows==$TESTFLOWS_VERSION echo PATH=$PATH >>$GITHUB_ENV - - name: Set image tag if called by another workflow - if: ${{ github.event.inputs.docker_image }} + - name: Set image tag if not given + if: ${{ !contains(inputs.docker_image, ':') }} id: set_version run: | python3 ./tests/ci/version_helper.py | tee /tmp/version_info source /tmp/version_info - echo "::set-output name=docker_image::${{ github.event.inputs.docker_image }}:${{ github.event.pull_request.number || 0 }}-$CLICKHOUSE_VERSION_STRING" + echo "::set-output name=docker_image::${{ inputs.docker_image }}:${{ github.event.pull_request.number || 0 }}-$CLICKHOUSE_VERSION_STRING" echo "::set-output name=commit_sha::$CLICKHOUSE_VERSION_GITHASH" - name: Run Grype Scan @@ -59,7 +59,7 @@ jobs: - name: Transform and Upload Grype Results env: S3_BUCKET: "altinity-build-artifacts" - COMMIT_SHA: ${{ steps.set_version.outputs.commit_sha || github.sha }} + COMMIT_SHA: ${{ steps.set_version.outputs.commit_sha || github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} PR_NUMBER: ${{ github.event.pull_request.number || 0 }} DOCKER_IMAGE: ${{ steps.set_version.outputs.docker_image || inputs.docker_image }} run: | @@ -69,7 +69,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: grype-results-${{ github.run_id }}-${{ github.job }} + name: grype-results-${{ hashFiles('raw.log') }} path: | result.json nice.log.txt diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index e633cc0868b5..54a765dc86aa 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -176,17 +176,17 @@ jobs: runner_type: altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} GrypeScanServerImage: - needs: [DockerServerImage] + needs: [RunConfig, DockerServerImage] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/grype_scan.yml with: - docker_image: "altinityinfra/clickhouse-server" + docker_image: "altinityinfra/clickhouse-server:${{ github.event.pull_request.number || 0 }}-${{ fromJson(needs.RunConfig.outputs.data).version }}" GrypeScanKeeperImage: - needs: [DockerKeeperImage] + needs: [RunConfig, DockerKeeperImage] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/grype_scan.yml with: - docker_image: "altinityinfra/clickhouse-keeper" + docker_image: "altinityinfra/clickhouse-keeper:${{ github.event.pull_request.number || 0 }}-${{ fromJson(needs.RunConfig.outputs.data).version }}" ############################################################################################ ##################################### BUILD REPORTER ####################################### ############################################################################################ From d89d757e4195fe4da83cfea7cfa282f322df4b1c Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Mon, 10 Mar 2025 13:42:08 -0400 Subject: [PATCH 07/38] install awscli with pip --- .github/workflows/grype_scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 0238a51a8475..062be588a9f0 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -19,7 +19,7 @@ on: jobs: grype_scan: name: Grype Scan - runs-on: [self-hosted, altinity-on-demand, altinity-type-cax11, altinity-image-x86-app-docker-ce] + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64] steps: - name: Checkout repository uses: actions/checkout@v4 @@ -35,7 +35,7 @@ jobs: python3 -m venv venv source venv/bin/activate pip install --upgrade requests chardet urllib3 - pip install testflows==$TESTFLOWS_VERSION + pip install testflows==$TESTFLOWS_VERSION awscli==1.33.28 echo PATH=$PATH >>$GITHUB_ENV - name: Set image tag if not given From 0f1a522b38bda441581b15b230abf27d918a560a Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Mon, 10 Mar 2025 19:27:34 -0400 Subject: [PATCH 08/38] add aws secrets --- .github/workflows/grype_scan.yml | 5 +++++ .github/workflows/release_branches.yml | 2 ++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 062be588a9f0..b000dbecb935 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -15,6 +15,11 @@ on: description: 'Docker image. If no tag, it will be determined by version_helper.py' required: true type: string +env: + PYTHONUNBUFFERED: 1 + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} jobs: grype_scan: diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 54a765dc86aa..44d3202b6bef 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -179,12 +179,14 @@ jobs: needs: [RunConfig, DockerServerImage] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/grype_scan.yml + secrets: inherit with: docker_image: "altinityinfra/clickhouse-server:${{ github.event.pull_request.number || 0 }}-${{ fromJson(needs.RunConfig.outputs.data).version }}" GrypeScanKeeperImage: needs: [RunConfig, DockerKeeperImage] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/grype_scan.yml + secrets: inherit with: docker_image: "altinityinfra/clickhouse-keeper:${{ github.event.pull_request.number || 0 }}-${{ fromJson(needs.RunConfig.outputs.data).version }}" ############################################################################################ From 324ca9755833609c7b7a34adbdc62cbf5589ef03 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Mon, 10 Mar 2025 22:32:07 -0400 Subject: [PATCH 09/38] Combine grype jobs with matrix, clean docker image name for log upload --- .github/grype/transform_and_upload_results_s3.sh | 2 ++ .github/workflows/release_branches.yml | 16 ++++++---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/grype/transform_and_upload_results_s3.sh b/.github/grype/transform_and_upload_results_s3.sh index 0690b0f81340..403ede2e1dd7 100755 --- a/.github/grype/transform_and_upload_results_s3.sh +++ b/.github/grype/transform_and_upload_results_s3.sh @@ -1,3 +1,5 @@ +DOCKER_IMAGE=$(echo "$DOCKER_IMAGE" | sed 's/[\/:]/_/g') + tfs --no-colors transform nice raw.log nice.log.txt tfs --no-colors report results -a "https://s3.amazonaws.com/$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" raw.log - --copyright "Altinity LTD" | tfs --no-colors document convert > results.html diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 44d3202b6bef..febf0a5024cc 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -175,20 +175,16 @@ jobs: test_name: Docker keeper image runner_type: altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} - GrypeScanServerImage: - needs: [RunConfig, DockerServerImage] + GrypeScan: + needs: [RunConfig, DockerServerImage, DockerKeeperImage] if: ${{ !failure() && !cancelled() }} + strategy: + matrix: + image: [server, keeper] uses: ./.github/workflows/grype_scan.yml secrets: inherit with: - docker_image: "altinityinfra/clickhouse-server:${{ github.event.pull_request.number || 0 }}-${{ fromJson(needs.RunConfig.outputs.data).version }}" - GrypeScanKeeperImage: - needs: [RunConfig, DockerKeeperImage] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/grype_scan.yml - secrets: inherit - with: - docker_image: "altinityinfra/clickhouse-keeper:${{ github.event.pull_request.number || 0 }}-${{ fromJson(needs.RunConfig.outputs.data).version }}" + docker_image: altinityinfra/clickhouse-${{ matrix.image }}:${{ github.event.pull_request.number || 0 }}-${{ fromJson(needs.RunConfig.outputs.data).version }} ############################################################################################ ##################################### BUILD REPORTER ####################################### ############################################################################################ From 1d25532d40a4cae527c04a62b4213deb76763329 Mon Sep 17 00:00:00 2001 From: Stuart <146047128+strtgbb@users.noreply.github.com> Date: Tue, 11 Mar 2025 14:49:58 -0400 Subject: [PATCH 10/38] Add Grype results to ci_running --- .../grype/transform_and_upload_results_s3.sh | 7 +++++-- .github/workflows/grype_scan.yml | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/grype/transform_and_upload_results_s3.sh b/.github/grype/transform_and_upload_results_s3.sh index 403ede2e1dd7..8cb8aeb4124c 100755 --- a/.github/grype/transform_and_upload_results_s3.sh +++ b/.github/grype/transform_and_upload_results_s3.sh @@ -1,9 +1,12 @@ DOCKER_IMAGE=$(echo "$DOCKER_IMAGE" | sed 's/[\/:]/_/g') +S3_PATH="s3://$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" +HTTPS_S3_PATH="https://s3.amazonaws.com/$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" +export HTTPS_S3_PATH + tfs --no-colors transform nice raw.log nice.log.txt -tfs --no-colors report results -a "https://s3.amazonaws.com/$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" raw.log - --copyright "Altinity LTD" | tfs --no-colors document convert > results.html +tfs --no-colors report results -a $HTTPS_S3_PATH raw.log - --copyright "Altinity LTD" | tfs --no-colors document convert > results.html -S3_PATH="s3://$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" aws s3 cp --no-progress nice.log.txt $S3_PATH/nice.log.txt --content-type "text/plain; charset=utf-8" || echo "nice log file not found". aws s3 cp --no-progress results.html $S3_PATH/results.html || echo "results file not found". aws s3 cp --no-progress raw.log $S3_PATH/raw.log || echo "raw.log file not found". diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index b000dbecb935..826aab518b81 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -62,6 +62,7 @@ jobs: python3 -u ./.github/grype/parse_vulnerabilities_grype.py -o nice --no-colors --log raw.log --test-to-end - name: Transform and Upload Grype Results + id: upload_results env: S3_BUCKET: "altinity-build-artifacts" COMMIT_SHA: ${{ steps.set_version.outputs.commit_sha || github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} @@ -69,6 +70,22 @@ jobs: DOCKER_IMAGE: ${{ steps.set_version.outputs.docker_image || inputs.docker_image }} run: | ./.github/grype/transform_and_upload_results_s3.sh + echo "::set-output name=https_s3_path=$HTTPS_S3_PATH" + + - name: Set commit status + uses: actions/github-script@v4 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.payload.pull_request.head.sha, + state: 'success', + target_url: '${{ steps.upload_results.outputs.https_s3_path }}/results.html', + description: '', + context: 'Grype Scan ${{ steps.set_version.outputs.docker_image || inputs.docker_image }}' + }) - name: Upload artifacts if: always() From 60d625f5716150ce1c658e07f02a717b21b65dbb Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Thu, 27 Mar 2025 17:47:13 -0400 Subject: [PATCH 11/38] fix version of actions/github-script --- .github/workflows/grype_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 826aab518b81..62d945ad6333 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -73,7 +73,7 @@ jobs: echo "::set-output name=https_s3_path=$HTTPS_S3_PATH" - name: Set commit status - uses: actions/github-script@v4 + uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | From 6fc4cee4c353a266962e605fd9d265c78028414a Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Fri, 28 Mar 2025 09:23:42 -0400 Subject: [PATCH 12/38] fix --- .github/grype/transform_and_upload_results_s3.sh | 2 +- .github/workflows/grype_scan.yml | 5 ++--- .github/workflows/release_branches.yml | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/grype/transform_and_upload_results_s3.sh b/.github/grype/transform_and_upload_results_s3.sh index 8cb8aeb4124c..330b8fbc20f3 100755 --- a/.github/grype/transform_and_upload_results_s3.sh +++ b/.github/grype/transform_and_upload_results_s3.sh @@ -2,7 +2,7 @@ DOCKER_IMAGE=$(echo "$DOCKER_IMAGE" | sed 's/[\/:]/_/g') S3_PATH="s3://$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" HTTPS_S3_PATH="https://s3.amazonaws.com/$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" -export HTTPS_S3_PATH +echo "::set-output name=https_s3_path=$HTTPS_S3_PATH" tfs --no-colors transform nice raw.log nice.log.txt tfs --no-colors report results -a $HTTPS_S3_PATH raw.log - --copyright "Altinity LTD" | tfs --no-colors document convert > results.html diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 62d945ad6333..3ddab1841734 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -70,7 +70,6 @@ jobs: DOCKER_IMAGE: ${{ steps.set_version.outputs.docker_image || inputs.docker_image }} run: | ./.github/grype/transform_and_upload_results_s3.sh - echo "::set-output name=https_s3_path=$HTTPS_S3_PATH" - name: Set commit status uses: actions/github-script@v7 @@ -80,10 +79,10 @@ jobs: github.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, - sha: context.payload.pull_request.head.sha, + sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}, state: 'success', target_url: '${{ steps.upload_results.outputs.https_s3_path }}/results.html', - description: '', + description: 'TODO', context: 'Grype Scan ${{ steps.set_version.outputs.docker_image || inputs.docker_image }}' }) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index febf0a5024cc..9368524ff1f7 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -179,6 +179,7 @@ jobs: needs: [RunConfig, DockerServerImage, DockerKeeperImage] if: ${{ !failure() && !cancelled() }} strategy: + fail-fast: false matrix: image: [server, keeper] uses: ./.github/workflows/grype_scan.yml From cb30c1c2be72d160e4b02c98c23b0b052a23ad56 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Fri, 28 Mar 2025 14:23:30 -0400 Subject: [PATCH 13/38] fix --- .github/workflows/grype_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 3ddab1841734..d3fd7b67953d 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -79,7 +79,7 @@ jobs: github.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, - sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}, + sha: '${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}', state: 'success', target_url: '${{ steps.upload_results.outputs.https_s3_path }}/results.html', description: 'TODO', From 318de6be02fbb9f79615549ee0ac6746eb783619 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Fri, 28 Mar 2025 18:12:12 -0400 Subject: [PATCH 14/38] fix --- .github/grype/transform_and_upload_results_s3.sh | 2 +- .github/workflows/grype_scan.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/grype/transform_and_upload_results_s3.sh b/.github/grype/transform_and_upload_results_s3.sh index 330b8fbc20f3..a74291700403 100755 --- a/.github/grype/transform_and_upload_results_s3.sh +++ b/.github/grype/transform_and_upload_results_s3.sh @@ -2,7 +2,7 @@ DOCKER_IMAGE=$(echo "$DOCKER_IMAGE" | sed 's/[\/:]/_/g') S3_PATH="s3://$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" HTTPS_S3_PATH="https://s3.amazonaws.com/$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" -echo "::set-output name=https_s3_path=$HTTPS_S3_PATH" +export $HTTPS_S3_PATH tfs --no-colors transform nice raw.log nice.log.txt tfs --no-colors report results -a $HTTPS_S3_PATH raw.log - --copyright "Altinity LTD" | tfs --no-colors document convert > results.html diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index d3fd7b67953d..27ecc9383d08 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -70,6 +70,7 @@ jobs: DOCKER_IMAGE: ${{ steps.set_version.outputs.docker_image || inputs.docker_image }} run: | ./.github/grype/transform_and_upload_results_s3.sh + echo "::set-output name=https_s3_path::$HTTPS_S3_PATH" - name: Set commit status uses: actions/github-script@v7 From 4851778e2641da2ba123e88747bf00804fe4762d Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Fri, 28 Mar 2025 20:05:05 -0400 Subject: [PATCH 15/38] fix --- .github/grype/transform_and_upload_results_s3.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/grype/transform_and_upload_results_s3.sh b/.github/grype/transform_and_upload_results_s3.sh index a74291700403..8cb8aeb4124c 100755 --- a/.github/grype/transform_and_upload_results_s3.sh +++ b/.github/grype/transform_and_upload_results_s3.sh @@ -2,7 +2,7 @@ DOCKER_IMAGE=$(echo "$DOCKER_IMAGE" | sed 's/[\/:]/_/g') S3_PATH="s3://$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" HTTPS_S3_PATH="https://s3.amazonaws.com/$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" -export $HTTPS_S3_PATH +export HTTPS_S3_PATH tfs --no-colors transform nice raw.log nice.log.txt tfs --no-colors report results -a $HTTPS_S3_PATH raw.log - --copyright "Altinity LTD" | tfs --no-colors document convert > results.html From 83b44686e3b57020b224a23a4e78813df0d35fef Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Fri, 28 Mar 2025 20:09:28 -0400 Subject: [PATCH 16/38] fix --- .github/workflows/grype_scan.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 27ecc9383d08..64be88a45cf3 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -49,8 +49,8 @@ jobs: run: | python3 ./tests/ci/version_helper.py | tee /tmp/version_info source /tmp/version_info - echo "::set-output name=docker_image::${{ inputs.docker_image }}:${{ github.event.pull_request.number || 0 }}-$CLICKHOUSE_VERSION_STRING" - echo "::set-output name=commit_sha::$CLICKHOUSE_VERSION_GITHASH" + echo "docker_image=${{ inputs.docker_image }}:${{ github.event.pull_request.number || 0 }}-$CLICKHOUSE_VERSION_STRING" >> $GITHUB_OUTPUT + echo "commit_sha=$CLICKHOUSE_VERSION_GITHASH" >> $GITHUB_OUTPUT - name: Run Grype Scan run: | @@ -70,7 +70,7 @@ jobs: DOCKER_IMAGE: ${{ steps.set_version.outputs.docker_image || inputs.docker_image }} run: | ./.github/grype/transform_and_upload_results_s3.sh - echo "::set-output name=https_s3_path::$HTTPS_S3_PATH" + echo "https_s3_path=$HTTPS_S3_PATH" >> $GITHUB_OUTPUT - name: Set commit status uses: actions/github-script@v7 From 9753b5cf50e786afbdbfc8a431d234b98e33941a Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Fri, 28 Mar 2025 22:23:46 -0400 Subject: [PATCH 17/38] add grype step summary and fix --- .../grype/transform_and_upload_results_s3.sh | 2 +- .github/workflows/grype_scan.yml | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/grype/transform_and_upload_results_s3.sh b/.github/grype/transform_and_upload_results_s3.sh index 8cb8aeb4124c..7a10b02887ef 100755 --- a/.github/grype/transform_and_upload_results_s3.sh +++ b/.github/grype/transform_and_upload_results_s3.sh @@ -2,7 +2,7 @@ DOCKER_IMAGE=$(echo "$DOCKER_IMAGE" | sed 's/[\/:]/_/g') S3_PATH="s3://$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" HTTPS_S3_PATH="https://s3.amazonaws.com/$S3_BUCKET/$PR_NUMBER/$COMMIT_SHA/grype/$DOCKER_IMAGE" -export HTTPS_S3_PATH +echo "https_s3_path=$HTTPS_S3_PATH" >> $GITHUB_OUTPUT tfs --no-colors transform nice raw.log nice.log.txt tfs --no-colors report results -a $HTTPS_S3_PATH raw.log - --copyright "Altinity LTD" | tfs --no-colors document convert > results.html diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 64be88a45cf3..86e0f728a96d 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -70,20 +70,33 @@ jobs: DOCKER_IMAGE: ${{ steps.set_version.outputs.docker_image || inputs.docker_image }} run: | ./.github/grype/transform_and_upload_results_s3.sh - echo "https_s3_path=$HTTPS_S3_PATH" >> $GITHUB_OUTPUT + + - name: Create step summary + run: | + echo "| Severity | Count |" >> $GITHUB_STEP_SUMMARY + echo "|------------|-------|" >> $GITHUB_STEP_SUMMARY + jq -r ' + .matches | + map(.vulnerability.severity) | + group_by(.) | + map({severity: .[0], count: length}) | + sort_by(.severity) | + map("| \(.severity) | \(.count) |") | + .[] + ' result.json >> $GITHUB_STEP_SUMMARY - name: Set commit status uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - github.repos.createCommitStatus({ + github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, sha: '${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}', state: 'success', target_url: '${{ steps.upload_results.outputs.https_s3_path }}/results.html', - description: 'TODO', + description: 'Grype Scan Completed', context: 'Grype Scan ${{ steps.set_version.outputs.docker_image || inputs.docker_image }}' }) From 289e6c99c0798328e316d951805f85c99f2cf3bc Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Sat, 29 Mar 2025 00:45:31 -0400 Subject: [PATCH 18/38] log grype's detected distro --- .github/workflows/grype_scan.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 86e0f728a96d..a1562bb82ced 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -85,6 +85,8 @@ jobs: .[] ' result.json >> $GITHUB_STEP_SUMMARY + jq -r '.distro | "Distro: \(.name):\(.version)"' result.json >> $GITHUB_STEP_SUMMARY + - name: Set commit status uses: actions/github-script@v7 with: From b980fb84dfd8289a1bfbc4a338e1ba4d71cf6a23 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Mon, 31 Mar 2025 08:53:59 -0400 Subject: [PATCH 19/38] better summary --- .github/workflows/grype_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index a1562bb82ced..1d56359f68bf 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -73,6 +73,7 @@ jobs: - name: Create step summary run: | + jq -r '.distro | "**Distro**: \(.name):\(.version)"' result.json >> $GITHUB_STEP_SUMMARY echo "| Severity | Count |" >> $GITHUB_STEP_SUMMARY echo "|------------|-------|" >> $GITHUB_STEP_SUMMARY jq -r ' @@ -85,7 +86,6 @@ jobs: .[] ' result.json >> $GITHUB_STEP_SUMMARY - jq -r '.distro | "Distro: \(.name):\(.version)"' result.json >> $GITHUB_STEP_SUMMARY - name: Set commit status uses: actions/github-script@v7 From 09ddbab2a1d4b929c89b10efd6f085c8ce72fac6 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Mon, 31 Mar 2025 15:54:32 -0400 Subject: [PATCH 20/38] Add scan for alpine server --- .github/workflows/release_branches.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 9368524ff1f7..3768a4dd5270 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -181,11 +181,17 @@ jobs: strategy: fail-fast: false matrix: - image: [server, keeper] + include: + - image: server + suffix: '' + - image: server + suffix: '-alpine' + - image: keeper + suffix: '' uses: ./.github/workflows/grype_scan.yml secrets: inherit with: - docker_image: altinityinfra/clickhouse-${{ matrix.image }}:${{ github.event.pull_request.number || 0 }}-${{ fromJson(needs.RunConfig.outputs.data).version }} + docker_image: altinityinfra/clickhouse-${{ matrix.image }}:${{ github.event.pull_request.number || 0 }}-${{ fromJson(needs.RunConfig.outputs.data).version }}${{ matrix.suffix }} ############################################################################################ ##################################### BUILD REPORTER ####################################### ############################################################################################ From 41debf940b932feda2d11357046d42d3b673dfc9 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Mon, 31 Mar 2025 16:15:46 -0400 Subject: [PATCH 21/38] Add high/critical stats to grype job outputs --- .github/workflows/grype_scan.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 1d56359f68bf..e4faef6f0db3 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -72,6 +72,7 @@ jobs: ./.github/grype/transform_and_upload_results_s3.sh - name: Create step summary + id: create_summary run: | jq -r '.distro | "**Distro**: \(.name):\(.version)"' result.json >> $GITHUB_STEP_SUMMARY echo "| Severity | Count |" >> $GITHUB_STEP_SUMMARY @@ -86,6 +87,17 @@ jobs: .[] ' result.json >> $GITHUB_STEP_SUMMARY + HIGH_COUNT=$(jq -r '.matches | map(.vulnerability.severity) | map(select(. == "High")) | length' result.json) + CRITICAL_COUNT=$(jq -r '.matches | map(.vulnerability.severity) | map(select(. == "Critical")) | length' result.json) + TOTAL_HIGH_CRITICAL=$((HIGH_COUNT + CRITICAL_COUNT)) + echo "total_high_critical=$TOTAL_HIGH_CRITICAL" >> $GITHUB_OUTPUT + + if [ $TOTAL_HIGH_CRITICAL -gt 0 ]; then + echo "## High and Critical vulnerabilities found" >> $GITHUB_STEP_SUMMARY + echo "```" >> $GITHUB_STEP_SUMMARY + tfs --no-colors show tests raw.log | grep -Pi "High|Critical" >> $GITHUB_STEP_SUMMARY + echo "```" >> $GITHUB_STEP_SUMMARY + fi - name: Set commit status uses: actions/github-script@v7 @@ -96,9 +108,9 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, sha: '${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}', - state: 'success', + state: ${{ steps.create_summary.outputs.total_high_critical > 0 && 'failure' || 'success' }}, target_url: '${{ steps.upload_results.outputs.https_s3_path }}/results.html', - description: 'Grype Scan Completed', + description: 'Grype Scan Completed with ${{ steps.create_summary.outputs.total_high_critical }} high/critical vulnerabilities', context: 'Grype Scan ${{ steps.set_version.outputs.docker_image || inputs.docker_image }}' }) From ae78ade2e8def8fe9e0aa526afc4e32309696a46 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Tue, 1 Apr 2025 17:10:35 -0400 Subject: [PATCH 22/38] Make parse_vulnerabilities_grype status more reliable --- .github/grype/parse_vulnerabilities_grype.py | 2 +- .github/workflows/grype_scan.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/grype/parse_vulnerabilities_grype.py b/.github/grype/parse_vulnerabilities_grype.py index 455d9ea2d39d..fec2ef3bfac7 100644 --- a/.github/grype/parse_vulnerabilities_grype.py +++ b/.github/grype/parse_vulnerabilities_grype.py @@ -21,7 +21,7 @@ def docker_vulnerabilities(self): ): note(vulnerability) critical_levels = set(["HIGH", "CRITICAL"]) - if vulnerability['vulnerability']["severity"] in critical_levels: + if vulnerability['vulnerability']["severity"].upper() in critical_levels: with Then( f"Found vulnerability of {vulnerability['vulnerability']['severity']} severity" ): diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index e4faef6f0db3..31a2aae8d6b8 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -62,6 +62,7 @@ jobs: python3 -u ./.github/grype/parse_vulnerabilities_grype.py -o nice --no-colors --log raw.log --test-to-end - name: Transform and Upload Grype Results + if: always() id: upload_results env: S3_BUCKET: "altinity-build-artifacts" @@ -72,6 +73,7 @@ jobs: ./.github/grype/transform_and_upload_results_s3.sh - name: Create step summary + if: always() id: create_summary run: | jq -r '.distro | "**Distro**: \(.name):\(.version)"' result.json >> $GITHUB_STEP_SUMMARY @@ -100,6 +102,7 @@ jobs: fi - name: Set commit status + if: always() uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} From 96e21e600502505b8b9b1a1b566059eddc2e2096 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Tue, 1 Apr 2025 18:56:38 -0400 Subject: [PATCH 23/38] add cve results to combined report --- .github/create_combined_ci_report.py | 46 ++++++++++++++++++++++++-- .github/workflows/release_branches.yml | 2 +- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/.github/create_combined_ci_report.py b/.github/create_combined_ci_report.py index 07b6a80763bb..a039162eea4b 100755 --- a/.github/create_combined_ci_report.py +++ b/.github/create_combined_ci_report.py @@ -6,6 +6,7 @@ import json import requests +import pandas as pd from clickhouse_driver import Client import boto3 from botocore.exceptions import NoCredentialsError @@ -123,6 +124,35 @@ def get_regression_fails(client: Client, job_url: str): df["job_name"] = df["job_name"].str.title() return df +def get_cves(pr_number, commit_sha): + s3_client = boto3.client("s3", endpoint_url=os.getenv("S3_URL")) + s3_path = f"s3://{S3_BUCKET}/{pr_number}/{commit_sha}/grype/" + + results = [] + + response = s3_client.list_objects_v2(Bucket=S3_BUCKET, Prefix=s3_path, Delimiter='/') + grype_result_dirs = [content['Prefix'] for content in response.get('CommonPrefixes', [])] + + for path in grype_result_dirs: + file_key = f"{path}result.json" + file_response = s3_client.get_object(Bucket=S3_BUCKET, Key=file_key) + content = file_response["Body"].read().decode("utf-8") + results.append(json.loads(content)) + + tables = [] + for scan_result in results: + for match in scan_result["matches"]: + tables.append( + { + "docker_image": scan_result["source"]["target"]["userInput"], + "identifier": match["vulnerability"]["id"], + "severity": match["vulnerability"]["severity"], + "namespace": match["vulnerability"]["namespace"], + } + ) + + return pd.DataFrame(tables) + def url_to_html_link(url: str) -> str: if not url: @@ -177,6 +207,9 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--known-fails", type=str, help="Path to the file with known fails" ) + parser.add_argument( + "--cves", action="store_true", help="Get CVEs from Grype results" + ) parser.add_argument( "--mark-preview", action="store_true", help="Mark the report as a preview" ) @@ -216,6 +249,7 @@ def main(): "checks_known_fails": [], "checks_errors": get_checks_errors(db_client, args.actions_run_url), "regression_fails": get_regression_fails(db_client, args.actions_run_url), + "docker_images_cves": [] if not args.cves else get_cves(args.pr_number, args.commit_sha), } if args.known_fails: @@ -231,6 +265,10 @@ def main(): db_client, args.actions_run_url, known_fails ) + high_cve_count = 0 + if fail_results['docker_images_cves']: + high_cve_count = fail_results['docker_images_cves'].value_counts()[['High', 'Critical']].sum() + combined_report = ( ci_running_report.replace("ClickHouse CI running for", "Combined CI Report for") .replace( @@ -242,6 +280,7 @@ def main():
  • Checks Errors ({len(fail_results['checks_errors'])})
  • Checks New Fails ({len(fail_results['checks_fails'])})
  • Regression New Fails ({len(fail_results['regression_fails'])})
    • +
  • Docker Images CVEs ({high_cve_count})
  • Checks Known Fails ({len(fail_results['checks_known_fails'])})
    • @@ -262,8 +301,11 @@ def main():

    Regression New Fails

    {format_results_as_html_table(fail_results['regression_fails'])} +

    Docker Images CVEs

    +{"

    Not Checked

    " if not args.cves else format_results_as_html_table(fail_results['docker_images_cves'])} +

    Checks Known Fails

    -{format_results_as_html_table(fail_results['checks_known_fails'])} +{"

    Not Checked

    " if not args.known_fails else format_results_as_html_table(fail_results['checks_known_fails'])} """, 1, ) @@ -276,7 +318,7 @@ def main(): exit(0) # Upload the report to S3 - s3_client = boto3.client("s3") + s3_client = boto3.client("s3", endpoint_url=os.getenv("S3_URL")) try: s3_client.put_object( diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 87a7551fa200..142ec8a95fff 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -598,7 +598,7 @@ jobs: run: | pip install clickhouse-driver==0.2.8 numpy==1.26.4 pandas==2.2.0 - REPORT_LINK=$(python3 .github/create_combined_ci_report.py --pr-number $PR_NUMBER --commit-sha $COMMIT_SHA --actions-run-url $ACTIONS_RUN_URL --known-fails tests/broken_tests.json) + REPORT_LINK=$(python3 .github/create_combined_ci_report.py --pr-number $PR_NUMBER --commit-sha $COMMIT_SHA --actions-run-url $ACTIONS_RUN_URL --known-fails tests/broken_tests.json --cves) IS_VALID_URL=$(echo $REPORT_LINK | grep -E '^https?://') if [[ -n $IS_VALID_URL ]]; then From 6de79b66fbbc90a06806c35901c6ce8c1c4a1f4c Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Wed, 2 Apr 2025 14:21:38 -0400 Subject: [PATCH 24/38] fix report errors --- .github/create_combined_ci_report.py | 2 +- .github/workflows/grype_scan.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/create_combined_ci_report.py b/.github/create_combined_ci_report.py index a039162eea4b..d300ea28287c 100755 --- a/.github/create_combined_ci_report.py +++ b/.github/create_combined_ci_report.py @@ -266,7 +266,7 @@ def main(): ) high_cve_count = 0 - if fail_results['docker_images_cves']: + if len(fail_results['docker_images_cves']) > 0: high_cve_count = fail_results['docker_images_cves'].value_counts()[['High', 'Critical']].sum() combined_report = ( diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 31a2aae8d6b8..5d036b83eafd 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -111,7 +111,7 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, sha: '${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}', - state: ${{ steps.create_summary.outputs.total_high_critical > 0 && 'failure' || 'success' }}, + state: '${{ steps.create_summary.outputs.total_high_critical > 0 && 'failure' || 'success' }}', target_url: '${{ steps.upload_results.outputs.https_s3_path }}/results.html', description: 'Grype Scan Completed with ${{ steps.create_summary.outputs.total_high_critical }} high/critical vulnerabilities', context: 'Grype Scan ${{ steps.set_version.outputs.docker_image || inputs.docker_image }}' From 115967521eda66744548733c63f1e593a537f1e2 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Wed, 2 Apr 2025 14:28:26 -0400 Subject: [PATCH 25/38] fix quoting of backticks --- .github/workflows/grype_scan.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 5d036b83eafd..4a359954d5bd 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -95,10 +95,10 @@ jobs: echo "total_high_critical=$TOTAL_HIGH_CRITICAL" >> $GITHUB_OUTPUT if [ $TOTAL_HIGH_CRITICAL -gt 0 ]; then - echo "## High and Critical vulnerabilities found" >> $GITHUB_STEP_SUMMARY - echo "```" >> $GITHUB_STEP_SUMMARY - tfs --no-colors show tests raw.log | grep -Pi "High|Critical" >> $GITHUB_STEP_SUMMARY - echo "```" >> $GITHUB_STEP_SUMMARY + echo '## High and Critical vulnerabilities found' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + tfs --no-colors show tests raw.log | grep -Pi 'High|Critical' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY fi - name: Set commit status From c2356a8ee77b27d0e8124388d931e0e0cffb305c Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Wed, 2 Apr 2025 19:38:00 -0400 Subject: [PATCH 26/38] Fix and upgrade cve tables --- .github/create_combined_ci_report.py | 44 +++++++++++++++++++++------- .github/workflows/grype_scan.yml | 3 +- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/.github/create_combined_ci_report.py b/.github/create_combined_ci_report.py index d300ea28287c..fade619b0ffd 100755 --- a/.github/create_combined_ci_report.py +++ b/.github/create_combined_ci_report.py @@ -124,14 +124,19 @@ def get_regression_fails(client: Client, job_url: str): df["job_name"] = df["job_name"].str.title() return df + def get_cves(pr_number, commit_sha): s3_client = boto3.client("s3", endpoint_url=os.getenv("S3_URL")) s3_path = f"s3://{S3_BUCKET}/{pr_number}/{commit_sha}/grype/" results = [] - response = s3_client.list_objects_v2(Bucket=S3_BUCKET, Prefix=s3_path, Delimiter='/') - grype_result_dirs = [content['Prefix'] for content in response.get('CommonPrefixes', [])] + response = s3_client.list_objects_v2( + Bucket=S3_BUCKET, Prefix=s3_path, Delimiter="/" + ) + grype_result_dirs = [ + content["Prefix"] for content in response.get("CommonPrefixes", []) + ] for path in grype_result_dirs: file_key = f"{path}result.json" @@ -139,19 +144,26 @@ def get_cves(pr_number, commit_sha): content = file_response["Body"].read().decode("utf-8") results.append(json.loads(content)) - tables = [] + rows = [] for scan_result in results: for match in scan_result["matches"]: - tables.append( + rows.append( { "docker_image": scan_result["source"]["target"]["userInput"], - "identifier": match["vulnerability"]["id"], "severity": match["vulnerability"]["severity"], + "identifier": match["vulnerability"]["id"], "namespace": match["vulnerability"]["namespace"], } ) - return pd.DataFrame(tables) + df = pd.DataFrame(rows) + df = df.sort_values( + by="severity", + key=lambda col: col.str.lower().map( + {"critical": 1, "high": 2, "medium": 3, "low": 4, "negligible": 5} + ), + ) + return df def url_to_html_link(url: str) -> str: @@ -178,6 +190,9 @@ def format_results_as_html_table(results) -> str: formatters={ "Results Link": url_to_html_link, "Test Name": format_test_name_for_linewrap, + "Identifier": lambda s: url_to_html_link( + "https://nvd.nist.gov/vuln/detail/" + s + ), }, escape=False, ) # tbody/thead tags interfere with the table sorting script @@ -249,7 +264,9 @@ def main(): "checks_known_fails": [], "checks_errors": get_checks_errors(db_client, args.actions_run_url), "regression_fails": get_regression_fails(db_client, args.actions_run_url), - "docker_images_cves": [] if not args.cves else get_cves(args.pr_number, args.commit_sha), + "docker_images_cves": ( + [] if not args.cves else get_cves(args.pr_number, args.commit_sha) + ), } if args.known_fails: @@ -266,8 +283,13 @@ def main(): ) high_cve_count = 0 - if len(fail_results['docker_images_cves']) > 0: - high_cve_count = fail_results['docker_images_cves'].value_counts()[['High', 'Critical']].sum() + if len(fail_results["docker_images_cves"]) > 0: + high_cve_count = ( + fail_results["docker_images_cves"]["severity"] + .str.lower() + .isin(("high", "critical")) + .sum() + ) combined_report = ( ci_running_report.replace("ClickHouse CI running for", "Combined CI Report for") @@ -280,8 +302,8 @@ def main():
  • Checks Errors ({len(fail_results['checks_errors'])})
  • Checks New Fails ({len(fail_results['checks_fails'])})
  • Regression New Fails ({len(fail_results['regression_fails'])})
    • -
  • Docker Images CVEs ({high_cve_count})
    • -
  • Checks Known Fails ({len(fail_results['checks_known_fails'])})
    • +
  • Docker Images CVEs ({'N/A' if not args.cves else f'{high_cve_count} high/critical)'}
    • +
  • Checks Known Fails ({'N/A' if not args.known_fails else len(fail_results['checks_known_fails'])})

    • CI Jobs Status

    diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 4a359954d5bd..6df5759bfd59 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -95,9 +95,10 @@ jobs: echo "total_high_critical=$TOTAL_HIGH_CRITICAL" >> $GITHUB_OUTPUT if [ $TOTAL_HIGH_CRITICAL -gt 0 ]; then + tfs --no-colors show tests raw.log | tee vuln_list.txt echo '## High and Critical vulnerabilities found' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY - tfs --no-colors show tests raw.log | grep -Pi 'High|Critical' >> $GITHUB_STEP_SUMMARY + grep -Pi 'High|Critical' vuln_list.txt >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY fi From 9ebbad379c6df9af12d46a70a159f1dcbd24e20b Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Wed, 2 Apr 2025 19:48:27 -0400 Subject: [PATCH 27/38] fix call to list_objects_v2 --- .github/create_combined_ci_report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/create_combined_ci_report.py b/.github/create_combined_ci_report.py index fade619b0ffd..1f68d46bb3bd 100755 --- a/.github/create_combined_ci_report.py +++ b/.github/create_combined_ci_report.py @@ -127,12 +127,12 @@ def get_regression_fails(client: Client, job_url: str): def get_cves(pr_number, commit_sha): s3_client = boto3.client("s3", endpoint_url=os.getenv("S3_URL")) - s3_path = f"s3://{S3_BUCKET}/{pr_number}/{commit_sha}/grype/" + s3_prefix = f"{pr_number}/{commit_sha}/grype/" results = [] response = s3_client.list_objects_v2( - Bucket=S3_BUCKET, Prefix=s3_path, Delimiter="/" + Bucket=S3_BUCKET, Prefix=s3_prefix, Delimiter="/" ) grype_result_dirs = [ content["Prefix"] for content in response.get("CommonPrefixes", []) From 06315a8b714300005cc094408c18f4caa3455ae5 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Thu, 3 Apr 2025 10:20:02 -0400 Subject: [PATCH 28/38] fixes and tweaks --- .github/create_combined_ci_report.py | 2 +- .github/workflows/grype_scan.yml | 29 +++++++++++++++------------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/create_combined_ci_report.py b/.github/create_combined_ci_report.py index 1f68d46bb3bd..e035b6a9da09 100755 --- a/.github/create_combined_ci_report.py +++ b/.github/create_combined_ci_report.py @@ -156,7 +156,7 @@ def get_cves(pr_number, commit_sha): } ) - df = pd.DataFrame(rows) + df = pd.DataFrame(rows).drop_duplicates() df = df.sort_values( by="severity", key=lambda col: col.str.lower().map( diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 6df5759bfd59..00f3ea6d314b 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -77,17 +77,21 @@ jobs: id: create_summary run: | jq -r '.distro | "**Distro**: \(.name):\(.version)"' result.json >> $GITHUB_STEP_SUMMARY - echo "| Severity | Count |" >> $GITHUB_STEP_SUMMARY - echo "|------------|-------|" >> $GITHUB_STEP_SUMMARY - jq -r ' - .matches | - map(.vulnerability.severity) | - group_by(.) | - map({severity: .[0], count: length}) | - sort_by(.severity) | - map("| \(.severity) | \(.count) |") | - .[] - ' result.json >> $GITHUB_STEP_SUMMARY + if jq -e '.matches | length == 0' result.json > /dev/null; then + echo "No results" >> $GITHUB_STEP_SUMMARY + else + echo "| Severity | Count |" >> $GITHUB_STEP_SUMMARY + echo "|------------|-------|" >> $GITHUB_STEP_SUMMARY + jq -r ' + .matches | + map(.vulnerability.severity) | + group_by(.) | + map({severity: .[0], count: length}) | + sort_by(.severity) | + map("| \(.severity) | \(.count) |") | + .[] + ' result.json >> $GITHUB_STEP_SUMMARY + fi HIGH_COUNT=$(jq -r '.matches | map(.vulnerability.severity) | map(select(. == "High")) | length' result.json) CRITICAL_COUNT=$(jq -r '.matches | map(.vulnerability.severity) | map(select(. == "Critical")) | length' result.json) @@ -95,10 +99,9 @@ jobs: echo "total_high_critical=$TOTAL_HIGH_CRITICAL" >> $GITHUB_OUTPUT if [ $TOTAL_HIGH_CRITICAL -gt 0 ]; then - tfs --no-colors show tests raw.log | tee vuln_list.txt echo '## High and Critical vulnerabilities found' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY - grep -Pi 'High|Critical' vuln_list.txt >> $GITHUB_STEP_SUMMARY + cat raw.log | tfs --no-colors show tests | grep -Pi 'High|Critical' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY fi From a27e5cb440da924f324b8c2113bbe647db232caf Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Thu, 3 Apr 2025 11:27:28 -0400 Subject: [PATCH 29/38] add arrows to indicate that columns are sortable --- .github/create_combined_ci_report.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/create_combined_ci_report.py b/.github/create_combined_ci_report.py index e035b6a9da09..2acd17d9ffd3 100755 --- a/.github/create_combined_ci_report.py +++ b/.github/create_combined_ci_report.py @@ -293,6 +293,7 @@ def main(): combined_report = ( ci_running_report.replace("ClickHouse CI running for", "Combined CI Report for") + .replace('', "th::before { content: '⇅ '}\n") .replace( "", f"""

    Table of Contents

    From 37ccdc152b83af4d9630690675bf8f56aea26cb1 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Thu, 3 Apr 2025 13:53:57 -0400 Subject: [PATCH 30/38] tweak summary --- .github/workflows/grype_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/grype_scan.yml b/.github/workflows/grype_scan.yml index 00f3ea6d314b..1414129fd666 100644 --- a/.github/workflows/grype_scan.yml +++ b/.github/workflows/grype_scan.yml @@ -78,7 +78,7 @@ jobs: run: | jq -r '.distro | "**Distro**: \(.name):\(.version)"' result.json >> $GITHUB_STEP_SUMMARY if jq -e '.matches | length == 0' result.json > /dev/null; then - echo "No results" >> $GITHUB_STEP_SUMMARY + echo "No CVEs" >> $GITHUB_STEP_SUMMARY else echo "| Severity | Count |" >> $GITHUB_STEP_SUMMARY echo "|------------|-------|" >> $GITHUB_STEP_SUMMARY From 2ee22df53eb875e6cf14a0cbe63a1560817b9d98 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Thu, 3 Apr 2025 18:10:46 -0400 Subject: [PATCH 31/38] fix pandas errors --- .github/create_combined_ci_report.py | 3 +++ .github/workflows/release_branches.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/create_combined_ci_report.py b/.github/create_combined_ci_report.py index 2acd17d9ffd3..3b3570281de4 100755 --- a/.github/create_combined_ci_report.py +++ b/.github/create_combined_ci_report.py @@ -156,6 +156,9 @@ def get_cves(pr_number, commit_sha): } ) + if len(rows) == 0: + return pd.DataFrame() + df = pd.DataFrame(rows).drop_duplicates() df = df.sort_values( by="severity", diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 142ec8a95fff..8f29c50e4e6c 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -596,7 +596,7 @@ jobs: ACTIONS_RUN_URL: ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }} shell: bash run: | - pip install clickhouse-driver==0.2.8 numpy==1.26.4 pandas==2.2.0 + pip install clickhouse-driver==0.2.8 numpy==1.26.4 pandas==2.0.3 REPORT_LINK=$(python3 .github/create_combined_ci_report.py --pr-number $PR_NUMBER --commit-sha $COMMIT_SHA --actions-run-url $ACTIONS_RUN_URL --known-fails tests/broken_tests.json --cves) From 414a1de8e9381e8bf91539d412c58c714451d02d Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Thu, 3 Apr 2025 20:20:04 -0400 Subject: [PATCH 32/38] fix finishcheck --- .github/workflows/release_branches.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 8f29c50e4e6c..387c3cf1a7d6 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -565,6 +565,7 @@ jobs: - CompatibilityCheckAarch64 - RegressionTestsRelease - RegressionTestsAarch64 + - GrypeScan - SignRelease runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] steps: @@ -599,6 +600,7 @@ jobs: pip install clickhouse-driver==0.2.8 numpy==1.26.4 pandas==2.0.3 REPORT_LINK=$(python3 .github/create_combined_ci_report.py --pr-number $PR_NUMBER --commit-sha $COMMIT_SHA --actions-run-url $ACTIONS_RUN_URL --known-fails tests/broken_tests.json --cves) + echo $REPORT_LINK IS_VALID_URL=$(echo $REPORT_LINK | grep -E '^https?://') if [[ -n $IS_VALID_URL ]]; then From a09f5d27bf50dfa4e855ad6afcceb36bd53c3fa0 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Tue, 8 Apr 2025 10:53:32 -0400 Subject: [PATCH 33/38] hotfix ci test report --- .github/create_combined_ci_report.py | 204 ++++++++++++++++++------- .github/workflows/release_branches.yml | 3 +- 2 files changed, 154 insertions(+), 53 deletions(-) diff --git a/.github/create_combined_ci_report.py b/.github/create_combined_ci_report.py index 4d90171b03d4..20617a651f27 100755 --- a/.github/create_combined_ci_report.py +++ b/.github/create_combined_ci_report.py @@ -9,6 +9,7 @@ from clickhouse_driver import Client import boto3 from botocore.exceptions import NoCredentialsError +import pandas as pd DATABASE_HOST_VAR = "CHECKS_DATABASE_HOST" DATABASE_USER_VAR = "CHECKS_DATABASE_USER" @@ -16,6 +17,106 @@ S3_BUCKET = "altinity-build-artifacts" +css = """ +/* Base colors inspired by Altinity */ +:root { + --altinity-blue: #007bff; + --altinity-dark-blue: #0056b3; + --altinity-light-gray: #f8f9fa; + --altinity-gray: #6c757d; + --altinity-white: #ffffff; +} + +/* Body and heading fonts */ +body { + font-family: "DejaVu Sans", "Noto Sans", Arial, sans-serif; + font-size: 0.9rem; + background-color: var(--altinity-light-gray); + color: var(--altinity-gray); + padding: 2rem; +} + +h1, h2, h3, h4, h5, h6 { + color: var(--altinity-dark-blue); +} + +/* General table styling */ +table { + min-width: min(900px, 98vw); + margin: 1rem 0; + border-collapse: collapse; + background-color: var(--altinity-white); + box-shadow: 0 0 8px rgba(0, 0, 0, 0.05); +} + +/* Table header styling */ +th { + background-color: var(--altinity-blue); + color: var(--altinity-white); + padding: 10px 16px; + text-align: left; + border-bottom: 2px solid var(--altinity-dark-blue); + white-space: nowrap; +} + +/* Table body row styling */ +tr:nth-child(even) { + background-color: var(--altinity-light-gray); +} + +tr:hover { + background-color: var(--altinity-dark-blue); + color: var(--altinity-white); +} + +/* Table cell styling */ +td { + padding: 8px 8px; + border-bottom: 1px solid var(--altinity-gray); +} + +""" + + +def get_commit_statuses(sha: str) -> pd.DataFrame: + """ + Fetch commit statuses for a given SHA and return as a pandas DataFrame. + + Args: + sha (str): Commit SHA to fetch statuses for. + + Returns: + pd.DataFrame: DataFrame containing all statuses. + """ + headers = { + "Authorization": f"token {os.getenv('GITHUB_TOKEN')}", + "Accept": "application/vnd.github.v3+json", + } + + url = f"https://api.github.com/repos/Altinity/ClickHouse/commits/{sha}/statuses" + response = requests.get(url, headers=headers) + + if response.status_code != 200: + raise Exception( + f"Failed to fetch statuses: {response.status_code} {response.text}" + ) + + data = response.json() + + # Parse relevant fields + parsed = [ + { + "test_name": item["context"], + "test_status": item["state"], + # "description": item["description"], + "results_link": item["target_url"], + } + for item in data + ] + + return pd.DataFrame(parsed) + + def get_checks_fails(client: Client, job_url: str): """ Get tests that did not succeed for the given job URL. @@ -137,24 +238,29 @@ def format_test_name_for_linewrap(text: str) -> str: return text.replace(".py::", "/") +def format_test_status(text: str) -> str: + """Format the test status for better readability.""" + color = ( + "red" + if text.lower().startswith("fail") + else "orange" if text.lower() == "error" else "green" + ) + return f'{text}' + + def format_results_as_html_table(results) -> str: if len(results) == 0: return "

    Nothing to report

    " results.columns = [col.replace("_", " ").title() for col in results.columns] - html = ( - results.to_html( - index=False, - formatters={ - "Results Link": url_to_html_link, - "Test Name": format_test_name_for_linewrap, - }, - escape=False, - ) # tbody/thead tags interfere with the table sorting script - .replace("\n", "") - .replace("\n", "") - .replace("\n", "") - .replace("\n", "") - .replace('
    ", - f"""

    Table of Contents

    + title = "CI Test Report" + + html_report = f""" + + + + + + + {title} + + +

    {title}

    +

    Generated from GitHub Actions

    + +

    Table of Contents

    {'

    This is a preview. FinishCheck has not completed.

    ' if args.mark_preview else ""} -

    CI Jobs Status

    -
    """, - 1, - ) - .replace( - "
    ", - f""" +

    CI Jobs Status

    +{format_results_as_html_table(fail_results['job_statuses'])}

    Checks Errors

    {format_results_as_html_table(fail_results['checks_errors'])} @@ -263,31 +362,34 @@ def main():

    Checks Known Fails

    {format_results_as_html_table(fail_results['checks_known_fails'])} -""", - 1, - ) - ) - report_path = Path("combined_report.html") - report_path.write_text(combined_report, encoding="utf-8") + + + +""" + + report_path = Path("ci_test_report.html") + report_path.write_text(html_report, encoding="utf-8") if args.no_upload: print(f"Report saved to {report_path}") exit(0) + report_destination_key = f"{args.pr_number}/{args.commit_sha}/ci_test_report.html" + # Upload the report to S3 s3_client = boto3.client("s3") try: s3_client.put_object( Bucket=S3_BUCKET, - Key=f"{args.pr_number}/{args.commit_sha}/combined_report.html", - Body=combined_report, + Key=report_destination_key, + Body=html_report, ContentType="text/html; charset=utf-8", ) except NoCredentialsError: print("Credentials not available for S3 upload.") - print(report_destination_url) + print(f"https://s3.amazonaws.com/{S3_BUCKET}/" + report_destination_key) if __name__ == "__main__": diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 2696c7e36b4a..65fda816f081 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -503,8 +503,7 @@ jobs: ACTIONS_RUN_URL: ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }} shell: bash run: | - pip install clickhouse-driver==0.2.8 numpy==1.26.4 pandas==2.2.0 - + pip install clickhouse-driver==0.2.8 numpy==1.26.4 pandas==2.0.3 REPORT_LINK=$(python3 .github/create_combined_ci_report.py --pr-number $PR_NUMBER --commit-sha $COMMIT_SHA --actions-run-url $ACTIONS_RUN_URL --known-fails tests/broken_tests.json) IS_VALID_URL=$(echo $REPORT_LINK | grep -E '^https?://') From 81a2f7430054f9359ea839e46bc7087e1d513eaa Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Tue, 8 Apr 2025 10:57:35 -0400 Subject: [PATCH 34/38] make sure github token is available --- .github/workflows/release_branches.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 65fda816f081..f68212fe6bc0 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -501,6 +501,7 @@ jobs: COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} PR_NUMBER: ${{ github.event.number }} ACTIONS_RUN_URL: ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash run: | pip install clickhouse-driver==0.2.8 numpy==1.26.4 pandas==2.0.3 From 6d02df3eeef153cc099a88c058754701412e0308 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Tue, 8 Apr 2025 12:18:44 -0400 Subject: [PATCH 35/38] tweaks --- .github/create_combined_ci_report.py | 22 +++++++++++++++++++--- .github/workflows/release_branches.yml | 3 +++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/create_combined_ci_report.py b/.github/create_combined_ci_report.py index 20617a651f27..6d5eca407329 100755 --- a/.github/create_combined_ci_report.py +++ b/.github/create_combined_ci_report.py @@ -58,6 +58,10 @@ border-bottom: 2px solid var(--altinity-dark-blue); white-space: nowrap; } +th.hth { + border-bottom: 1px solid var(--altinity-gray); + border-right: 2px solid var(--altinity-dark-blue); +} /* Table body row styling */ tr:nth-child(even) { @@ -108,13 +112,17 @@ def get_commit_statuses(sha: str) -> pd.DataFrame: { "test_name": item["context"], "test_status": item["state"], - # "description": item["description"], + "message": item["description"], "results_link": item["target_url"], } for item in data ] - return pd.DataFrame(parsed) + return ( + pd.DataFrame(parsed) + .sort_values(by=["test_status", "test_name"], ascending=[True, True]) + .reset_index(drop=True) + ) def get_checks_fails(client: Client, job_url: str): @@ -259,6 +267,7 @@ def format_results_as_html_table(results) -> str: "Test Name": format_test_name_for_linewrap, "Test Status": format_test_status, "Check Status": format_test_status, + "Message": lambda m: m.replace("\n", " "), }, escape=False, ) @@ -336,7 +345,14 @@ def main():

    {title}

    -

    Generated from GitHub Actions

    + + + + + + + +
    Task{args.actions_run_url.split('/')[-1]}
    Commit{args.commit_sha}

    Table of Contents

    {'

    This is a preview. FinishCheck has not completed.

    ' if args.mark_preview else ""} diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index f68212fe6bc0..f7f21dd01058 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -505,7 +505,10 @@ jobs: shell: bash run: | pip install clickhouse-driver==0.2.8 numpy==1.26.4 pandas==2.0.3 + + set +e REPORT_LINK=$(python3 .github/create_combined_ci_report.py --pr-number $PR_NUMBER --commit-sha $COMMIT_SHA --actions-run-url $ACTIONS_RUN_URL --known-fails tests/broken_tests.json) + echo $REPORT_LINK IS_VALID_URL=$(echo $REPORT_LINK | grep -E '^https?://') if [[ -n $IS_VALID_URL ]]; then From d3cf6e0e60fcff0f13660933e69ce62e40078002 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Tue, 8 Apr 2025 12:56:24 -0400 Subject: [PATCH 36/38] update css --- .github/create_combined_ci_report.py | 49 +++++++++++++++++----------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/.github/create_combined_ci_report.py b/.github/create_combined_ci_report.py index 6d5eca407329..e010d96bf4b6 100755 --- a/.github/create_combined_ci_report.py +++ b/.github/create_combined_ci_report.py @@ -20,24 +20,26 @@ css = """ /* Base colors inspired by Altinity */ :root { - --altinity-blue: #007bff; - --altinity-dark-blue: #0056b3; - --altinity-light-gray: #f8f9fa; + --altinity-background: #000D45; + --altinity-accent: #189DCF; + --altinity-highlight: #FFC600; --altinity-gray: #6c757d; + --altinity-light-gray: #f8f9fa; --altinity-white: #ffffff; } /* Body and heading fonts */ body { - font-family: "DejaVu Sans", "Noto Sans", Arial, sans-serif; - font-size: 0.9rem; - background-color: var(--altinity-light-gray); - color: var(--altinity-gray); + font-family: Arimo, "Proxima Nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 1rem; + background-color: var(--altinity-background); + color: var(--altinity-light-gray); padding: 2rem; } h1, h2, h3, h4, h5, h6 { - color: var(--altinity-dark-blue); + font-family: Figtree, "Proxima Nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + color: var(--altinity-white); } /* General table styling */ @@ -46,37 +48,45 @@ margin: 1rem 0; border-collapse: collapse; background-color: var(--altinity-white); + border-color: var(--altinity-accent); box-shadow: 0 0 8px rgba(0, 0, 0, 0.05); + color: var(--altinity-background); } /* Table header styling */ th { - background-color: var(--altinity-blue); + background-color: var(--altinity-accent); color: var(--altinity-white); padding: 10px 16px; text-align: left; - border-bottom: 2px solid var(--altinity-dark-blue); + border-bottom: 2px solid var(--altinity-background); white-space: nowrap; } th.hth { - border-bottom: 1px solid var(--altinity-gray); - border-right: 2px solid var(--altinity-dark-blue); + border-bottom: 1px solid var(--altinity-accent); + border-right: 2px solid var(--altinity-background); } /* Table body row styling */ -tr:nth-child(even) { - background-color: var(--altinity-light-gray); -} - tr:hover { - background-color: var(--altinity-dark-blue); - color: var(--altinity-white); + background-color: var(--altinity-light-gray); } /* Table cell styling */ td { padding: 8px 8px; - border-bottom: 1px solid var(--altinity-gray); + border-color: var(--altinity-accent); + border-bottom: 1px solid var(--altinity-accent); +} + +/* Link styling */ +a { + color: var(--altinity-accent); + text-decoration: none; +} +a:hover { + color: var(--altinity-highlight); + text-decoration: underline; } """ @@ -267,6 +277,7 @@ def format_results_as_html_table(results) -> str: "Test Name": format_test_name_for_linewrap, "Test Status": format_test_status, "Check Status": format_test_status, + "Status": format_test_status, "Message": lambda m: m.replace("\n", " "), }, escape=False, From 4ba14be92fe09943f82aed9cd105f5547bc186fe Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Tue, 8 Apr 2025 13:13:33 -0400 Subject: [PATCH 37/38] sortable tables --- .github/create_combined_ci_report.py | 234 +++++++++++++++++++-------- 1 file changed, 163 insertions(+), 71 deletions(-) diff --git a/.github/create_combined_ci_report.py b/.github/create_combined_ci_report.py index e010d96bf4b6..5621255288b0 100755 --- a/.github/create_combined_ci_report.py +++ b/.github/create_combined_ci_report.py @@ -18,79 +18,170 @@ css = """ -/* Base colors inspired by Altinity */ -:root { - --altinity-background: #000D45; - --altinity-accent: #189DCF; - --altinity-highlight: #FFC600; - --altinity-gray: #6c757d; - --altinity-light-gray: #f8f9fa; - --altinity-white: #ffffff; -} - -/* Body and heading fonts */ -body { - font-family: Arimo, "Proxima Nova", "Helvetica Neue", Helvetica, Arial, sans-serif; - font-size: 1rem; - background-color: var(--altinity-background); - color: var(--altinity-light-gray); - padding: 2rem; -} - -h1, h2, h3, h4, h5, h6 { - font-family: Figtree, "Proxima Nova", "Helvetica Neue", Helvetica, Arial, sans-serif; - color: var(--altinity-white); -} - -/* General table styling */ -table { - min-width: min(900px, 98vw); - margin: 1rem 0; - border-collapse: collapse; - background-color: var(--altinity-white); - border-color: var(--altinity-accent); - box-shadow: 0 0 8px rgba(0, 0, 0, 0.05); - color: var(--altinity-background); -} - -/* Table header styling */ -th { - background-color: var(--altinity-accent); - color: var(--altinity-white); - padding: 10px 16px; - text-align: left; - border-bottom: 2px solid var(--altinity-background); - white-space: nowrap; -} -th.hth { + /* Base colors inspired by Altinity */ + :root { + --altinity-background: #000D45; + --altinity-accent: #189DCF; + --altinity-highlight: #FFC600; + --altinity-gray: #6c757d; + --altinity-light-gray: #f8f9fa; + --altinity-white: #ffffff; + } + + /* Body and heading fonts */ + body { + font-family: Arimo, "Proxima Nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 1rem; + background-color: var(--altinity-background); + color: var(--altinity-light-gray); + padding: 2rem; + } + + h1, h2, h3, h4, h5, h6 { + font-family: Figtree, "Proxima Nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + color: var(--altinity-white); + } + + /* General table styling */ + table { + min-width: min(900px, 98vw); + margin: 1rem 0; + border-collapse: collapse; + background-color: var(--altinity-white); + border-color: var(--altinity-accent); + box-shadow: 0 0 8px rgba(0, 0, 0, 0.05); + color: var(--altinity-background); + } + + /* Table header styling */ + th { + background-color: var(--altinity-accent); + color: var(--altinity-white); + padding: 10px 16px; + text-align: left; + border-bottom: 2px solid var(--altinity-background); + white-space: nowrap; + } + th.hth { + border-bottom: 1px solid var(--altinity-accent); + border-right: 2px solid var(--altinity-background); + } + + /* Table header sorting styling */ + th { + cursor: pointer; + } + th.no-sort { + pointer-events: none; + } + th::after, + th::before { + transition: color 0.2s ease-in-out; + font-size: 1.2em; + color: transparent; + } + th::after { + margin-left: 3px; + content: '\\025B8'; + } + th:hover::after { + color: inherit; + } + th.dir-d::after { + color: inherit; + content: '\\025BE'; + } + th.dir-u::after { + color: inherit; + content: '\\025B4'; + } + + /* Table body row styling */ + tr:hover { + background-color: var(--altinity-light-gray); + } + + /* Table cell styling */ + td { + padding: 8px 8px; + border-color: var(--altinity-accent); border-bottom: 1px solid var(--altinity-accent); - border-right: 2px solid var(--altinity-background); -} - -/* Table body row styling */ -tr:hover { - background-color: var(--altinity-light-gray); -} - -/* Table cell styling */ -td { - padding: 8px 8px; - border-color: var(--altinity-accent); - border-bottom: 1px solid var(--altinity-accent); -} - -/* Link styling */ -a { - color: var(--altinity-accent); - text-decoration: none; -} -a:hover { - color: var(--altinity-highlight); - text-decoration: underline; -} + } + /* Link styling */ + a { + color: var(--altinity-accent); + text-decoration: none; + } + a:hover { + color: var(--altinity-highlight); + text-decoration: underline; + } """ +script = """ + +""" def get_commit_statuses(sha: str) -> pd.DataFrame: """ @@ -358,10 +449,10 @@ def main():

    {title}

    - + - +
    Task{args.actions_run_url.split('/')[-1]}Task{args.actions_run_url.split('/')[-1]}
    Commit{args.commit_sha}Commit{args.commit_sha}
    @@ -390,6 +481,7 @@ def main():

    Checks Known Fails

    {format_results_as_html_table(fail_results['checks_known_fails'])} +{script} """ From 5a1921b678ff2d72735156bf05668d082f5bd6f8 Mon Sep 17 00:00:00 2001 From: Your Name <146047128+strtgbb@users.noreply.github.com> Date: Tue, 8 Apr 2025 14:21:26 -0400 Subject: [PATCH 38/38] cleanup --- .github/create_combined_ci_report.py | 79 +++++++++++++++------------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/.github/create_combined_ci_report.py b/.github/create_combined_ci_report.py index 5621255288b0..fb34ad3b44bb 100755 --- a/.github/create_combined_ci_report.py +++ b/.github/create_combined_ci_report.py @@ -4,6 +4,7 @@ from pathlib import Path from itertools import combinations import json +from datetime import datetime import requests from clickhouse_driver import Client @@ -20,47 +21,48 @@ css = """ /* Base colors inspired by Altinity */ :root { - --altinity-background: #000D45; - --altinity-accent: #189DCF; - --altinity-highlight: #FFC600; - --altinity-gray: #6c757d; - --altinity-light-gray: #f8f9fa; - --altinity-white: #ffffff; + --altinity-background: #000D45; + --altinity-accent: #189DCF; + --altinity-highlight: #FFC600; + --altinity-gray: #6c757d; + --altinity-light-gray: #f8f9fa; + --altinity-white: #ffffff; } /* Body and heading fonts */ body { - font-family: Arimo, "Proxima Nova", "Helvetica Neue", Helvetica, Arial, sans-serif; - font-size: 1rem; - background-color: var(--altinity-background); - color: var(--altinity-light-gray); - padding: 2rem; + font-family: Arimo, "Proxima Nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 1rem; + background-color: var(--altinity-background); + color: var(--altinity-light-gray); + padding: 2rem; } h1, h2, h3, h4, h5, h6 { - font-family: Figtree, "Proxima Nova", "Helvetica Neue", Helvetica, Arial, sans-serif; - color: var(--altinity-white); + font-family: Figtree, "Proxima Nova", "Helvetica Neue", Helvetica, Arial, sans-serif; + color: var(--altinity-white); } /* General table styling */ table { - min-width: min(900px, 98vw); - margin: 1rem 0; - border-collapse: collapse; - background-color: var(--altinity-white); - border-color: var(--altinity-accent); - box-shadow: 0 0 8px rgba(0, 0, 0, 0.05); - color: var(--altinity-background); + min-width: min(900px, 98vw); + margin: 1rem 0; + border-collapse: collapse; + background-color: var(--altinity-white); + border: 1px solid var(--altinity-accent); + box-shadow: 0 0 8px rgba(0, 0, 0, 0.05); + color: var(--altinity-background); } /* Table header styling */ th { - background-color: var(--altinity-accent); - color: var(--altinity-white); - padding: 10px 16px; - text-align: left; - border-bottom: 2px solid var(--altinity-background); - white-space: nowrap; + background-color: var(--altinity-accent); + color: var(--altinity-white); + padding: 10px 16px; + text-align: left; + border: none; + border-bottom: 2px solid var(--altinity-background); + white-space: nowrap; } th.hth { border-bottom: 1px solid var(--altinity-accent); @@ -98,24 +100,23 @@ /* Table body row styling */ tr:hover { - background-color: var(--altinity-light-gray); + background-color: var(--altinity-light-gray); } /* Table cell styling */ td { - padding: 8px 8px; - border-color: var(--altinity-accent); - border-bottom: 1px solid var(--altinity-accent); + padding: 8px 8px; + border: 1px solid var(--altinity-accent); } /* Link styling */ a { - color: var(--altinity-accent); - text-decoration: none; + color: var(--altinity-accent); + text-decoration: none; } a:hover { - color: var(--altinity-highlight); - text-decoration: underline; + color: var(--altinity-highlight); + text-decoration: underline; } """ @@ -183,6 +184,7 @@ """ + def get_commit_statuses(sha: str) -> pd.DataFrame: """ Fetch commit statuses for a given SHA and return as a pandas DataFrame. @@ -264,7 +266,7 @@ def get_checks_known_fails(client: Client, job_url: str, known_fails: dict): len(df.columns) - 1, "reason", df["test_name"] - .cat.remove_unused_categories() + .astype(str) .apply( lambda test_name: known_fails[test_name].get("reason", "No reason given") ), @@ -372,7 +374,7 @@ def format_results_as_html_table(results) -> str: "Message": lambda m: m.replace("\n", " "), }, escape=False, - ) + ).replace(' border="1"', "") return html @@ -447,13 +449,16 @@ def main():

    {title}

    - +
    + + +
    Task{args.actions_run_url.split('/')[-1]}
    Commit{args.commit_sha}
    Date{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')} UTC

    Table of Contents