Skip to content

CLOUDP-315271: Onboard Kundukto to CI #3862

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions .github/workflows/code-health.yml
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,27 @@ jobs:
with:
go-version-file: 'go.mod'
- run: make check-templates
purls_check:
runs-on: ubuntu-latest
steps:
- uses: GitHubSecurityLab/actions-permissions/monitor@v1
with:
config: ${{ vars.PERMISSIONS_CONFIG }}
- name: Checkout repository
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
- name: Generate purls
run: make gen-purls > /dev/null
- name: Check for uncommitted changes in purls.txt
run: |
if ! git diff --quiet --exit-code build/package/purls.txt; then
echo "build/package/purls.txt is out of date. Please run 'make gen-purls' and commit the result."
git --no-pager diff build/package/purls.txt
exit 1
fi
verify_image:
name: Build docker image
runs-on: ubuntu-latest
Expand Down
7 changes: 6 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ addcopy: ## Add missing license to files
@scripts/add-copy.sh

.PHONY: generate
generate: gen-docs gen-mocks gen-api-commands ## Generate docs, mocks, code, api commands, all auto generated assets
generate: gen-docs gen-mocks gen-api-commands gen-purls ## Generate docs, mocks, code, api commands, all auto generated assets

.PHONY: apply-overlay
apply-overlay: ## Apply overlay on openapi spec
Expand Down Expand Up @@ -149,6 +149,11 @@ gen-docs: gen-docs-metadata ## Generate docs for atlascli commands
@echo "==> Generating docs"
go run -ldflags "$(LINKER_FLAGS)" ./tools/cmd/docs

.PHONY: gen-purls
gen-purls: build ## Generate list of purls
@echo "==> Generating purls"
@go version -m ./bin/atlas | awk '$$1 == "dep" || $$1 == "=>" { print "pkg:golang/" $$2 "@" $$3 }' > build/package/purls.txt

.PHONY: build
build: ## Generate an atlas binary in ./bin
@echo "==> Building $(ATLAS_BINARY_NAME) binary"
Expand Down
67 changes: 67 additions & 0 deletions build/ci/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,56 @@ functions:
params:
<<: *go_options
binary: build/package/generate-notices.sh
"generate sbom":
- command: ec2.assume_role
params:
role_arn: ${ecr_role_arn}
- command: subprocess.exec
params:
<<: *go_options
include_expansions_in_env:
- workdir
binary: make
args:
- gen-purls
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we shouldn't do it anymore, we should make sure the file is fully updated

- command: subprocess.exec
params:
<<: *go_options
include_expansions_in_env:
- AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY
- AWS_SESSION_TOKEN
- workdir
binary: build/package/generate-sbom.sh
"write kondukto credentials":
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should merge "write kondukto credentials" and "run silkbomb" and make sure to add a rm kondukto_credentials.env in the end

- command: ec2.assume_role
params:
role_arn: ${kondukto_role_arn}
- command: shell.exec
params:
silent: true
shell: bash
include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
script: |
set -e
kondukto_token=$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)
echo "KONDUKTO_TOKEN=$kondukto_token" > ${workdir}/kondukto_credentials.env
"run silkbomb":
- command: shell.exec
params:
shell: bash
script: |
docker run \
--pull=always \
--platform="linux/amd64" \
--rm \
--env-file ${workdir}/kondukto_credentials.env \
-v ${workdir}:/workdir \
901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 \
upload \
--sbom-in /workdir/src/github.com/mongodb/mongodb-atlas-cli/compliance/sbom.json \
--repo mongodb_mongodb-atlas-cli \
--branch ${branch_name}
"package":
- command: github.generate_token
params:
Expand Down Expand Up @@ -317,6 +367,11 @@ tasks:
permissions: public-read
content_type: ${content_type|application/octet-stream}
display_name: unsigned
- name: generate_and_upload_sbom
commands:
- func: "generate sbom"
- func: "write kondukto credentials"
- func: "run silkbomb"
- name: package_goreleaser
tags: ["packaging"]
depends_on:
Expand Down Expand Up @@ -528,6 +583,8 @@ buildvariants:
depends_on:
- name: package_msi
variant: "go_atlascli_msi_snapshot"
- name: generate_and_upload_sbom
variant: ssdlc
- name: publish_atlascli_snapshot
display_name: "Publish AtlasCLI Snapshot"
run_on:
Expand All @@ -553,6 +610,8 @@ buildvariants:
depends_on:
- name: package_msi
variant: release_atlascli_msi
- name: generate_and_upload_sbom
variant: ssdlc
- name: copybara
display_name: "Copybara"
git_tag_only: true
Expand Down Expand Up @@ -605,3 +664,11 @@ buildvariants:
- ubuntu2004-small
tasks:
- name: .smoke-test .generate .repo .atlascli
- name: ssdlc
display_name: Compliance [ssdlc]
run_on:
- ubuntu2204-small
expansions:
<<: *go_linux_version
tasks:
- name: generate_and_upload_sbom
3 changes: 2 additions & 1 deletion build/package/.goreleaser.yml
Original file line number Diff line number Diff line change
Expand Up @@ -142,4 +142,5 @@ release:
name_template: "MongoDB Atlas CLI {{.Version}}"
extra_files:
- glob: ./bin/*.msi
version: 2
- glob: compliance/**/*
version: 2
34 changes: 34 additions & 0 deletions build/package/generate-sbom.sh
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

general design comment, can we commit the purl file and make it a CI check (GH or EVG no preference) that is kept up to date, this is similar to how other repos manage this file

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

check the library owners check which is kind of similar

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would't go for this approach we would jump from 2 files storing dependency information (library_owners.json and go.mod) to 3 files (library_owners.json, go.mod and purls.txt).

I would prefer for the tooling to translate dependencies into purls and later json on the fly as needed.

I'm inclined to even remove library_owners.json, given only our team has code in atlasCLI since the kubernetes plugin extraction.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm ok to remove lib owners (same reason) but I see value on purls being committed given shipped dependencies are not the same as the ones in go.mod and this raises awareness when adding new libs and the implications, this also comes with the comment that I'd like if purls are auto generated by the precommit hook, similar to mms

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have no strong opinion here. Having the purl file committed could definitely help with making dependencies more visible, but I see @fmenezes's point about not wanting to manage yet another file.

I can create a make command to generate purls from the binary and set up a GH action to check that it's up-to-date similar to how we do the docs check.

@fmenezes, any strong feelings about this idea?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sure that is fine, we can commit it but bear in mind the extra checks

Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/usr/bin/env bash

# Copyright 2025 MongoDB Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -Eeou pipefail

export WORKDIR=${workdir:?}

# Authenticate Docker to AWS ECR
aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com

cd "$WORKDIR/src/github.com/mongodb/mongodb-atlas-cli"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we need this we can set working dir at evergreen function

mkdir ./compliance
Copy link
Collaborator

@fmenezes fmenezes May 1, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: for empty folders you can commit a complicance/.gitkeep empty file to keep folder on source code


echo "Generating SBOM..."
docker run --rm \
-v "$WORKDIR/src/github.com/mongodb/mongodb-atlas-cli:/pwd" \
901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 \
update \
--purls /pwd/build/package/purls.txt \
--sbom-out /pwd/compliance/sbom.json

122 changes: 122 additions & 0 deletions build/package/purls.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
pkg:golang/cloud.google.com/go/[email protected]
pkg:golang/cloud.google.com/go/auth/[email protected]
pkg:golang/cloud.google.com/go/compute/[email protected]
pkg:golang/cloud.google.com/go/[email protected]
pkg:golang/cloud.google.com/go/[email protected]
pkg:golang/cloud.google.com/go/[email protected]
pkg:golang/github.com/AlecAivazis/survey/[email protected]
pkg:golang/github.com/Azure/azure-sdk-for-go/sdk/[email protected]
pkg:golang/github.com/Azure/azure-sdk-for-go/sdk/[email protected]
pkg:golang/github.com/Azure/azure-sdk-for-go/sdk/[email protected]
pkg:golang/github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/[email protected]
pkg:golang/github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/[email protected]
pkg:golang/github.com/AzureAD/[email protected]
pkg:golang/github.com/Masterminds/semver/[email protected]
pkg:golang/github.com/PaesslerAG/[email protected]
pkg:golang/github.com/PaesslerAG/[email protected]
pkg:golang/github.com/ProtonMail/[email protected]
pkg:golang/github.com/STARRY-S/[email protected]
pkg:golang/github.com/andybalholm/[email protected]
pkg:golang/github.com/aws/[email protected]
pkg:golang/github.com/aws/aws-sdk-go-v2/[email protected]
pkg:golang/github.com/aws/aws-sdk-go-v2/[email protected]
pkg:golang/github.com/aws/aws-sdk-go-v2/feature/ec2/[email protected]
pkg:golang/github.com/aws/aws-sdk-go-v2/internal/[email protected]
pkg:golang/github.com/aws/aws-sdk-go-v2/internal/endpoints/[email protected]
pkg:golang/github.com/aws/aws-sdk-go-v2/internal/[email protected]
pkg:golang/github.com/aws/aws-sdk-go-v2/service/internal/[email protected]
pkg:golang/github.com/aws/aws-sdk-go-v2/service/internal/[email protected]
pkg:golang/github.com/aws/aws-sdk-go-v2/service/[email protected]
pkg:golang/github.com/aws/aws-sdk-go-v2/service/[email protected]
pkg:golang/github.com/aws/aws-sdk-go-v2/service/[email protected]
pkg:golang/github.com/aws/aws-sdk-go-v2/service/[email protected]
pkg:golang/github.com/aws/[email protected]
pkg:golang/github.com/bodgit/[email protected]
pkg:golang/github.com/bodgit/[email protected]
pkg:golang/github.com/bodgit/[email protected]
pkg:golang/github.com/briandowns/[email protected]
pkg:golang/github.com/cloudflare/[email protected]
pkg:golang/github.com/denisbrodbeck/[email protected]
pkg:golang/github.com/dsnet/[email protected]
pkg:golang/github.com/ebitengine/[email protected]
pkg:golang/github.com/fatih/[email protected]
pkg:golang/github.com/felixge/[email protected]
pkg:golang/github.com/fsnotify/[email protected]
pkg:golang/github.com/go-logr/[email protected]
pkg:golang/github.com/go-logr/[email protected]
pkg:golang/github.com/go-viper/mapstructure/[email protected]
pkg:golang/github.com/golang-jwt/jwt/[email protected]
pkg:golang/github.com/golang/[email protected]
pkg:golang/github.com/golang/[email protected]
pkg:golang/github.com/google/go-github/[email protected]
pkg:golang/github.com/google/[email protected]
pkg:golang/github.com/google/[email protected]
pkg:golang/github.com/google/[email protected]
pkg:golang/github.com/googleapis/[email protected]
pkg:golang/github.com/googleapis/gax-go/[email protected]
pkg:golang/github.com/hashicorp/[email protected]
pkg:golang/github.com/hashicorp/[email protected]
pkg:golang/github.com/hashicorp/golang-lru/[email protected]
pkg:golang/github.com/iancoleman/[email protected]
pkg:golang/github.com/kballard/[email protected]
pkg:golang/github.com/klauspost/[email protected]
pkg:golang/github.com/klauspost/[email protected]
pkg:golang/github.com/kylelemons/[email protected]
pkg:golang/github.com/mattn/[email protected]
pkg:golang/github.com/mattn/[email protected]
pkg:golang/github.com/mgutz/[email protected]
pkg:golang/github.com/mholt/[email protected]
pkg:golang/github.com/minio/[email protected]
pkg:golang/github.com/mongodb-forks/[email protected]
pkg:golang/github.com/montanaflynn/[email protected]
pkg:golang/github.com/nwaples/rardecode/[email protected]
pkg:golang/github.com/pelletier/[email protected]
pkg:golang/github.com/pelletier/go-toml/[email protected]
pkg:golang/github.com/pierrec/lz4/[email protected]
pkg:golang/github.com/pkg/[email protected]
pkg:golang/github.com/sagikazarmark/[email protected]
pkg:golang/github.com/shirou/gopsutil/[email protected]
pkg:golang/github.com/sorairolake/[email protected]
pkg:golang/github.com/sourcegraph/[email protected]
pkg:golang/github.com/spf13/[email protected]
pkg:golang/github.com/spf13/[email protected]
pkg:golang/github.com/spf13/[email protected]
pkg:golang/github.com/spf13/[email protected]
pkg:golang/github.com/spf13/[email protected]
pkg:golang/github.com/subosito/[email protected]
pkg:golang/github.com/tangzero/[email protected]
pkg:golang/github.com/therootcompany/[email protected]
pkg:golang/github.com/tklauser/[email protected]
pkg:golang/github.com/ulikunitz/[email protected]
pkg:golang/github.com/xdg-go/[email protected]
pkg:golang/github.com/xdg-go/[email protected]
pkg:golang/github.com/xdg-go/[email protected]
pkg:golang/github.com/youmark/[email protected]
pkg:golang/go.mongodb.org/[email protected]
pkg:golang/go.mongodb.org/atlas-sdk/[email protected]
pkg:golang/go.mongodb.org/atlas-sdk/[email protected]
pkg:golang/go.mongodb.org/[email protected]
pkg:golang/go.opentelemetry.io/auto/[email protected]
pkg:golang/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/[email protected]
pkg:golang/go.opentelemetry.io/contrib/instrumentation/net/http/[email protected]
pkg:golang/go.opentelemetry.io/[email protected]
pkg:golang/go.opentelemetry.io/otel/[email protected]
pkg:golang/go.opentelemetry.io/otel/[email protected]
pkg:golang/[email protected]
pkg:golang/golang.org/x/[email protected]
pkg:golang/golang.org/x/[email protected]
pkg:golang/golang.org/x/[email protected]
pkg:golang/golang.org/x/[email protected]
pkg:golang/golang.org/x/[email protected]
pkg:golang/golang.org/x/[email protected]
pkg:golang/golang.org/x/[email protected]
pkg:golang/golang.org/x/[email protected]
pkg:golang/golang.org/x/[email protected]
pkg:golang/golang.org/x/[email protected]
pkg:golang/google.golang.org/[email protected]
pkg:golang/google.golang.org/[email protected]
pkg:golang/google.golang.org/genproto/googleapis/[email protected]
pkg:golang/google.golang.org/genproto/googleapis/[email protected]
pkg:golang/google.golang.org/[email protected]
pkg:golang/google.golang.org/[email protected]
pkg:golang/gopkg.in/[email protected]
Loading