From ac63fb2333ba2fc4e5356a12525db47fda2f517a Mon Sep 17 00:00:00 2001 From: Filipe C Menezes Date: Mon, 26 Aug 2024 11:50:36 +0100 Subject: [PATCH] CLOUDP-269871 Allow image build to be triggered on demand --- build/ci/evergreen.yml | 264 +++++++++++++++++++++++++++++++++- build/ci/evergreen_common.yml | 221 ---------------------------- build/ci/periodic_build.yml | 126 ---------------- 3 files changed, 263 insertions(+), 348 deletions(-) delete mode 100644 build/ci/evergreen_common.yml delete mode 100644 build/ci/periodic_build.yml diff --git a/build/ci/evergreen.yml b/build/ci/evergreen.yml index 58ead4ff3d..e289672171 100644 --- a/build/ci/evergreen.yml +++ b/build/ci/evergreen.yml @@ -18,7 +18,6 @@ command_type: system pre_error_fails_task: true include: - filename: build/ci/release.yml - - filename: build/ci/evergreen_common.yml variables: - &go_linux_version @@ -50,6 +49,165 @@ post: - command: attach.xunit_results params: files: ["src/github.com/mongodb/mongodb-atlas-cli/*.xml"] +functions: + "clone": + - command: git.get_project + type: setup + params: + directory: src/github.com/mongodb/mongodb-atlas-cli + "build": + - command: subprocess.exec + type: test + params: + <<: *go_options + command: make build + "install mongodb database tools": + - command: shell.exec + type: setup + params: + <<: *go_options + working_dir: src/github.com/mongodb/mongodb-atlas-cli/bin + shell: bash + env: + <<: *go_env + MONGODB_DATABASE_TOOLS_VERSION: 100.10.0 + script: | + set -Eeou pipefail + if [[ "$(uname -s)" == "Darwin"* ]]; then + curl -sLo mongodb-database-tools.zip "https://fastdl.mongodb.org/tools/db/mongodb-database-tools-macos-arm64-$MONGODB_DATABASE_TOOLS_VERSION.zip" + unzip mongodb-database-tools.zip + mv mongodb-database-tools-*/bin/* . + rm -rf mongodb-database-tools-* mongodb-database-tools.zip + else + mkdir -p mongodb-database-tools + curl -sfL "https://fastdl.mongodb.org/tools/db/mongodb-database-tools-rhel70-x86_64-$MONGODB_DATABASE_TOOLS_VERSION.tgz" | tar zx -C mongodb-database-tools --strip-components 1 + mv mongodb-database-tools/bin/* . + rm -rf mongodb-database-tools + fi + "e2e test": + - command: subprocess.exec + type: test + params: + <<: *go_options + include_expansions_in_env: + - go_base_path + - workdir + - MCLI_ORG_ID + - MCLI_PROJECT_ID + - MCLI_PRIVATE_API_KEY + - MCLI_PUBLIC_API_KEY + - MCLI_SERVICE + - TEST_CMD + - E2E_TAGS + - E2E_TEST_BUCKET + - E2E_CLOUD_ROLE_ID + - MCLI_OPS_MANAGER_URL + - OM_VERSION + - LOCAL_KEY + - KMIP_CA + - KMIP_CERT + - GCP_CREDENTIALS + - AWS_ACCESS_KEY + - AWS_SECRET_ACCESS_KEY + - AZURE_TENANT_ID + - AZURE_CLIENT_ID + - AZURE_CLIENT_SECRET + - E2E_TIMEOUT + - E2E_SERVERLESS_INSTANCE_NAME + - E2E_PARALLEL + - IDENTITY_PROVIDER_ID + - revision + env: + <<: *go_env + MCLI_SKIP_UPDATE_CHECK: "yes" + TEST_CMD: gotestsum --junitfile e2e-tests.xml --format standard-verbose -- + command: make e2e-test + "install gotestsum": + - command: shell.exec + type: setup + params: + <<: *go_options + working_dir: src/github.com/mongodb/mongodb-atlas-cli/bin + shell: bash + script: | + set -Eeou pipefail + export PATH="$ADD_PATH:$PATH" + if [[ "$(uname -s)" == "Darwin"* ]]; then + URL=https://github.com/gotestyourself/gotestsum/releases/download/v${gotestsum_ver}/gotestsum_${gotestsum_ver}_darwin_arm64.tar.gz + else + URL=https://github.com/gotestyourself/gotestsum/releases/download/v${gotestsum_ver}/gotestsum_${gotestsum_ver}_linux_amd64.tar.gz + fi + curl -sfL $URL | tar zx + "generate evergreen": + - command: subprocess.exec + type: test + params: + <<: *go_options + include_expansions_in_env: + - go_base_path + - workdir + - task_type + binary: build/ci/evergreen-generate-tasks.sh + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_files_include_filter: + - src/github.com/mongodb/mongodb-atlas-cli/tasks.json + remote_file: ${project}/dist/${revision}_${created_at}/atlascli/${task_type}/ + bucket: mongodb-mongocli-build + permissions: public-read + content_type: ${content_type|application/json} + display_name: internal-bucket + - command: generate.tasks + params: + files: + - tasks.json + "increase inotify limits": + - command: shell.exec + params: + shell: bash + script: | + set -e + if [[ "$(uname -s)" == "Darwin"* ]]; then + echo "No need to increase inotify limits on macOS" + else + cat /proc/sys/fs/inotify/max_user_watches + cat /proc/sys/fs/inotify/max_user_instances + echo 1024 | sudo tee /proc/sys/fs/inotify/max_user_instances + cat /proc/sys/fs/inotify/max_user_instances + fi + "install packer": + - command: shell.exec + params: + <<: *go_options + working_dir: src/github.com/mongodb/mongodb-atlas-cli/bin + env: + <<: *go_env + PACKER_VERSION: 1.11.2 + shell: bash + script: | + set -Eeou pipefail + curl -sLo packer.zip https://releases.hashicorp.com/packer/$PACKER_VERSION/packer_$(echo $PACKER_VERSION)_linux_amd64.zip + unzip packer.zip + rm packer.zip LICENSE.txt + packer -v + "run packer": + - command: shell.exec + params: + <<: *go_options + working_dir: src/github.com/mongodb/mongodb-atlas-cli/build/ci/packer + env: + <<: *go_env + AZURE_APP_ID: ${azure_app_id} + AZURE_DISPLAY_NAME: ${azure_display_name} + AZURE_PASSWORD: ${azure_password} + AZURE_TENANT: ${azure_tenant} + AZURE_SUBSCRIPTION_ID: ${azure_subscription_id} + shell: bash + script: | + packer init ${file} + packer build -force ${file} tasks: - name: compile tags: ["code_health"] @@ -1086,6 +1244,77 @@ tasks: MCLI_SERVICE: cloud E2E_TAGS: atlas,deployments,atlasclusters E2E_TIMEOUT: 3h + - name: atlas_deployments_local_noauth_e2e + tags: ["e2e","deployments","local","noauth"] + must_have_test_results: true + exec_timeout_secs: 11400 # 3 hours 10 minutes + commands: + - func: "install gotestsum" + - func: "install mongodb database tools" + - func: "increase inotify limits" + - func: "e2e test" + timeout_secs: 11400 # 3 hours 10 minutes + vars: + MCLI_ORG_ID: ${atlas_org_id} + MCLI_PROJECT_ID: ${atlas_project_id} + MCLI_PRIVATE_API_KEY: ${atlas_private_api_key} + MCLI_PUBLIC_API_KEY: ${atlas_public_api_key} + MCLI_OPS_MANAGER_URL: ${mcli_ops_manager_url} + MCLI_SERVICE: cloud + E2E_TAGS: atlas,deployments,local,noauth + E2E_TIMEOUT: 3h + - name: atlas_deployments_local_nocli_e2e + tags: ["e2e","deployments","local","nocli"] + must_have_test_results: true + exec_timeout_secs: 11400 # 3 hours 10 minutes + commands: + - func: "install gotestsum" + - func: "install mongodb database tools" + - func: "increase inotify limits" + - func: "e2e test" + timeout_secs: 11400 # 3 hours 10 minutes + vars: + MCLI_ORG_ID: ${atlas_org_id} + MCLI_PROJECT_ID: ${atlas_project_id} + MCLI_PRIVATE_API_KEY: ${atlas_private_api_key} + MCLI_PUBLIC_API_KEY: ${atlas_public_api_key} + MCLI_OPS_MANAGER_URL: ${mcli_ops_manager_url} + MCLI_SERVICE: cloud + E2E_TAGS: atlas,deployments,local,nocli + E2E_TIMEOUT: 3h + - name: atlas_deployments_local_auth_e2e + tags: ["e2e","deployments","local","auth"] + must_have_test_results: true + exec_timeout_secs: 11400 # 3 hours 10 minutes + commands: + - func: "install gotestsum" + - func: "install mongodb database tools" + - func: "increase inotify limits" + - func: "e2e test" + timeout_secs: 11400 # 3 hours 10 minutes + vars: + MCLI_ORG_ID: ${atlas_org_id} + MCLI_PROJECT_ID: ${atlas_project_id} + MCLI_PRIVATE_API_KEY: ${atlas_private_api_key} + MCLI_PUBLIC_API_KEY: ${atlas_public_api_key} + MCLI_OPS_MANAGER_URL: ${mcli_ops_manager_url} + MCLI_SERVICE: cloud + E2E_TAGS: atlas,deployments,local,auth + E2E_TIMEOUT: 3h + - name: build_win11_image + tags: ["packer", "windows", "win11"] + commands: + - func: "install packer" + - func: "run packer" + vars: + file: "windows-11-azure.pkr.hcl" + - name: build_win10_image + tags: ["packer", "windows", "win10"] + commands: + - func: "install packer" + - func: "run packer" + vars: + file: "windows-10-azure.pkr.hcl" buildvariants: - name: code_health display_name: "Code Health" @@ -1236,6 +1465,39 @@ buildvariants: <<: *go_linux_version tasks: - name: ".e2e .atlas .plugin" + - name: e2e_local_deployments_rhel + display_name: "E2E Local Deployments Tests (rhel/podman)" + allowed_requesters: ["patch", "ad_hoc", "github_pr"] + tags: + - localdev + run_on: + - rhel90-small + expansions: + <<: *go_linux_version + tasks: + - name: ".e2e .deployments .local" + - name: e2e_local_deployments_macos + display_name: "E2E Local Deployments Tests (macos/docker)" + allowed_requesters: ["patch", "ad_hoc", "github_pr"] + tags: + - localdev + run_on: + - macos-14-arm64-docker + expansions: + <<: *go_linux_version + tasks: + - name: ".e2e .deployments .local" + - name: build_images + display_name: "Build Windows Images" + allowed_requesters: ["patch", "ad_hoc", "github_pr"] + tags: + - packer + run_on: + - rhel80-small + expansions: + <<: *go_linux_version + tasks: + - name: ".packer .windows" patch_aliases: - alias: "localdev" variant_tags: ["localdev"] diff --git a/build/ci/evergreen_common.yml b/build/ci/evergreen_common.yml deleted file mode 100644 index 828114400a..0000000000 --- a/build/ci/evergreen_common.yml +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright 2024 MongoDB Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -variables: - - &go_env - XDG_CONFIG_HOME: ${go_base_path}${workdir} - GO111MODULE: "on" - GOROOT: ${go_root} - GOPATH: ${go_base_path}${workdir} - ADD_PATH: "${go_bin}:${go_base_path}${workdir}/bin:${go_base_path}${workdir}/src/github.com/mongodb/mongodb-atlas-cli/bin" - GOPROXY: ${go_proxy} - - &go_options - add_to_path: - - ${go_bin} - - ${go_base_path}${workdir}/bin - - ${go_base_path}${workdir}/src/github.com/mongodb/mongodb-atlas-cli/bin - include_expansions_in_env: - - go_base_path - - go_proxy - - workdir - working_dir: src/github.com/mongodb/mongodb-atlas-cli - env: - <<: *go_env -functions: - "clone": - - command: git.get_project - type: setup - params: - directory: src/github.com/mongodb/mongodb-atlas-cli - "build": - - command: subprocess.exec - type: test - params: - <<: *go_options - command: make build - "install mongodb database tools": - - command: shell.exec - type: setup - params: - <<: *go_options - working_dir: src/github.com/mongodb/mongodb-atlas-cli/bin - shell: bash - env: - <<: *go_env - MONGODB_DATABASE_TOOLS_VERSION: 100.10.0 - script: | - set -Eeou pipefail - if [[ "$(uname -s)" == "Darwin"* ]]; then - curl -sLo mongodb-database-tools.zip "https://fastdl.mongodb.org/tools/db/mongodb-database-tools-macos-arm64-$MONGODB_DATABASE_TOOLS_VERSION.zip" - unzip mongodb-database-tools.zip - mv mongodb-database-tools-*/bin/* . - rm -rf mongodb-database-tools-* mongodb-database-tools.zip - else - mkdir -p mongodb-database-tools - curl -sfL "https://fastdl.mongodb.org/tools/db/mongodb-database-tools-rhel70-x86_64-$MONGODB_DATABASE_TOOLS_VERSION.tgz" | tar zx -C mongodb-database-tools --strip-components 1 - mv mongodb-database-tools/bin/* . - rm -rf mongodb-database-tools - fi - "e2e test": - - command: subprocess.exec - type: test - params: - <<: *go_options - include_expansions_in_env: - - go_base_path - - workdir - - MCLI_ORG_ID - - MCLI_PROJECT_ID - - MCLI_PRIVATE_API_KEY - - MCLI_PUBLIC_API_KEY - - MCLI_SERVICE - - TEST_CMD - - E2E_TAGS - - E2E_TEST_BUCKET - - E2E_CLOUD_ROLE_ID - - MCLI_OPS_MANAGER_URL - - OM_VERSION - - LOCAL_KEY - - KMIP_CA - - KMIP_CERT - - GCP_CREDENTIALS - - AWS_ACCESS_KEY - - AWS_SECRET_ACCESS_KEY - - AZURE_TENANT_ID - - AZURE_CLIENT_ID - - AZURE_CLIENT_SECRET - - E2E_TIMEOUT - - E2E_SERVERLESS_INSTANCE_NAME - - E2E_PARALLEL - - IDENTITY_PROVIDER_ID - - revision - env: - <<: *go_env - MCLI_SKIP_UPDATE_CHECK: "yes" - TEST_CMD: gotestsum --junitfile e2e-tests.xml --format standard-verbose -- - command: make e2e-test - "install gotestsum": - - command: shell.exec - type: setup - params: - <<: *go_options - working_dir: src/github.com/mongodb/mongodb-atlas-cli/bin - shell: bash - script: | - set -Eeou pipefail - export PATH="$ADD_PATH:$PATH" - if [[ "$(uname -s)" == "Darwin"* ]]; then - URL=https://github.com/gotestyourself/gotestsum/releases/download/v${gotestsum_ver}/gotestsum_${gotestsum_ver}_darwin_arm64.tar.gz - else - URL=https://github.com/gotestyourself/gotestsum/releases/download/v${gotestsum_ver}/gotestsum_${gotestsum_ver}_linux_amd64.tar.gz - fi - curl -sfL $URL | tar zx - "generate evergreen": - - command: subprocess.exec - type: test - params: - <<: *go_options - include_expansions_in_env: - - go_base_path - - workdir - - task_type - binary: build/ci/evergreen-generate-tasks.sh - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_files_include_filter: - - src/github.com/mongodb/mongodb-atlas-cli/tasks.json - remote_file: ${project}/dist/${revision}_${created_at}/atlascli/${task_type}/ - bucket: mongodb-mongocli-build - permissions: public-read - content_type: ${content_type|application/json} - display_name: internal-bucket - - command: generate.tasks - params: - files: - - tasks.json - "increase inotify limits": - - command: shell.exec - params: - shell: bash - script: | - set -e - if [[ "$(uname -s)" == "Darwin"* ]]; then - echo "No need to increase inotify limits on macOS" - else - cat /proc/sys/fs/inotify/max_user_watches - cat /proc/sys/fs/inotify/max_user_instances - echo 1024 | sudo tee /proc/sys/fs/inotify/max_user_instances - cat /proc/sys/fs/inotify/max_user_instances - fi -tasks: - - name: atlas_deployments_local_noauth_e2e - tags: ["e2e","deployments","local","noauth"] - must_have_test_results: true - exec_timeout_secs: 11400 # 3 hours 10 minutes - commands: - - func: "install gotestsum" - - func: "install mongodb database tools" - - func: "increase inotify limits" - - func: "e2e test" - timeout_secs: 11400 # 3 hours 10 minutes - vars: - MCLI_ORG_ID: ${atlas_org_id} - MCLI_PROJECT_ID: ${atlas_project_id} - MCLI_PRIVATE_API_KEY: ${atlas_private_api_key} - MCLI_PUBLIC_API_KEY: ${atlas_public_api_key} - MCLI_OPS_MANAGER_URL: ${mcli_ops_manager_url} - MCLI_SERVICE: cloud - E2E_TAGS: atlas,deployments,local,noauth - E2E_TIMEOUT: 3h - - name: atlas_deployments_local_nocli_e2e - tags: ["e2e","deployments","local","nocli"] - must_have_test_results: true - exec_timeout_secs: 11400 # 3 hours 10 minutes - commands: - - func: "install gotestsum" - - func: "install mongodb database tools" - - func: "increase inotify limits" - - func: "e2e test" - timeout_secs: 11400 # 3 hours 10 minutes - vars: - MCLI_ORG_ID: ${atlas_org_id} - MCLI_PROJECT_ID: ${atlas_project_id} - MCLI_PRIVATE_API_KEY: ${atlas_private_api_key} - MCLI_PUBLIC_API_KEY: ${atlas_public_api_key} - MCLI_OPS_MANAGER_URL: ${mcli_ops_manager_url} - MCLI_SERVICE: cloud - E2E_TAGS: atlas,deployments,local,nocli - E2E_TIMEOUT: 3h - - name: atlas_deployments_local_auth_e2e - tags: ["e2e","deployments","local","auth"] - must_have_test_results: true - exec_timeout_secs: 11400 # 3 hours 10 minutes - commands: - - func: "install gotestsum" - - func: "install mongodb database tools" - - func: "increase inotify limits" - - func: "e2e test" - timeout_secs: 11400 # 3 hours 10 minutes - vars: - MCLI_ORG_ID: ${atlas_org_id} - MCLI_PROJECT_ID: ${atlas_project_id} - MCLI_PRIVATE_API_KEY: ${atlas_private_api_key} - MCLI_PUBLIC_API_KEY: ${atlas_public_api_key} - MCLI_OPS_MANAGER_URL: ${mcli_ops_manager_url} - MCLI_SERVICE: cloud - E2E_TAGS: atlas,deployments,local,auth - E2E_TIMEOUT: 3h diff --git a/build/ci/periodic_build.yml b/build/ci/periodic_build.yml deleted file mode 100644 index 8a809dd4a7..0000000000 --- a/build/ci/periodic_build.yml +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2024 MongoDB Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -exec_timeout_secs: 4200 # automatically fail any task if it takes longer than 1h10m to finish. -include: - - filename: build/ci/evergreen_common.yml - -variables: - - &go_linux_version - go_root: "/opt/golang/go1.22" - go_bin: "/opt/golang/go1.22/bin" - go_base_path: "" - - &go_env - XDG_CONFIG_HOME: ${go_base_path}${workdir} - GO111MODULE: "on" - GOROOT: ${go_root} - GOPATH: ${go_base_path}${workdir} - ADD_PATH: "${go_bin}:${go_base_path}${workdir}/bin:${go_base_path}${workdir}/src/github.com/mongodb/mongodb-atlas-cli/bin" - GOPROXY: ${go_proxy} - - &go_options - add_to_path: - - ${go_bin} - - ${go_base_path}${workdir}/bin - - ${go_base_path}${workdir}/src/github.com/mongodb/mongodb-atlas-cli/bin - include_expansions_in_env: - - go_base_path - - go_proxy - - workdir - working_dir: src/github.com/mongodb/mongodb-atlas-cli - env: - <<: *go_env -pre: - - func: "clone" -post: - - command: attach.xunit_results - params: - files: ["src/github.com/mongodb/mongodb-atlas-cli/*.xml"] -functions: - "install packer": - - command: shell.exec - params: - <<: *go_options - working_dir: src/github.com/mongodb/mongodb-atlas-cli/bin - env: - <<: *go_env - PACKER_VERSION: 1.11.2 - shell: bash - script: | - set -Eeou pipefail - curl -sLo packer.zip https://releases.hashicorp.com/packer/$PACKER_VERSION/packer_$(echo $PACKER_VERSION)_linux_amd64.zip - unzip packer.zip - rm packer.zip LICENSE.txt - packer -v - "run packer": - - command: shell.exec - params: - <<: *go_options - working_dir: src/github.com/mongodb/mongodb-atlas-cli/build/ci/packer - env: - <<: *go_env - AZURE_APP_ID: ${azure_app_id} - AZURE_DISPLAY_NAME: ${azure_display_name} - AZURE_PASSWORD: ${azure_password} - AZURE_TENANT: ${azure_tenant} - AZURE_SUBSCRIPTION_ID: ${azure_subscription_id} - shell: bash - script: | - packer init ${file} - packer build -force ${file} -tasks: - - name: build_win11_image - tags: ["packer", "windows", "win11"] - commands: - - func: "install packer" - - func: "run packer" - vars: - file: "windows-11-azure.pkr.hcl" - - name: build_win10_image - tags: ["packer", "windows", "win10"] - commands: - - func: "install packer" - - func: "run packer" - vars: - file: "windows-10-azure.pkr.hcl" -buildvariants: - - name: e2e_local_deployments_rhel - display_name: "E2E Local Deployments Tests (rhel/podman)" - tags: - - localdev - run_on: - - rhel90-small - expansions: - <<: *go_linux_version - tasks: - - name: ".e2e .deployments .local" - - name: e2e_local_deployments_macos - display_name: "E2E Local Deployments Tests (macos/docker)" - tags: - - localdev - run_on: - - macos-14-arm64-docker - expansions: - <<: *go_linux_version - tasks: - - name: ".e2e .deployments .local" - - name: build_images - display_name: "Build Windows Images" - expansions: - <<: *go_linux_version - tags: - - packer - run_on: - - rhel80-small - tasks: - - name: ".packer .windows"