diff --git a/.github/workflows/e2e-rosa-test.yml b/.github/workflows/e2e-rosa-test.yml new file mode 100644 index 0000000000..0036545893 --- /dev/null +++ b/.github/workflows/e2e-rosa-test.yml @@ -0,0 +1,382 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT + +name: Reusable ROSA E2E Test + +env: + TERRAFORM_AWS_ASSUME_ROLE_DURATION: 14400 # 4 hours + +on: + workflow_call: + inputs: + terraform_dir: + required: true + type: string + job_id: + required: true + type: string + test_props: + required: true + type: string + test_repo_name: + required: true + type: string + test_repo_url: + required: true + type: string + test_repo_branch: + required: true + type: string + cloudwatch_agent_repository: + required: true + type: string + cloudwatch_agent_tag: + required: true + type: string + cloudwatch_agent_operator_repository: + required: true + type: string + cloudwatch_agent_target_allocator_repository: + required: false + type: string + cloudwatch_agent_operator_tag: + required: true + type: string + region: + required: true + type: string + helm_charts_branch: + required: true + type: string + terraform_assume_role: + required: true + type: string + agent_config: + required: true + type: string + prometheus_config: + required: false + type: string + default: "" + otel_config: + required: false + type: string + default: "" + sample_app: + required: true + type: string + s3_integration_bucket: + default: ${{vars.S3_INTEGRATION_BUCKET}} + type: string + +jobs: + CreateCluster: + name: "Create ROSA HCP Cluster" + runs-on: ubuntu-latest + outputs: + cluster_id: ${{ steps.terraform.outputs.cluster_id }} + cluster_name: ${{ steps.terraform.outputs.cluster_name }} + strategy: + fail-fast: false + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + repository: ${{inputs.test_repo_name}} + ref: ${{inputs.test_repo_branch}} + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ inputs.terraform_assume_role}} + aws-region: ${{ inputs.region }} + role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} + - name: Install Terraform + uses: hashicorp/setup-terraform@v3 + - name: Verify Terraform version + run: terraform --version + + - name: Terraform apply + id: terraform + uses: nick-fields/retry@v2 + env: + TF_VAR_rhcs_token: ${{secrets.RHCS_ACCOUNT_TOKEN}} + TF_VAR_billing_account_id: ${{secrets.ROSA_BILLING_ACCOUNT}} + TF_VAR_aws_region: ${{inputs.region}} + TF_VAR_cluster_name: cwa-rosa-test-${{github.run_id}}-${{github.run_attempt}} + with: + max_attempts: 1 + timeout_minutes: 80 + retry_wait_seconds: 5 + command: | + cd terraform/rosa/e2e + + terraform init + if terraform apply --auto-approve; then + echo "cluster_id=$(terraform output -raw cluster_id)" >> $GITHUB_OUTPUT + echo "cluster_name=$(terraform output -raw cluster_name)" >> $GITHUB_OUTPUT + aws s3 cp terraform.tfstate s3://${{inputs.s3_integration_bucket}}/integration-test/local-stack-terraform-state/${{inputs.github_sha}}/terraform.tfstate + else + terraform destroy --auto-approve && exit 1 + fi + + ValidateCluster: + needs: [CreateCluster] + name: "Validate Cluster" + runs-on: ubuntu-latest + strategy: + fail-fast: false + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + repository: ${{inputs.test_repo_name}} + ref: ${{inputs.test_repo_branch}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ inputs.terraform_assume_role}} + aws-region: ${{ inputs.region }} + role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} + + - name: Login ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Install OpenShift CLI + uses: redhat-actions/oc-installer@v1 + + - name: Install ROSA CLI + shell: bash + run: | + curl -LO "https://mirror.openshift.com/pub/openshift-v4/clients/rosa/latest/rosa-linux.tar.gz" + tar -xvf rosa-linux.tar.gz + sudo mv rosa /usr/local/bin/rosa + chmod +x /usr/local/bin/rosa + rm -f rosa-linux.tar.gz + rosa version + + - name: Get Secrets + uses: aws-actions/aws-secretsmanager-get-secrets@v2 + with: + parse-json-secrets: true + secret-ids: | + ,${{needs.CreateCluster.outputs.cluster_name}}-htpasswd + + - name: Login to OpenShift + uses: redhat-actions/oc-login@v1 + with: + # URL to your OpenShift cluster. + openshift_server_url: ${{ env.OPENSHIFT_SERVER }} + openshift_username: ${{ env.OPENSHIFT_USERNAME }} + openshift_password: ${{ env.OPENSHIFT_PASSWORD }} + + - name: Validate Cluster and Host ROSA Setup + working-directory: test/e2e/rosa + env: + CLUSTER_NAME: ${{needs.CreateCluster.outputs.cluster_name}} + run: go test -computeType=ROSA -eksClusterName=${{needs.CreateCluster.outputs.cluster_name}} -eksDeploymentStrategy="DAEMON" -region=${{env.AWS_REGION}} -v + PatchCluster: + needs: [CreateCluster,ValidateCluster] + name: "Patch ROSA Cluster" + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + repository: ${{inputs.test_repo_name}} + ref: ${{inputs.test_repo_branch}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ inputs.terraform_assume_role}} + aws-region: ${{ inputs.region }} + role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} + + - name: Install OpenShift CLI + uses: redhat-actions/oc-installer@v1 + + - name: Install ROSA CLI + shell: bash + run: | + curl -LO "https://mirror.openshift.com/pub/openshift-v4/clients/rosa/latest/rosa-linux.tar.gz" + tar -xvf rosa-linux.tar.gz + sudo mv rosa /usr/local/bin/rosa + chmod +x /usr/local/bin/rosa + rm -f rosa-linux.tar.gz + rosa version + + - name: Get Secrets + uses: aws-actions/aws-secretsmanager-get-secrets@v2 + with: + parse-json-secrets: true + secret-ids: | + ,${{needs.CreateCluster.outputs.cluster_name}}-htpasswd + + - name: Login to OpenShift + uses: redhat-actions/oc-login@v1 + with: + # URL to your OpenShift cluster. + openshift_server_url: ${{ env.OPENSHIFT_SERVER }} + openshift_username: ${{ env.OPENSHIFT_USERNAME }} + openshift_password: ${{ env.OPENSHIFT_PASSWORD }} + + - name: Install SSM Agent + working-directory: terraform/rosa/patching + run : | + oc apply -f ./patch.yaml + + RosaTest: + needs: [CreateCluster,ValidateCluster,PatchCluster] + name: ${{matrix.arrays.testName}} + runs-on: ubuntu-latest + strategy: + max-parallel: 1 + fail-fast: false + matrix: + arrays: ${{ fromJson(inputs.test_props) }} + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + repository: ${{inputs.test_repo_name}} + ref: ${{inputs.test_repo_branch}} + + - uses: actions/checkout@v3 + with: + repository: "aws-observability/helm-charts" + ref: ${{inputs.helm_charts_branch}} + path: "terraform/eks/e2e/helm-charts"# have to fix this later + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ inputs.terraform_assume_role}} + aws-region: ${{ inputs.region }} + role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} + + - name: Install Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Verify Terraform version + run: terraform --version + + - name: Install OpenShift CLI + uses: redhat-actions/oc-installer@v1 + + - name: Install ROSA CLI + shell: bash + run: | + curl -LO "https://mirror.openshift.com/pub/openshift-v4/clients/rosa/latest/rosa-linux.tar.gz" + tar -xvf rosa-linux.tar.gz + sudo mv rosa /usr/local/bin/rosa + chmod +x /usr/local/bin/rosa + rm -f rosa-linux.tar.gz + rosa version + + - name: Get Secrets + uses: aws-actions/aws-secretsmanager-get-secrets@v2 + with: + parse-json-secrets: true + secret-ids: | + ,${{needs.CreateCluster.outputs.cluster_name}}-htpasswd + + - name: Login to OpenShift + uses: redhat-actions/oc-login@v1 + with: + # URL to your OpenShift cluster. + openshift_server_url: ${{ env.OPENSHIFT_SERVER }} + openshift_username: ${{ env.OPENSHIFT_USERNAME }} + openshift_password: ${{ env.OPENSHIFT_PASSWORD }} + + - name: ${{matrix.arrays.testName}} + working-directory: ${{matrix.arrays.test_dir}} + run: | + go test --computeType=ROSA \ + --eksClusterName=${{needs.CreateCluster.outputs.cluster_name}} \ + --eksDeploymentStrategy="DAEMON" \ + --region=${{env.AWS_REGION}} \ + --helm_charts_branch=${{inputs.helm_charts_branch}} \ + --cloudwatch_agent_repository=${{inputs.cloudwatch_agent_repository}} \ + --cloudwatch_agent_tag=${{inputs.cloudwatch_agent_tag}} \ + --cloudwatch_agent_repository_url=${{inputs.cloudwatch_agent_repository_url}} \ + --cloudwatch_agent_operator_repository=${{inputs.cloudwatch_agent_operator_repository}} \ + --cloudwatch_agent_operator_tag=${{inputs.cloudwatch_agent_operator_tag}} \ + --cloudwatch_agent_operator_repository_url=${{inputs.cloudwatch_agent_operator_repository_url}} \ + --cloudwatch_agent_target_allocator_repository=${{inputs.cloudwatch_agent_target_allocator_repository}} \ + --cloudwatch_agent_target_allocator_tag=${{inputs.cloudwatch_agent_target_allocator_tag}} \ + --cloudwatch_agent_target_allocator_repository_url=${{inputs.cloudwatch_agent_target_allocator_repository_url}} \ + --sample_app=${{matrix.arrays.sample_app_path}} \ + --timeout 30m \ + --v + + - name: Cleanup + working-directory: ${{matrix.arrays.test_dir}} + if: always() + run: | + go test --computeType=ROSA \ + --destroy \ + --eksClusterName=${{needs.CreateCluster.outputs.cluster_name}} \ + --eksDeploymentStrategy="DAEMON" \ + --region=${{env.AWS_REGION}} \ + --sample_app=${{matrix.arrays.sample_app_path}} \ + --timeout 30m \ + --v + DestroyCluster: + name: "Destroy ROSA HCP Cluster" + if: ${{ always() && needs.CreateCluster.result== 'success' }} + needs: [CreateCluster, ValidateCluster, RosaTest] + runs-on: ubuntu-latest + strategy: + fail-fast: false + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + with: + repository: ${{inputs.test_repo_name}} + ref: ${{inputs.test_repo_branch}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ inputs.terraform_assume_role}} + aws-region: ${{ inputs.region }} + role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} + + - name: Copy state + working-directory: terraform/rosa/e2e + run: aws s3 cp s3://${{inputs.s3_integration_bucket}}/integration-test/local-stack-terraform-state/${{inputs.github_sha}}/terraform.tfstate . + + - name: Install Terraform + uses: hashicorp/setup-terraform@v3 + - name: Verify Terraform version + run: terraform --version + + - name: Terraform destroy + id: terraform + uses: nick-fields/retry@v2 + env: + TF_VAR_rhcs_token: ${{secrets.RHCS_ACCOUNT_TOKEN}} + TF_VAR_billing_account_id: ${{secrets.ROSA_BILLING_ACCOUNT}} + TF_VAR_aws_region: ${{inputs.region}} + TF_VAR_cluster_name: ${{needs.CreateCluster.outputs.cluster_name}} + with: + max_attempts: 1 + timeout_minutes: 80 + retry_wait_seconds: 5 + command: | + cd terraform/rosa/e2e + terraform init + terraform destroy --auto-approve || exit 0 + diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 2ea9b5075a..67b7ef4c74 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -8,7 +8,7 @@ env: ECR_INTEGRATION_TEST_REPO: "cwagent-integration-test" CWA_GITHUB_TEST_REPO_NAME: "aws/amazon-cloudwatch-agent-test" CWA_GITHUB_TEST_REPO_URL: "https://github.com/aws/amazon-cloudwatch-agent-test.git" - CWA_GITHUB_TEST_REPO_BRANCH: "main" + CWA_GITHUB_TEST_REPO_BRANCH: "e2e-rosa" TERRAFORM_AWS_ASSUME_ROLE_ITAR: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} TERRAFORM_AWS_ASSUME_ROLE_CN: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} OPERATOR_GITHUB_REPO_NAME: "aws/amazon-cloudwatch-agent-operator" @@ -16,6 +16,7 @@ env: on: schedule: - cron: '0 17 * * 1' + workflow_dispatch: inputs: region: @@ -32,7 +33,7 @@ on: required: false type: string description: 'Branch of the helm charts to test' - default: 'main' + default: 'rosa' concurrency: group: ${{ github.workflow }}-${{ github.ref_name }}-parent @@ -124,13 +125,34 @@ jobs: echo "ECR_INTEGRATION_TEST_REPO: ${{ steps.set-outputs.outputs.ECR_INTEGRATION_TEST_REPO }}" echo "ECR_OPERATOR_REPO: ${{ steps.set-outputs.outputs.ECR_OPERATOR_REPO }}" echo "ECR_TARGET_ALLOCATOR_REPO: ${{ steps.set-outputs.outputs.ECR_TARGET_ALLOCATOR_REPO }}" - + RosaBaseTest: + needs: [ GetLatestOperatorCommitSHA, GenerateTestMatrix, OutputEnvVariables ] + name: 'Rosa Base Test' + uses: ./.github/workflows/e2e-rosa-test.yml + secrets: inherit + with: + terraform_dir: terraform/rosa/e2e + job_id: rosa-test + test_props: ${{ needs.GenerateTestMatrix.outputs.rosa_e2e_matrix}} + test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} + test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} + test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} + cloudwatch_agent_repository: ${{ needs.OutputEnvVariables.outputs.ECR_INTEGRATION_TEST_REPO }} + cloudwatch_agent_tag: ${{ github.sha }} + cloudwatch_agent_operator_repository: ${{ needs.OutputEnvVariables.outputs.ECR_OPERATOR_REPO }} + cloudwatch_agent_operator_tag: ${{ needs.GetLatestOperatorCommitSHA.outputs.operator_commit_sha }} + region: ${{ inputs.region}} + helm_charts_branch: ${{ inputs.helm-charts-branch || 'main' }} + terraform_assume_role: ${{vars.TERRAFORM_AWS_ASSUME_ROLE}} + agent_config: resources/cwagent_configs/jvm_tomcat.json + sample_app: resources/sample_apps/tomcat.yaml GenerateTestMatrix: needs: [BuildAgent, BuildOperator] name: 'GenerateTestMatrix' runs-on: ubuntu-latest outputs: eks_e2e_jmx_matrix: ${{ steps.set-matrix.outputs.eks_e2e_jmx_matrix }} + rosa_e2e_matrix: ${{steps.set-matrix.outputs.rosa_e2e_matrix}} steps: - uses: actions/checkout@v3 with: @@ -147,10 +169,12 @@ jobs: run: | go run generator/test_case_generator.go -e2e echo "::set-output name=eks_e2e_jmx_matrix::$(echo $(cat generator/resources/eks_e2e_jmx_complete_test_matrix.json))" + echo "::set-output name=rosa_e2e_matrix::$(echo $(cat generator/resources/rosa_e2e_cluster_complete_test_matrix.json))" - name: Echo test plan matrix run: | echo "eks_e2e_jmx_matrix: ${{ steps.set-matrix.outputs.eks_e2e_jmx_matrix }}" + echo "rosa_e2e_matrix: ${{ steps.set-matrix.outputs.rosa_e2e_matrix }}" EKSE2EJVMTomcatTest: needs: [ GetLatestOperatorCommitSHA, GenerateTestMatrix, OutputEnvVariables ]