From 9610e1ca8b394bcf07cb222f4cabfb1daaccbb49 Mon Sep 17 00:00:00 2001 From: Can Sun Date: Wed, 31 May 2023 18:39:48 -0700 Subject: [PATCH 01/12] workflow fix --- .github/workflows/build.yml | 2 ++ .github/workflows/integration-tests.yml | 13 ++++--------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 44eebda..fa94830 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,6 +2,8 @@ name: Build on: pull_request: + branches: + - main jobs: build: diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 74a6ec9..6358179 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,11 +1,10 @@ name: Integration Tests on: - workflow_run: - workflows: - - Build - types: - - completed + pull_request: + branches: + - main + needs: build jobs: test: @@ -16,10 +15,6 @@ jobs: contents: write actions: read - # Run only if a successful PR request build workflow triggered the integration tests - if: > - github.event.workflow_run.event == 'pull_request' && - github.event.workflow_run.conclusion == 'success' steps: # Set up and install required dependencies From aea3db7f2ddec51ec3ad129b73dc461f901369c9 Mon Sep 17 00:00:00 2001 From: Can Sun Date: Wed, 31 May 2023 18:50:16 -0700 Subject: [PATCH 02/12] Fix the workflow sequences --- .github/workflows/integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 6358179..0269d65 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -4,7 +4,7 @@ on: pull_request: branches: - main - needs: build + needs: [build] jobs: test: From f787b93b62cbbdb3c64b3ff4125209af6968f7a7 Mon Sep 17 00:00:00 2001 From: Can Sun Date: Wed, 31 May 2023 18:54:00 -0700 Subject: [PATCH 03/12] Move needs to jobs --- .github/workflows/integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 0269d65..67f2dd9 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -4,10 +4,10 @@ on: pull_request: branches: - main - needs: [build] jobs: test: + needs: build runs-on: ubuntu-latest permissions: pull-requests: write From f80d077e249fd00c93e60040d798f348be85dbad Mon Sep 17 00:00:00 2001 From: Can Sun Date: Wed, 31 May 2023 19:01:22 -0700 Subject: [PATCH 04/12] Fix the name of workflow --- .github/workflows/integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 67f2dd9..f1654e5 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -7,7 +7,7 @@ on: jobs: test: - needs: build + needs: Build runs-on: ubuntu-latest permissions: pull-requests: write From 0e2fc6f55d04b2e64a7480adeddacd383a9fc656 Mon Sep 17 00:00:00 2001 From: Can Sun Date: Wed, 31 May 2023 19:11:22 -0700 Subject: [PATCH 05/12] More fixes --- .github/workflows/integration-tests.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index f1654e5..15a0462 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -4,10 +4,15 @@ on: pull_request: branches: - main + workflow_run: + workflows: + - Build + types: + - completed jobs: test: - needs: Build + needs: build runs-on: ubuntu-latest permissions: pull-requests: write From cec0e5459ff38e341ea93a8020eb6bc3754830cd Mon Sep 17 00:00:00 2001 From: Can Sun Date: Wed, 31 May 2023 19:15:24 -0700 Subject: [PATCH 06/12] Fix more --- .github/workflows/integration-tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 15a0462..06cde80 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -11,6 +11,11 @@ on: - completed jobs: + check: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 test: needs: build runs-on: ubuntu-latest From fe6725a8ba13b1e6584b7626ac9cc50ffc42b5c2 Mon Sep 17 00:00:00 2001 From: Can Sun Date: Wed, 31 May 2023 19:18:18 -0700 Subject: [PATCH 07/12] Remove needs from workflow --- .github/workflows/integration-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 06cde80..3aa190b 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -17,7 +17,6 @@ jobs: - name: Checkout code uses: actions/checkout@v2 test: - needs: build runs-on: ubuntu-latest permissions: pull-requests: write From 4f87efd9a5275999edc50d1053579c4e26548626 Mon Sep 17 00:00:00 2001 From: Can Sun Date: Wed, 31 May 2023 19:22:57 -0700 Subject: [PATCH 08/12] Remove unecessary integration test yml --- .github/workflows/build.yml | 63 +++++++++++++++++++- .github/workflows/integration-tests.yml | 77 ------------------------- 2 files changed, 62 insertions(+), 78 deletions(-) delete mode 100644 .github/workflows/integration-tests.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fa94830..9075e34 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: Build +name: Build and Test on: pull_request: @@ -52,3 +52,64 @@ jobs: with: name: processedArtifacts path: processedArtifacts/ + + integration_test: + runs-on: ubuntu-latest + needs: build + permissions: + pull-requests: write + id-token: write + contents: write + actions: read + + steps: + + # Set up and install required dependencies + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade boto3 + + # Configure AWS credentials + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-region: us-west-2 + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + role-session-name: GithubSession + + # Download processed artifacts generated from build workflow + - name: 'Download Build Artifacts' + uses: actions/github-script@v3.1.0 + with: + script: | + var artifacts = await github.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{github.event.workflow_run.id }}, + }); + var matchArtifact = artifacts.data.artifacts.filter((artifact) => { + return artifact.name == "processedArtifacts" + })[0]; + var download = await github.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + var fs = require('fs'); + fs.writeFileSync('${{github.workspace}}/processedArtifacts.zip', Buffer.from(download.data)); + + # Unzip the build artifacts and run integration tests + - name: Run Integration Tests + run: | + unzip processedArtifacts.zip + cd integration_test + chmod +x run-spark-integration-test + ./run-spark-integration-test diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml deleted file mode 100644 index 3aa190b..0000000 --- a/.github/workflows/integration-tests.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Integration Tests - -on: - pull_request: - branches: - - main - workflow_run: - workflows: - - Build - types: - - completed - -jobs: - check: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - test: - runs-on: ubuntu-latest - permissions: - pull-requests: write - id-token: write - contents: write - actions: read - - steps: - - # Set up and install required dependencies - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install --upgrade boto3 - - # Configure AWS credentials - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-region: us-west-2 - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - role-session-name: GithubSession - - # Download processed artifacts generated from build workflow - - name: 'Download Build Artifacts' - uses: actions/github-script@v3.1.0 - with: - script: | - var artifacts = await github.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: ${{github.event.workflow_run.id }}, - }); - var matchArtifact = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == "processedArtifacts" - })[0]; - var download = await github.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - var fs = require('fs'); - fs.writeFileSync('${{github.workspace}}/processedArtifacts.zip', Buffer.from(download.data)); - - # Unzip the build artifacts and run integration tests - - name: Run Integration Tests - run: | - unzip processedArtifacts.zip - cd integration_test - chmod +x run-spark-integration-test - ./run-spark-integration-test From 5459ee73f2a2c81dc96a72c71e908fc71763b102 Mon Sep 17 00:00:00 2001 From: Can Sun Date: Wed, 31 May 2023 19:31:39 -0700 Subject: [PATCH 09/12] Add manual approval step to run the tests --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9075e34..f4fb32a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,11 @@ name: Build and Test on: + workflow_dispatch: + permissions: + # Restrict to users with write access + push: write + pull_request: branches: - main From 1135fce1272c0ff0fd51fa5302f15148b81806a0 Mon Sep 17 00:00:00 2001 From: Can Sun Date: Wed, 31 May 2023 19:37:42 -0700 Subject: [PATCH 10/12] Remove the dispatch --- .github/workflows/build.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f4fb32a..9075e34 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,11 +1,6 @@ name: Build and Test on: - workflow_dispatch: - permissions: - # Restrict to users with write access - push: write - pull_request: branches: - main From 2a0cc0b27fd4558da13b83397b98f36138477881 Mon Sep 17 00:00:00 2001 From: Can Sun Date: Wed, 31 May 2023 19:43:33 -0700 Subject: [PATCH 11/12] upgrade all v2 actions to v3 --- .github/workflows/build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9075e34..cc63921 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,9 +16,9 @@ jobs: steps: # Set up and Install required dependencies - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} @@ -28,7 +28,7 @@ jobs: pip install tox tox-gh-actions # Check out code from PR and initiate build process - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build run: | echo Build started on `date` @@ -48,7 +48,7 @@ jobs: cp -R pyspark-sdk/dist processedArtifacts # Upload processed artifacts to be used for running the integration tests in subsequent integration tests workflow - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: processedArtifacts path: processedArtifacts/ @@ -65,9 +65,9 @@ jobs: steps: # Set up and install required dependencies - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} @@ -78,7 +78,7 @@ jobs: # Configure AWS credentials - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 + uses: aws-actions/configure-aws-credentials@v3 with: aws-region: us-west-2 role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} From 4bcd3e69a53c76e8e77c5ff4850963c83d4a3fcd Mon Sep 17 00:00:00 2001 From: Can Sun Date: Wed, 31 May 2023 19:54:00 -0700 Subject: [PATCH 12/12] bump up the github script action version --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc63921..c70ad86 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -78,7 +78,7 @@ jobs: # Configure AWS credentials - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v3 + uses: aws-actions/configure-aws-credentials@v2 with: aws-region: us-west-2 role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} @@ -86,7 +86,7 @@ jobs: # Download processed artifacts generated from build workflow - name: 'Download Build Artifacts' - uses: actions/github-script@v3.1.0 + uses: actions/github-script@v6 with: script: | var artifacts = await github.actions.listWorkflowRunArtifacts({