diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 44eebda..c70ad86 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,7 +1,9 @@ -name: Build +name: Build and Test on: pull_request: + branches: + - main jobs: build: @@ -14,9 +16,9 @@ jobs: steps: # Set up and Install required dependencies - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} @@ -26,7 +28,7 @@ jobs: pip install tox tox-gh-actions # Check out code from PR and initiate build process - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build run: | echo Build started on `date` @@ -46,7 +48,68 @@ jobs: cp -R pyspark-sdk/dist processedArtifacts # Upload processed artifacts to be used for running the integration tests in subsequent integration tests workflow - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: processedArtifacts path: processedArtifacts/ + + integration_test: + runs-on: ubuntu-latest + needs: build + permissions: + pull-requests: write + id-token: write + contents: write + actions: read + + steps: + + # Set up and install required dependencies + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade boto3 + + # Configure AWS credentials + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-region: us-west-2 + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + role-session-name: GithubSession + + # Download processed artifacts generated from build workflow + - name: 'Download Build Artifacts' + uses: actions/github-script@v6 + with: + script: | + var artifacts = await github.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{github.event.workflow_run.id }}, + }); + var matchArtifact = artifacts.data.artifacts.filter((artifact) => { + return artifact.name == "processedArtifacts" + })[0]; + var download = await github.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + var fs = require('fs'); + fs.writeFileSync('${{github.workspace}}/processedArtifacts.zip', Buffer.from(download.data)); + + # Unzip the build artifacts and run integration tests + - name: Run Integration Tests + run: | + unzip processedArtifacts.zip + cd integration_test + chmod +x run-spark-integration-test + ./run-spark-integration-test diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml deleted file mode 100644 index 74a6ec9..0000000 --- a/.github/workflows/integration-tests.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: Integration Tests - -on: - workflow_run: - workflows: - - Build - types: - - completed - -jobs: - test: - runs-on: ubuntu-latest - permissions: - pull-requests: write - id-token: write - contents: write - actions: read - - # Run only if a successful PR request build workflow triggered the integration tests - if: > - github.event.workflow_run.event == 'pull_request' && - github.event.workflow_run.conclusion == 'success' - steps: - - # Set up and install required dependencies - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install --upgrade boto3 - - # Configure AWS credentials - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-region: us-west-2 - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - role-session-name: GithubSession - - # Download processed artifacts generated from build workflow - - name: 'Download Build Artifacts' - uses: actions/github-script@v3.1.0 - with: - script: | - var artifacts = await github.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: ${{github.event.workflow_run.id }}, - }); - var matchArtifact = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == "processedArtifacts" - })[0]; - var download = await github.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - var fs = require('fs'); - fs.writeFileSync('${{github.workspace}}/processedArtifacts.zip', Buffer.from(download.data)); - - # Unzip the build artifacts and run integration tests - - name: Run Integration Tests - run: | - unzip processedArtifacts.zip - cd integration_test - chmod +x run-spark-integration-test - ./run-spark-integration-test