workflow fix

.github/workflows/build.yml

@@ -1,7 +1,9 @@
-name: Build
+name: Build and Test
 
 on:
   pull_request:
+    branches:
+      - main
 
 jobs:
   build:
@@ -14,9 +16,9 @@ jobs:
     steps:
 
       # Set up and Install required dependencies
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -26,7 +28,7 @@ jobs:
           pip install tox tox-gh-actions
 
       # Check out code from PR and initiate build process
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Build
         run: |
           echo Build started on `date`
@@ -46,7 +48,68 @@ jobs:
           cp -R pyspark-sdk/dist processedArtifacts
 
       # Upload processed artifacts to be used for running the integration tests in subsequent integration tests workflow
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: processedArtifacts
           path: processedArtifacts/
+
+  integration_test:
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      pull-requests: write
+      id-token: write
+      contents: write
+      actions: read
+
+    steps:
+
+      # Set up and install required dependencies
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade boto3
+
+      # Configure AWS credentials
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-region: us-west-2
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          role-session-name: GithubSession
+
+      # Download processed artifacts generated from build workflow
+      - name: 'Download Build Artifacts'
+        uses: actions/github-script@v6
+        with:
+          script: |
+            var artifacts = await github.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: ${{github.event.workflow_run.id }},
+            });
+            var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "processedArtifacts"
+            })[0];
+            var download = await github.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            var fs = require('fs');
+            fs.writeFileSync('${{github.workspace}}/processedArtifacts.zip', Buffer.from(download.data));
+
+      # Unzip the build artifacts and run integration tests
+      - name: Run Integration Tests
+        run: |
+          unzip processedArtifacts.zip
+          cd integration_test
+          chmod +x run-spark-integration-test
+          ./run-spark-integration-test