build.yml

name: Build

on:
  push:
    branches:
      - main
      - develop
      - version*
    tags:
      - build*
  pull_request:
    branches:
      - main
      - develop
      - version*

env:
  GOGETCMD: "go get -v -t -d ./..."
  GOTESTCMD: "go test -timeout 240s -v ./..."
  GOPRIVATE: github.com/stackql/*
  GH_ACCESS_TOKEN: ${{ secrets.ACTIONS_PRIVATE_PACKAGE_SECRET }}
  PLANCACHEENABLED: "true"
  GOLANG_VERSION: 1.25.3
  STACKQL_CORE_REPOSITORY: ${{ vars.STACKQL_CORE_REPOSITORY != '' && vars.STACKQL_CORE_REPOSITORY || 'stackql/stackql' }}
  STACKQL_CORE_REF: ${{ vars.STACKQL_CORE_REF != '' && vars.STACKQL_CORE_REF || 'main' }}

jobs:


  test_python_package_build:
    # id: test_python_package_build
    name: Test Python Package Build
    runs-on: ubuntu-22.04
    timeout-minutes: ${{ vars.DEFAULT_JOB_TIMEOUT_MIN == '' && 120 || vars.DEFAULT_JOB_TIMEOUT_MIN }}
    steps:

    - name: Check out code into the Go module directory
      uses: actions/checkout@v4.1.1
      with:
        repository: ${{ env.STACKQL_CORE_REPOSITORY }}
        ref: ${{ env.STACKQL_CORE_REF }}
        token: ${{ secrets.CI_STACKQL_PACKAGE_DOWNLOAD_TOKEN  }}
        path: stackql-core-pkg

    - name: Setup Python
      uses: actions/setup-python@v5.0.0
      with:
        cache: pip
        python-version: '3.12' 

    - name: Install Poetry
      uses: snok/install-poetry@v1
      with:
        version: 1.8.3
        virtualenvs-create: true
        virtualenvs-in-project: false
        virtualenvs-path: stackql-core-pkg/my-custom-path
        installer-parallel: true

    
    - name: Build package
      working-directory: stackql-core-pkg
      run: |
        cicd/util/01-build-robot-lib.sh
    
    - name: Upload python package artifact
      uses: actions/upload-artifact@v4.3.1
      with:
        name: python-package-dist-folder
        path: stackql-core-pkg/test/dist

  winbuild:
    name: Windows Build
    runs-on: windows-latest
    steps:
    - name: Set up Go 1.x
      uses: actions/setup-go@v2
      with:
        go-version: ^${{ env.GOLANG_VERSION }}
      id: go

    - name: Get rid of dissruptive line endings before checkout
      run: |
        git config --global core.autocrlf false

    - name: Check out code into the Go module directory
      uses: actions/checkout@v2

    - name: Get dependencies
      run: |
        go env -w GOPRIVATE="github.com/stackql/*"
        git config --global url."https://$env:GHACCESSTOKEN@github.com/".insteadOf "https://github.com/"
        git --no-pager config --list
        go get -v -t -d ./...
      env:
        GOGETCMD: ${{env.GOGETCMD}}
        CGO_ENABLED: 1
        GHACCESSTOKEN: ${{env.GH_ACCESS_TOKEN}}
        GOPRIVATE: ${{env.GOPRIVATE}}

    - name: Generate Build Flags and Build
      env:
        BUILDCOMMITSHA: ${{github.sha}}
        BUILDBRANCH: ${{github.ref}}
        BUILDPLATFORM: ${{runner.os}}
        BUILDPATCHVERSION: ${{github.run_number}}
        CGO_ENABLED: 1
        GH_ACCESS_TOKEN: ${{env.GH_ACCESS_TOKEN}}
        GOPRIVATE: ${{env.GOPRIVATE}}
      run: |
        git config --global url.https://$env:GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/
        $BuildCommitSHA = $env:BUILDCOMMITSHA
        $BuildShortCommitSHA = "$BuildCommitSHA".Substring(0,6)
        $BuildDate = (Get-Date -UFormat "%a %b %e %H:%M:%S UTC %Y").ToString()
        $BuildPlatform = $env:BUILDPLATFORM
        $PlanCacheEnabled = $env:PLANCACHEENABLED	
            
        Write-Output $BuildCommitSHA
        Write-Output $BuildShortCommitSHA
        Write-Output $BuildDate
        Write-Output $BuildPlatform
        
        go build -x -v `
          -o build/anysdk ./cmd/interrogate

    - name: Test
      if: success()
      run: go test -timeout 240s -v ./...

  linuxbuild:
    needs: test_python_package_build
    name: Linux Build
    runs-on: ubuntu-latest
    steps:
    - name: Set up Go 1.x
      uses: actions/setup-go@v2
      with:
        go-version: ^${{ env.GOLANG_VERSION }}
      id: go

    - name: Check out code into the Go module directory
      uses: actions/checkout@v2
    
    - name: API sprawl check
      run: |
        if ./cicd/scripts/hack/check_api_growth.sh; then
          echo "API sprawl check passed"
        else
          echo "API sprawl check failed - please review for details"
          # exit 1
        fi
    
    - name: Setup system dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y jq
    
    - name: Setup Python
      uses: actions/setup-python@v5.0.0
      with:
        # cache: pip # this requires requirements in source control
        python-version: '3.12'
    
    - name: Install python dependencies
      run: |
        echo "Installing python dependencies"
        pip3 install -r cicd/testing-requirements.txt
    
    - name: Download python package dist folder
      uses: actions/download-artifact@v4.1.2
      with:
        name: python-package-dist-folder
        path: test/dist

    - name: Install python testing package
      run: |
        echo "Inspecting python package"
        for file in test/dist/*.whl; do
          pip3 install "$file" --force-reinstall
        done 
      
    - name: Get dependencies
      run: |
        git config --global url.https://$GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/
        $GOGETCMD
      env:
        GOGETCMD: ${{env.GOGETCMD}}
        GH_ACCESS_TOKEN: ${{env.GH_ACCESS_TOKEN}}
        GOPRIVATE: ${{env.GOPRIVATE}}

    - name: Generate Build Flags and Build
      env:
        BUILDCOMMITSHA: ${{github.sha}}
        BUILDBRANCH: ${{github.ref}}
        BUILDPLATFORM: ${{runner.os}}
        BUILDPATCHVERSION: ${{github.run_number}}
      run: |
        BUILDSHORTCOMMITSHA=$(echo $BUILDCOMMITSHA | cut -c 1-7)
        BUILDDATE=$(date)
        echo "BUILDBRANCH: ${BUILDBRANCH}"
        echo "BUILDCOMMITSHA: ${BUILDCOMMITSHA}"
        echo "BUILDSHORTCOMMITSHA: ${BUILDSHORTCOMMITSHA}"
        echo "BUILDDATE: ${BUILDDATE}"
        echo "BUILDPLATFORM: ${BUILDPLATFORM}"

        go build -x -v \
          -o build/anysdk ./cmd/interrogate
      
    - name: Test
      if: success()
      run: go test -timeout 240s -v ./...
    
    - name: Trivial CLI Test
      if: success()
      run: |
        result="$(build/anysdk const | jq -r '.ExtensionKeyAlwaysRequired')"
        if [ "$result" != "x-alwaysRequired" ]; then
          echo "Trivial CLI Test Failed with unexpected result: $result"
          exit 1
        else
          echo "Trivial CLI Test passed with expected result: $result"
        fi

    - name: Download core
      uses: actions/checkout@v2
      with:
        repository: ${{ env.STACKQL_CORE_REPOSITORY }}
        ref: ${{ env.STACKQL_CORE_REF }}
        path: stackql-core
    
    - name: Create materials for core tests
      working-directory: stackql-core
      run: |
        pip3 install -r cicd/requirements.txt
        openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365
        openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365
        openssl req -x509 -keyout test/server/mtls/credentials/pg_rubbish_key.pem -out test/server/mtls/credentials/pg_rubbish_cert.pem -config test/server/mtls/openssl.cnf -days 365 
        python3 test/python/stackql_test_tooling/registry_rewrite.py \
          --srcdir $(pwd)/test/registry/src \
          --destdir $(pwd)/test/registry-mocked/src
    
    - name: Create mock routed services for local tests
      run: |
        python3 stackql-core/test/python/stackql_test_tooling/registry_rewrite.py \
          --srcdir $(pwd)/test/registry-simple/src \
          --destdir $(pwd)/test/registry-mocked/src
    
    - name: Start Core Test Mocks
      working-directory: stackql-core
      run: |
        pgrep -f flask | xargs kill -9 || true
        flask --app=./test/python/stackql_test_tooling/flask/gcp/app          run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1080 & 
        flask --app=./test/python/stackql_test_tooling/flask/oauth2/token_srv run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 2091 & 
        flask --app=./test/python/stackql_test_tooling/flask/aws/app run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1091 & 
    
    - name: Run core mocked testing
      working-directory: stackql-core
      run: |
        export GCP_SERVICE_ACCOUNT_KEY="$(cat test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json)"
        bucketsListIDs="$(${{ github.workspace }}/build/anysdk query \
          --svc-file-path="test/registry-mocked/src/googleapis.com/v0.1.2/services/storage-v1.yaml" \
          --tls.allowInsecure \
          --prov-file-path="test/registry-mocked/src/googleapis.com/v0.1.2/provider.yaml" \
          --resource buckets \
          --method list \
          --parameters '{ "project": "stackql-demo" }' \
          | jq -r '.items[].id')" 
        matchingBuckets="$(echo "${bucketsListIDs}" | grep "stackql-demo" )"
        if [ "${matchingBuckets}" = "" ]; then
          echo "Core Test Failed with no matching buckets"
          exit 1
        else
          echo "Core Test passed with matching buckets: $matchingBuckets"
        fi
    
    - name: Run local templated openssl mutate test
      run: |
        rm -rf test/tmp/*.pem
        ${{ github.workspace }}/build/anysdk query \
          --svc-file-path="test/registry/src/local_openssl/v0.1.0/services/keys.yaml"  \
          --prov-file-path="test/registry/src/local_openssl/v0.1.0/provider.yaml" \
          --resource rsa \
          --method create_key_pair \
          --parameters '{ 
            "config_file":   "test/openssl/openssl.cnf",
            "key_out_file":  "test/tmp/key.pem",
            "cert_out_file": "test/tmp/cert.pem",
            "days":          90
          }'
        endDateFound="$(openssl x509 -in test/tmp/cert.pem -noout -dates | grep "notAfter")"
        if [ "${endDateFound}" = "" ]; then
          echo "Core Test Failed with no matching end date"
          exit 1
        else
          echo "Core Test passed with matching end date info: $endDateFound"
        fi
      
    - name: Mocked CLI HTTP templated aws ec2 volumes response transform test
      run: |
        export AWS_SECRET_ACCESS_KEY="totally-fake"
        export AWS_ACCESS_KEY_ID="totally-fake"
        volumeIDs="$(${{ github.workspace }}/build/anysdk query \
          --svc-file-path="test/registry-mocked/src/aws/v0.1.0/services/ec2.yaml" \
          --tls.allowInsecure \
          --prov-file-path="test/registry-mocked/src/aws/v0.1.0/provider.yaml" \
          --resource volumes_presented \
          --method describeVolumes \
          --parameters '{ "region": "ap-southeast-2" }' \
          | jq -r '.line_items[].volume_id')" 
        matchingVolumes="$(echo "${volumeIDs}" | grep "vol-00100000000000000" )"
        if [ "${matchingVolumes}" = "" ]; then
          echo "Mocked CLI HTTP templated Test Failed with no matching buckets"
          exit 1
        else
          echo "Mocked CLI HTTP templated Test passed with matching buckets: $matchingVolumes"
        fi

    - name: Run local templated openssl select test
      run: |
        response="$(${{ github.workspace }}/build/anysdk query \
          --svc-file-path='test/registry/src/local_openssl/v0.1.0/services/keys.yaml'  \
          --prov-file-path='test/registry/src/local_openssl/v0.1.0/provider.yaml' \
          --resource x509 \
          --method describe_certificate \
          --parameters '{
            "cert_file": "test/tmp/cert.pem"
          }')"
        publicKeyAlgorithm="$(echo "$response" | jq -r '.public_key_algorithm')"
        if [ "${publicKeyAlgorithm}" != "rsaEncryption" ]; then
          echo "Core Test Failed with unexpected public key algorithm '$publicKeyAlgorithm'"
          exit 1
        else
          echo "Core Test passed with matching public key algorithm: '$publicKeyAlgorithm'"
        fi

    - name: Prepare for robot test run
      run: |
        pgrep -f flask | xargs kill -9 || true
        python ${{ github.workspace }}/stackql-core/test/python/stackql_test_tooling/registry_rewrite.py --srcdir "${{ github.workspace }}/test/registry/src" --destdir "${{ github.workspace }}/test/registry-mocked/src"
        openssl req -x509 -keyout ${{ github.workspace }}/test/credentials/pg_server_key.pem  -out ${{ github.workspace }}/test/credentials/pg_server_cert.pem   -config ${{ github.workspace }}/stackql-core/test/server/mtls/openssl.cnf -days 365
        openssl req -x509 -keyout ${{ github.workspace }}/test/credentials/pg_client_key.pem  -out ${{ github.workspace }}/test/credentials/pg_client_cert.pem   -config ${{ github.workspace }}/stackql-core/test/server/mtls/openssl.cnf -days 365
        openssl req -x509 -keyout ${{ github.workspace }}/test/credentials/pg_rubbish_key.pem -out ${{ github.workspace }}/test/credentials/pg_rubbish_cert.pem  -config ${{ github.workspace }}/stackql-core/test/server/mtls/openssl.cnf -days 365

    - name: Run mocked robot tests
      env:
        AWS_ACCESS_KEY_ID: dummy
        AWS_SECRET_ACCESS_KEY: dummy
      run: |
        export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/test/python"
        robot -d test/robot/reports/mocked test/robot/cli/mocked

    - name: Output from mocked robot tests
      if: always()
      run: |
        cat test/robot/reports/mocked/output.xml

    - name: Run AOT CLI robot tests
      run: |
        export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/test/python"
        robot -d test/robot/reports/aot test/robot/cli/aot

    - name: Output from AOT CLI robot tests
      if: always()
      run: |
        cat test/robot/reports/aot/output.xml

    - name: Run Interrogate CLI robot tests
      run: |
        export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/test/python"
        robot -d test/robot/reports/interrogate test/robot/cli/interrogate

    - name: Output from Interrogate CLI robot tests
      if: always()
      run: |
        cat test/robot/reports/interrogate/output.xml

  macosbuild:
    name: MacOS Build
    runs-on: macos-latest
    steps:
    - name: Set up Go 1.x
      uses: actions/setup-go@v2
      with:
        go-version: ^${{ env.GOLANG_VERSION }}
      id: go

    - name: Check out code into the Go module directory
      uses: actions/checkout@v2

    - name: Get dependencies
      run: |
        git config --global url.https://$GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/
        $GOGETCMD
      env:
        GOGETCMD: ${{env.GOGETCMD}}
        CGO_ENABLED: 1
        GH_ACCESS_TOKEN: ${{env.GH_ACCESS_TOKEN}}
        GOPRIVATE: ${{env.GOPRIVATE}}

    - name: Generate Build Flags and Build
      env:
        BUILDCOMMITSHA: ${{github.sha}}
        BUILDBRANCH: ${{github.ref}}
        BUILDPLATFORM: ${{runner.os}}
        BUILDPATCHVERSION: ${{github.run_number}}
        CGO_ENABLED: 1
      run: |
        BUILDSHORTCOMMITSHA=$(echo $BUILDCOMMITSHA | cut -c 1-7)
        BUILDDATE=$(date)
        echo "BUILDBRANCH: ${BUILDBRANCH}"
        echo "BUILDCOMMITSHA: ${BUILDCOMMITSHA}"
        echo "BUILDSHORTCOMMITSHA: ${BUILDSHORTCOMMITSHA}"
        echo "BUILDDATE: ${BUILDDATE}"
        echo "BUILDPLATFORM: ${BUILDPLATFORM}"

        go build -x -v \
          -o build/anysdk ./cmd/interrogate

    - name: Test
      if: success()
      run: go test -timeout 240s -v ./...

  macosarmbuild:
    name: MacOS ARM Build
    runs-on: macos-latest
    steps:
    - name: Set up Go 1.x
      uses: actions/setup-go@v2
      with:
        go-version: ^${{ env.GOLANG_VERSION }}
      id: go

    - name: Check out code into the Go module directory
      uses: actions/checkout@v2

    - name: Get dependencies
      run: |
        git config --global url.https://$GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/
        $GOGETCMD
      env:
        GOGETCMD: ${{env.GOGETCMD}}
        CGO_ENABLED: 1
        GH_ACCESS_TOKEN: ${{env.GH_ACCESS_TOKEN}}
        GOPRIVATE: ${{env.GOPRIVATE}}

    - name: Generate Build Flags and Build
      env:
        BUILDCOMMITSHA: ${{github.sha}}
        BUILDBRANCH: ${{github.ref}}
        BUILDPLATFORM: "darwin_arm64"
        BUILDPATCHVERSION: ${{github.run_number}}
        CGO_ENABLED: 1
      run: |
        BUILDSHORTCOMMITSHA=$(echo $BUILDCOMMITSHA | cut -c 1-7)
        BUILDDATE=$(date)
        echo "BUILDBRANCH: ${BUILDBRANCH}"
        echo "BUILDCOMMITSHA: ${BUILDCOMMITSHA}"
        echo "BUILDSHORTCOMMITSHA: ${BUILDSHORTCOMMITSHA}"
        echo "BUILDDATE: ${BUILDDATE}"
        echo "BUILDPLATFORM: ${BUILDPLATFORM}"

        GOOS=darwin GOARCH=arm64 \
        go build -x -v \
          -o build/anysdk ./cmd/interrogate