Deploy APL #4039
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy APL | |
| on: | |
| workflow_call: | |
| inputs: | |
| lke_tier: | |
| description: 'Linode Kubernetes Engine Tier' | |
| type: string | |
| default: standard | |
| kubernetes_version: | |
| description: 'Kubernetes version' | |
| type: string | |
| default: '1.34' | |
| linode_types: | |
| description: 'Linode instance types' | |
| type: string | |
| default: g6-dedicated-8 | |
| install_profile: | |
| description: 'APL installation profile' | |
| type: string | |
| default: minimal-with-team | |
| domain_zone: | |
| description: 'Select Domain Zone' | |
| type: string | |
| default: DNS-Integration | |
| kms: | |
| description: 'Should APL encrypt secrets in values repo (DNS or KMS is turned on)?' | |
| type: string | |
| default: age | |
| certificate: | |
| description: 'Select certificate issuer' | |
| type: string | |
| default: letsencrypt_production | |
| is_pre_installed: | |
| description: Fake if Otomi is pre-installed by Installer | |
| type: string | |
| default: 'false' | |
| enableWiz: | |
| description: 'Check the box to enable Wiz integration' | |
| type: boolean | |
| default: true | |
| workflow_dispatch: | |
| inputs: | |
| lke_tier: | |
| description: 'Linode Kubernetes Engine Tier' | |
| type: choice | |
| default: standard | |
| options: | |
| - standard | |
| - enterprise | |
| kubernetes_version: | |
| description: 'Kubernetes version (select version compatible with lke_tier)' | |
| type: choice | |
| options: | |
| - 'v1.31.9+lke7' | |
| - '1.32' | |
| - '1.33' | |
| - '1.34' | |
| default: '1.34' | |
| linode_types: | |
| description: 'Linode instance types' | |
| type: choice | |
| options: | |
| - g6-dedicated-4 | |
| - g6-dedicated-8 | |
| - g6-dedicated-16 | |
| - g6-dedicated-32 | |
| - g6-dedicated-48 | |
| - g6-dedicated-50 | |
| - g6-dedicated-56 | |
| default: g6-dedicated-4 | |
| install_profile: | |
| description: APL installation profile | |
| default: minimal-with-team | |
| type: choice | |
| options: | |
| - minimal | |
| - minimal-with-team | |
| - full | |
| - no-apl | |
| domain_zone: | |
| type: choice | |
| description: 'Select Domain Zone' | |
| options: | |
| - Zone-1 | |
| - Zone-2 | |
| - Random | |
| - DNS-Integration | |
| kms: | |
| type: choice | |
| description: Should APL encrypt secrets in values repo (DNS or KMS is turned on)? | |
| options: | |
| - age | |
| - no_kms | |
| default: age | |
| certificate: | |
| type: choice | |
| description: Select certificate issuer | |
| options: | |
| - gen_custom_ca | |
| - letsencrypt_staging | |
| - letsencrypt_production | |
| default: letsencrypt_production | |
| is_pre_installed: | |
| type: choice | |
| description: Fake if Otomi is pre-installed by Installer | |
| options: | |
| - 'true' | |
| - 'false' | |
| default: 'true' | |
| disableORCS: | |
| description: 'Check the box if you dont want to pull from ORCS' | |
| type: boolean | |
| default: false | |
| enableWiz: | |
| description: 'Check the box to enable Wiz integration' | |
| type: boolean | |
| default: true | |
| env: | |
| CACHE_REGISTRY: ghcr.io | |
| CACHE_REPO: linode/apl-core | |
| REPO: linode/apl-core | |
| GIT_USER: svcAPLBot | |
| CHECK_CONTEXT: continuous-integration/integration-test | |
| COMMIT_ID: '${{ github.event.pull_request.head.sha || github.sha }}' | |
| COMMIT_REF: '${{ github.event.pull_request.head.ref || github.ref }}' | |
| BOT_EMAIL: ${{ vars.BOT_EMAIL }} | |
| BOT_USERNAME: ${{ vars.BOT_USERNAME }} | |
| DEV_DOMAINS: ${{ secrets.DEV_DOMAINS }} | |
| LKE_TIER: ${{ inputs.lke_tier }} | |
| LKE_VERSION: ${{ inputs.kubernetes_version }} | |
| LKE_CP_ACL_IPV4: '0.0.0.0/0' | |
| jobs: | |
| preprocess-input: | |
| name: Preprocess input variables | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Print user input | |
| run: | | |
| echo "ref: $COMMIT_REF" | |
| echo 'install_profile: ${{ inputs.install_profile }}' | |
| echo 'lke_tier: ${{ inputs.lke_tier }}' | |
| echo 'kubernetes_version: ${{ inputs.kubernetes_version }}' | |
| echo 'kms: ${{ inputs.kms }}' | |
| echo 'domain_zone: ${{ inputs.domain_zone }}' | |
| echo 'certificate: ${{ inputs.certificate }}' | |
| echo 'is_pre_installed: ${{ inputs.is_pre_installed }}' | |
| echo 'disableORCS: ${{ inputs.disableORCS }}' | |
| preprocess-linode-input: | |
| needs: preprocess-input | |
| name: Preprocess input variables for linode | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install the Linode CLI | |
| uses: linode/action-linode-cli@v1 | |
| with: | |
| token: ${{ secrets.LINODE_TOKEN }} | |
| - name: Check if cluster is running | |
| run: | | |
| case "${{ inputs.domain_zone }}" in | |
| "Zone-1") LINODE_CLUSTER_NAME=${{ github.actor }}-1 ;; | |
| "Zone-2") LINODE_CLUSTER_NAME=${{ github.actor }}-2 ;; | |
| "Random") LINODE_CLUSTER_NAME=${{ github.actor }}-$RANDOM ;; | |
| "DNS-Integration") LINODE_CLUSTER_NAME=apl-test-${{ inputs.install_profile }} ;; | |
| esac | |
| [[ ${{ inputs.install_profile }} == 'no-apl' ]] && LINODE_CLUSTER_NAME=$LINODE_CLUSTER_NAME-no-apl | |
| if [[ $(linode-cli lke clusters-list --json | jq --arg name "$LINODE_CLUSTER_NAME" '[.[] | select(.label == $name)] | length > 0') == "true" ]]; then | |
| echo "An LKE cluster with the same name ($LINODE_CLUSTER_NAME) already exists." | |
| echo "Visit https://cloud.linode.com/kubernetes/clusters to delete your cluster" | |
| echo "Exiting workflow..." | |
| exit 1 | |
| fi | |
| run-integration-test-linode: | |
| name: Run integration test on linode cluster | |
| needs: preprocess-linode-input | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install the Linode CLI | |
| uses: linode/action-linode-cli@v1 | |
| with: | |
| token: ${{ secrets.LINODE_TOKEN }} | |
| - name: Set k8s cluster name | |
| run: | | |
| case "${{ inputs.domain_zone }}" in | |
| "Zone-1") LINODE_CLUSTER_NAME=${{ github.actor }}-1 ;; | |
| "Zone-2") LINODE_CLUSTER_NAME=${{ github.actor }}-2 ;; | |
| "Random") LINODE_CLUSTER_NAME=${{ github.actor }}-$RANDOM ;; | |
| "DNS-Integration") LINODE_CLUSTER_NAME=apl-test-${{ inputs.install_profile }} ;; | |
| esac | |
| [[ ${{ inputs.install_profile }} == 'no-apl' ]] && LINODE_CLUSTER_NAME=$LINODE_CLUSTER_NAME-no-apl | |
| echo LINODE_CLUSTER_NAME=$LINODE_CLUSTER_NAME >> $GITHUB_ENV | |
| - name: Determine domain name to use for scheduled integration test | |
| env: | |
| EDGEDNS_ZONE: ${{ secrets.EDGEDNS_ZONE }} | |
| if: ${{ inputs.domain_zone == 'DNS-Integration' && inputs.install_profile != 'no-apl'}} | |
| run: | | |
| RAND=$(openssl rand -hex 4) | |
| DOMAIN="integration-${RAND}.${EDGEDNS_ZONE}" | |
| echo "::add-mask::$DOMAIN" | |
| echo DOMAIN=$DOMAIN >> $GITHUB_ENV | |
| - name: Determine domain name | |
| if: ${{ inputs.domain_zone != 'DNS-Integration' && inputs.install_profile != 'no-apl' }} | |
| env: | |
| EDGEDNS_ZONE: ${{ secrets.EDGEDNS_ZONE }} | |
| run: | | |
| # Mapping of domain_zone to domain names | |
| case "${{ inputs.domain_zone }}" in | |
| "Zone-1") DOMAIN=$(jq -r '."${{ github.actor }}"[0]' <<< ${{ env.DEV_DOMAINS }}) ;; | |
| "Zone-2") DOMAIN=$(jq -r '."${{ github.actor }}"[1]' <<< ${{ env.DEV_DOMAINS }}) ;; | |
| "Random") DOMAIN="$(openssl rand -hex 4)$(date +"%d%m%y").${EDGEDNS_ZONE}" ;; | |
| esac | |
| echo "::add-mask::$DOMAIN" | |
| echo DOMAIN=$DOMAIN >> $GITHUB_ENV | |
| - name: Create k8s cluster for testing | |
| run: | | |
| if [[ '${{ env.LKE_TIER }}' == 'enterprise' ]]; then | |
| tier_arg="--tier enterprise" | |
| export LINODE_CLI_API_VERSION=v4beta | |
| ha="" | |
| else | |
| tier_arg="" | |
| ha="--control_plane.high_availability true" | |
| fi | |
| cp_acl="--control_plane.acl.enabled true --control_plane.acl.addresses.ipv4 $LKE_CP_ACL_IPV4" | |
| linode-cli lke cluster-create \ | |
| --label ${{ env.LINODE_CLUSTER_NAME }} \ | |
| --region nl-ams \ | |
| --k8s_version ${{ env.LKE_VERSION }} \ | |
| --node_pools.type ${{ inputs.linode_types }} \ | |
| --node_pools.count 3 \ | |
| --node_pools.autoscaler.enabled true \ | |
| --node_pools.autoscaler.max 3 \ | |
| --node_pools.autoscaler.min 3 \ | |
| --tags testing \ | |
| --no-defaults \ | |
| $ha $tier_arg $cp_acl | |
| - name: Retrieve cluster id | |
| run: echo "LINODE_CLUSTER_ID=$(linode-cli lke clusters-list --json | jq -ce '.[] | select(.label | startswith("${{ env.LINODE_CLUSTER_NAME }}")) | .id')" >> $GITHUB_ENV | |
| - name: Wait for cluster to be ready | |
| run: | | |
| echo "Waiting for the cluster to be active..." | |
| while :; do | |
| rawOutput=$(linode-cli lke pools-list ${{ env.LINODE_CLUSTER_ID }} --json) | |
| allReady=$(echo "$rawOutput" | jq -r 'map(.nodes[] | .status == "ready") | all') | |
| echo "All nodes ready: $allReady" | |
| if [ "$allReady" == "true" ]; then | |
| echo "Cluster is ready" | |
| break | |
| fi | |
| sleep 30 | |
| done | |
| - name: Save kubectl config with auth token | |
| if: ${{ inputs.install_profile != 'no-apl' }} | |
| run: | | |
| echo "Waiting for kubeconfig..." | |
| while :; do | |
| linode-cli get-kubeconfig --label "${{ env.LINODE_CLUSTER_NAME }}" 2> /dev/null && break | |
| echo "still waiting..." | |
| sleep 10 | |
| done | |
| echo LINODE_CLUSTER_CONTEXT=`kubectl config current-context` >> $GITHUB_ENV | |
| - name: Checkout | |
| if: ${{ inputs.install_profile != 'no-apl' }} | |
| uses: actions/checkout@v5 | |
| - name: Set up Wiz | |
| if: ${{ inputs.enableWiz }} | |
| run: | | |
| # Use cluster information from environment variables | |
| CLUSTER_NAME="${{ env.LINODE_CLUSTER_NAME }}" | |
| CLUSTER_ID="${{ env.LINODE_CLUSTER_ID }}" | |
| echo "Setting up Wiz for cluster: $CLUSTER_NAME (ID: $CLUSTER_ID)" | |
| # Create YAML configuration files from templates | |
| KUB_INTEGRATION_PATH="./${CLUSTER_NAME}_kub_integration.yaml" | |
| ADMISSION_PATH="./${CLUSTER_NAME}_admission_control.yaml" | |
| sed "s/__WIZ_CLIENT_ID__/${{ secrets.WIZ_CLIENT_ID }}/g; s/__WIZ_CLIENT_TOKEN__/${{ secrets.WIZ_CLIENT_TOKEN }}/g; s/<connectorName>/${CLUSTER_NAME}/g; s/<clusterExternalId>/${CLUSTER_ID}/g" .github/templates/wiz-kubernetes-integration.yaml > "$KUB_INTEGRATION_PATH" | |
| sed "s/__WIZ_CLIENT_ID__/${{ secrets.WIZ_CLIENT_ID }}/g; s/__WIZ_CLIENT_TOKEN__/${{ secrets.WIZ_CLIENT_TOKEN }}/g; s/<connectorName>/${CLUSTER_NAME}/g; s/<clusterExternalId>/${CLUSTER_ID}/g" .github/templates/wiz-admission-control.yaml > "$ADMISSION_PATH" | |
| # Verify cluster connectivity | |
| echo "Verifying cluster connectivity..." | |
| kubectl get nodes | |
| # Add Helm repo for Wiz | |
| helm repo add wiz-sec https://charts.wiz.io/ | |
| helm repo update | |
| # Create namespace if not exists | |
| kubectl create namespace wiz --dry-run=client -o yaml | kubectl apply -f - | |
| # Set release name | |
| RELEASE_NAME="wiz-${CLUSTER_NAME//_/-}" | |
| RELEASE_NAME="${RELEASE_NAME,,}" # Convert to lowercase | |
| NAMESPACE="wiz" | |
| # Install Wiz components | |
| echo "🚀 Installing Wiz Kubernetes Integration..." | |
| helm install "$RELEASE_NAME" wiz-sec/wiz-kubernetes-integration --values "$KUB_INTEGRATION_PATH" -n "$NAMESPACE" | |
| echo "🚀 Installing Wiz Admission Controller..." | |
| helm install wiz-lke-ac wiz-sec/wiz-admission-controller --values "$ADMISSION_PATH" -n "$NAMESPACE" --wait | |
| echo "✅ Wiz deployment for cluster $CLUSTER_NAME completed." | |
| - name: APL install | |
| if: ${{ inputs.install_profile != 'no-apl' }} | |
| env: | |
| LETSENCRYPT_STAGING: ${{ secrets.LETSENCRYPT_STAGING }} | |
| LETSENCRYPT_PRODUCTION: ${{ secrets.LETSENCRYPT_PRODUCTION }} | |
| EDGEDNS_ACCESS_TOKEN: ${{ secrets.EDGEDNS_ACCESS_TOKEN }} | |
| EDGEDNS_CLIENT_TOKEN: ${{ secrets.EDGEDNS_CLIENT_TOKEN }} | |
| EDGEDNS_CLIENT_SECRET: ${{ secrets.EDGEDNS_CLIENT_SECRET }} | |
| EDGEDNS_ZONE: ${{ secrets.EDGEDNS_ZONE }} | |
| EDGEDNS_HOST: ${{ secrets.EDGEDNS_HOST }} | |
| run: | | |
| touch values.yaml | |
| adminPassword="$(head /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 24)" | |
| additional_args="" | |
| [[ '${{ inputs.certificate }}' == 'letsencrypt_staging' ]] && echo "$LETSENCRYPT_STAGING" >> values.yaml | |
| [[ '${{ inputs.certificate }}' == 'letsencrypt_production' ]] && echo "$LETSENCRYPT_PRODUCTION" >> values.yaml | |
| [[ '${{ inputs.kms }}' == 'age' ]] && additional_args+=" --set kms.sops.provider=age" | |
| [[ '${{ inputs.is_pre_installed }}' == 'true' ]] && additional_args+=" --set otomi.isPreInstalled=true" | |
| if [[ '${{ inputs.disableORCS }}' == 'true' ]]; then | |
| additional_args+=" --set otomi.useORCS=false" | |
| if [[ ! ${GITHUB_REF##*/} =~ ^v[0-9].+$ ]]; then | |
| # If a pipeline installs APL from the semver tag then pull container image from DockerHub | |
| # Use cache registry otherwise | |
| kubectl create secret docker-registry reg-otomi-github \ | |
| --docker-server=${{ env.CACHE_REGISTRY }} \ | |
| --docker-username=${{ env.BOT_USERNAME }} \ | |
| --docker-password='${{ secrets.BOT_PULL_TOKEN }}' | |
| additional_args+=" --set imageName=${{ env.CACHE_REGISTRY }}/${{ env.CACHE_REPO }} --set imagePullSecretNames[0]=reg-otomi-github" | |
| fi | |
| fi | |
| install_args="otomi chart/apl --wait --wait-for-jobs --timeout 90m0s \ | |
| --values tests/integration/${{ inputs.install_profile }}.yaml \ | |
| --values values.yaml \ | |
| --set cluster.provider=linode \ | |
| --set cluster.linode.tier=${{ env.LKE_TIER }} \ | |
| --set cluster.name=aplinstall${{ env.LINODE_CLUSTER_ID }} \ | |
| --set cluster.k8sContext=${{ env.LINODE_CLUSTER_CONTEXT }} \ | |
| --set cluster.domainSuffix=${{ env.DOMAIN }} \ | |
| --set dns.domainFilters[0]=${{ env.DOMAIN }} \ | |
| --set dns.provider.akamai.clientSecret=${EDGEDNS_CLIENT_SECRET} \ | |
| --set dns.provider.akamai.host=${EDGEDNS_HOST} \ | |
| --set dns.provider.akamai.accessToken=${EDGEDNS_ACCESS_TOKEN} \ | |
| --set dns.provider.akamai.clientToken=${EDGEDNS_CLIENT_TOKEN} \ | |
| --set otomi.hasExternalDNS=true \ | |
| --set otomi.adminPassword=$adminPassword \ | |
| --set otomi.version=${GITHUB_REF##*/} \ | |
| --set apps.linode-cfw.enabled=true \ | |
| $additional_args" | |
| helm install $install_args & | |
| HELM_PID=$! | |
| sleep 120 | |
| # While helm is installing we can crete the wildcard dns record | |
| while true; do | |
| PUB_IP=$(kubectl get svc ingress-nginx-platform-controller -n ingress -ojson | jq '.status.loadBalancer.ingress[0].ip' -r) | |
| if [[ -n "$PUB_IP" ]]; then | |
| echo "::add-mask::$PUB_IP" | |
| echo PUB_IP=$PUB_IP >> $GITHUB_ENV | |
| break | |
| else | |
| echo "Waiting for ingress-nginx-platform-controller IP..." | |
| sleep 5 | |
| fi | |
| done | |
| pip3 install edgegrid-python requests | |
| python3 bin/edgedns_A_record.py create "*.${DOMAIN}" $PUB_IP || \ | |
| (echo "Will try to recreate it" && \ | |
| python3 bin/edgedns_A_record.py delete "*.${DOMAIN}" && \ | |
| python3 bin/edgedns_A_record.py create "*.${DOMAIN}" $PUB_IP) | |
| wait $HELM_PID | |
| - name: Gather k8s events on failure | |
| if: failure() | |
| run: | | |
| kubectl get events --sort-by='.lastTimestamp' -A | |
| - name: Gather k8s pods on failure | |
| if: failure() | |
| run: | | |
| kubectl get pods -A -o wide | |
| - name: Gather APL logs on failure | |
| if: failure() | |
| run: | | |
| kubectl logs jobs/otomi-apl --tail 150 | |
| - name: Remove the test cluster | |
| if: ${{ always() && inputs.domain_zone == 'DNS-Integration' }} | |
| run: | | |
| linode-cli lke cluster-delete ${{ env.LINODE_CLUSTER_ID }} | |
| - name: Delete Domain | |
| if: ${{ always() && inputs.domain_zone == 'DNS-Integration' }} | |
| env: | |
| EDGEDNS_ACCESS_TOKEN: ${{ secrets.EDGEDNS_ACCESS_TOKEN }} | |
| EDGEDNS_CLIENT_TOKEN: ${{ secrets.EDGEDNS_CLIENT_TOKEN }} | |
| EDGEDNS_CLIENT_SECRET: ${{ secrets.EDGEDNS_CLIENT_SECRET }} | |
| EDGEDNS_ZONE: ${{ secrets.EDGEDNS_ZONE }} | |
| EDGEDNS_HOST: ${{ secrets.EDGEDNS_HOST }} | |
| run: | | |
| python3 bin/edgedns_A_record.py delete "*.${DOMAIN}" |