feat: implement diracx tasks support (#261) #1040
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Basic Tests | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| branches: | |
| - master | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| jobs: | |
| detect-docs-only: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| outputs: | |
| docs-only: ${{ steps.check.outputs.docs-only }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check for docs-only changes | |
| id: check | |
| run: | | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| BASE_BRANCH="${{ github.base_ref }}" | |
| git fetch origin "$BASE_BRANCH" | |
| CHANGED_FILES=$(git diff --name-only "origin/$BASE_BRANCH"...HEAD) | |
| else | |
| CHANGED_FILES=$(git diff --name-only HEAD~1) | |
| fi | |
| DOCS_ONLY=true | |
| if [ -z "$CHANGED_FILES" ]; then | |
| DOCS_ONLY=false | |
| fi | |
| while IFS= read -r file; do | |
| if [[ "$file" != docs/* ]]; then | |
| DOCS_ONLY=false | |
| break | |
| fi | |
| done <<< "$CHANGED_FILES" | |
| echo "docs-only=$DOCS_ONLY" >> "$GITHUB_OUTPUT" | |
| pre-commit: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: prefix-dev/setup-pixi@v0.9.5 | |
| with: | |
| cache: false | |
| environments: pre-commit | |
| - uses: pre-commit/action@v3.0.1 | |
| - uses: pre-commit-ci/lite-action@v1.1.0 | |
| if: always() | |
| run-demo: | |
| needs: detect-docs-only | |
| if: needs.detect-docs-only.outputs.docs-only != 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Start demo | |
| run: | | |
| ./run_demo.sh --exit-when-done | |
| - name: Modify refresh token expiry and upgrade | |
| run: | | |
| # Modify the values.yaml to change the refresh token expiry | |
| sed -i 's/DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES: "[0-9]*"/DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES: "12345"/' .demo/values.yaml | |
| # Run helm upgrade to apply the changes | |
| export KUBECONFIG=$PWD/.demo/kube.conf | |
| export HELM_DATA_HOME=$PWD/.demo/helm_data | |
| $PWD/.demo/helm upgrade --debug diracx-demo $PWD/diracx --values .demo/values.yaml | |
| - name: Verify environment variable is set correctly | |
| run: | | |
| export KUBECONFIG=$PWD/.demo/kube.conf | |
| # Wait for the deployment to be ready after upgrade | |
| $PWD/.demo/kubectl rollout status deployment/diracx-demo --timeout=300s | |
| # Get the environment variable value from the deployment | |
| env_value=$(.demo/kubectl exec deployments/diracx-demo -c diracx -- env | grep DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES | awk -F '=' {'print $NF'}) | |
| if [ "$env_value" != "12345" ]; then | |
| echo "ERROR: Environment variable DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES is not set to 12345, got: $env_value" | |
| exit 1 | |
| fi | |
| echo "SUCCESS: Environment variable DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES is correctly set to 12345" | |
| - name: Debugging information | |
| if: always() | |
| run: curl https://raw.githubusercontent.com/DIRACGrid/diracx/refs/heads/main/.github/scripts/collect-k8s-debug-info.sh | bash -s -- "$PWD/.demo" | |
| - name: Check for success | |
| run: | | |
| if [ ! -f ".demo/.success" ]; then | |
| echo "Demo failed" | |
| cat ".demo/.failed" | |
| exit 1 | |
| fi | |
| # Same as run-demo except mount the sources inside the container | |
| # Uses ci_values.yaml to override image tags to "dev" | |
| run-demo-mount-sources: | |
| needs: detect-docs-only | |
| if: needs.detect-docs-only.outputs.docs-only != 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Clone source | |
| run: | | |
| git clone https://github.com/DIRACGrid/diracx.git /tmp/diracx | |
| git clone https://github.com/DIRACGrid/diracx-web.git /tmp/diracx-web | |
| git clone https://github.com/DIRACGrid/DIRAC.git /tmp/DIRAC | |
| # We have to copy the code to another directory | |
| # and make it a git repository by itself because otherwise the | |
| # root in the pyproject do not make sense once mounted | |
| # in the containers. | |
| # The directory needs to be named after the module, | |
| # otherwise the script will try to mount it in a location which | |
| # kubernetes does not expect, and we get an error | |
| cp -r /tmp/DIRAC/dirac-common /tmp/DIRACCommon | |
| sed -i 's@\.\.@.@g' /tmp/DIRACCommon/pyproject.toml | |
| git init /tmp/DIRACCommon/ | |
| git config --global user.email "so_annoying@example.com" | |
| git config --global user.name "so_annoying" | |
| git -C /tmp/DIRACCommon add . | |
| git -C /tmp/DIRACCommon commit -m 'init' | |
| # We need a tag with a big version because the diracx packages | |
| # specify a minimal DIRACCommon version. So for pip to resolve | |
| # that, we need a large number | |
| git -C /tmp/DIRACCommon tag -a 99.99.99 -m 99.99.99 | |
| - name: Start demo | |
| run: | | |
| ./run_demo.sh --exit-when-done --ci-values demo/ci_values.yaml /tmp/diracx /tmp/DIRACCommon/ /tmp/diracx-web | |
| - name: Modify refresh token expiry and upgrade | |
| run: | | |
| # Modify the values.yaml to change the refresh token expiry | |
| sed -i 's/DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES: "[0-9]*"/DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES: "12345"/' .demo/values.yaml | |
| # Run helm upgrade to apply the changes | |
| export KUBECONFIG=$PWD/.demo/kube.conf | |
| export HELM_DATA_HOME=$PWD/.demo/helm_data | |
| $PWD/.demo/helm upgrade --debug diracx-demo $PWD/diracx --values .demo/values.yaml --values demo/ci_values.yaml | |
| - name: Verify environment variable is set correctly | |
| run: | | |
| export KUBECONFIG=$PWD/.demo/kube.conf | |
| # Wait for the deployment to be ready after upgrade | |
| $PWD/.demo/kubectl rollout status deployment/diracx-demo --timeout=300s | |
| # Get the environment variable value from the deployment | |
| env_value=$(.demo/kubectl exec deployments/diracx-demo -c diracx -- env | grep DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES | awk -F '=' {'print $NF'}) | |
| if [ "$env_value" != "12345" ]; then | |
| echo "ERROR: Environment variable DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES is not set to 12345, got: $env_value" | |
| exit 1 | |
| fi | |
| echo "SUCCESS: Environment variable DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES is correctly set to 12345" | |
| - name: Debugging information | |
| run: | | |
| export KUBECONFIG=$PWD/.demo/kube.conf | |
| .demo/kubectl get pods | |
| for pod_name in $(.demo/kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do | |
| echo "${pod_name}" | |
| .demo/kubectl describe pod/"${pod_name}" || true | |
| for container_name in $(.demo/kubectl get pods $pod_name -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}'); do | |
| echo $pod_name $container_name | |
| .demo/kubectl logs "${pod_name}" -c "${container_name}" || true | |
| done | |
| done | |
| - name: Check for success | |
| run: | | |
| if [ ! -f ".demo/.success" ]; then | |
| echo "Demo failed" | |
| cat ".demo/.failed" | |
| exit 1 | |
| fi |