Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
241 changes: 181 additions & 60 deletions .github/workflows/operator-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,8 @@
name: Operator CI

on:
workflow_dispatch: {}
pull_request:
branches:
- main
paths:
- operator/**
- containers/operator.Dockerfile
Expand Down Expand Up @@ -55,18 +54,22 @@ jobs:
# Standard E2E tests on all supported K8s versions
k8s-version: ["1.31.14", "1.32.11", "1.33.7", "1.34.3", "1.35.0"]
test-suite: ["e2e"]
make-targets: ["setup-kind-cluster e2e-tests"]
include:
# Deployment policy tests on 15-node cluster (K8s 1.34 only)
# Deployment policy tests on 15-node cluster (K8s 1.35 only)
- k8s-version: "1.35.0"
test-suite: deployment-policy
kind-config: k8s-tests/chainsaw/deployment-policy/kind-config.yaml
make-target: deployment-policy-tests
make-targets: "setup-kind-cluster deployment-policy-tests"
# CLI e2e tests on K8s 1.34 only
- k8s-version: "1.35.0"
test-suite: cli-e2e
make-target: cli-e2e-tests
make-targets: "setup-kind-cluster cli-e2e-tests"
- k8s-version: "1.35.0"
test-suite: unit-tests
make-targets: "vet lint unit-tests"
fail-fast: false # Continue testing other versions if one fails
name: ${{ matrix.test-suite }}-tests (k8s-${{ matrix.k8s-version }})
name: ${{ matrix.test-suite }} (k8s-${{ matrix.k8s-version }})
steps:
- uses: actions/checkout@v4
with:
Expand All @@ -78,16 +81,18 @@ jobs:
go-version: ${{ env.GO_VERSION }}
cache-dependency-path: operator/go.sum
- name: Log in to the Container registry
if: matrix.test-suite != 'unit-tests' # unit tests don't need a container registry login
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Create Kubernetes KinD Cluster v${{ matrix.k8s-version }}
if: matrix.test-suite != 'unit-tests' # unit tests don't need a kind cluster
id: kind
uses: helm/kind-action@v1
with:
version: v0.30.0
version: v0.31.0
node_image: kindest/node:v${{ matrix.k8s-version }}
config: ${{ matrix.kind-config || 'operator/config/local-dev/kind-config.yaml' }}
cluster_name: kind
Expand Down Expand Up @@ -119,35 +124,113 @@ jobs:
- name: Run ${{ matrix.test-suite }} tests
run: |
cd operator
if [ "${{ matrix.test-suite }}" = "e2e" ]; then
make setup-kind-cluster
make test
elif [ "${{ matrix.test-suite }}" = "cli-e2e" ]; then
make setup-kind-cluster
make ${{ matrix.make-target }}
else
make ${{ matrix.make-target }}
fi
# Upload coverage to Coveralls using goveralls (Go-specific tool)
- name: Upload coverage to Coveralls
if: matrix.test-suite == 'e2e' && matrix.k8s-version == '1.35.0'
make ${{ matrix.make-targets }} merge-coverage
# Save coverage artifacts from any test suite that generates them
- name: Upload coverage artifact
if: hashFiles('operator/reporting/cover.out') != ''
uses: actions/upload-artifact@v4
with:
name: coverage-${{ matrix.test-suite }}-k8s-${{ matrix.k8s-version }}
path: operator/reporting/cover.out
retention-days: 1
if-no-files-found: ignore

# Merge coverage from all test suites and upload to Coveralls
upload-coverage:
runs-on: ubuntu-latest
needs: [tests]
# Only upload coverage for PRs and main branch pushes, not for tags
if: success() && !startsWith(github.ref, 'refs/tags/')
steps:
- uses: actions/checkout@v4
- name: Setup Go ${{ env.GO_VERSION }}
uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- name: Download all coverage artifacts
uses: actions/download-artifact@v4
with:
pattern: coverage-*
path: coverage-artifacts
merge-multiple: false
- name: Merge coverage files
run: |
cd operator
mkdir -p reporting
# Combine all coverage files
for file in ../coverage-artifacts/*/cover.out; do
if [ -f "$file" ]; then
echo "Merging coverage from $file"
cat "$file" >> reporting/all-cover.out
fi
done
# Create final merged coverage file
echo "mode: set" > reporting/cover.out
tail -n +2 reporting/all-cover.out | sed '/mode: set/d' >> reporting/cover.out
# Show total coverage
echo "📊 Total Combined Coverage:"
go tool cover -func reporting/cover.out | grep total
- name: Upload to Coveralls
uses: coverallsapp/github-action@v2
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
file: operator/reporting/cover.out
format: golang

# Build multi-platform container image and push to registry
build-and-push-operator:
# Compute image tags and version metadata once for reuse
compute-metadata:
runs-on: ubuntu-latest
needs: [tests] # Don't run the build and push if tests fail
# Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
needs: [tests]
outputs:
git-sha: ${{ steps.meta.outputs.git-sha }}
version: ${{ steps.meta.outputs.version }}
tags: ${{ steps.meta.outputs.tags }}
steps:
- uses: actions/checkout@v4
- name: Fetch all tags
run: git fetch --tags --force
- name: Compute metadata
id: meta
run: |
export GIT_SHA=$(git rev-parse --short ${{ github.sha }})
echo "git-sha=${GIT_SHA}" >> $GITHUB_OUTPUT

case ${{ github.ref_type }} in
branch)
export VERSION=$(git tag --list 'operator*' --sort=-v:refname | head -n 1 | cut -d/ -f2)+${GIT_SHA}
TAGS="${GIT_SHA} $(echo "${VERSION}" | tr + -)"
;;
tag)
export VERSION=$(echo "${{ github.ref_name }}" | cut -f 2 -d /)
TAGS="${GIT_SHA} ${VERSION} latest"
;;
*)
echo "Unknown ref type: ${{ github.ref_type }}"
exit 1
;;
esac

echo "version=${VERSION}" >> $GITHUB_OUTPUT
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "📦 Version: ${VERSION}"
echo "🏷️ Tags: ${TAGS}"

# Build container images on native architecture runners (much faster than QEMU)
build-operator:
runs-on: ${{ matrix.runner }}
needs: [compute-metadata]
strategy:
matrix:
include:
- platform: linux/amd64
runner: ubuntu-latest
- platform: linux/arm64
runner: ubuntu-24.04-arm
permissions:
contents: read
packages: write
attestations: write
id-token: write
#
steps:
- name: Checkout repository
uses: actions/checkout@v4
Expand All @@ -159,60 +242,98 @@ jobs:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

# Setup for multi-platform builds (linux/amd64, linux/arm64)
- name: Set up QEMU
uses: docker/setup-qemu-action@v3

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

# Build and tag container image based on git ref type
- name: Build the operator container image
# Build and tag container image for single platform on native hardware
- name: Build the operator container image (${{ matrix.platform }})
id: build
env:
platforms: ${{ env.PLATFORMS }}
GIT_SHA: ${{ needs.compute-metadata.outputs.git-sha }}
VERSION: ${{ needs.compute-metadata.outputs.version }}
run: |
apt-get update && apt-get install -y make git jq
sudo apt-get update && sudo apt-get install -y jq
cd operator
# if this is a tag build, use the tag as the version, otherwise use the sha
git fetch --all
export GIT_SHA=$(git rev-parse --short ${{ github.sha }})
TAGS="-t ${REGISTRY@L}/${{env.IMAGE_NAME}}/operator:${GIT_SHA}"
case ${{ github.ref_type }} in
branch)
# The last tag + current git sha
export OPERATOR_VERSION=$(git tag --list 'operator*' --sort=-v:refname | head -n 1 | cut -d/ -f2)+${GIT_SHA}
TAGS="$TAGS -t ${REGISTRY@L}/${{env.IMAGE_NAME}}/operator:$(echo "${OPERATOR_VERSION}" | tr + -)"
;;
tag)
# The version part of the tag
export OPERATOR_VERSION=$(echo "${{ github.ref_name }}" | cut -f 2 -d /)
TAGS="$TAGS -t ${REGISTRY@L}/${{env.IMAGE_NAME}}/operator:${OPERATOR_VERSION} -t ${REGISTRY@L}/${{env.IMAGE_NAME}}/operator:latest"
;;
*)
echo "Unkown type ${{ github.ref_type }}"
exit 1
;;
esac
PLATFORM_TAG=$(echo "${{ matrix.platform }}" | tr '/' '-')

# Lowercase for Docker compliance
IMAGE_NAME=$(echo "${{env.IMAGE_NAME}}" | tr '[:upper:]' '[:lower:]')
REGISTRY=$(echo "${{env.REGISTRY}}" | tr '[:upper:]' '[:lower:]')

# Build platform-specific tags for all target tags
TAGS=""
for TAG in ${{ needs.compute-metadata.outputs.tags }}; do
TAGS="$TAGS -t ${REGISTRY}/${IMAGE_NAME}/operator:${TAG}-${PLATFORM_TAG}"
done

set -x
docker buildx build \
--build-arg GIT_SHA=${GIT_SHA} \
--build-arg VERSION=${OPERATOR_VERSION} \
--build-arg GO_VERSION=${GO_VERSION} \
--build-arg VERSION=${VERSION} \
--build-arg GO_VERSION=${{ env.GO_VERSION }} \
--push \
--platform ${{ env.PLATFORMS }} \
--platform ${{ matrix.platform }} \
--provenance=false \
${TAGS@L} \
--metadata-file=metadata.json \
-f ../containers/operator.Dockerfile .
cat metadata.json

echo "digest=$(cat metadata.json | jq -r .\"containerimage.digest\")" >> $GITHUB_OUTPUT
cat $GITHUB_OUTPUT

# Create multi-platform manifest from individual architecture builds
create-manifest:
runs-on: ubuntu-latest
needs: [compute-metadata, build-operator]
permissions:
contents: read
packages: write
attestations: write
id-token: write
steps:
- name: Log in to the Container registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

# Create and push multi-platform manifests, then delete platform-specific tags
- name: Create manifests and cleanup
id: manifest
run: |
sudo apt-get update && sudo apt-get install -y jq

# Lowercase for Docker compliance
IMAGE_NAME=$(echo "${{env.IMAGE_NAME}}" | tr '[:upper:]' '[:lower:]')
REGISTRY=$(echo "${{env.REGISTRY}}" | tr '[:upper:]' '[:lower:]')

# Create manifest for each tag combining amd64 and arm64 images
for TAG in ${{ needs.compute-metadata.outputs.tags }}; do
FULL_TAG="${REGISTRY}/${IMAGE_NAME}/operator:${TAG}"
echo "📦 Creating manifest for $FULL_TAG"
docker manifest create $FULL_TAG \
${FULL_TAG}-linux-amd64 \
${FULL_TAG}-linux-arm64
docker manifest push $FULL_TAG
echo "✅ Pushed $FULL_TAG"
done

# Get digest of the main tag (git sha) for attestation
MAIN_TAG="${REGISTRY}/${IMAGE_NAME}/operator:${{ needs.compute-metadata.outputs.git-sha }}"
DIGEST=$(docker manifest inspect $MAIN_TAG | jq -r '.manifests[0].digest')
echo "digest=$DIGEST" >> $GITHUB_OUTPUT
echo "subject-name=${REGISTRY}/${IMAGE_NAME}/operator" >> $GITHUB_OUTPUT

# Note: Platform-specific tags (e.g., v1.0.0-linux-amd64) are left in registry
# as intermediate artifacts. Users should pull the multi-platform manifest tags.
# GitHub Container Registry doesn't easily support programmatic tag deletion.
echo "✅ Multi-platform manifests created successfully"

# Generate supply chain security attestation
# Generate supply chain security attestation for the multi-platform manifest
- name: Generate artifact attestation
uses: actions/attest-build-provenance@v2
with:
subject-name: ${{ env.REGISTRY }}/${{env.IMAGE_NAME}}/operator
subject-digest: ${{ steps.build.outputs.digest }}
subject-name: ${{ steps.manifest.outputs.subject-name }}
subject-digest: ${{ steps.manifest.outputs.digest }}
push-to-registry: true
2 changes: 1 addition & 1 deletion chart/templates/deploymentpolicy-crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ spec:
listKind: DeploymentPolicyList
plural: deploymentpolicies
singular: deploymentpolicy
scope: Namespaced
scope: Cluster
versions:
- name: v1alpha1
schema:
Expand Down
1 change: 1 addition & 0 deletions chart/templates/validating-webhook.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ webhooks:
operations:
- CREATE
- UPDATE
- DELETE
resources:
- deploymentpolicies
scope: '*'
Expand Down
4 changes: 1 addition & 3 deletions docs/deployment_policy.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ apiVersion: skyhook.nvidia.com/v1alpha1
kind: DeploymentPolicy
metadata:
name: my-policy
namespace: skyhook
spec:
# Default applies to nodes that don't match any compartment
default:
Expand Down Expand Up @@ -260,7 +259,7 @@ spec:
```

**Behavior**:
- DeploymentPolicy must be in the **same namespace**
- DeploymentPolicy is **cluster-scoped** (not namespaced)
- Each node is assigned to a compartment based on selectors
- Nodes not matching any compartment use the `default` settings

Expand All @@ -284,7 +283,6 @@ apiVersion: skyhook.nvidia.com/v1alpha1
kind: DeploymentPolicy
metadata:
name: legacy-equivalent
namespace: skyhook
spec:
default:
budget:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ apiVersion: skyhook.nvidia.com/v1alpha1
kind: DeploymentPolicy
metadata:
name: helm-test-policy
namespace: skyhook
spec:
default:
budget:
Expand Down
Loading
Loading