feat: build images in cluster with kaniko (#9)

* feat(skaffold): update config to build with kaniko
* feat: add just recipes to build with kaniko

* chore: placeholder docker-config.json example
* chore: update gitignore
* chore: add gcp_project_id to envrc example
* chore: add test containerfile
* chore: update deployment image
* fix: scipod now built by kaniko in cluster

Author: cameronraysmith

.envrc.example

@@ -1,4 +1,5 @@
 export CONTAINER_BUILDER=podman # path to container management executable
+export GCP_PROJECT_ID=project-111111 # gcp project id
 export GITHUB_USERNAME=username # github username associated to uploading startup scripts as github gists
 export GITHUB_ORG_NAME=sciexp # name of the github org or user containing the github repository with code for development
 export GITHUB_REPO_NAME=scidev # name of a github repository with a conda environment yaml file

.github/workflows/ci.yaml

@@ -46,20 +46,3 @@ jobs:
           containerfile-path: "containers/Containerfile.${{ github.job }}"
           push-condition: ${{ github.event_name != 'pull_request' }}
           github-token: ${{ secrets.GITHUB_TOKEN }}
-
-  scipod:
-    needs: [scan,scidev]
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - uses: actions/checkout@v4
-      - name: Tag Build and Push Testing Container
-        uses: ./.github/actions/tag-build-push-container
-        with:
-          registry: ghcr.io
-          image-name: ${{ github.repository_owner }}/${{ github.job }}
-          containerfile-path: "containers/Containerfile.${{ github.job }}"
-          push-condition: ${{ github.event_name != 'pull_request' }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,2 +1,4 @@
 .direnv
 .envrc
+kaniko-key.json
+docker-config.json

cluster/resources/deployment.cue

@@ -21,10 +21,14 @@ deployment: scidev: {
 			metadata: labels: app: "scidev"
 			spec: {
 				containers: [{
-					name:            "dev"
-					image:           "ghcr.io/sciexp/scidev"
+					name:            "scidev"
+					image:           "ghcr.io/sciexp/scipod"
 					imagePullPolicy: "IfNotPresent"
-					command: ["/bin/sh", "-c", "sleep infinity"]
+					command: [
+						"/bin/sh",
+						"-c",
+						"sleep infinity",
+					]
 					resources: {
 						requests: {
 							cpu:    "16"
@@ -41,15 +45,13 @@ deployment: scidev: {
 						mountPath: "/workspace"
 					}]
 				}]
-
 				nodeSelector: {
 					"gpu-type": "nvidia-tesla-t4"
-					"spot":	    "true"
+					spot:       "false"
 				}
-
 				volumes: [{
 					name: "scidev"
-					persistentVolumeClaim: claimName: "scidev-claim"
+					persistentVolumeClaim: claimName: "scidev"
 				}]
 			}
 		}

cluster/resources/deployment.yaml

@@ -20,7 +20,7 @@ spec:
     spec:
       containers:
         - name: scidev
-          image: ghcr.io/sciexp/scidev
+          image: ghcr.io/sciexp/scipod
           imagePullPolicy: IfNotPresent
           command:
             - /bin/sh

containers/Containerfile.test

@@ -0,0 +1,3 @@
+FROM debian:stable-slim
+
+ENTRYPOINT [ "/bin/sh" ]

docker-config.example.json

@@ -0,0 +1,7 @@
+{
+  "auths": {
+    "ghcr.io": {
+      "auth": "`echo "username:ghp_TOKEN" | base64`"
+    }
+  }
+}

justfile

@@ -23,6 +23,7 @@ builder := env_var_or_default('BUILDER', 'podman')
 container_user := "runner"
 container_home := "/home" / container_user
 container_work := container_home / "work"
+gcp_project_id := env_var_or_default('GCP_PROJECT_ID', 'sciexp')
 git_username := env_var_or_default('GITHUB_USERNAME', 'sciexp')
 git_org_name := env_var_or_default('GITHUB_ORG_NAME', 'sciexp')
 git_repo_name := env_var_or_default('GITHUB_REPO_NAME', 'scidev')
@@ -134,6 +135,10 @@ info:
 render:
   skaffold render -t latest
 
+# Build image with skaffold
+build:
+  skaffold build
+
 # Deploy latest container_image in current kube context (invert: terminate)
 deploy:
   skaffold deploy -t latest
@@ -154,6 +159,26 @@ terminate:
 delete:
   skaffold delete
 
+kaniko_service_account_email := "kaniko-" + git_repo_name + "@" + gcp_project_id + ".iam.gserviceaccount.com"
+
+# Create kaniko service account and download key
+get-kaniko-credentials:
+  gcloud iam service-accounts describe {{kaniko_service_account_email}} || \
+  gcloud iam service-accounts create kaniko-{{git_repo_name}} --display-name="kaniko {{git_repo_name}} service account" && \
+  gcloud projects add-iam-policy-binding {{gcp_project_id}} \
+    --member=serviceAccount:{{kaniko_service_account_email}} \
+    --role=roles/artifactregistry.createOnPushWriter \
+    --role=roles/artifactregistry.repositories.uploadArtifacts
+  gcloud iam service-accounts keys create ./kaniko-key.json \
+  --iam-account {{kaniko_service_account_email}}
+
+# Create container regcred from docker-config.json to use with pullSecretName: regcred if private
+create-regcred:
+  kubectl create secret generic regcred \
+  --from-file=.dockerconfigjson=./docker-config.json \
+  --type=kubernetes.io/dockerconfigjson \
+  --dry-run=client -o yaml > ./regcred-secret.yaml
+
 #---------------------
 # container management
 #---------------------

skaffold.cue

@@ -9,10 +9,51 @@ config: {
 		"cluster/resources/deployment.yaml",
 	]
 	deploy: kubectl: defaultNamespace: "scidev"
-	build: artifacts: [{
-		image: "ghcr.io/sciexp/scidev"
-		docker: dockerfile: "containers/Containerfile.scidev"
-	}]
+	build: {
+		artifacts: [{
+			// - image: ghcr.io/sciexp/scidev
+			//   docker:
+			//     dockerfile: containers/Containerfile.scidev
+			// - image: us-central1-docker.pkg.dev/sciexp/scidev/scipod
+			//   docker:
+			//     dockerfile: containers/Containerfile.scipod
+			image: "ghcr.io/sciexp/scipod"
+			kaniko: {
+				dockerfile: "containers/Containerfile.scipod"
+				cache: {
+					ttl:             "168h"
+					cacheCopyLayers: true
+				}
+			}
+		}]
+		tagPolicy: sha256: {}
+
+		cluster: {
+			namespace: "scidev"
+			// With GCP credentials
+			// pullSecretPath: ./kaniko-key.json
+			// pullSecretName: kaniko-secret
+			// if private, with docker-style credentials
+			// pullSecretName: regcred
+			// randomPullSecret: true
+			// docker-style push credentials
+			dockerConfig: {
+				path: "./docker-config.json"
+			}
+			resources: {
+				requests: {
+					cpu:    "8"
+					memory: "16Gi"
+				}
+				limits: {
+					cpu:    "32"
+					memory: "180Gi"
+				}
+			}
+			concurrency: 5
+		}
+	}
+
 	profiles: [{
 		name: "dev"
 		deploy: statusCheckDeadlineSeconds: 960

skaffold.yaml

@@ -10,9 +10,41 @@ deploy:
     defaultNamespace: scidev
 build:
   artifacts:
-  - image: ghcr.io/sciexp/scidev
-    docker:
-      dockerfile: containers/Containerfile.scidev
+  # - image: ghcr.io/sciexp/scidev
+  #   docker:
+  #     dockerfile: containers/Containerfile.scidev
+  # - image: us-central1-docker.pkg.dev/sciexp/scidev/scipod
+  #   docker:
+  #     dockerfile: containers/Containerfile.scipod
+  - image: ghcr.io/sciexp/scipod
+    kaniko:
+      dockerfile: containers/Containerfile.scipod
+      cache:
+        ttl: 168h
+        cacheCopyLayers: true
+  tagPolicy:
+    sha256: {}
+  
+  cluster:
+    namespace: scidev
+    # With GCP credentials
+    # pullSecretPath: ./kaniko-key.json
+    # pullSecretName: kaniko-secret
+    # if private, with docker-style credentials
+    # pullSecretName: regcred
+    # randomPullSecret: true
+    # docker-style push credentials
+    dockerConfig:
+      path: ./docker-config.json
+    resources:
+      requests:
+        cpu: "8"
+        memory: 16Gi
+      limits:
+        cpu: "32"
+        memory: 180Gi
+    concurrency: 5
+  
 profiles:
   - name: dev
     deploy: