diff --git a/kubernetes/README.md b/kubernetes/README.md index 56e96797..ac70666e 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -9,3 +9,20 @@ kubectl apply -k . These files assume that the cluster needs to be created in the `buildbarn` namespace. Storage is backed by persistent volumes. + + +**Prototyping/Quickstart** +To run against the cluster (for protyping & debugging), forward port 8980 in one terminal +```sh +while true; do kubectl --n buildbarn port-forward services/frontend :8980; done +``` + +**Debugging**: +Check Pods are up and get logs within the pods +```sh +kubectl get pods -n buildbarn +kubectl logs -n buildbarn +``` + +**Note** +- Yaml files here work for microk8s and k3s, but will not work straight out of the box for kubeadm \ No newline at end of file diff --git a/kubernetes/config/storage.yaml b/kubernetes/config/storage.yaml index b97c57ac..c4104c0d 100644 --- a/kubernetes/config/storage.yaml +++ b/kubernetes/config/storage.yaml @@ -75,6 +75,43 @@ data: getAuthorizer: { allow: {} }, putAuthorizer: { allow: {} }, }, + fileSystemAccessCache: { + backend: { + 'local': { + keyLocationMapOnBlockDevice: { + file: { + path: '/storage-fsac/key_location_map', + sizeBytes: 1024 * 1024, + }, + }, + keyLocationMapMaximumGetAttempts: 8, + keyLocationMapMaximumPutAttempts: 32, + oldBlocks: 8, + currentBlocks: 24, + newBlocks: 1, + blocksOnBlockDevice: { + source: { + file: { + path: '/storage-fsac/blocks', + sizeBytes: 20 * 1024 * 1024, + }, + }, + spareBlocks: 3, + dataIntegrityValidationCache: { + cacheSize: 10000, + cacheDuration: '14400s', + cacheReplacementPolicy: 'LEAST_RECENTLY_USED', + }, + }, + persistent: { + stateDirectoryPath: '/storage-fsac/persistent_state', + minimumEpochInterval: '300s', + }, + }, + }, + getAuthorizer: { allow: {} }, + putAuthorizer: { allow: {} }, + }, } kind: ConfigMap metadata: diff --git a/kubernetes/config/worker-ubuntu22-04.yaml b/kubernetes/config/worker-ubuntu22-04.yaml index 920fc761..c4856c45 100644 --- a/kubernetes/config/worker-ubuntu22-04.yaml +++ b/kubernetes/config/worker-ubuntu22-04.yaml @@ -4,7 +4,48 @@ data: local common = import 'common.libsonnet'; { - blobstore: common.blobstore, + blobstore: { + actionCache: common.blobstore.actionCache, + contentAddressableStorage: { + readCaching: { + slow: common.blobstore.contentAddressableStorage, + fast: { + 'local': { + keyLocationMapOnBlockDevice: { + file: { + path: '/storage-worker-cas/key_location_map', + sizeBytes: 100 * 1024 * 1024, + }, + }, + keyLocationMapMaximumGetAttempts: 8, + keyLocationMapMaximumPutAttempts: 32, + oldBlocks: 8, + currentBlocks: 24, + newBlocks: 1, + blocksOnBlockDevice: { + source: { + file: { + path: '/storage-worker-cas/blocks', + sizeBytes: 8 * 1024 * 1024 * 1024, + }, + }, + spareBlocks: 3, + dataIntegrityValidationCache: { + cacheSize: 10000, + cacheDuration: '14400s', + cacheReplacementPolicy: 'LEAST_RECENTLY_USED', + }, + }, + persistent: { + stateDirectoryPath: '/storage-worker-cas/persistent_state', + minimumEpochInterval: '300s', + }, + }, + }, + replicator: { deduplicating: { 'local': {} } }, + }, + }, + }, browserUrl: common.browserUrl, maximumMessageSizeBytes: common.maximumMessageSizeBytes, scheduler: { address: 'scheduler:8983' }, @@ -12,16 +53,24 @@ data: setUmask: { umask: 0 }, }, buildDirectories: [{ - native: { - buildDirectoryPath: '/worker/build', - cacheDirectoryPath: '/worker/cache', - maximumCacheFileCount: 10000, - maximumCacheSizeBytes: 1024 * 1024 * 1024, - cacheReplacementPolicy: 'LEAST_RECENTLY_USED', + virtual: { + maximumExecutionTimeoutCompensation: "3600s", + shuffleDirectoryListings: true, + mount: { + mountPath: '/worker/build', + fuse: { + directoryEntryValidity: "300s", + inodeAttributeValidity: "300s", + allowOther: true, + directMount: true, + }, + }, }, runners: [{ endpoint: { address: 'unix:///worker/runner' }, concurrency: 8, + maximumFilePoolFileCount: 10000, + maximumFilePoolSizeBytes: 8*1024*1024*1024, platform: { properties: [ { name: 'OSFamily', value: 'Linux' }, @@ -34,12 +83,64 @@ data: }, }], }], + filePool: { + blockDevice: { + file: { + path: '/worker/filepool', + sizeBytes:8 * 8*1024*1024*1024, // concurrency * maximumFilePoolSizeBytes + }, + }, + }, outputUploadConcurrency: 11, directoryCache: { maximumCount: 1000, maximumSizeBytes: 1000 * 1024, cacheReplacementPolicy: 'LEAST_RECENTLY_USED', }, + prefetching: { + fileSystemAccessCache:{ + readCaching: { + slow: common.blobstore.contentAddressableStorage, + fast: { + 'local': { + keyLocationMapOnBlockDevice: { + file: { + path: '/storage-worker-cas/key_location_map', + sizeBytes: 100 * 1024 * 1024, + }, + }, + keyLocationMapMaximumGetAttempts: 8, + keyLocationMapMaximumPutAttempts: 32, + oldBlocks: 8, + currentBlocks: 24, + newBlocks: 1, + blocksOnBlockDevice: { + source: { + file: { + path: '/storage-worker-cas/blocks', + sizeBytes: 8 * 1024 * 1024 * 1024, + }, + }, + spareBlocks: 3, + dataIntegrityValidationCache: { + cacheSize: 10000, + cacheDuration: '14400s', + cacheReplacementPolicy: 'LEAST_RECENTLY_USED', + }, + }, + persistent: { + stateDirectoryPath: '/storage-worker-cas/persistent_state', + minimumEpochInterval: '300s', + }, + }, + }, + replicator: { 'local' : {} }, + }, + }, + bloomFilterBitsPerPath: 14, + bloomFilterMaximumSizeBytes: 65536, + downloadConcurrency: 24, + }, } kind: ConfigMap metadata: diff --git a/kubernetes/storage.yaml b/kubernetes/storage.yaml index 68a0cfe2..4c2b859d 100644 --- a/kubernetes/storage.yaml +++ b/kubernetes/storage.yaml @@ -30,18 +30,22 @@ spec: name: cas - mountPath: /storage-ac name: ac + - mountPath: /storage-fsac + name: fsac initContainers: - name: volume-init image: busybox:1.31.1-uclibc command: - sh - -c - - mkdir -m 0700 -p /storage-cas/persistent_state /storage-ac/persistent_state + - mkdir -m 0700 -p /storage-cas/persistent_state /storage-ac/persistent_state /storage-fsac/persistent_state volumeMounts: - mountPath: /storage-cas name: cas - mountPath: /storage-ac name: ac + - mountPath: /storage-fsac + name: fsac volumes: - name: configs projected: @@ -73,6 +77,14 @@ spec: resources: requests: storage: 1Gi + - metadata: + name: fsac + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 12Gi --- apiVersion: v1 kind: Service diff --git a/kubernetes/worker-ubuntu22-04.yaml b/kubernetes/worker-ubuntu22-04.yaml index bbec852c..36563c66 100644 --- a/kubernetes/worker-ubuntu22-04.yaml +++ b/kubernetes/worker-ubuntu22-04.yaml @@ -7,7 +7,7 @@ metadata: prometheus.io/port: "80" prometheus.io/scrape: "true" spec: - replicas: 8 + replicas: 2 selector: matchLabels: app: worker @@ -28,7 +28,12 @@ spec: name: configs readOnly: true - mountPath: /worker + mountPropagation: Bidirectional name: worker + - mountPath: /storage-worker-cas + name: storage-worker-cas + securityContext: + privileged: true env: - name: NODE_NAME valueFrom: @@ -42,14 +47,16 @@ spec: image: ghcr.io/catthehacker/ubuntu:act-22.04@sha256:5f9c35c25db1d51a8ddaae5c0ba8d3c163c5e9a4a6cc97acd409ac7eae239448 name: runner securityContext: - runAsUser: 65534 - allowPrivilegeEscalation: false + privileged: true volumeMounts: - mountPath: /config/ name: configs readOnly: true - mountPath: /worker + mountPropagation: HostToContainer name: worker + - mountPath: /storage-worker-cas + name: storage-worker-cas - mountPath: /bb name: bb-runner readOnly: true @@ -64,10 +71,15 @@ spec: command: - sh - -c - - mkdir -pm 0777 /worker/build && mkdir -pm 0700 /worker/cache && chmod 0777 /worker + - mkdir -pm 0777 /worker/build && mkdir -pm 0700 /worker/cache && mkdir -pm 0777 /storage-worker-cas/persistent_state && mkdir -pm 0777 /worker-fuse && chmod 0777 /worker volumeMounts: - mountPath: /worker + mountPropagation: HostToContainer name: worker + - mountPath: /storage-worker-cas + name: storage-worker-cas + securityContext: + privileged: true volumes: - name: bb-runner emptyDir: {} @@ -89,5 +101,9 @@ spec: items: - key: common.libsonnet path: common.libsonnet - - emptyDir: {} - name: worker + - name: worker + hostPath: + path: /worker-fuse + type: Directory + - name: storage-worker-cas + emptyDir: {}