Skip to content

Commit dac3c57

Browse files
shacharr-googleshacharr-google
committed
Add GKE DaemonSet for TimeSync configuration
This commit adds a daemonset which is the counterpart of the compute engine scripts found in the gce folder. It is creating a GKE cluster with chrony and PTP-KVM enabled clock synchronization, waiting until chrony is synchronized before allowing workload to run on the node.
1 parent d74642c commit dac3c57

11 files changed

+322
-0
lines changed

‎.checkov.baseline‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,31 @@
4646
]
4747
}
4848
]
49+
},
50+
{
51+
"file": "/time-sync-example/gke/serviceaccount.yaml",
52+
"findings": [
53+
{
54+
"resource": "ServiceAccount.default.node-initializer-sa",
55+
"check_ids": [
56+
"CKV_K8S_20",
57+
"CKV_K8S_21"
58+
]
59+
}
60+
]
61+
},
62+
{
63+
"file": "/time-sync-example/gke/daemon-set.yaml",
64+
"findings": [
65+
{
66+
"resource": "DaemonSet.default.node-initializer",
67+
"check_ids": [
68+
"CKV_K8S_17",
69+
"CKV_K8S_18",
70+
"CKV_K8S_19"
71+
]
72+
}
73+
]
4974
}
5075
]
5176
}

‎.jscpd.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"threshold": 5
3+
}

‎time-sync-example/gke/cluster-role-binding.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRoleBinding
3+
metadata:
4+
name: node-initializer-role-binding
5+
roleRef:
6+
apiGroup: rbac.authorization.k8s.io
7+
kind: ClusterRole
8+
name: node-initializer-role
9+
subjects:
10+
- kind: ServiceAccount
11+
name: node-initializer-sa
12+
namespace: default

‎time-sync-example/gke/cluster-role.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRole
3+
metadata:
4+
name: node-initializer-role
5+
rules:
6+
- apiGroups:
7+
- ""
8+
resources:
9+
- nodes
10+
verbs:
11+
- get
12+
- list
13+
- watch
14+
- patch
15+
- update

‎time-sync-example/gke/cm-entrypoint.yaml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: entrypoint
5+
labels:
6+
app: default-init
7+
data:
8+
entrypoint.sh: |
9+
#!/usr/bin/env bash
10+
11+
set -euo pipefail
12+
13+
DEBIAN_FRONTEND=noninteractive
14+
ROOT_MOUNT_DIR="${ROOT_MOUNT_DIR:-/root}"
15+
16+
echo "Loading PTP-KVM Kernel module"
17+
chroot "${ROOT_MOUNT_DIR}" modprobe ptp_kvm
18+
echo "ptp_kvm" >${ROOT_MOUNT_DIR}/etc/modules-load.d/ptp_kvm.conf
19+
20+
21+
echo "Configuring chrony to leverage PTP-KVM"
22+
grep -v "^log " ${ROOT_MOUNT_DIR}/etc/chrony/chrony.conf > /tmp/chrony_new.conf && mv /tmp/chrony_new.conf ${ROOT_MOUNT_DIR}/etc/chrony/chrony.conf
23+
grep -v "^server " ${ROOT_MOUNT_DIR}/etc/chrony/chrony.conf > /tmp/chrony_new.conf && mv /tmp/chrony_new.conf ${ROOT_MOUNT_DIR}/etc/chrony/chrony.conf
24+
grep -v "^pool " ${ROOT_MOUNT_DIR}/etc/chrony/chrony.conf > /tmp/chrony_new.conf && mv /tmp/chrony_new.conf ${ROOT_MOUNT_DIR}/etc/chrony/chrony.conf
25+
grep "^refclock " ${ROOT_MOUNT_DIR}/etc/chrony/chrony.conf > /dev/null || echo "refclock PHC /dev/ptp_kvm poll -1 prefer" >> ${ROOT_MOUNT_DIR}/etc/chrony/chrony.conf
26+
grep "^log " ${ROOT_MOUNT_DIR}/etc/chrony/chrony.conf > /dev/null || echo "log rawmeasurements refclocks selection statistics tracking" >> ${ROOT_MOUNT_DIR}/etc/chrony/chrony.conf
27+
grep "^logdir " ${ROOT_MOUNT_DIR}/etc/chrony/chrony.conf > /dev/null || echo "logdir /var/log/chrony" >> ${ROOT_MOUNT_DIR}/etc/chrony/chrony.conf
28+
chroot "${ROOT_MOUNT_DIR}" mkdir -p /var/log/chrony
29+
chroot "${ROOT_MOUNT_DIR}" chown ntp: /var/log/chrony || echo "Chown failed - OK on Ubuntu"
30+
31+
echo "Restarting chrony"
32+
chroot "${ROOT_MOUNT_DIR}" systemctl restart chronyd
33+
34+
function untaint() {
35+
echo "Node name is: ${NODE_NAME}"
36+
echo "Finding the taint"
37+
38+
TAINT_KEY=startup-taint.cluster-autoscaler.kubernetes.io/node-initializer
39+
TAINT_VALUE=true
40+
TAINT_EFFECT=NoSchedule
41+
42+
LABEL_KEY=node-initializer
43+
LABEL_VALUE=done
44+
45+
# In case that the untaint patch failed due to a conflict, keep retrying until the taint cannot be found.
46+
while true
47+
do
48+
taint=$(${ROOT_MOUNT_DIR}/home/kubernetes/bin/kubectl get node "${NODE_NAME}" \
49+
-o jsonpath="{.spec.taints[?(@.key==\"${TAINT_KEY}\")]}")
50+
51+
if [[ -n "$taint" ]]; then
52+
echo "Found the taint. Untainting."
53+
${ROOT_MOUNT_DIR}/home/kubernetes/bin/kubectl taint nodes ${NODE_NAME} \
54+
${TAINT_KEY}:${TAINT_EFFECT}-
55+
${ROOT_MOUNT_DIR}/home/kubernetes/bin/kubectl label nodes ${NODE_NAME} \
56+
${LABEL_KEY}=${LABEL_VALUE}
57+
else
58+
echo "Didn't find the taint. Nothing to do."
59+
return
60+
fi
61+
done
62+
}
63+
# Give chrony a second to stabilize the clock
64+
sleep 1
65+
untaint

‎time-sync-example/gke/create_gke_cluster.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/bin/bash
2+
3+
if [ -z "$1" ]; then
4+
echo "Usage: create_gke_cluster.sh <project_id> <region> <cluster_name>" >&2
5+
exit 1
6+
fi
7+
8+
PROJECT_ID="$1"
9+
REGION="$2"
10+
CLUSTER_NAME="$3"
11+
MACHINE_TYPE="c3-standard-4"
12+
13+
gcloud beta container --project "$PROJECT_ID" clusters create \
14+
"$CLUSTER_NAME" --region "$REGION" --release-channel "regular" \
15+
--machine-type "$MACHINE_TYPE" --image-type "COS_CONTAINERD" \
16+
--node-taints startup-taint.cluster-autoscaler.kubernetes.io/node-initializer=true:NoSchedule \
17+
--logging=SYSTEM,WORKLOAD --monitoring=SYSTEM,STORAGE,POD,DEPLOYMENT,STATEFULSET,DAEMONSET,HPA,JOBSET,CADVISOR,KUBELET,DCGM
18+
19+
gcloud container clusters get-credentials "$CLUSTER_NAME" \
20+
--location="$REGION" --project "$PROJECT_ID"
21+
22+
# Add a service account the daemonset will use to untaint the node
23+
# when chrony is synced.
24+
kubectl apply -f serviceaccount.yaml
25+
# Bind the service account to a role allowed to untain nodes
26+
kubectl apply -f cluster-role.yaml
27+
kubectl apply -f cluster-role-binding.yaml
28+
# Add a config map with entrypoint script configuring chrony
29+
kubectl apply -f cm-entrypoint.yaml
30+
# Finally, configure daemonset with chrony configuration,
31+
# node untaint and monitoring
32+
kubectl apply -f daemonset.yaml

‎time-sync-example/gke/create_gke_metric.sh‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/bash
2+
3+
if [ -z "$1" ]; then
4+
echo "Usage: create_gke_metric.sh <project_id>" >&2
5+
exit 1
6+
fi
7+
8+
PROJECT_ID="$1"
9+
PROJECT_NUMBER=$(gcloud projects describe "$PROJECT_ID" --format="value(projectNumber)")
10+
SERVICE_ACCOUNT_EMAIL=${PROJECT_NUMBER}[email protected]
11+
12+
gcloud projects add-iam-policy-binding "${PROJECT_ID}" \
13+
--member="serviceAccount:${SERVICE_ACCOUNT_EMAIL}" \
14+
--role="roles/logging.logWriter"
15+
16+
gcloud projects add-iam-policy-binding "${PROJECT_ID}" \
17+
--member="serviceAccount:${SERVICE_ACCOUNT_EMAIL}" \
18+
--role="roles/compute.instanceAdmin"
19+
20+
gcloud projects add-iam-policy-binding "${PROJECT_ID}" \
21+
--member="serviceAccount:${SERVICE_ACCOUNT_EMAIL}" \
22+
--role="roles/monitoring.metricWriter"
23+
24+
gcloud logging --project "$PROJECT_ID" metrics create phc-clock-max-error --config-from-file=gke-clock-metric.json
25+
gcloud monitoring --project "$PROJECT_ID" dashboards create --config-from-file=gke-metrics-dashboard.json

‎time-sync-example/gke/daemonset.yaml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
apiVersion: apps/v1
2+
kind: DaemonSet
3+
metadata:
4+
name: node-initializer
5+
labels:
6+
app: default-init
7+
annotations:
8+
checkov.io/skip1: CKV_K8S_21:This service account is part of the node-initializer app.
9+
spec:
10+
selector:
11+
matchLabels:
12+
app: default-init
13+
updateStrategy:
14+
type: RollingUpdate
15+
template:
16+
metadata:
17+
labels:
18+
name: node-initializer
19+
app: default-init
20+
spec:
21+
serviceAccount: node-initializer-sa
22+
# Startup taints are meant to be used when there is an operation that has to complete before
23+
# any pods can run on the node
24+
tolerations: # The daemonset has the toleration to get scheduled on the new nodes
25+
- key: "startup-taint.cluster-autoscaler.kubernetes.io/node-initializer" # NOTE: In GKE 1.27, use ignore-taint instead of startup-taint
26+
operator: "Equal"
27+
value: "true"
28+
effect: "NoSchedule"
29+
- key: "kubernetes.io/arch"
30+
operator: "Equal"
31+
value: "arm64"
32+
effect: "NoSchedule"
33+
hostPID: true
34+
hostIPC: true
35+
hostNetwork: true
36+
volumes:
37+
- name: root-mount
38+
hostPath:
39+
path: /
40+
- name: entrypoint
41+
configMap:
42+
name: entrypoint
43+
defaultMode: 0744
44+
initContainers:
45+
- image: ubuntu:22.04
46+
name: node-initializer
47+
command: ["/scripts/entrypoint.sh"]
48+
env:
49+
- name: ROOT_MOUNT_DIR
50+
value: /root
51+
- name: NODE_NAME
52+
valueFrom:
53+
fieldRef:
54+
fieldPath: spec.nodeName
55+
securityContext:
56+
privileged: true
57+
volumeMounts:
58+
- name: root-mount
59+
mountPath: /root
60+
- name: entrypoint
61+
mountPath: /scripts
62+
- name: logshipper
63+
image: ubuntu:22.04
64+
restartPolicy: Always
65+
command: ['sh', '-c', 'tail -F ${ROOT_MOUNT_DIR}/var/log/chrony/tracking.log | xargs -L 1 echo ${NODE_NAME}: ']
66+
env:
67+
- name: ROOT_MOUNT_DIR
68+
value: /root
69+
- name: NODE_NAME
70+
valueFrom:
71+
fieldRef:
72+
fieldPath: spec.nodeName
73+
volumeMounts:
74+
- name: root-mount
75+
mountPath: /root
76+
containers:
77+
- image: "k8s.gcr.io/pause:3.3"
78+
name: pause

‎time-sync-example/gke/gke-clock-metric.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"name": "phc-clock-max-error",
3+
"description": "Maximum error of the VM clock from the host clock exposed by ptp_kvm",
4+
"filter": "resource.type=\"k8s_container\"\n resource.labels.namespace_name=\"default\"\n resource.labels.container_name=\"logshipper\"",
5+
"metricDescriptor": {
6+
"metricKind": "DELTA",
7+
"valueType": "DISTRIBUTION",
8+
"unit": "s",
9+
"labels": [
10+
{
11+
"key": "resourceName",
12+
"valueType": "STRING",
13+
"description": "The name of the resource extracted from the log payload"
14+
}
15+
]
16+
},
17+
"valueExtractor": "REGEXP_EXTRACT(textPayload, \"^.*PHC0.* ([-\\\\d\\\\.eE]+)$\")",
18+
"labelExtractors": { "resourceName" : "REGEXP_EXTRACT(textPayload, \"(.*): .*\")"},
19+
"bucketOptions": {
20+
"explicitBuckets": {
21+
"bounds": [
22+
0.0, 5.0E-7, 1.0E-6, 2.0E-6, 4.0E-6, 5.0E-6, 8.0E-6, 1.0E-5, 2.0E-5, 1.0E-4, 0.001, 0.01, 0.1, 1.0
23+
]
24+
}
25+
}
26+
}

‎time-sync-example/gke/gke-metrics-dashboard.json‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
{
2+
"displayName": "Overall Clock Accuracy",
3+
"dashboardFilters": [],
4+
"labels": {},
5+
"mosaicLayout": {
6+
"columns": 48,
7+
"tiles": [
8+
{
9+
"width": 28,
10+
"height": 28,
11+
"widget": {
12+
"xyChart": {
13+
"dataSets": [
14+
{
15+
"plotType": "LINE",
16+
"targetAxis": "Y1",
17+
"timeSeriesQuery": {
18+
"prometheusQuery": "label_replace(\nhistogram_quantile(1, sum by (le, resourceName) (\nincrease(logging_googleapis_com:phc_clock_max_error_bucket{monitored_resource=\"k8s_container\"}[1m])\n)) * 1000000000,\n\"instance_name\", \"$1\", \"resourceName\", \"(.*)\"\n) +\non(instance_name) (\ncompute_googleapis_com:instance_clock_accuracy_ptp_kvm_nanosecond_accuracy{monitored_resource=\"gce_instance\"}\n)\n",
19+
"unitOverride": "ns"
20+
}
21+
}
22+
],
23+
"chartOptions": {
24+
"displayHorizontal": false,
25+
"mode": "COLOR"
26+
},
27+
"yAxis": {
28+
"label": "Clock Accuracy",
29+
"scale": "LINEAR"
30+
},
31+
"thresholds": []
32+
}
33+
}
34+
}
35+
]
36+
}
37+
}

0 commit comments

Comments
 (0)