Skip to content

Commit e340d5d

Browse files
authored
feat: add Tilt support for local development (#102)
This will allow for more rapid development as tilt supports hot reload on code changes.
1 parent b5cca41 commit e340d5d

14 files changed

+531
-4
lines changed

.gitignore

+4-1
Original file line numberDiff line numberDiff line change
@@ -48,4 +48,7 @@ test/e2e/data/infrastructure-oci/v1beta1/cluster-template-multiple-node-nsg.yaml
4848
test/e2e/data/infrastructure-oci/v1beta1/cluster-template-local-vcn-peering.yaml
4949
test/e2e/data/infrastructure-oci/v1beta1/cluster-template-cluster-class.yaml
5050
test/e2e/data/infrastructure-oci/v1beta1/cluster-template-remote-vcn-peering.yaml
51-
test/e2e/data/infrastructure-oci/v1beta1/cluster-template-externally-managed-vcn.yaml
51+
test/e2e/data/infrastructure-oci/v1beta1/cluster-template-externally-managed-vcn.yaml
52+
53+
# tilt
54+
tilt-settings.json

Makefile

+58-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME)
2525
TAG ?= dev
2626
ARCH ?= amd64
2727
ALL_ARCH = amd64 arm64
28-
28+
TOOLS_DIR := hack/tools
2929
TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/bin)
3030

3131
GINKGO_VER := v1.16.5
@@ -60,6 +60,21 @@ EXP_MACHINE_POOL ?= false
6060
# Set build time variables including version details
6161
LDFLAGS := $(shell source ./hack/version.sh; version::ldflags)
6262

63+
# curl retries
64+
CURL_RETRIES=3
65+
66+
ENVSUBST_VER := v2.0.0-20210730161058-179042472c46
67+
ENVSUBST_BIN := envsubst
68+
ENVSUBST := $(TOOLS_BIN_DIR)/$(ENVSUBST_BIN)-$(ENVSUBST_VER)
69+
70+
KUSTOMIZE_VER := v4.5.2
71+
KUSTOMIZE_BIN := kustomize
72+
KUSTOMIZE := $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER)
73+
74+
KUBECTL_VER := v1.22.9
75+
KUBECTL_BIN := kubectl
76+
KUBECTL := $(TOOLS_BIN_DIR)/$(KUBECTL_BIN)-$(KUBECTL_VER)
77+
6378
# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
6479
ifeq (,$(shell go env GOBIN))
6580
GOBIN=$(shell go env GOPATH)/bin
@@ -326,3 +341,45 @@ release-metadata: $(RELEASE_DIR)
326341
.PHONY: clean-release
327342
clean-release:
328343
rm -rf $(RELEASE_DIR)
344+
345+
GO_INSTALL = ./scripts/go_install.sh
346+
GOOS := $(shell go env GOOS)
347+
GOARCH := $(shell go env GOARCH)
348+
349+
.PHONY: install-tools # populate hack/tools/bin
350+
install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL)
351+
352+
envsubst: $(ENVSUBST) ## Build a local copy of envsubst.
353+
kustomize: $(KUSTOMIZE) ## Build a local copy of kustomize.
354+
kubectl: $(KUBECTL) ## Build a local copy of kubectl.
355+
356+
$(ENVSUBST): ## Build envsubst from tools folder.
357+
GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) github.com/drone/envsubst/v2/cmd/envsubst $(ENVSUBST_BIN) $(ENVSUBST_VER)
358+
359+
$(KUSTOMIZE): ## Build kustomize from tools folder.
360+
GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) sigs.k8s.io/kustomize/kustomize/v4 $(KUSTOMIZE_BIN) $(KUSTOMIZE_VER)
361+
362+
$(KUBECTL): ## Build kubectl from tools folder.
363+
mkdir -p $(TOOLS_BIN_DIR)
364+
rm -f "$(TOOLS_BIN_DIR)/$(KUBECTL_BIN)*"
365+
curl --retry $(CURL_RETRIES) -fsL https://storage.googleapis.com/kubernetes-release/release/$(KUBECTL_VER)/bin/$(GOOS)/$(GOARCH)/kubectl -o $(KUBECTL)
366+
ln -sf $(KUBECTL) $(TOOLS_BIN_DIR)/$(KUBECTL_BIN)
367+
chmod +x $(KUBECTL) $(TOOLS_BIN_DIR)/$(KUBECTL_BIN)
368+
369+
.PHONY: $(ENVSUBST_BIN)
370+
$(ENVSUBST_BIN): $(ENVSUBST)
371+
372+
.PHONY: $(KUBECTL_BIN)
373+
$(KUBECTL_BIN): $(KUBECTL)
374+
375+
## --------------------------------------
376+
## Tilt / Kind
377+
## --------------------------------------
378+
379+
.PHONY: kind-create
380+
kind-create: $(KUBECTL) ## Create kind cluster if needed.
381+
./scripts/kind-with-registry.sh
382+
383+
.PHONY: tilt-up
384+
tilt-up: install-tools kind-create ## Start tilt and build kind cluster if needed.
385+
EXP_CLUSTER_RESOURCE_SET=true EXP_MACHINE_POOL=true tilt up

Tiltfile

+215
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
2+
envsubst_cmd = "./hack/tools/bin/envsubst"
3+
kubectl_cmd = "./hack/tools/bin/kubectl"
4+
helm_cmd = "./hack/tools/bin/helm"
5+
tools_bin = "./hack/tools/bin"
6+
7+
update_settings(k8s_upsert_timeout_secs = 60) # on first tilt up, often can take longer than 30 seconds
8+
9+
#Add tools to path
10+
os.putenv("PATH", os.getenv("PATH") + ":" + tools_bin)
11+
12+
keys = ["OCI_TENANCY_ID", "OCI_USER_ID", "OCI_CREDENTIALS_FINGERPRINT", "OCI_REGION", "OCI_CREDENTIALS_KEY_PATH"]
13+
14+
# set defaults
15+
settings = {
16+
"allowed_contexts": [
17+
"kind-capoci",
18+
],
19+
"deploy_cert_manager": True,
20+
"preload_images_for_kind": True,
21+
"kind_cluster_name": "capoci",
22+
"capi_version": "v1.1.3",
23+
"cert_manager_version": "v1.1.0",
24+
"kubernetes_version": "v1.22.9",
25+
}
26+
27+
# global settings
28+
settings.update(read_json(
29+
"tilt-settings.json",
30+
default = {},
31+
))
32+
33+
if settings.get("trigger_mode") == "manual":
34+
trigger_mode(TRIGGER_MODE_MANUAL)
35+
36+
if "allowed_contexts" in settings:
37+
allow_k8s_contexts(settings.get("allowed_contexts"))
38+
39+
if "default_registry" in settings:
40+
default_registry(settings.get("default_registry"))
41+
42+
tilt_helper_dockerfile_header = """
43+
# Tilt image
44+
FROM golang:1.17 as tilt-helper
45+
# Support live reloading with Tilt
46+
RUN wget --output-document /restart.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/restart.sh && \
47+
wget --output-document /start.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/start.sh && \
48+
chmod +x /start.sh && chmod +x /restart.sh
49+
"""
50+
51+
tilt_dockerfile_header = """
52+
FROM gcr.io/distroless/base:debug as tilt
53+
WORKDIR /
54+
COPY --from=tilt-helper /start.sh .
55+
COPY --from=tilt-helper /restart.sh .
56+
COPY manager .
57+
"""
58+
59+
def validate_auth():
60+
substitutions = settings.get("kustomize_substitutions", {})
61+
os.environ.update(substitutions)
62+
for sub in substitutions:
63+
if sub[-4:] == "_B64":
64+
os.environ[sub[:-4]] = base64_decode(os.environ[sub])
65+
print("{} was not specified in tilt-settings.json, attempting to load {}".format(base64_decode(os.environ[sub]), sub))
66+
missing = [k for k in keys if not os.environ.get(k)]
67+
if missing:
68+
fail("missing kustomize_substitutions keys {} in tilt-setting.json".format(missing))
69+
70+
# Users may define their own Tilt customizations in tilt.d. This directory is excluded from git and these files will
71+
# not be checked in to version control.
72+
def include_user_tilt_files():
73+
user_tiltfiles = listdir("tilt.d")
74+
for f in user_tiltfiles:
75+
include(f)
76+
77+
# deploy CAPI
78+
def deploy_capi():
79+
version = settings.get("capi_version")
80+
capi_uri = "https://github.com/kubernetes-sigs/cluster-api/releases/download/{}/cluster-api-components.yaml".format(version)
81+
cmd = "curl -sSL {} | {} | {} apply -f -".format(capi_uri, envsubst_cmd, kubectl_cmd)
82+
local(cmd, quiet = False)
83+
if settings.get("extra_args"):
84+
extra_args = settings.get("extra_args")
85+
if extra_args.get("core"):
86+
core_extra_args = extra_args.get("core")
87+
if core_extra_args:
88+
for namespace in ["capi-system", "capi-webhook-system"]:
89+
patch_args_with_extra_args(namespace, "capi-controller-manager", core_extra_args)
90+
if extra_args.get("kubeadm-bootstrap"):
91+
kb_extra_args = extra_args.get("kubeadm-bootstrap")
92+
if kb_extra_args:
93+
patch_args_with_extra_args("capi-kubeadm-bootstrap-system", "capi-kubeadm-bootstrap-controller-manager", kb_extra_args)
94+
95+
def patch_args_with_extra_args(namespace, name, extra_args):
96+
args_str = str(local("{} get deployments {} -n {} -o jsonpath={{.spec.template.spec.containers[1].args}}".format(kubectl_cmd, name, namespace)))
97+
args_to_add = [arg for arg in extra_args if arg not in args_str]
98+
if args_to_add:
99+
args = args_str[1:-1].split()
100+
args.extend(args_to_add)
101+
patch = [{
102+
"op": "replace",
103+
"path": "/spec/template/spec/containers/1/args",
104+
"value": args,
105+
}]
106+
local("{} patch deployment {} -n {} --type json -p='{}'".format(kubectl_cmd, name, namespace, str(encode_json(patch)).replace("\n", "")))
107+
108+
# Build CAPOCI and add feature gates
109+
def capoci():
110+
# Apply the kustomized yaml for this provider
111+
yaml = str(kustomizesub("./config/default"))
112+
113+
# add extra_args if they are defined
114+
if settings.get("extra_args"):
115+
oci_extra_args = settings.get("extra_args").get("oci")
116+
if oci_extra_args:
117+
yaml_dict = decode_yaml_stream(yaml)
118+
append_arg_for_container_in_deployment(yaml_dict, "capoci-controller-manager", "capoci-system", "cluster-api-oci-controller", oci_extra_args)
119+
yaml = str(encode_yaml_stream(yaml_dict))
120+
yaml = fixup_yaml_empty_arrays(yaml)
121+
122+
# Set up a local_resource build of the provider's manager binary.
123+
local_resource(
124+
"manager",
125+
cmd = 'mkdir -p .tiltbuild;CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags \'-extldflags "-static"\' -o .tiltbuild/manager',
126+
deps = ["api", "cloud", "config", "controllers", "exp", "feature", "pkg", "go.mod", "go.sum", "main.go", "auth-config.yaml"],
127+
labels = ["cluster-api"],
128+
)
129+
130+
dockerfile_contents = "\n".join([
131+
tilt_helper_dockerfile_header,
132+
tilt_dockerfile_header,
133+
])
134+
135+
entrypoint = ["sh", "/start.sh", "/manager"]
136+
extra_args = settings.get("extra_args")
137+
if extra_args:
138+
entrypoint.extend(extra_args)
139+
140+
# Set up an image build for the provider. The live update configuration syncs the output from the local_resource
141+
# build into the container.
142+
docker_build(
143+
ref = "ghcr.io/oracle/cluster-api-oci-controller-amd64:dev",
144+
context = "./.tiltbuild/",
145+
dockerfile_contents = dockerfile_contents,
146+
target = "tilt",
147+
entrypoint = entrypoint,
148+
only = "manager",
149+
live_update = [
150+
sync(".tiltbuild/manager", "/manager"),
151+
run("sh /restart.sh"),
152+
],
153+
ignore = ["templates"],
154+
network = "host",
155+
)
156+
157+
#secret_settings(disable_scrub=True)
158+
k8s_yaml(blob(yaml))
159+
160+
161+
def append_arg_for_container_in_deployment(yaml_stream, name, namespace, contains_image_name, args):
162+
for item in yaml_stream:
163+
if item["kind"] == "Deployment" and item.get("metadata").get("name") == name and item.get("metadata").get("namespace") == namespace:
164+
containers = item.get("spec").get("template").get("spec").get("containers")
165+
for container in containers:
166+
if contains_image_name in container.get("image"):
167+
container.get("args").extend(args)
168+
169+
def fixup_yaml_empty_arrays(yaml_str):
170+
yaml_str = yaml_str.replace("conditions: null", "conditions: []")
171+
return yaml_str.replace("storedVersions: null", "storedVersions: []")
172+
173+
def waitforsystem():
174+
local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-kubeadm-bootstrap-system")
175+
local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-kubeadm-control-plane-system")
176+
local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-system")
177+
178+
def base64_encode(to_encode):
179+
encode_blob = local("echo '{}' | tr -d '\n' | base64 - | tr -d '\n'".format(to_encode), quiet = True, echo_off = True)
180+
return str(encode_blob)
181+
182+
def base64_encode_file(path_to_encode):
183+
encode_blob = local("cat {} | tr -d '\n' | base64 - | tr -d '\n'".format(path_to_encode), quiet = True)
184+
return str(encode_blob)
185+
186+
def read_file_from_path(path_to_read):
187+
str_blob = local("cat {} | tr -d '\n'".format(path_to_read), quiet = True)
188+
return str(str_blob)
189+
190+
def base64_decode(to_decode):
191+
decode_blob = local("echo '{}' | base64 --decode -".format(to_decode), quiet = True, echo_off = True)
192+
return str(decode_blob)
193+
194+
def kustomizesub(folder):
195+
yaml = local('hack/kustomize-sub.sh {}'.format(folder), quiet=True)
196+
return yaml
197+
198+
##############################
199+
# Actual work happens here
200+
##############################
201+
202+
validate_auth()
203+
204+
include_user_tilt_files()
205+
206+
load("ext://cert_manager", "deploy_cert_manager")
207+
208+
if settings.get("deploy_cert_manager"):
209+
deploy_cert_manager()
210+
211+
deploy_capi()
212+
213+
capoci()
214+
215+
waitforsystem()

config/manager/manager.yaml

-2
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@ spec:
2222
labels:
2323
control-plane: controller-manager
2424
spec:
25-
securityContext:
26-
runAsNonRoot: true
2725
containers:
2826
- command:
2927
- /manager

hack/kustomize-sub.sh

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
2+
#!/bin/bash
3+
# Copyright 2021 The Kubernetes Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -o errexit
18+
set -o nounset
19+
set -o pipefail
20+
21+
root=$(dirname "${BASH_SOURCE[0]}")
22+
$root/tools/bin/kustomize build $1 | $root/tools/bin/envsubst

0 commit comments

Comments
 (0)