Skip to content

Commit a952d58

Browse files
mortenljmortenlj
authored
Workload Image (#598)
* Add .dockerignore to reduce docker context * Initial Tiltfile for local development * Tiltfile: Auto-generate liberator charts on changes * Tiltfile: Ensure naiserator stays in the context from tilt (re)load * Get Image from external Image resource if not present in spec (WIP) * Tiltfile: Hardcode single directory to avoid creating new tempdir all the time * Tiltfile: Make naiserator startup optional This allows for running naiserator in debugger instead of managed by tilt * Add mise to project and set up tilt tasks * Allow for long timeouts when running with tilt This allows for slow stepping through debugger etc. * Use image from Image resource if no image on Application/Naisjob If there is no image on the resource when reconciling, look for a matching Image resource and use the image from there. The chosen image is set on Status.EffectiveImage when resource is saved to the cluster. * Improve mise tilt tasks * Add synchronizer test for applications with external image * Use liberator from workload-image branch * Latest liberator on main * A bit too long for tests to run ... * Avoid sync for apps that have not been synced with effective image yet When this feature goes out, all apps will have an empty effective image, and would have been considered changed with the code as it was before. This is unfortunate, as we only want to sync when there is an actual change, and in these cases we can wait for the hash to change before syncing and setting effective image. * Give controller runtime a logger
1 parent bd2ef1a commit a952d58

24 files changed

+531
-116
lines changed

‎.dockerignore

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
.idea/
2+
naiserator.iml
3+
vendor/
4+
cover.out
5+
./naiserator
6+
/bin/
7+
*.proto
8+
/.testbin/
9+
*.DS_Store
10+
.direnv
11+
hack
12+
charts
13+
.github
14+
README.md

‎.mise-tasks/tilt/_default

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/usr/bin/env bash
2+
#MISE description="Launch tilt for local development, confined to selected context"
3+
#USAGE flag "-p --product <product>" help="The product used to launch local kubernetes" default="kind" { choices "kind" "minikube" "docker-desktop" "k3d" }
4+
#USAGE flag "-n --namespace <namespace>" help="The namespace to use" default="default"
5+
6+
# shellcheck disable=SC2154
7+
8+
context_name="${usage_product}-tilt-naiserator"
9+
10+
ctlptl create cluster "${usage_product}" --registry=ctlptl-registry --name "${context_name}"
11+
12+
kubie exec "${context_name}" "${usage_namespace}" tilt up --stream

‎.mise-tasks/tilt/cleanup

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/usr/bin/env bash
2+
#MISE description="Launch tilt for local development, confined to selected context"
3+
#USAGE flag "-p --product <product>" help="The product used to launch local kubernetes" default="kind" { choices "kind" "minikube" "docker-desktop" "k3d" }
4+
#USAGE flag "-n --namespace <namespace>" help="The namespace to use" default="default"
5+
6+
# shellcheck disable=SC2154
7+
8+
context_name="${usage_product}-tilt-naiserator"
9+
10+
kubie exec "${context_name}" "${usage_namespace}" tilt down
11+
12+
ctlptl delete cluster "${context_name}"

‎.mise-tasks/tilt/down

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/usr/bin/env bash
2+
#MISE description="Launch tilt for local development, confined to selected context"
3+
#USAGE flag "-p --product <product>" help="The product used to launch local kubernetes" default="kind" { choices "kind" "minikube" "docker-desktop" "k3d" }
4+
#USAGE flag "-n --namespace <namespace>" help="The namespace to use" default="default"
5+
6+
# shellcheck disable=SC2154
7+
8+
context_name="${usage_product}-tilt-naiserator"
9+
10+
kubie exec "${context_name}" "${usage_namespace}" tilt down

‎Tiltfile

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
load('ext://cert_manager', 'deploy_cert_manager')
2+
load('ext://helm_resource', 'helm_resource', 'helm_repo')
3+
load('ext://local_output', 'local_output')
4+
5+
APP_NAME = "naiserator"
6+
7+
8+
def ignore_rules():
9+
return str(read_file(".dockerignore")).split("\n")
10+
11+
12+
deploy_cert_manager()
13+
14+
helm_repo('aiven', 'https://aiven.github.io/aiven-charts')
15+
helm_resource('aiven-operator-crds', 'aiven/aiven-operator-crds', resource_deps=['aiven'], pod_readiness="ignore")
16+
17+
helm_repo('prometheus', 'https://prometheus-community.github.io/helm-charts')
18+
helm_resource('prometheus-operator-crds', 'prometheus/prometheus-operator-crds', resource_deps=['prometheus'],
19+
pod_readiness="ignore")
20+
21+
# Load liberator charts, assuming liberator checked out next to naiserator
22+
# Automatically generate updated CRDs from the liberator code when the apis change, and apply them to the cluster
23+
local_resource(
24+
"liberator-chart",
25+
cmd="make generate",
26+
dir="../liberator",
27+
ignore=["../liberator/**/zz_generated.deepcopy.go"],
28+
deps=["../liberator/pkg/apis"],
29+
)
30+
k8s_yaml(helm("../liberator/charts", name="nais-crds"))
31+
liberator_objects = [
32+
"aivenapplications.aiven.nais.io:CustomResourceDefinition:default",
33+
"bigquerydatasets.google.nais.io:CustomResourceDefinition:default",
34+
"streams.kafka.nais.io:CustomResourceDefinition:default",
35+
"topics.kafka.nais.io:CustomResourceDefinition:default",
36+
"applications.nais.io:CustomResourceDefinition:default",
37+
"azureadapplications.nais.io:CustomResourceDefinition:default",
38+
"idportenclients.nais.io:CustomResourceDefinition:default",
39+
"images.nais.io:CustomResourceDefinition:default",
40+
"jwkers.nais.io:CustomResourceDefinition:default",
41+
"maskinportenclients.nais.io:CustomResourceDefinition:default",
42+
"naisjobs.nais.io:CustomResourceDefinition:default",
43+
]
44+
k8s_resource(
45+
new_name="nais-crds",
46+
objects=liberator_objects,
47+
resource_deps=["liberator-chart"],
48+
)
49+
50+
# Create a tempdir for naiserator configs
51+
naiserator_dir = "/tmp/tilt-naiserator"
52+
local("mkdir -p {}".format(naiserator_dir))
53+
54+
# Copy tilt spesific naiserator config to tempdir for naiserator to use
55+
local_resource(
56+
"naiserator-config",
57+
cmd="cp ./hack/tilt-naiserator-config.yaml {}/naiserator.yaml".format(naiserator_dir),
58+
deps=["hack/tilt-naiserator-config.yaml"],
59+
)
60+
61+
# Ensure we save the current kube context to a file for naiserator
62+
# This is so we don't accidentally switch context if other tools change the current context after startup
63+
# Falls apart if the Tiltfile is updated, as that copies the kubeconfig again.
64+
# See https://github.com/tilt-dev/tilt/issues/6295
65+
local_resource(
66+
"naiserator-kubeconfig",
67+
cmd="kubectl config view --minify --flatten > {}/kubeconfig".format(naiserator_dir),
68+
)
69+
70+
# Start naiserator locally, so changes are detected and rebuilt automatically
71+
local_resource(
72+
"naiserator",
73+
cmd="go build -o cmd/naiserator/naiserator ./cmd/naiserator",
74+
serve_cmd="{}/cmd/naiserator/naiserator --kubeconfig={}/kubeconfig".format(config.main_dir, naiserator_dir),
75+
deps=["cmd/naiserator/naiserator.go", "go.mod", "go.sum", "pkg", "/tmp/naiserator.yaml"],
76+
resource_deps=["nais-crds", "aiven-operator-crds", "prometheus-operator-crds", "naiserator-config",
77+
"naiserator-kubeconfig"],
78+
ignore=ignore_rules(),
79+
serve_dir=naiserator_dir,
80+
auto_init=False,
81+
trigger_mode=TRIGGER_MODE_MANUAL,
82+
)

‎cmd/naiserator/main.go

+2
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
ctrl "sigs.k8s.io/controller-runtime"
2727
"sigs.k8s.io/controller-runtime/pkg/cache"
2828
"sigs.k8s.io/controller-runtime/pkg/client"
29+
ctrl_log "sigs.k8s.io/controller-runtime/pkg/log"
2930
kubemetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
3031
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
3132
)
@@ -122,6 +123,7 @@ func run() error {
122123

123124
metrics.Register(kubemetrics.Registry)
124125
logSink := &logrus2logr.Logrus2Logr{Logger: log.StandardLogger()}
126+
ctrl_log.SetLogger(logr.New(logSink))
125127
mgr, err := ctrl.NewManager(kconfig, ctrl.Options{
126128
Cache: cache.Options{
127129
SyncPeriod: &cfg.Informer.FullSyncInterval,

‎go.mod

+24-8
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ require (
1414
github.com/magiconair/properties v1.8.9
1515
github.com/mitchellh/hashstructure v1.1.0
1616
github.com/mitchellh/mapstructure v1.5.0
17-
github.com/nais/liberator v0.0.0-20250219141329-f224bfab7a8d
17+
github.com/nais/liberator v0.0.0-20250314120052-1cb863cb99a7
1818
github.com/novln/docker-parser v1.0.0
1919
github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.74.0
2020
github.com/prometheus/client_golang v1.21.0
@@ -34,18 +34,21 @@ require (
3434
require (
3535
github.com/beorn7/perks v1.0.1 // indirect
3636
github.com/cespare/xxhash/v2 v2.3.0 // indirect
37+
github.com/chigopher/pathlib v0.19.1 // indirect
3738
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
3839
github.com/eapache/go-resiliency v1.3.0 // indirect
3940
github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
4041
github.com/eapache/queue v1.1.0 // indirect
4142
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
4243
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
4344
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
45+
github.com/fatih/color v1.18.0 // indirect
4446
github.com/fsnotify/fsnotify v1.8.0 // indirect
4547
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
4648
github.com/go-openapi/jsonpointer v0.21.0 // indirect
4749
github.com/go-openapi/jsonreference v0.21.0 // indirect
4850
github.com/go-openapi/swag v0.23.0 // indirect
51+
github.com/gobuffalo/flect v1.0.3 // indirect
4952
github.com/gogo/protobuf v1.3.2 // indirect
5053
github.com/golang/snappy v0.0.4 // indirect
5154
github.com/google/btree v1.1.3 // indirect
@@ -54,45 +57,57 @@ require (
5457
github.com/hashicorp/errwrap v1.1.0 // indirect
5558
github.com/hashicorp/go-uuid v1.0.3 // indirect
5659
github.com/hashicorp/hcl v1.0.0 // indirect
60+
github.com/huandu/xstrings v1.4.0 // indirect
61+
github.com/iancoleman/strcase v0.3.0 // indirect
62+
github.com/inconshreveable/mousetrap v1.1.0 // indirect
5763
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
5864
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
5965
github.com/jcmturner/gofork v1.7.6 // indirect
6066
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
6167
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
68+
github.com/jinzhu/copier v0.4.0 // indirect
6269
github.com/josharian/intern v1.0.0 // indirect
6370
github.com/json-iterator/go v1.1.12 // indirect
6471
github.com/klauspost/compress v1.17.11 // indirect
6572
github.com/mailru/easyjson v0.9.0 // indirect
73+
github.com/mattn/go-colorable v0.1.14 // indirect
74+
github.com/mattn/go-isatty v0.0.20 // indirect
75+
github.com/mitchellh/go-homedir v1.1.0 // indirect
6676
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
6777
github.com/modern-go/reflect2 v1.0.2 // indirect
6878
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
6979
github.com/onsi/ginkgo/v2 v2.22.1 // indirect
7080
github.com/onsi/gomega v1.36.2 // indirect
71-
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
81+
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
7282
github.com/pierrec/lz4/v4 v4.1.17 // indirect
7383
github.com/pkg/errors v0.9.1 // indirect
7484
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
7585
github.com/prometheus/client_model v0.6.1 // indirect
7686
github.com/prometheus/common v0.62.0 // indirect
7787
github.com/prometheus/procfs v0.15.1 // indirect
7888
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
79-
github.com/sagikazarmark/locafero v0.4.0 // indirect
89+
github.com/rs/zerolog v1.33.0 // indirect
90+
github.com/sagikazarmark/locafero v0.7.0 // indirect
8091
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
8192
github.com/sourcegraph/conc v0.3.0 // indirect
82-
github.com/spf13/afero v1.11.0 // indirect
83-
github.com/spf13/cast v1.6.0 // indirect
93+
github.com/spf13/afero v1.12.0 // indirect
94+
github.com/spf13/cast v1.7.1 // indirect
95+
github.com/spf13/cobra v1.8.1 // indirect
8496
github.com/subosito/gotenv v1.6.0 // indirect
97+
github.com/vektra/mockery/v2 v2.53.1 // indirect
8598
github.com/x448/float16 v0.8.4 // indirect
8699
go.uber.org/multierr v1.11.0 // indirect
87-
golang.org/x/crypto v0.32.0 // indirect
88-
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
89-
golang.org/x/net v0.34.0 // indirect
100+
golang.org/x/crypto v0.35.0 // indirect
101+
golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac // indirect
102+
golang.org/x/mod v0.23.0 // indirect
103+
golang.org/x/net v0.36.0 // indirect
90104
golang.org/x/oauth2 v0.26.0 // indirect
91105
golang.org/x/sync v0.11.0 // indirect
92106
golang.org/x/sys v0.30.0 // indirect
93107
golang.org/x/term v0.29.0 // indirect
94108
golang.org/x/text v0.22.0 // indirect
95109
golang.org/x/time v0.10.0 // indirect
110+
golang.org/x/tools v0.30.0 // indirect
96111
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
97112
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
98113
gopkg.in/inf.v0 v0.9.1 // indirect
@@ -101,6 +116,7 @@ require (
101116
k8s.io/apiextensions-apiserver v0.32.1 // indirect
102117
k8s.io/klog/v2 v2.130.1 // indirect
103118
k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 // indirect
119+
sigs.k8s.io/controller-tools v0.17.2 // indirect
104120
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
105121
sigs.k8s.io/structured-merge-diff/v4 v4.5.0 // indirect
106122
sigs.k8s.io/yaml v1.4.0 // indirect

0 commit comments

Comments
 (0)