Skip to content

Commit 4d8e79e

Browse files
stttssttts
committed
Switch to builder focused cluster support
Signed-off-by: Dr. Stefan Schimanski <[email protected]>
1 parent d023d3e commit 4d8e79e

File tree

13 files changed

+452
-373
lines changed

13 files changed

+452
-373
lines changed

‎pkg/builder/controller.go

+193-79
Large diffs are not rendered by default.

‎pkg/config/controller.go

+11-4
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,15 @@ type Controller struct {
4747
// Defaults to true, which means the controller will use leader election.
4848
NeedLeaderElection *bool
4949

50-
// WatchProviderClusters indicates whether the controller should
51-
// only watch clusters that are engaged by the cluster provider. Defaults to false
52-
// if no provider is set, and to true if a provider is set.
53-
WatchProviderClusters *bool
50+
// EngageWithDefaultCluster indicates whether the controller should engage
51+
// with the default cluster. This default to false if a cluster provider
52+
// is configured, and to true otherwise.
53+
//
54+
// This is an experimental feature and is subject to change.
55+
EngageWithDefaultCluster *bool
56+
57+
// EngageWithProvidedClusters indicates whether the controller should engage
58+
// with the provided clusters of the manager. This defaults to true if a
59+
// cluster provider is set, and to false otherwise.
60+
EngageWithProviderClusters *bool
5461
}

‎pkg/controller/controller.go

+10-14
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@ import (
2424
"github.com/go-logr/logr"
2525
"k8s.io/client-go/util/workqueue"
2626
"k8s.io/klog/v2"
27-
"k8s.io/utils/ptr"
28-
2927
"sigs.k8s.io/controller-runtime/pkg/handler"
3028
"sigs.k8s.io/controller-runtime/pkg/internal/controller"
3129
"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -64,10 +62,16 @@ type Options struct {
6462
// to each reconciliation via the context field.
6563
LogConstructor func(request *reconcile.Request) logr.Logger
6664

67-
// WatchProviderClusters indicates whether the controller should
68-
// only watch clusters that are engaged by the cluster provider. Defaults to false
69-
// if no provider is set, and to true if a provider is set.
70-
WatchProviderClusters *bool
65+
// EngageWithDefaultCluster indicates whether the controller should engage
66+
// with the default cluster of a manager. This defaults to false through the
67+
// global controller options of the manager if a cluster provider is set,
68+
// and to true otherwise. Here it can be overridden.
69+
EngageWithDefaultCluster *bool
70+
// EngageWithProvidedClusters indicates whether the controller should engage
71+
// with the provided clusters of a manager. This defaults to true through the
72+
// global controller options of the manager if a cluster provider is set,
73+
// and to false otherwise. Here it can be overridden.
74+
EngageWithProviderClusters *bool
7175
}
7276

7377
// Controller implements a Kubernetes API. A Controller manages a work queue fed reconcile.Requests
@@ -161,13 +165,6 @@ func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller
161165
options.NeedLeaderElection = mgr.GetControllerOptions().NeedLeaderElection
162166
}
163167

164-
if options.WatchProviderClusters == nil {
165-
options.WatchProviderClusters = mgr.GetControllerOptions().WatchProviderClusters
166-
if options.WatchProviderClusters == nil { // should never happen
167-
options.WatchProviderClusters = ptr.To(false)
168-
}
169-
}
170-
171168
// Create controller with dependencies set
172169
return &controller.Controller{
173170
Do: options.Reconciler,
@@ -182,7 +179,6 @@ func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller
182179
LogConstructor: options.LogConstructor,
183180
RecoverPanic: options.RecoverPanic,
184181
LeaderElected: options.NeedLeaderElection,
185-
WatchProviderClusters: *options.WatchProviderClusters,
186182
}, nil
187183
}
188184

‎pkg/controller/multicluster.go

+106
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controller
18+
19+
import (
20+
"context"
21+
"sync"
22+
23+
"sigs.k8s.io/controller-runtime/pkg/cluster"
24+
)
25+
26+
// MultiClusterController is a Controller that is aware of the Cluster it is
27+
// running in. It engage and disengage clusters dynamically, starting the
28+
// watches and stopping them.
29+
type MultiClusterController interface {
30+
cluster.AwareRunnable
31+
Controller
32+
}
33+
34+
// ClusterWatcher starts watches for a given Cluster. The ctx should be
35+
// used to cancel the watch when the Cluster is disengaged.
36+
type ClusterWatcher interface {
37+
Watch(ctx context.Context, cl cluster.Cluster) error
38+
}
39+
40+
// NewMultiClusterController creates a new MultiClusterController for the given
41+
// controller with the given ClusterWatcher.
42+
func NewMultiClusterController(c Controller, watcher ClusterWatcher) MultiClusterController {
43+
return &multiClusterController{
44+
Controller: c,
45+
watcher: watcher,
46+
clusters: map[string]struct{}{},
47+
}
48+
}
49+
50+
type multiClusterController struct {
51+
Controller
52+
watcher ClusterWatcher
53+
54+
lock sync.Mutex
55+
clusters map[string]struct{}
56+
}
57+
58+
// Engage gets called when the runnable should start operations for the given Cluster.
59+
func (c *multiClusterController) Engage(clusterCtx context.Context, cl cluster.Cluster) error {
60+
c.lock.Lock()
61+
defer c.lock.Unlock()
62+
63+
if _, ok := c.clusters[cl.Name()]; ok {
64+
return nil
65+
}
66+
67+
// pass through in case the controller itself is cluster aware
68+
if ctrl, ok := c.Controller.(cluster.AwareRunnable); ok {
69+
if err := ctrl.Engage(clusterCtx, cl); err != nil {
70+
return err
71+
}
72+
}
73+
74+
// start watches on the cluster
75+
if err := c.watcher.Watch(clusterCtx, cl); err != nil {
76+
if ctrl, ok := c.Controller.(cluster.AwareRunnable); ok {
77+
if err := ctrl.Disengage(clusterCtx, cl); err != nil {
78+
return err
79+
}
80+
}
81+
return err
82+
}
83+
c.clusters[cl.Name()] = struct{}{}
84+
85+
return nil
86+
}
87+
88+
// Disengage gets called when the runnable should stop operations for the given Cluster.
89+
func (c *multiClusterController) Disengage(ctx context.Context, cl cluster.Cluster) error {
90+
c.lock.Lock()
91+
defer c.lock.Unlock()
92+
93+
if _, ok := c.clusters[cl.Name()]; !ok {
94+
return nil
95+
}
96+
delete(c.clusters, cl.Name())
97+
98+
// pass through in case the controller itself is cluster aware
99+
if ctrl, ok := c.Controller.(cluster.AwareRunnable); ok {
100+
if err := ctrl.Disengage(ctx, cl); err != nil {
101+
return err
102+
}
103+
}
104+
105+
return nil
106+
}

‎pkg/handler/cluster.go

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package handler
18+
19+
import (
20+
"context"
21+
"time"
22+
23+
"k8s.io/client-go/util/workqueue"
24+
"sigs.k8s.io/controller-runtime/pkg/event"
25+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
26+
)
27+
28+
// ForCluster wraps an EventHandler and adds the cluster name to the reconcile.Requests.
29+
func ForCluster(clusterName string, h EventHandler) EventHandler {
30+
return &clusterAwareHandler{
31+
clusterName: clusterName,
32+
handler: h,
33+
}
34+
}
35+
36+
type clusterAwareHandler struct {
37+
handler EventHandler
38+
clusterName string
39+
}
40+
41+
var _ EventHandler = &clusterAwareHandler{}
42+
43+
func (c *clusterAwareHandler) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) {
44+
c.handler.Create(ctx, evt, &clusterWorkqueue{RateLimitingInterface: q, clusterName: c.clusterName})
45+
}
46+
47+
func (c *clusterAwareHandler) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
48+
c.handler.Update(ctx, evt, &clusterWorkqueue{RateLimitingInterface: q, clusterName: c.clusterName})
49+
}
50+
51+
func (c *clusterAwareHandler) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
52+
c.handler.Delete(ctx, evt, &clusterWorkqueue{RateLimitingInterface: q, clusterName: c.clusterName})
53+
}
54+
55+
func (c *clusterAwareHandler) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) {
56+
c.handler.Generic(ctx, evt, &clusterWorkqueue{RateLimitingInterface: q, clusterName: c.clusterName})
57+
}
58+
59+
// clusterWorkqueue is a wrapper around a RateLimitingInterface that adds the
60+
// cluster name to the reconcile.Requests
61+
type clusterWorkqueue struct {
62+
workqueue.RateLimitingInterface
63+
clusterName string
64+
}
65+
66+
func (q *clusterWorkqueue) AddAfter(item interface{}, duration time.Duration) {
67+
req := item.(reconcile.Request)
68+
req.ClusterName = q.clusterName
69+
q.RateLimitingInterface.AddAfter(req, duration)
70+
}
71+
72+
func (q *clusterWorkqueue) Add(item interface{}) {
73+
req := item.(reconcile.Request)
74+
req.ClusterName = q.clusterName
75+
q.RateLimitingInterface.Add(req)
76+
}
77+
78+
func (q *clusterWorkqueue) AddRateLimited(item interface{}) {
79+
req := item.(reconcile.Request)
80+
req.ClusterName = q.clusterName
81+
q.RateLimitingInterface.AddRateLimited(req)
82+
}

‎pkg/handler/enqueue.go

+21-61
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import (
2121

2222
"k8s.io/apimachinery/pkg/types"
2323
"k8s.io/client-go/util/workqueue"
24-
"sigs.k8s.io/controller-runtime/pkg/cluster"
2524
"sigs.k8s.io/controller-runtime/pkg/event"
2625
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
2726
"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -36,53 +35,33 @@ var _ EventHandler = &EnqueueRequestForObject{}
3635
// EnqueueRequestForObject enqueues a Request containing the Name and Namespace of the object that is the source of the Event.
3736
// (e.g. the created / deleted / updated objects Name and Namespace). handler.EnqueueRequestForObject is used by almost all
3837
// Controllers that have associated Resources (e.g. CRDs) to reconcile the associated Resource.
39-
type EnqueueRequestForObject struct {
40-
cluster cluster.Cluster
41-
}
38+
type EnqueueRequestForObject struct{}
4239

4340
// Create implements EventHandler.
4441
func (e *EnqueueRequestForObject) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) {
4542
if evt.Object == nil {
4643
enqueueLog.Error(nil, "CreateEvent received with no metadata", "event", evt)
4744
return
4845
}
49-
var clusterName string
50-
if e.cluster != nil {
51-
clusterName = e.cluster.Name()
52-
}
53-
q.Add(reconcile.Request{
54-
ClusterName: clusterName,
55-
NamespacedName: types.NamespacedName{
56-
Name: evt.Object.GetName(),
57-
Namespace: evt.Object.GetNamespace(),
58-
},
59-
})
46+
q.Add(reconcile.Request{NamespacedName: types.NamespacedName{
47+
Name: evt.Object.GetName(),
48+
Namespace: evt.Object.GetNamespace(),
49+
}})
6050
}
6151

6252
// Update implements EventHandler.
6353
func (e *EnqueueRequestForObject) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
64-
var clusterName string
65-
if e.cluster != nil {
66-
clusterName = e.cluster.Name()
67-
}
68-
6954
switch {
7055
case evt.ObjectNew != nil:
71-
q.Add(reconcile.Request{
72-
ClusterName: clusterName,
73-
NamespacedName: types.NamespacedName{
74-
Name: evt.ObjectNew.GetName(),
75-
Namespace: evt.ObjectNew.GetNamespace(),
76-
},
77-
})
56+
q.Add(reconcile.Request{NamespacedName: types.NamespacedName{
57+
Name: evt.ObjectNew.GetName(),
58+
Namespace: evt.ObjectNew.GetNamespace(),
59+
}})
7860
case evt.ObjectOld != nil:
79-
q.Add(reconcile.Request{
80-
ClusterName: clusterName,
81-
NamespacedName: types.NamespacedName{
82-
Name: evt.ObjectOld.GetName(),
83-
Namespace: evt.ObjectOld.GetNamespace(),
84-
},
85-
})
61+
q.Add(reconcile.Request{NamespacedName: types.NamespacedName{
62+
Name: evt.ObjectOld.GetName(),
63+
Namespace: evt.ObjectOld.GetNamespace(),
64+
}})
8665
default:
8766
enqueueLog.Error(nil, "UpdateEvent received with no metadata", "event", evt)
8867
}
@@ -94,17 +73,10 @@ func (e *EnqueueRequestForObject) Delete(ctx context.Context, evt event.DeleteEv
9473
enqueueLog.Error(nil, "DeleteEvent received with no metadata", "event", evt)
9574
return
9675
}
97-
var clusterName string
98-
if e.cluster != nil {
99-
clusterName = e.cluster.Name()
100-
}
101-
q.Add(reconcile.Request{
102-
ClusterName: clusterName,
103-
NamespacedName: types.NamespacedName{
104-
Name: evt.Object.GetName(),
105-
Namespace: evt.Object.GetNamespace(),
106-
},
107-
})
76+
q.Add(reconcile.Request{NamespacedName: types.NamespacedName{
77+
Name: evt.Object.GetName(),
78+
Namespace: evt.Object.GetNamespace(),
79+
}})
10880
}
10981

11082
// Generic implements EventHandler.
@@ -113,20 +85,8 @@ func (e *EnqueueRequestForObject) Generic(ctx context.Context, evt event.Generic
11385
enqueueLog.Error(nil, "GenericEvent received with no metadata", "event", evt)
11486
return
11587
}
116-
var clusterName string
117-
if e.cluster != nil {
118-
clusterName = e.cluster.Name()
119-
}
120-
q.Add(reconcile.Request{
121-
ClusterName: clusterName,
122-
NamespacedName: types.NamespacedName{
123-
Name: evt.Object.GetName(),
124-
Namespace: evt.Object.GetNamespace(),
125-
},
126-
})
127-
}
128-
129-
// DeepCopyFor implements cluster.AwareDeepCopy[EventHandler].
130-
func (e *EnqueueRequestForObject) DeepCopyFor(c cluster.Cluster) DeepCopyableEventHandler {
131-
return &EnqueueRequestForObject{cluster: c}
88+
q.Add(reconcile.Request{NamespacedName: types.NamespacedName{
89+
Name: evt.Object.GetName(),
90+
Namespace: evt.Object.GetNamespace(),
91+
}})
13292
}

0 commit comments

Comments
 (0)