Skip to content

Commit 3d41090

Browse files
sbueringersbueringer
authored
✨ Add CRD migrator, deprecate clusterctl upgrade CRD storage version migration (#11889)
* Add CRD migrator * Fixes * Mark test helper correctly * Reduce CRD RBAC * Add e2e test coverage * Fixup * Fixups * Fixup e2e test * Condense RBAC, fix copyright
1 parent 67245f5 commit 3d41090

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+3312
-63
lines changed

‎Makefile

+18-1
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,22 @@ generate-manifests-core: $(CONTROLLER_GEN) $(KUSTOMIZE) ## Generate manifests e.
310310
paths=./util/test/builder/... \
311311
crd:crdVersions=v1 \
312312
output:crd:dir=./util/test/builder/crd
313+
$(CONTROLLER_GEN) \
314+
paths=./controllers/crdmigrator/test/t1/... \
315+
crd:crdVersions=v1 \
316+
output:crd:dir=./controllers/crdmigrator/test/t1/crd
317+
$(CONTROLLER_GEN) \
318+
paths=./controllers/crdmigrator/test/t2/... \
319+
crd:crdVersions=v1 \
320+
output:crd:dir=./controllers/crdmigrator/test/t2/crd
321+
$(CONTROLLER_GEN) \
322+
paths=./controllers/crdmigrator/test/t3/... \
323+
crd:crdVersions=v1 \
324+
output:crd:dir=./controllers/crdmigrator/test/t3/crd
325+
$(CONTROLLER_GEN) \
326+
paths=./controllers/crdmigrator/test/t4/... \
327+
crd:crdVersions=v1 \
328+
output:crd:dir=./controllers/crdmigrator/test/t4/crd
313329

314330
.PHONY: generate-manifests-kubeadm-bootstrap
315331
generate-manifests-kubeadm-bootstrap: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc. for kubeadm bootstrap
@@ -384,7 +400,8 @@ generate-go-deepcopy-core: $(CONTROLLER_GEN) ## Generate deepcopy go code for co
384400
paths=./$(EXP_DIR)/runtime/hooks/api/... \
385401
paths=./internal/runtime/test/... \
386402
paths=./cmd/clusterctl/... \
387-
paths=./util/test/builder/...
403+
paths=./util/test/builder/... \
404+
paths=./controllers/crdmigrator/test/...
388405

389406
.PHONY: generate-go-deepcopy-kubeadm-bootstrap
390407
generate-go-deepcopy-kubeadm-bootstrap: $(CONTROLLER_GEN) ## Generate deepcopy go code for kubeadm bootstrap

‎api/v1beta1/common_types.go

+3
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,9 @@ const (
197197
// VariableDefinitionFromInline indicates a patch or variable was defined in the `.spec` of a ClusterClass
198198
// rather than from an external patch extension.
199199
VariableDefinitionFromInline = "inline"
200+
201+
// CRDMigrationObservedGenerationAnnotation indicates on a CRD for which generation CRD migration is completed.
202+
CRDMigrationObservedGenerationAnnotation = "crd-migration.cluster.x-k8s.io/observed-generation"
200203
)
201204

202205
// MachineSetPreflightCheck defines a valid MachineSet preflight check.

‎bootstrap/kubeadm/config/rbac/role.yaml

+29
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,25 @@ rules:
2424
verbs:
2525
- create
2626
- patch
27+
- apiGroups:
28+
- apiextensions.k8s.io
29+
resources:
30+
- customresourcedefinitions
31+
verbs:
32+
- get
33+
- list
34+
- watch
35+
- apiGroups:
36+
- apiextensions.k8s.io
37+
resourceNames:
38+
- kubeadmconfigs.bootstrap.cluster.x-k8s.io
39+
- kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io
40+
resources:
41+
- customresourcedefinitions
42+
- customresourcedefinitions/status
43+
verbs:
44+
- patch
45+
- update
2746
- apiGroups:
2847
- authentication.k8s.io
2948
resources:
@@ -50,6 +69,16 @@ rules:
5069
- patch
5170
- update
5271
- watch
72+
- apiGroups:
73+
- bootstrap.cluster.x-k8s.io
74+
resources:
75+
- kubeadmconfigtemplates
76+
verbs:
77+
- get
78+
- list
79+
- patch
80+
- update
81+
- watch
5382
- apiGroups:
5483
- cluster.x-k8s.io
5584
resources:

‎bootstrap/kubeadm/main.go

+33
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727

2828
"github.com/spf13/pflag"
2929
corev1 "k8s.io/api/core/v1"
30+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
3031
"k8s.io/apimachinery/pkg/labels"
3132
"k8s.io/apimachinery/pkg/runtime"
3233
"k8s.io/apimachinery/pkg/selection"
@@ -50,6 +51,7 @@ import (
5051
kubeadmbootstrapcontrollers "sigs.k8s.io/cluster-api/bootstrap/kubeadm/controllers"
5152
"sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/webhooks"
5253
"sigs.k8s.io/cluster-api/controllers/clustercache"
54+
"sigs.k8s.io/cluster-api/controllers/crdmigrator"
5355
"sigs.k8s.io/cluster-api/controllers/remote"
5456
expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
5557
"sigs.k8s.io/cluster-api/feature"
@@ -90,11 +92,13 @@ var (
9092
clusterConcurrency int
9193
clusterCacheConcurrency int
9294
kubeadmConfigConcurrency int
95+
skipCRDMigrationPhases []string
9396
tokenTTL time.Duration
9497
)
9598

9699
func init() {
97100
_ = clientgoscheme.AddToScheme(scheme)
101+
_ = apiextensionsv1.AddToScheme(scheme)
98102
_ = clusterv1.AddToScheme(scheme)
99103
_ = expv1.AddToScheme(scheme)
100104
_ = bootstrapv1alpha3.AddToScheme(scheme)
@@ -140,6 +144,9 @@ func InitFlags(fs *pflag.FlagSet) {
140144
fs.IntVar(&kubeadmConfigConcurrency, "kubeadmconfig-concurrency", 10,
141145
"Number of kubeadm configs to process simultaneously")
142146

147+
fs.StringArrayVar(&skipCRDMigrationPhases, "skip-crd-migration-phases", []string{},
148+
"List of CRD migration phases to skip. Valid values are: StorageVersionMigration, CleanupManagedFields.")
149+
143150
fs.DurationVar(&syncPeriod, "sync-period", 10*time.Minute,
144151
"The minimum interval at which watched resources are reconciled (e.g. 15m)")
145152

@@ -181,6 +188,11 @@ func InitFlags(fs *pflag.FlagSet) {
181188
// Add RBAC for the authorized diagnostics endpoint.
182189
// +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create
183190
// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create
191+
// ADD CRD RBAC for CRD Migrator.
192+
// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch
193+
// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions;customresourcedefinitions/status,verbs=update;patch,resourceNames=kubeadmconfigs.bootstrap.cluster.x-k8s.io;kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io
194+
// ADD CR RBAC for CRD Migrator.
195+
// +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=kubeadmconfigtemplates,verbs=get;list;watch;patch;update
184196

185197
func main() {
186198
InitFlags(pflag.CommandLine)
@@ -340,6 +352,27 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) {
340352
os.Exit(1)
341353
}
342354

355+
crdMigratorSkipPhases := []crdmigrator.Phase{}
356+
for _, p := range skipCRDMigrationPhases {
357+
crdMigratorSkipPhases = append(crdMigratorSkipPhases, crdmigrator.Phase(p))
358+
}
359+
if err := (&crdmigrator.CRDMigrator{
360+
Client: mgr.GetClient(),
361+
APIReader: mgr.GetAPIReader(),
362+
SkipCRDMigrationPhases: crdMigratorSkipPhases,
363+
// Note: The kubebuilder RBAC markers above has to be kept in sync
364+
// with the CRDs that should be migrated by this provider.
365+
Config: map[client.Object]crdmigrator.ByObjectConfig{
366+
&bootstrapv1.KubeadmConfig{}: {UseCache: true},
367+
&bootstrapv1.KubeadmConfigTemplate{}: {UseCache: false},
368+
},
369+
// The CRDMigrator is run with only concurrency 1 to ensure we don't overwhelm the apiserver by patching a
370+
// lot of CRs concurrently.
371+
}).SetupWithManager(ctx, mgr, concurrency(1)); err != nil {
372+
setupLog.Error(err, "Unable to create controller", "controller", "CRDMigrator")
373+
os.Exit(1)
374+
}
375+
343376
if err := (&kubeadmbootstrapcontrollers.KubeadmConfigReconciler{
344377
Client: mgr.GetClient(),
345378
SecretCachingClient: secretCachingClient,

‎cmd/clusterctl/client/cluster/upgrader.go

+29-22
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,9 @@ type UpgradePlan struct {
5757

5858
// UpgradeOptions defines the options used to upgrade installation.
5959
type UpgradeOptions struct {
60-
WaitProviders bool
61-
WaitProviderTimeout time.Duration
60+
WaitProviders bool
61+
WaitProviderTimeout time.Duration
62+
EnableCRDStorageVersionMigration bool
6263
}
6364

6465
// isPartialUpgrade returns true if at least one upgradeItem in the plan does not have a target version.
@@ -363,28 +364,30 @@ func (u *providerUpgrader) doUpgrade(ctx context.Context, upgradePlan *UpgradePl
363364
return providers[a].GetProviderType().Order() < providers[b].GetProviderType().Order()
364365
})
365366

366-
// Migrate CRs to latest CRD storage version, if necessary.
367-
// Note: We have to do this before the providers are scaled down or deleted
368-
// so conversion webhooks still work.
369-
for _, upgradeItem := range providers {
370-
// If there is not a specified next version, skip it (we are already up-to-date).
371-
if upgradeItem.NextVersion == "" {
372-
continue
373-
}
367+
if opts.EnableCRDStorageVersionMigration {
368+
// Migrate CRs to latest CRD storage version, if necessary.
369+
// Note: We have to do this before the providers are scaled down or deleted
370+
// so conversion webhooks still work.
371+
for _, upgradeItem := range providers {
372+
// If there is not a specified next version, skip it (we are already up-to-date).
373+
if upgradeItem.NextVersion == "" {
374+
continue
375+
}
374376

375-
// Gets the provider components for the target version.
376-
components, err := u.getUpgradeComponents(ctx, upgradeItem)
377-
if err != nil {
378-
return err
379-
}
377+
// Gets the provider components for the target version.
378+
components, err := u.getUpgradeComponents(ctx, upgradeItem)
379+
if err != nil {
380+
return err
381+
}
380382

381-
c, err := u.proxy.NewClient(ctx)
382-
if err != nil {
383-
return err
384-
}
383+
c, err := u.proxy.NewClient(ctx)
384+
if err != nil {
385+
return err
386+
}
385387

386-
if err := NewCRDMigrator(c).Run(ctx, components.Objs()); err != nil {
387-
return err
388+
if err := NewCRDMigrator(c).Run(ctx, components.Objs()); err != nil {
389+
return err
390+
}
388391
}
389392
}
390393

@@ -448,7 +451,11 @@ func (u *providerUpgrader) doUpgrade(ctx context.Context, upgradePlan *UpgradePl
448451
}
449452
}
450453

451-
return waitForProvidersReady(ctx, InstallOptions(opts), installQueue, u.proxy)
454+
installOpts := InstallOptions{
455+
WaitProviders: opts.WaitProviders,
456+
WaitProviderTimeout: opts.WaitProviderTimeout,
457+
}
458+
return waitForProvidersReady(ctx, installOpts, installQueue, u.proxy)
452459
}
453460

454461
func (u *providerUpgrader) scaleDownProvider(ctx context.Context, provider clusterctlv1.Provider) error {

‎cmd/clusterctl/client/upgrade.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,9 @@ type ApplyUpgradeOptions struct {
123123

124124
// WaitProviderTimeout sets the timeout per provider upgrade.
125125
WaitProviderTimeout time.Duration
126+
127+
// EnableCRDStorageVersionMigration enables storage version migration of CRDs.
128+
EnableCRDStorageVersionMigration bool
126129
}
127130

128131
func (c *clusterctlClient) ApplyUpgrade(ctx context.Context, options ApplyUpgradeOptions) error {
@@ -171,8 +174,9 @@ func (c *clusterctlClient) ApplyUpgrade(ctx context.Context, options ApplyUpgrad
171174
len(options.AddonProviders) > 0
172175

173176
opts := cluster.UpgradeOptions{
174-
WaitProviders: options.WaitProviders,
175-
WaitProviderTimeout: options.WaitProviderTimeout,
177+
WaitProviders: options.WaitProviders,
178+
WaitProviderTimeout: options.WaitProviderTimeout,
179+
EnableCRDStorageVersionMigration: options.EnableCRDStorageVersionMigration,
176180
}
177181

178182
// If we are upgrading a specific set of providers only, process the providers and call ApplyCustomPlan.

‎cmd/clusterctl/cmd/upgrade_apply.go

+29-23
Original file line numberDiff line numberDiff line change
@@ -28,18 +28,19 @@ import (
2828
)
2929

3030
type upgradeApplyOptions struct {
31-
kubeconfig string
32-
kubeconfigContext string
33-
contract string
34-
coreProvider string
35-
bootstrapProviders []string
36-
controlPlaneProviders []string
37-
infrastructureProviders []string
38-
ipamProviders []string
39-
runtimeExtensionProviders []string
40-
addonProviders []string
41-
waitProviders bool
42-
waitProviderTimeout int
31+
kubeconfig string
32+
kubeconfigContext string
33+
contract string
34+
coreProvider string
35+
bootstrapProviders []string
36+
controlPlaneProviders []string
37+
infrastructureProviders []string
38+
ipamProviders []string
39+
runtimeExtensionProviders []string
40+
addonProviders []string
41+
waitProviders bool
42+
waitProviderTimeout int
43+
enableCRDStorageVersionMigration bool
4344
}
4445

4546
var ua = &upgradeApplyOptions{}
@@ -94,6 +95,10 @@ func init() {
9495
"Wait for providers to be upgraded.")
9596
upgradeApplyCmd.Flags().IntVar(&ua.waitProviderTimeout, "wait-provider-timeout", 5*60,
9697
"Wait timeout per provider upgrade in seconds. This value is ignored if --wait-providers is false")
98+
upgradeApplyCmd.Flags().BoolVar(&ua.enableCRDStorageVersionMigration, "enable-crd-storage-version-migration", false,
99+
"Enable CRD storage version migration")
100+
_ = upgradeApplyCmd.Flags().MarkDeprecated("enable-crd-storage-version-migration",
101+
"Storage version migration during upgrades has been deprecated and will be removed in Cluster API v1.13")
97102
}
98103

99104
func runUpgradeApply() error {
@@ -120,16 +125,17 @@ func runUpgradeApply() error {
120125
}
121126

122127
return c.ApplyUpgrade(ctx, client.ApplyUpgradeOptions{
123-
Kubeconfig: client.Kubeconfig{Path: ua.kubeconfig, Context: ua.kubeconfigContext},
124-
Contract: ua.contract,
125-
CoreProvider: ua.coreProvider,
126-
BootstrapProviders: ua.bootstrapProviders,
127-
ControlPlaneProviders: ua.controlPlaneProviders,
128-
InfrastructureProviders: ua.infrastructureProviders,
129-
IPAMProviders: ua.ipamProviders,
130-
RuntimeExtensionProviders: ua.runtimeExtensionProviders,
131-
AddonProviders: ua.addonProviders,
132-
WaitProviders: ua.waitProviders,
133-
WaitProviderTimeout: time.Duration(ua.waitProviderTimeout) * time.Second,
128+
Kubeconfig: client.Kubeconfig{Path: ua.kubeconfig, Context: ua.kubeconfigContext},
129+
Contract: ua.contract,
130+
CoreProvider: ua.coreProvider,
131+
BootstrapProviders: ua.bootstrapProviders,
132+
ControlPlaneProviders: ua.controlPlaneProviders,
133+
InfrastructureProviders: ua.infrastructureProviders,
134+
IPAMProviders: ua.ipamProviders,
135+
RuntimeExtensionProviders: ua.runtimeExtensionProviders,
136+
AddonProviders: ua.addonProviders,
137+
WaitProviders: ua.waitProviders,
138+
WaitProviderTimeout: time.Duration(ua.waitProviderTimeout) * time.Second,
139+
EnableCRDStorageVersionMigration: ua.enableCRDStorageVersionMigration,
134140
})
135141
}

‎config/rbac/role.yaml

+26-8
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,28 @@ rules:
7373
- get
7474
- list
7575
- watch
76+
- apiGroups:
77+
- apiextensions.k8s.io
78+
resourceNames:
79+
- clusterclasses.cluster.x-k8s.io
80+
- clusterresourcesetbindings.addons.cluster.x-k8s.io
81+
- clusterresourcesets.addons.cluster.x-k8s.io
82+
- clusters.cluster.x-k8s.io
83+
- extensionconfigs.runtime.cluster.x-k8s.io
84+
- ipaddressclaims.ipam.cluster.x-k8s.io
85+
- ipaddresses.ipam.cluster.x-k8s.io
86+
- machinedeployments.cluster.x-k8s.io
87+
- machinedrainrules.cluster.x-k8s.io
88+
- machinehealthchecks.cluster.x-k8s.io
89+
- machinepools.cluster.x-k8s.io
90+
- machines.cluster.x-k8s.io
91+
- machinesets.cluster.x-k8s.io
92+
resources:
93+
- customresourcedefinitions
94+
- customresourcedefinitions/status
95+
verbs:
96+
- patch
97+
- update
7698
- apiGroups:
7799
- authentication.k8s.io
78100
resources:
@@ -93,6 +115,7 @@ rules:
93115
- clusters
94116
- clusters/finalizers
95117
- clusters/status
118+
- machinedrainrules
96119
- machinehealthchecks/finalizers
97120
- machinehealthchecks/status
98121
verbs:
@@ -125,21 +148,16 @@ rules:
125148
- patch
126149
- update
127150
- watch
128-
- apiGroups:
129-
- cluster.x-k8s.io
130-
resources:
131-
- machinedrainrules
132-
verbs:
133-
- get
134-
- list
135-
- watch
136151
- apiGroups:
137152
- ipam.cluster.x-k8s.io
138153
resources:
139154
- ipaddressclaims
155+
- ipaddresses
140156
verbs:
141157
- get
142158
- list
159+
- patch
160+
- update
143161
- watch
144162
- apiGroups:
145163
- runtime.cluster.x-k8s.io

0 commit comments

Comments
 (0)