Skip to content

Commit 37900d9

Browse files
burgerdev3u13r
burgerdev
and
3u13r
committed
kubernetes: set feature gate ControlPlaneKubeletLocalMode
Co-Authored-By: Leonard Cohnen <[email protected]>
1 parent 304dc79 commit 37900d9

File tree

4 files changed

+89
-55
lines changed

4 files changed

+89
-55
lines changed

‎bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ go_library(
2828
"@io_k8s_kubelet//config/v1beta1",
2929
"@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3",
3030
"@io_k8s_kubernetes//cmd/kubeadm/app/constants",
31+
"@org_golang_x_mod//semver",
3132
],
3233
)
3334

‎bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/certificate"
1313
"github.com/edgelesssys/constellation/v2/internal/constants"
1414
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
15+
"golang.org/x/mod/semver"
1516
corev1 "k8s.io/api/core/v1"
1617
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1718
kubeletconf "k8s.io/kubelet/config/v1beta1"
@@ -38,7 +39,7 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl
3839
cloudProvider = "external"
3940
}
4041

41-
return KubeadmInitYAML{
42+
initConfig := KubeadmInitYAML{
4243
InitConfiguration: kubeadm.InitConfiguration{
4344
TypeMeta: metav1.TypeMeta{
4445
APIVersion: kubeadm.SchemeGroupVersion.String(),
@@ -157,6 +158,11 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl
157158
TLSPrivateKeyFile: certificate.KeyFilename,
158159
},
159160
}
161+
162+
if semver.Compare(clusterVersion, "v1.31.0") >= 0 {
163+
initConfig.ClusterConfiguration.FeatureGates = map[string]bool{"ControlPlaneKubeletLocalMode": true}
164+
}
165+
return initConfig
160166
}
161167

162168
// JoinConfiguration returns a new kubeadm join configuration.

‎internal/constellation/kubecmd/kubecmd.go

Lines changed: 78 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,18 @@ func (k *KubeCmd) UpgradeKubernetesVersion(ctx context.Context, kubernetesVersio
131131
)
132132
}
133133

134+
// TODO(burgerdev): remove after releasing v2.19
135+
// Workaround for https://github.com/kubernetes/kubernetes/issues/127316: force kubelet to
136+
// connect to the local API server.
137+
if err := k.patchKubeadmConfig(ctx, func(cc *kubeadm.ClusterConfiguration) {
138+
if cc.FeatureGates == nil {
139+
cc.FeatureGates = map[string]bool{}
140+
}
141+
cc.FeatureGates["ControlPlaneKubeletLocalMode"] = true
142+
}); err != nil {
143+
return fmt.Errorf("setting FeatureGate ControlPlaneKubeletLocalMode: %w", err)
144+
}
145+
134146
versionConfig, ok := versions.VersionConfigs[kubernetesVersion]
135147
if !ok {
136148
return fmt.Errorf("skipping Kubernetes upgrade: %w", compatibility.NewInvalidUpgradeError(
@@ -236,65 +248,32 @@ func (k *KubeCmd) ApplyJoinConfig(ctx context.Context, newAttestConfig config.At
236248
// ExtendClusterConfigCertSANs extends the ClusterConfig stored under "kube-system/kubeadm-config" with the given SANs.
237249
// Empty strings are ignored, existing SANs are preserved.
238250
func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error {
239-
var kubeadmConfig *corev1.ConfigMap
240-
if err := k.retryAction(ctx, func(ctx context.Context) error {
241-
var err error
242-
kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap)
243-
return err
244-
}); err != nil {
245-
return fmt.Errorf("retrieving current kubeadm-config: %w", err)
246-
}
247-
248-
clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey]
249-
if !ok {
250-
return errors.New("ClusterConfiguration missing from kubeadm-config")
251-
}
252-
253-
var clusterConfiguration kubeadm.ClusterConfiguration
254-
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil {
255-
return fmt.Errorf("decoding cluster configuration data: %w", err)
256-
}
257-
258-
existingSANs := make(map[string]struct{})
259-
for _, existingSAN := range clusterConfiguration.APIServer.CertSANs {
260-
existingSANs[existingSAN] = struct{}{}
261-
}
262-
263-
var missingSANs []string
264-
for _, san := range alternativeNames {
265-
if san == "" {
266-
continue // skip empty SANs
251+
if err := k.patchKubeadmConfig(ctx, func(clusterConfiguration *kubeadm.ClusterConfiguration) {
252+
existingSANs := make(map[string]struct{})
253+
for _, existingSAN := range clusterConfiguration.APIServer.CertSANs {
254+
existingSANs[existingSAN] = struct{}{}
267255
}
268-
if _, ok := existingSANs[san]; !ok {
269-
missingSANs = append(missingSANs, san)
270-
existingSANs[san] = struct{}{} // make sure we don't add the same SAN twice
271-
}
272-
}
273256

274-
if len(missingSANs) == 0 {
275-
k.log.Debug("No new SANs to add to the cluster's apiserver SAN field")
276-
return nil
277-
}
278-
k.log.Debug("Extending the cluster's apiserver SAN field", "certSANs", strings.Join(missingSANs, ", "))
279-
280-
clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...)
281-
sort.Strings(clusterConfiguration.APIServer.CertSANs)
257+
var missingSANs []string
258+
for _, san := range alternativeNames {
259+
if san == "" {
260+
continue // skip empty SANs
261+
}
262+
if _, ok := existingSANs[san]; !ok {
263+
missingSANs = append(missingSANs, san)
264+
existingSANs[san] = struct{}{} // make sure we don't add the same SAN twice
265+
}
266+
}
282267

283-
opt := k8sjson.SerializerOptions{Yaml: true}
284-
serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt)
285-
encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion)
286-
newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration)
287-
if err != nil {
288-
return fmt.Errorf("marshaling ClusterConfiguration: %w", err)
289-
}
268+
if len(missingSANs) == 0 {
269+
k.log.Debug("No new SANs to add to the cluster's apiserver SAN field")
270+
}
271+
k.log.Debug("Extending the cluster's apiserver SAN field", "certSANs", strings.Join(missingSANs, ", "))
290272

291-
kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML)
292-
k.log.Debug("Triggering kubeadm config update now")
293-
if err = k.retryAction(ctx, func(ctx context.Context) error {
294-
_, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig)
295-
return err
273+
clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...)
274+
sort.Strings(clusterConfiguration.APIServer.CertSANs)
296275
}); err != nil {
297-
return fmt.Errorf("setting new kubeadm config: %w", err)
276+
return fmt.Errorf("extending ClusterConfig.CertSANs: %w", err)
298277
}
299278

300279
k.log.Debug("Successfully extended the cluster's apiserver SAN field")
@@ -462,6 +441,51 @@ func (k *KubeCmd) retryAction(ctx context.Context, action func(ctx context.Conte
462441
return retrier.Do(ctx)
463442
}
464443

444+
// patchKubeadmConfig fetches and unpacks the kube-system/kubeadm-config ClusterConfiguration entry,
445+
// runs doPatch on it and uploads the result.
446+
func (k *KubeCmd) patchKubeadmConfig(ctx context.Context, doPatch func(*kubeadm.ClusterConfiguration)) error {
447+
var kubeadmConfig *corev1.ConfigMap
448+
if err := k.retryAction(ctx, func(ctx context.Context) error {
449+
var err error
450+
kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap)
451+
return err
452+
}); err != nil {
453+
return fmt.Errorf("retrieving current kubeadm-config: %w", err)
454+
}
455+
456+
clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey]
457+
if !ok {
458+
return errors.New("ClusterConfiguration missing from kubeadm-config")
459+
}
460+
461+
var clusterConfiguration kubeadm.ClusterConfiguration
462+
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil {
463+
return fmt.Errorf("decoding cluster configuration data: %w", err)
464+
}
465+
466+
doPatch(&clusterConfiguration)
467+
468+
opt := k8sjson.SerializerOptions{Yaml: true}
469+
serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt)
470+
encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion)
471+
newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration)
472+
if err != nil {
473+
return fmt.Errorf("marshaling ClusterConfiguration: %w", err)
474+
}
475+
476+
kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML)
477+
k.log.Debug("Triggering kubeadm config update now")
478+
if err = k.retryAction(ctx, func(ctx context.Context) error {
479+
_, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig)
480+
return err
481+
}); err != nil {
482+
return fmt.Errorf("setting new kubeadm config: %w", err)
483+
}
484+
485+
k.log.Debug("Successfully patched the cluster's kubeadm-config")
486+
return nil
487+
}
488+
465489
func checkForApplyError(expected, actual updatev1alpha1.NodeVersion) error {
466490
var err error
467491
switch {

‎internal/constellation/kubecmd/kubecmd_test.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,9 @@ func TestUpgradeKubernetesVersion(t *testing.T) {
281281
}
282282
kubectl := &stubKubectl{
283283
unstructuredInterface: unstructuredClient,
284+
configMaps: map[string]*corev1.ConfigMap{
285+
constants.KubeadmConfigMap: {Data: map[string]string{"ClusterConfiguration": kubeadmClusterConfigurationV1Beta4}},
286+
},
284287
}
285288
if tc.customClientFn != nil {
286289
kubectl.unstructuredInterface = tc.customClientFn(nodeVersion)

0 commit comments

Comments
 (0)