From a2a25fd88e87661d900988f13a3729699014625c Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Thu, 10 Oct 2024 20:04:08 +0200 Subject: [PATCH] kubernetes: set feature gate ControlPlaneKubeletLocalMode Co-Authored-By: Leonard Cohnen --- .../internal/kubernetes/k8sapi/BUILD.bazel | 1 + .../kubernetes/k8sapi/kubeadm_config.go | 8 +- internal/constellation/kubecmd/kubecmd.go | 132 +++++++++++------- .../constellation/kubecmd/kubecmd_test.go | 3 + 4 files changed, 89 insertions(+), 55 deletions(-) diff --git a/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel b/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel index ef87085bd8..968a2b0114 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel +++ b/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel @@ -28,6 +28,7 @@ go_library( "@io_k8s_kubelet//config/v1beta1", "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3", "@io_k8s_kubernetes//cmd/kubeadm/app/constants", + "@org_golang_x_mod//semver", ], ) diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go index 430839a469..1ce387aeb4 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go @@ -12,6 +12,7 @@ import ( "github.com/edgelesssys/constellation/v2/bootstrapper/internal/certificate" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/kubernetes" + "golang.org/x/mod/semver" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletconf "k8s.io/kubelet/config/v1beta1" @@ -38,7 +39,7 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl cloudProvider = "external" } - return KubeadmInitYAML{ + initConfig := KubeadmInitYAML{ InitConfiguration: kubeadm.InitConfiguration{ TypeMeta: metav1.TypeMeta{ APIVersion: kubeadm.SchemeGroupVersion.String(), @@ -157,6 +158,11 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl TLSPrivateKeyFile: certificate.KeyFilename, }, } + + if semver.Compare(clusterVersion, "v1.31.0") >= 0 { + initConfig.ClusterConfiguration.FeatureGates = map[string]bool{"ControlPlaneKubeletLocalMode": true} + } + return initConfig } // JoinConfiguration returns a new kubeadm join configuration. diff --git a/internal/constellation/kubecmd/kubecmd.go b/internal/constellation/kubecmd/kubecmd.go index a193f2528e..dd3c4f9da7 100644 --- a/internal/constellation/kubecmd/kubecmd.go +++ b/internal/constellation/kubecmd/kubecmd.go @@ -131,6 +131,18 @@ func (k *KubeCmd) UpgradeKubernetesVersion(ctx context.Context, kubernetesVersio ) } + // TODO(burgerdev): remove after releasing v2.19 + // Workaround for https://github.com/kubernetes/kubernetes/issues/127316: force kubelet to + // connect to the local API server. + if err := k.patchKubeadmConfig(ctx, func(cc *kubeadm.ClusterConfiguration) { + if cc.FeatureGates == nil { + cc.FeatureGates = map[string]bool{} + } + cc.FeatureGates["ControlPlaneKubeletLocalMode"] = true + }); err != nil { + return fmt.Errorf("setting FeatureGate ControlPlaneKubeletLocalMode: %w", err) + } + versionConfig, ok := versions.VersionConfigs[kubernetesVersion] if !ok { return fmt.Errorf("skipping Kubernetes upgrade: %w", compatibility.NewInvalidUpgradeError( @@ -236,65 +248,32 @@ func (k *KubeCmd) ApplyJoinConfig(ctx context.Context, newAttestConfig config.At // ExtendClusterConfigCertSANs extends the ClusterConfig stored under "kube-system/kubeadm-config" with the given SANs. // Empty strings are ignored, existing SANs are preserved. func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error { - var kubeadmConfig *corev1.ConfigMap - if err := k.retryAction(ctx, func(ctx context.Context) error { - var err error - kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap) - return err - }); err != nil { - return fmt.Errorf("retrieving current kubeadm-config: %w", err) - } - - clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey] - if !ok { - return errors.New("ClusterConfiguration missing from kubeadm-config") - } - - var clusterConfiguration kubeadm.ClusterConfiguration - if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil { - return fmt.Errorf("decoding cluster configuration data: %w", err) - } - - existingSANs := make(map[string]struct{}) - for _, existingSAN := range clusterConfiguration.APIServer.CertSANs { - existingSANs[existingSAN] = struct{}{} - } - - var missingSANs []string - for _, san := range alternativeNames { - if san == "" { - continue // skip empty SANs + if err := k.patchKubeadmConfig(ctx, func(clusterConfiguration *kubeadm.ClusterConfiguration) { + existingSANs := make(map[string]struct{}) + for _, existingSAN := range clusterConfiguration.APIServer.CertSANs { + existingSANs[existingSAN] = struct{}{} } - if _, ok := existingSANs[san]; !ok { - missingSANs = append(missingSANs, san) - existingSANs[san] = struct{}{} // make sure we don't add the same SAN twice - } - } - if len(missingSANs) == 0 { - k.log.Debug("No new SANs to add to the cluster's apiserver SAN field") - return nil - } - k.log.Debug("Extending the cluster's apiserver SAN field", "certSANs", strings.Join(missingSANs, ", ")) - - clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...) - sort.Strings(clusterConfiguration.APIServer.CertSANs) + var missingSANs []string + for _, san := range alternativeNames { + if san == "" { + continue // skip empty SANs + } + if _, ok := existingSANs[san]; !ok { + missingSANs = append(missingSANs, san) + existingSANs[san] = struct{}{} // make sure we don't add the same SAN twice + } + } - opt := k8sjson.SerializerOptions{Yaml: true} - serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt) - encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion) - newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration) - if err != nil { - return fmt.Errorf("marshaling ClusterConfiguration: %w", err) - } + if len(missingSANs) == 0 { + k.log.Debug("No new SANs to add to the cluster's apiserver SAN field") + } + k.log.Debug("Extending the cluster's apiserver SAN field", "certSANs", strings.Join(missingSANs, ", ")) - kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML) - k.log.Debug("Triggering kubeadm config update now") - if err = k.retryAction(ctx, func(ctx context.Context) error { - _, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig) - return err + clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...) + sort.Strings(clusterConfiguration.APIServer.CertSANs) }); err != nil { - return fmt.Errorf("setting new kubeadm config: %w", err) + return fmt.Errorf("extending ClusterConfig.CertSANs: %w", err) } k.log.Debug("Successfully extended the cluster's apiserver SAN field") @@ -462,6 +441,51 @@ func (k *KubeCmd) retryAction(ctx context.Context, action func(ctx context.Conte return retrier.Do(ctx) } +// patchKubeadmConfig fetches and unpacks the kube-system/kubeadm-config ClusterConfiguration entry, +// runs doPatch on it and uploads the result. +func (k *KubeCmd) patchKubeadmConfig(ctx context.Context, doPatch func(*kubeadm.ClusterConfiguration)) error { + var kubeadmConfig *corev1.ConfigMap + if err := k.retryAction(ctx, func(ctx context.Context) error { + var err error + kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap) + return err + }); err != nil { + return fmt.Errorf("retrieving current kubeadm-config: %w", err) + } + + clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey] + if !ok { + return errors.New("ClusterConfiguration missing from kubeadm-config") + } + + var clusterConfiguration kubeadm.ClusterConfiguration + if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil { + return fmt.Errorf("decoding cluster configuration data: %w", err) + } + + doPatch(&clusterConfiguration) + + opt := k8sjson.SerializerOptions{Yaml: true} + serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt) + encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion) + newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration) + if err != nil { + return fmt.Errorf("marshaling ClusterConfiguration: %w", err) + } + + kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML) + k.log.Debug("Triggering kubeadm config update now") + if err = k.retryAction(ctx, func(ctx context.Context) error { + _, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig) + return err + }); err != nil { + return fmt.Errorf("setting new kubeadm config: %w", err) + } + + k.log.Debug("Successfully patched the cluster's kubeadm-config") + return nil +} + func checkForApplyError(expected, actual updatev1alpha1.NodeVersion) error { var err error switch { diff --git a/internal/constellation/kubecmd/kubecmd_test.go b/internal/constellation/kubecmd/kubecmd_test.go index 79eef03c34..c88dab9bda 100644 --- a/internal/constellation/kubecmd/kubecmd_test.go +++ b/internal/constellation/kubecmd/kubecmd_test.go @@ -281,6 +281,9 @@ func TestUpgradeKubernetesVersion(t *testing.T) { } kubectl := &stubKubectl{ unstructuredInterface: unstructuredClient, + configMaps: map[string]*corev1.ConfigMap{ + constants.KubeadmConfigMap: {Data: map[string]string{"ClusterConfiguration": kubeadmClusterConfigurationV1Beta4}}, + }, } if tc.customClientFn != nil { kubectl.unstructuredInterface = tc.customClientFn(nodeVersion)