Skip to content

Commit

Permalink
kubernetes: set feature gate ControlPlaneKubeletLocalMode
Browse files Browse the repository at this point in the history 
Co-Authored-By: Leonard Cohnen <[email protected]>
  • Loading branch information
@burgerdev @3u13r
burgerdev and 3u13r committed Feb 18, 2025
1 parent 304dc79 commit 37900d9
Show file tree
Hide file tree
Showing 4 changed files with 89 additions and 55 deletions.
1 change: 1 addition & 0 deletions bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ go_library(
"@io_k8s_kubelet//config/v1beta1",
"@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3",
"@io_k8s_kubernetes//cmd/kubeadm/app/constants",
"@org_golang_x_mod//semver",
],
)

Expand Down
8 changes: 7 additions & 1 deletion bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/certificate"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"golang.org/x/mod/semver"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeletconf "k8s.io/kubelet/config/v1beta1"
Expand All @@ -38,7 +39,7 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl
cloudProvider = "external"
}

return KubeadmInitYAML{
initConfig := KubeadmInitYAML{
InitConfiguration: kubeadm.InitConfiguration{
TypeMeta: metav1.TypeMeta{
APIVersion: kubeadm.SchemeGroupVersion.String(),
Expand Down Expand Up @@ -157,6 +158,11 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl
TLSPrivateKeyFile: certificate.KeyFilename,
},
}

if semver.Compare(clusterVersion, "v1.31.0") >= 0 {
initConfig.ClusterConfiguration.FeatureGates = map[string]bool{"ControlPlaneKubeletLocalMode": true}
}
return initConfig
}

// JoinConfiguration returns a new kubeadm join configuration.
Expand Down
132 changes: 78 additions & 54 deletions internal/constellation/kubecmd/kubecmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,18 @@ func (k *KubeCmd) UpgradeKubernetesVersion(ctx context.Context, kubernetesVersio
)
}

// TODO(burgerdev): remove after releasing v2.19
// Workaround for https://github.com/kubernetes/kubernetes/issues/127316: force kubelet to
// connect to the local API server.
if err := k.patchKubeadmConfig(ctx, func(cc *kubeadm.ClusterConfiguration) {
if cc.FeatureGates == nil {
cc.FeatureGates = map[string]bool{}
}
cc.FeatureGates["ControlPlaneKubeletLocalMode"] = true
}); err != nil {
return fmt.Errorf("setting FeatureGate ControlPlaneKubeletLocalMode: %w", err)
}

versionConfig, ok := versions.VersionConfigs[kubernetesVersion]
if !ok {
return fmt.Errorf("skipping Kubernetes upgrade: %w", compatibility.NewInvalidUpgradeError(
Expand Down Expand Up @@ -236,65 +248,32 @@ func (k *KubeCmd) ApplyJoinConfig(ctx context.Context, newAttestConfig config.At
// ExtendClusterConfigCertSANs extends the ClusterConfig stored under "kube-system/kubeadm-config" with the given SANs.
// Empty strings are ignored, existing SANs are preserved.
func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error {
var kubeadmConfig *corev1.ConfigMap
if err := k.retryAction(ctx, func(ctx context.Context) error {
var err error
kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap)
return err
}); err != nil {
return fmt.Errorf("retrieving current kubeadm-config: %w", err)
}

clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey]
if !ok {
return errors.New("ClusterConfiguration missing from kubeadm-config")
}

var clusterConfiguration kubeadm.ClusterConfiguration
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil {
return fmt.Errorf("decoding cluster configuration data: %w", err)
}

existingSANs := make(map[string]struct{})
for _, existingSAN := range clusterConfiguration.APIServer.CertSANs {
existingSANs[existingSAN] = struct{}{}
}

var missingSANs []string
for _, san := range alternativeNames {
if san == "" {
continue // skip empty SANs
if err := k.patchKubeadmConfig(ctx, func(clusterConfiguration *kubeadm.ClusterConfiguration) {
existingSANs := make(map[string]struct{})
for _, existingSAN := range clusterConfiguration.APIServer.CertSANs {
existingSANs[existingSAN] = struct{}{}
}
if _, ok := existingSANs[san]; !ok {
missingSANs = append(missingSANs, san)
existingSANs[san] = struct{}{} // make sure we don't add the same SAN twice
}
}

if len(missingSANs) == 0 {
k.log.Debug("No new SANs to add to the cluster's apiserver SAN field")
return nil
}
k.log.Debug("Extending the cluster's apiserver SAN field", "certSANs", strings.Join(missingSANs, ", "))

clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...)
sort.Strings(clusterConfiguration.APIServer.CertSANs)
var missingSANs []string
for _, san := range alternativeNames {
if san == "" {
continue // skip empty SANs
}
if _, ok := existingSANs[san]; !ok {
missingSANs = append(missingSANs, san)
existingSANs[san] = struct{}{} // make sure we don't add the same SAN twice
}
}

opt := k8sjson.SerializerOptions{Yaml: true}
serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt)
encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion)
newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration)
if err != nil {
return fmt.Errorf("marshaling ClusterConfiguration: %w", err)
}
if len(missingSANs) == 0 {
k.log.Debug("No new SANs to add to the cluster's apiserver SAN field")
}
k.log.Debug("Extending the cluster's apiserver SAN field", "certSANs", strings.Join(missingSANs, ", "))

kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML)
k.log.Debug("Triggering kubeadm config update now")
if err = k.retryAction(ctx, func(ctx context.Context) error {
_, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig)
return err
clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...)
sort.Strings(clusterConfiguration.APIServer.CertSANs)
}); err != nil {
return fmt.Errorf("setting new kubeadm config: %w", err)
return fmt.Errorf("extending ClusterConfig.CertSANs: %w", err)
}

k.log.Debug("Successfully extended the cluster's apiserver SAN field")
Expand Down Expand Up @@ -462,6 +441,51 @@ func (k *KubeCmd) retryAction(ctx context.Context, action func(ctx context.Conte
return retrier.Do(ctx)
}

// patchKubeadmConfig fetches and unpacks the kube-system/kubeadm-config ClusterConfiguration entry,
// runs doPatch on it and uploads the result.
func (k *KubeCmd) patchKubeadmConfig(ctx context.Context, doPatch func(*kubeadm.ClusterConfiguration)) error {
var kubeadmConfig *corev1.ConfigMap
if err := k.retryAction(ctx, func(ctx context.Context) error {
var err error
kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap)
return err
}); err != nil {
return fmt.Errorf("retrieving current kubeadm-config: %w", err)
}

clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey]
if !ok {
return errors.New("ClusterConfiguration missing from kubeadm-config")
}

var clusterConfiguration kubeadm.ClusterConfiguration
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil {
return fmt.Errorf("decoding cluster configuration data: %w", err)
}

doPatch(&clusterConfiguration)

opt := k8sjson.SerializerOptions{Yaml: true}
serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt)
encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion)
newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration)
if err != nil {
return fmt.Errorf("marshaling ClusterConfiguration: %w", err)
}

kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML)
k.log.Debug("Triggering kubeadm config update now")
if err = k.retryAction(ctx, func(ctx context.Context) error {
_, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig)
return err
}); err != nil {
return fmt.Errorf("setting new kubeadm config: %w", err)
}

k.log.Debug("Successfully patched the cluster's kubeadm-config")
return nil
}

func checkForApplyError(expected, actual updatev1alpha1.NodeVersion) error {
var err error
switch {
Expand Down
3 changes: 3 additions & 0 deletions internal/constellation/kubecmd/kubecmd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,9 @@ func TestUpgradeKubernetesVersion(t *testing.T) {
}
kubectl := &stubKubectl{
unstructuredInterface: unstructuredClient,
configMaps: map[string]*corev1.ConfigMap{
constants.KubeadmConfigMap: {Data: map[string]string{"ClusterConfiguration": kubeadmClusterConfigurationV1Beta4}},
},
}
if tc.customClientFn != nil {
kubectl.unstructuredInterface = tc.customClientFn(nodeVersion)
Expand Down

0 comments on commit 37900d9

Please sign in to comment.