From 13b3d38af444062be5f21641a10ec49571c07b8b Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Sun, 24 May 2026 15:49:45 +0300 Subject: [PATCH] feat(machined): allow kubelet kubeconfig tls-server-name override for KubePrism Add an optional `tlsServerName` field to `machine.features.kubePrism`. When set, the generated kubelet kubeconfig carries `clusters[0].cluster.tls-server-name` set to that value, so the kubelet uses it for SNI and certificate hostname verification while still dialing `https://127.0.0.1:` as the TCP destination. Without this knob, the endpoint in the kubelet kubeconfig is hardcoded to `https://127.0.0.1:` (see `internal/app/machined/pkg/controllers/secrets/kubelet.go`), which forces SNI=`127.0.0.1` over the proxied connection. In setups where KubePrism's upstream apiserver is reached through an SNI-routing L4 proxy (for example nginx-ingress in ssl-passthrough mode in front of a Kamaji-hosted apiserver), SNI=`127.0.0.1` does not match any route and the proxy serves a fallback certificate, so the kubelet rejects the connection with `cannot validate certificate for 127.0.0.1 because it doesn't contain any IP SANs`. Setting `tls-server-name` in the kubeconfig is the Kubernetes-native way to decouple the TCP destination from the TLS SNI/hostname check, and matches how kubeconfigs already support this field. When the field is empty (default), behavior is unchanged. Refs siderolabs/talos#13424 Signed-off-by: IvanHunters --- .../pkg/controllers/k8s/kubelet_service.go | 6 +- .../pkg/controllers/secrets/kubelet.go | 3 + .../pkg/controllers/secrets/kubelet_test.go | 77 +++++++++++++++++++ pkg/machinery/config/config/machine.go | 1 + .../types/v1alpha1/v1alpha1_features.go | 5 ++ .../config/types/v1alpha1/v1alpha1_types.go | 16 ++++ .../types/v1alpha1/v1alpha1_types_doc.go | 7 ++ pkg/machinery/resources/secrets/kubelet.go | 6 ++ .../configuration/v1alpha1/config.md | 1 + 9 files changed, 120 insertions(+), 2 deletions(-) diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_service.go b/internal/app/machined/pkg/controllers/k8s/kubelet_service.go index ec00e935817..802bb8112e6 100644 --- a/internal/app/machined/pkg/controllers/k8s/kubelet_service.go +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_service.go @@ -208,7 +208,7 @@ func (ctrl *KubeletServiceController) Run(ctx context.Context, r controller.Runt return err } - if err = ctrl.updateKubeconfig(secretSpec.Endpoint, secretSpec.AcceptedCAs, logger); err != nil { + if err = ctrl.updateKubeconfig(secretSpec.Endpoint, secretSpec.EndpointTLSServerName, secretSpec.AcceptedCAs, logger); err != nil { return err } @@ -306,6 +306,7 @@ func (ctrl *KubeletServiceController) writePKI(secretSpec *secrets.KubeletSpec) Clusters: map[string]*clientcmdapi.Cluster{ "local": { Server: secretSpec.Endpoint.String(), + TLSServerName: secretSpec.EndpointTLSServerName, CertificateAuthorityData: acceptedCAs, }, }, @@ -394,7 +395,7 @@ func (ctrl *KubeletServiceController) writeKubeletCredentialProviderConfig(cfgSp } // updateKubeconfig updates the kubeconfig of kubelet with the given endpoint if it exists. -func (ctrl *KubeletServiceController) updateKubeconfig(newEndpoint *url.URL, acceptedCAs []*talosx509.PEMEncodedCertificate, logger *zap.Logger) error { +func (ctrl *KubeletServiceController) updateKubeconfig(newEndpoint *url.URL, newTLSServerName string, acceptedCAs []*talosx509.PEMEncodedCertificate, logger *zap.Logger) error { config, err := clientcmd.LoadFromFile(constants.KubeletKubeconfig) if errors.Is(err, os.ErrNotExist) { return nil @@ -422,6 +423,7 @@ func (ctrl *KubeletServiceController) updateKubeconfig(newEndpoint *url.URL, acc } cluster.Server = newEndpoint.String() + cluster.TLSServerName = newTLSServerName cluster.CertificateAuthorityData = bytes.Join(xslices.Map(acceptedCAs, func(ca *talosx509.PEMEncodedCertificate) []byte { return ca.Crt }), nil) return clientcmd.WriteToFile(*config, constants.KubeletKubeconfig) diff --git a/internal/app/machined/pkg/controllers/secrets/kubelet.go b/internal/app/machined/pkg/controllers/secrets/kubelet.go index d7e77d291b1..fba345cc8b2 100644 --- a/internal/app/machined/pkg/controllers/secrets/kubelet.go +++ b/internal/app/machined/pkg/controllers/secrets/kubelet.go @@ -43,6 +43,8 @@ func NewKubeletController() *KubeletController { cfgProvider := cfg.Config() kubeletSecrets := res.TypedSpec() + kubeletSecrets.EndpointTLSServerName = "" + switch { case cfgProvider.Machine().Features().KubePrism().Enabled(): // use cluster endpoint for controlplane nodes with loadbalancer support @@ -52,6 +54,7 @@ func NewKubeletController() *KubeletController { } kubeletSecrets.Endpoint = localEndpoint + kubeletSecrets.EndpointTLSServerName = cfgProvider.Machine().Features().KubePrism().TLSServerName() case cfgProvider.Machine().Type().IsControlPlane(): // use localhost endpoint for controlplane nodes localEndpoint, err := url.Parse(fmt.Sprintf("https://localhost:%d", cfgProvider.Cluster().LocalAPIServerPort())) diff --git a/internal/app/machined/pkg/controllers/secrets/kubelet_test.go b/internal/app/machined/pkg/controllers/secrets/kubelet_test.go index 939419bad82..573107520ec 100644 --- a/internal/app/machined/pkg/controllers/secrets/kubelet_test.go +++ b/internal/app/machined/pkg/controllers/secrets/kubelet_test.go @@ -12,6 +12,7 @@ import ( "github.com/cosi-project/runtime/pkg/resource" "github.com/cosi-project/runtime/pkg/state" "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/go-pointer" "github.com/siderolabs/go-retry/retry" "github.com/stretchr/testify/suite" @@ -92,6 +93,82 @@ func (suite *KubeletSuite) TestReconcile() { suite.Assert().Equal([]*x509.PEMEncodedCertificate{{Crt: k8sCA.Crt}}, spec.AcceptedCAs) suite.Assert().Equal("abc", spec.BootstrapTokenID) suite.Assert().Equal("def", spec.BootstrapTokenSecret) + suite.Assert().Equal("", spec.EndpointTLSServerName) + + return nil + }, + ), + ) +} + +// TestReconcileKubePrismTLSServerName verifies that when KubePrism is enabled +// with a tlsServerName, the kubelet endpoint stays on loopback and the +// EndpointTLSServerName is propagated to the resource for kubeconfig generation. +func (suite *KubeletSuite) TestReconcileKubePrismTLSServerName() { + u, err := url.Parse("https://foo:6443") + suite.Require().NoError(err) + + ca, err := x509.NewSelfSignedCertificateAuthority(x509.RSA(false)) + suite.Require().NoError(err) + + k8sCA := x509.NewCertificateAndKeyFromCertificateAuthority(ca) + + cfg := config.NewMachineConfig( + container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineFeatures: &v1alpha1.FeaturesConfig{ + KubePrismSupport: &v1alpha1.KubePrism{ + ServerEnabled: pointer.To(true), + ServerPort: 7445, + ServerTLSServerName: "cluster-xyz.example.com", + }, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + URL: u, + }, + }, + ClusterCA: k8sCA, + BootstrapToken: "abc.def", + }, + }, + ), + ) + + suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg)) + + suite.Assert().NoError( + retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + kubeletSecrets, err := ctest.Get[*secrets.Kubelet]( + suite, + resource.NewMetadata( + secrets.NamespaceName, + secrets.KubeletType, + secrets.KubeletID, + resource.VersionUndefined, + ), + ) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + spec := kubeletSecrets.TypedSpec() + + if spec.EndpointTLSServerName != "cluster-xyz.example.com" { + return retry.ExpectedErrorf("EndpointTLSServerName not propagated yet: %q", spec.EndpointTLSServerName) + } + + suite.Assert().Equal("https://127.0.0.1:7445", spec.Endpoint.String()) + suite.Assert().Equal("cluster-xyz.example.com", spec.EndpointTLSServerName) return nil }, diff --git a/pkg/machinery/config/config/machine.go b/pkg/machinery/config/config/machine.go index 38ba2c38a70..707ce07efc7 100644 --- a/pkg/machinery/config/config/machine.go +++ b/pkg/machinery/config/config/machine.go @@ -384,6 +384,7 @@ type KubernetesTalosAPIAccess interface { type KubePrism interface { Enabled() bool Port() int + TLSServerName() string } // UdevConfig describes configuration for udev. diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_features.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_features.go index 57d5f6cc412..e3154ffb6be 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_features.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_features.go @@ -60,6 +60,11 @@ func (a *KubePrism) Port() int { return a.ServerPort } +// TLSServerName implements [config.KubePrism]. +func (a *KubePrism) TLSServerName() string { + return a.ServerTLSServerName +} + // HostDNSEnabled implements config.NetworkHostDNSConfig interface. func (h *HostDNSConfig) HostDNSEnabled() bool { return pointer.SafeDeref(h.HostDNSConfigEnabled) diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go index 557dbe706fb..7b847055dfb 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go @@ -2293,6 +2293,22 @@ type KubePrism struct { // description: | // KubePrism port. ServerPort int `yaml:"port,omitempty"` + // description: | + // Override the TLS server name (SNI) used by the kubelet when connecting to + // the KubePrism endpoint. + // + // KubePrism still listens on `127.0.0.1:` and the kubelet still dials + // that address, but the generated kubelet kubeconfig will carry + // `clusters[0].cluster.tls-server-name` set to this value, so the kubelet + // uses it for SNI and certificate hostname verification. + // + // This is useful when KubePrism's upstream apiserver is reached through an + // SNI-routing L4 proxy (for example nginx-ingress in ssl-passthrough mode in + // front of a Kamaji-hosted apiserver), where SNI=127.0.0.1 doesn't match any + // route and the proxy serves a fallback certificate. + // + // When empty (default), no `tls-server-name` is set and behavior is unchanged. + ServerTLSServerName string `yaml:"tlsServerName,omitempty"` } // ImageCacheConfig describes the configuration for the Image Cache feature. diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go index 19c17878b64..15b028c7425 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go @@ -1793,6 +1793,13 @@ func (KubePrism) Doc() *encoder.Doc { Description: "KubePrism port.", Comments: [3]string{"" /* encoder.HeadComment */, "KubePrism port." /* encoder.LineComment */, "" /* encoder.FootComment */}, }, + { + Name: "tlsServerName", + Type: "string", + Note: "", + Description: "Override the TLS server name (SNI) used by the kubelet when connecting to\nthe KubePrism endpoint.\n\nKubePrism still listens on `127.0.0.1:` and the kubelet still dials\nthat address, but the generated kubelet kubeconfig will carry\n`clusters[0].cluster.tls-server-name` set to this value, so the kubelet\nuses it for SNI and certificate hostname verification.\n\nThis is useful when KubePrism's upstream apiserver is reached through an\nSNI-routing L4 proxy (for example nginx-ingress in ssl-passthrough mode in\nfront of a Kamaji-hosted apiserver), where SNI=127.0.0.1 doesn't match any\nroute and the proxy serves a fallback certificate.\n\nWhen empty (default), no `tls-server-name` is set and behavior is unchanged.", + Comments: [3]string{"" /* encoder.HeadComment */, "Override the TLS server name (SNI) used by the kubelet when connecting to" /* encoder.LineComment */, "" /* encoder.FootComment */}, + }, }, } diff --git a/pkg/machinery/resources/secrets/kubelet.go b/pkg/machinery/resources/secrets/kubelet.go index 9a24fd7b499..13c9d591407 100644 --- a/pkg/machinery/resources/secrets/kubelet.go +++ b/pkg/machinery/resources/secrets/kubelet.go @@ -31,6 +31,12 @@ type Kubelet = typed.Resource[KubeletSpec, KubeletExtension] type KubeletSpec struct { Endpoint *url.URL `yaml:"endpoint" protobuf:"1"` + // EndpointTLSServerName, when non-empty, is propagated to the generated + // kubelet kubeconfig as `clusters[0].cluster.tls-server-name`, overriding + // the SNI/hostname the kubelet uses while still dialing Endpoint as the + // TCP destination. + EndpointTLSServerName string `yaml:"endpointTLSServerName,omitempty" protobuf:"6"` + AcceptedCAs []*x509.PEMEncodedCertificate `yaml:"acceptedCAs" protobuf:"5"` BootstrapTokenID string `yaml:"bootstrapTokenID" protobuf:"3"` diff --git a/website/content/v1.14/reference/configuration/v1alpha1/config.md b/website/content/v1.14/reference/configuration/v1alpha1/config.md index 9ec9c45cdea..a0b148028d3 100644 --- a/website/content/v1.14/reference/configuration/v1alpha1/config.md +++ b/website/content/v1.14/reference/configuration/v1alpha1/config.md @@ -701,6 +701,7 @@ KubePrism describes the configuration for the KubePrism load balancer. |-------|------|-------------|----------| |`enabled` |bool |Enable KubePrism support - will start local load balancing proxy. | | |`port` |int |KubePrism port. | | +|`tlsServerName` |string |
Override the TLS server name (SNI) used by the kubelet when connecting to the KubePrism endpoint.KubePrism still listens on `127.0.0.1:` and the kubelet still dials that address, but the generated kubelet kubeconfig will carry `clusters[0].cluster.tls-server-name` set to this value, so the kubelet uses it for SNI and certificate hostname verification.

This is useful when KubePrism's upstream apiserver is reached through an SNI-routing L4 proxy (for example nginx-ingress in ssl-passthrough mode in front of a Kamaji-hosted apiserver), where SNI=127.0.0.1 doesn't match any route and the proxy serves a fallback certificate.

When empty (default), no `tls-server-name` is set and behavior is unchanged.
| |