From 845f91de886208df4db8d030291d54699abf0422 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 24 Nov 2025 16:09:59 -0800 Subject: [PATCH 1/3] tcp_udp support in gateway api --- .../v1beta1/loadbalancerconfig_types.go | 2 +- config/crd/gateway/gateway-crds.yaml | 2 +- ...ay.k8s.aws_loadbalancerconfigurations.yaml | 2 +- .../crds/gateway-crds.yaml | 2 +- pkg/deploy/aga/listener_synthesizer.go | 22 +- pkg/deploy/elbv2/listener_manager.go | 11 +- pkg/gateway/model/mock_tg_builder.go | 2 +- pkg/gateway/model/model_build_listener.go | 79 ++++-- .../model/model_build_listener_test.go | 228 ++++++++++++----- pkg/gateway/model/model_build_target_group.go | 39 +-- ...odel_build_target_group_binding_network.go | 2 +- .../model/model_build_target_group_test.go | 230 ++++++------------ pkg/gateway/routeutils/backend_service.go | 5 +- 13 files changed, 342 insertions(+), 284 deletions(-) diff --git a/apis/gateway/v1beta1/loadbalancerconfig_types.go b/apis/gateway/v1beta1/loadbalancerconfig_types.go index 1b7599c06..96b92e025 100644 --- a/apis/gateway/v1beta1/loadbalancerconfig_types.go +++ b/apis/gateway/v1beta1/loadbalancerconfig_types.go @@ -152,7 +152,7 @@ type WAFv2Configuration struct { ACL string `json:"webACL"` } -// +kubebuilder:validation:Pattern="^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{4}|[1-9]\\d{0,3})?$" +// +kubebuilder:validation:Pattern="^(HTTP|HTTPS|TLS|TCP|UDP|TCP_UDP)?:(6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{4}|[1-9]\\d{0,3})?$" type ProtocolPort string type ListenerConfiguration struct { // protocolPort is identifier for the listener on load balancer. It should be of the form PROTOCOL:PORT diff --git a/config/crd/gateway/gateway-crds.yaml b/config/crd/gateway/gateway-crds.yaml index 2b49d19bb..07a3cf393 100644 --- a/config/crd/gateway/gateway-crds.yaml +++ b/config/crd/gateway/gateway-crds.yaml @@ -561,7 +561,7 @@ spec: protocolPort: description: protocolPort is identifier for the listener on load balancer. It should be of the form PROTOCOL:PORT - pattern: ^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$ + pattern: ^(HTTP|HTTPS|TLS|TCP|UDP|TCP_UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$ type: string sslPolicy: description: sslPolicy is the security policy that defines which diff --git a/config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml b/config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml index 342dcafd2..b548433b9 100644 --- a/config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml +++ b/config/crd/gateway/gateway.k8s.aws_loadbalancerconfigurations.yaml @@ -167,7 +167,7 @@ spec: protocolPort: description: protocolPort is identifier for the listener on load balancer. It should be of the form PROTOCOL:PORT - pattern: ^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$ + pattern: ^(HTTP|HTTPS|TLS|TCP|UDP|TCP_UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$ type: string sslPolicy: description: sslPolicy is the security policy that defines which diff --git a/helm/aws-load-balancer-controller/crds/gateway-crds.yaml b/helm/aws-load-balancer-controller/crds/gateway-crds.yaml index 2b49d19bb..07a3cf393 100644 --- a/helm/aws-load-balancer-controller/crds/gateway-crds.yaml +++ b/helm/aws-load-balancer-controller/crds/gateway-crds.yaml @@ -561,7 +561,7 @@ spec: protocolPort: description: protocolPort is identifier for the listener on load balancer. It should be of the form PROTOCOL:PORT - pattern: ^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$ + pattern: ^(HTTP|HTTPS|TLS|TCP|UDP|TCP_UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$ type: string sslPolicy: description: sslPolicy is the security policy that defines which diff --git a/pkg/deploy/aga/listener_synthesizer.go b/pkg/deploy/aga/listener_synthesizer.go index 7e003d140..a0a8f7f33 100644 --- a/pkg/deploy/aga/listener_synthesizer.go +++ b/pkg/deploy/aga/listener_synthesizer.go @@ -250,9 +250,9 @@ type resAndSDKListenerPair struct { // matchResAndSDKListeners matches resource listeners with SDK listeners using a multi-phase approach. // // The algorithm implements a two-phase matching process: -// 1. First phase (Exact Matching): Matches listeners with identical protocol and port ranges -// 2. Second phase (Similarity Matching): For remaining unmatched listeners, uses a similarity-based -// algorithm to find the best matches based on protocol and port range overlap +// 1. First phase (Exact Matching): Matches listeners with identical protocol and port ranges +// 2. Second phase (Similarity Matching): For remaining unmatched listeners, uses a similarity-based +// algorithm to find the best matches based on protocol and port range overlap // // Returns three groups: // - matchedResAndSDKListeners: pairs of resource and SDK listeners that will be updated @@ -288,7 +288,7 @@ func (s *listenerSynthesizer) matchResAndSDKListeners(resListeners []*agamodel.L // 3. Matches listeners with identical keys (exact protocol and port range matches) // 4. Returns matched pairs and remaining unmatched listeners // -// The key generation ensures that port ranges in different order but with identical +// The key generation ensures that port ranges in different order but with identical // values still match correctly. func (s *listenerSynthesizer) findExactMatches(resListeners []*agamodel.Listener, sdkListeners []*ListenerResource) ( []resAndSDKListenerPair, []*agamodel.Listener, []*ListenerResource) { @@ -473,17 +473,17 @@ func (s *listenerSynthesizer) findSimilarityMatches(resListeners []*agamodel.Lis // The scoring system uses these components: // // 1. Base Protocol Score: -// - If protocols match: +40 points (significant bonus) -// - If protocols don't match: 0 points (no bonus) +// - If protocols match: +40 points (significant bonus) +// - If protocols don't match: 0 points (no bonus) // // 2. Port Overlap Score: -// - Uses Jaccard similarity: (intersection / union) * 100 -// - Calculates the percentage of common ports between the two listeners -// - Converts port ranges into individual port sets for precise comparison +// - Uses Jaccard similarity: (intersection / union) * 100 +// - Calculates the percentage of common ports between the two listeners +// - Converts port ranges into individual port sets for precise comparison // // 3. Client Affinity Score: -// - If both listeners have client affinity specified and they match: +10 points -// - Otherwise: 0 points (no bonus) +// - If both listeners have client affinity specified and they match: +10 points +// - Otherwise: 0 points (no bonus) // // Note: In the future, we might need to add endpoint matching as well as one of the // score components so that we match the listeners with the most endpoint matches diff --git a/pkg/deploy/elbv2/listener_manager.go b/pkg/deploy/elbv2/listener_manager.go index 666cf3053..fea76efc9 100644 --- a/pkg/deploy/elbv2/listener_manager.go +++ b/pkg/deploy/elbv2/listener_manager.go @@ -31,11 +31,12 @@ var alpnNone = []string{ } var PROTOCOLS_SUPPORTING_LISTENER_ATTRIBUTES = map[elbv2model.Protocol]bool{ - elbv2model.ProtocolHTTP: true, - elbv2model.ProtocolHTTPS: true, - elbv2model.ProtocolTCP: true, - elbv2model.ProtocolUDP: false, - elbv2model.ProtocolTLS: false, + elbv2model.ProtocolHTTP: true, + elbv2model.ProtocolHTTPS: true, + elbv2model.ProtocolTCP: true, + elbv2model.ProtocolUDP: false, + elbv2model.ProtocolTLS: false, + elbv2model.ProtocolTCP_UDP: true, } // ListenerManager is responsible for create/update/delete Listener resources. diff --git a/pkg/gateway/model/mock_tg_builder.go b/pkg/gateway/model/mock_tg_builder.go index 21b080bb4..2e17bdc03 100644 --- a/pkg/gateway/model/mock_tg_builder.go +++ b/pkg/gateway/model/mock_tg_builder.go @@ -20,7 +20,7 @@ func (m *mockTargetGroupBuilder) getLocalFrontendNlbData() map[string]*elbv2mode } func (m *mockTargetGroupBuilder) buildTargetGroup(stack core.Stack, - gw *gwv1.Gateway, listenerPort int32, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) { + gw *gwv1.Gateway, listenerPort int32, listenerProtocol elbv2model.Protocol, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) { var tg *elbv2model.TargetGroup if len(m.tgs) > 0 { diff --git a/pkg/gateway/model/model_build_listener.go b/pkg/gateway/model/model_build_listener.go index fe7fd63d9..6b68136e8 100644 --- a/pkg/gateway/model/model_build_listener.go +++ b/pkg/gateway/model/model_build_listener.go @@ -58,7 +58,7 @@ func (l listenerBuilderImpl) buildListeners(ctx context.Context, stack core.Stac portsWithRoutes := sets.Int32KeySet(routes) // Materialise the listener only if listener has associated routes if len(gwLsPorts.Intersection(portsWithRoutes).List()) != 0 { - lbLsCfgs := mapLoadBalancerListenerConfigsByPort(lbCfg, gw.Spec.Listeners) + lbLsCfgs := mapLoadBalancerListenerConfigsByPort(lbCfg, gwLsCfgs) for _, port := range gwLsPorts.Intersection(portsWithRoutes).List() { ls, err := l.buildListener(ctx, stack, lb, gw, port, routes[port], lbCfg, gwLsCfgs[port], lbLsCfgs[port]) if err != nil { @@ -83,7 +83,7 @@ func (l listenerBuilderImpl) buildListeners(ctx context.Context, stack core.Stac return secrets, nil } -func (l listenerBuilderImpl) buildListener(ctx context.Context, stack core.Stack, lb *elbv2model.LoadBalancer, gw *gwv1.Gateway, port int32, routes []routeutils.RouteDescriptor, lbCfg elbv2gw.LoadBalancerConfiguration, gwLsCfg *gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.Listener, error) { +func (l listenerBuilderImpl) buildListener(ctx context.Context, stack core.Stack, lb *elbv2model.LoadBalancer, gw *gwv1.Gateway, port int32, routes []routeutils.RouteDescriptor, lbCfg elbv2gw.LoadBalancerConfiguration, gwLsCfg gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.Listener, error) { var listenerSpec *elbv2model.ListenerSpec var err error @@ -104,7 +104,7 @@ func (l listenerBuilderImpl) buildListener(ctx context.Context, stack core.Stack return elbv2model.NewListener(stack, lsResID, *listenerSpec), nil } -func (l listenerBuilderImpl) buildListenerSpec(ctx context.Context, lb *elbv2model.LoadBalancer, gw *gwv1.Gateway, port int32, lbCfg elbv2gw.LoadBalancerConfiguration, gwLsCfg *gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.ListenerSpec, error) { +func (l listenerBuilderImpl) buildListenerSpec(ctx context.Context, lb *elbv2model.LoadBalancer, gw *gwv1.Gateway, port int32, lbCfg elbv2gw.LoadBalancerConfiguration, gwLsCfg gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.ListenerSpec, error) { tags, err := l.buildListenerTags(lbCfg) if err != nil { return &elbv2model.ListenerSpec{}, err @@ -133,7 +133,7 @@ func (l listenerBuilderImpl) buildListenerSpec(ctx context.Context, lb *elbv2mod return listenerSpec, nil } -func (l listenerBuilderImpl) buildL7ListenerSpec(ctx context.Context, lb *elbv2model.LoadBalancer, gw *gwv1.Gateway, lbCfg elbv2gw.LoadBalancerConfiguration, port int32, gwLsCfg *gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.ListenerSpec, error) { +func (l listenerBuilderImpl) buildL7ListenerSpec(ctx context.Context, lb *elbv2model.LoadBalancer, gw *gwv1.Gateway, lbCfg elbv2gw.LoadBalancerConfiguration, port int32, gwLsCfg gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.ListenerSpec, error) { listenerSpec, err := l.buildListenerSpec(ctx, lb, gw, port, lbCfg, gwLsCfg, lbLsCfg) if err != nil { return &elbv2model.ListenerSpec{}, err @@ -147,7 +147,7 @@ func (l listenerBuilderImpl) buildL7ListenerSpec(ctx context.Context, lb *elbv2m return listenerSpec, nil } -func (l listenerBuilderImpl) buildL4ListenerSpec(ctx context.Context, stack core.Stack, lb *elbv2model.LoadBalancer, gw *gwv1.Gateway, lbCfg elbv2gw.LoadBalancerConfiguration, port int32, routes []routeutils.RouteDescriptor, gwLsCfg *gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.ListenerSpec, error) { +func (l listenerBuilderImpl) buildL4ListenerSpec(ctx context.Context, stack core.Stack, lb *elbv2model.LoadBalancer, gw *gwv1.Gateway, lbCfg elbv2gw.LoadBalancerConfiguration, port int32, routes []routeutils.RouteDescriptor, gwLsCfg gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.ListenerSpec, error) { listenerSpec, err := l.buildListenerSpec(ctx, lb, gw, port, lbCfg, gwLsCfg, lbLsCfg) if err != nil { return &elbv2model.ListenerSpec{}, err @@ -177,7 +177,7 @@ func (l listenerBuilderImpl) buildL4ListenerSpec(ctx context.Context, stack core return nil, nil } - arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, port, lb.Spec.IPAddressType, routeDescriptor, backend) + arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, port, listenerSpec.Protocol, lb.Spec.IPAddressType, routeDescriptor, backend) if tgErr != nil { return &elbv2model.ListenerSpec{}, tgErr } @@ -222,7 +222,7 @@ func (l listenerBuilderImpl) buildListenerRules(ctx context.Context, stack core. } targetGroupTuples := make([]elbv2model.TargetGroupTuple, 0, len(rule.GetBackends())) for _, backend := range rule.GetBackends() { - arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, port, ipAddressType, route, backend) + arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, port, ls.Spec.Protocol, ipAddressType, route, backend) if tgErr != nil { return nil, tgErr } @@ -313,7 +313,7 @@ func buildListenerAttributes(lsCfg *elbv2gw.ListenerConfiguration) ([]elbv2model return attributes, nil } -func (l listenerBuilderImpl) buildCertificates(ctx context.Context, gw *gwv1.Gateway, port int32, gwLsCfg *gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) ([]elbv2model.Certificate, error) { +func (l listenerBuilderImpl) buildCertificates(ctx context.Context, gw *gwv1.Gateway, port int32, gwLsCfg gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) ([]elbv2model.Certificate, error) { if !isSecureProtocol(gwLsCfg.protocol) { return []elbv2model.Certificate{}, nil } @@ -408,7 +408,7 @@ func buildL4ListenerDefaultActions(arn core.StringToken) []elbv2model.Action { } } -func (l listenerBuilderImpl) buildMutualAuthenticationAttributes(ctx context.Context, gwLsCfg *gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.MutualAuthenticationAttributes, error) { +func (l listenerBuilderImpl) buildMutualAuthenticationAttributes(ctx context.Context, gwLsCfg gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.MutualAuthenticationAttributes, error) { // Skip mTLS configuration for non-secure protocols if !isSecureProtocol(gwLsCfg.protocol) || lbLsCfg == nil || lbLsCfg.MutualAuthentication == nil { return nil, nil @@ -453,7 +453,7 @@ func (l listenerBuilderImpl) buildMutualAuthenticationAttributes(ctx context.Con }, nil } -func (l listenerBuilderImpl) buildSSLPolicy(gwLsCfg *gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*string, error) { +func (l listenerBuilderImpl) buildSSLPolicy(gwLsCfg gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*string, error) { if !isSecureProtocol(gwLsCfg.protocol) { return nil, nil } @@ -488,17 +488,34 @@ func buildListenerALPNPolicy(listenerProtocol elbv2model.Protocol, lbLsCfg *elbv } // mapGatewayListenerConfigsByPort creates a mapping of ports to listener configurations from the Gateway listeners. -func mapGatewayListenerConfigsByPort(gw *gwv1.Gateway, routes map[int32][]routeutils.RouteDescriptor) (map[int32]*gwListenerConfig, error) { - gwListenerConfigs := make(map[int32]*gwListenerConfig) +func mapGatewayListenerConfigsByPort(gw *gwv1.Gateway, routes map[int32][]routeutils.RouteDescriptor) (map[int32]gwListenerConfig, error) { + gwListenerConfigs := make(map[int32]gwListenerConfig) for _, listener := range gw.Spec.Listeners { port := int32(listener.Port) - protocol := listener.Protocol - if gwListenerConfigs[port] != nil && string(gwListenerConfigs[port].protocol) != string(protocol) { - return nil, fmt.Errorf("invalid listeners on gateway, listeners with same ports cannot have different protocols") + protocol := elbv2model.Protocol(listener.Protocol) + + _, hasPort := gwListenerConfigs[port] + if !hasPort { + gwListenerConfigs[port] = gwListenerConfig{ + protocol: protocol, + hostnames: sets.New[string](), + } } - if gwListenerConfigs[port] == nil { - gwListenerConfigs[port] = &gwListenerConfig{ - protocol: elbv2model.Protocol(protocol), + + if hasPort && gwListenerConfigs[port].protocol != protocol { + // Special case TCP_UDP (or TCP_QUIC) + + mergedValue, mergeErr := mergeProtocols(gwListenerConfigs[port].protocol, protocol) + + if mergeErr != nil { + return nil, fmt.Errorf("invalid listeners on gateway, listeners with same ports cannot have different protocols") + } + + // TODO this only works for TCP, UDP route merging. + // If we need to support TLS merging, then this will need + // to be updated. + gwListenerConfigs[port] = gwListenerConfig{ + protocol: mergedValue, hostnames: sets.New[string](), } } @@ -532,11 +549,11 @@ func mapGatewayListenerConfigsByPort(gw *gwv1.Gateway, routes map[int32][]routeu // mapLoadBalancerListenerConfigsByPort creates a mapping of ports to their corresponding // listener configurations from the LoadBalancer configuration. -func mapLoadBalancerListenerConfigsByPort(lbCfg elbv2gw.LoadBalancerConfiguration, gatewayListeners []gwv1.Listener) map[int32]*elbv2gw.ListenerConfiguration { +func mapLoadBalancerListenerConfigsByPort(lbCfg elbv2gw.LoadBalancerConfiguration, gatewayListeners map[int32]gwListenerConfig) map[int32]*elbv2gw.ListenerConfiguration { configuredListeners := sets.NewString() - for _, configuredListener := range gatewayListeners { - configuredListeners.Insert(generateListenerPortKey(configuredListener)) + for port, configuredListener := range gatewayListeners { + configuredListeners.Insert(generateListenerPortKey(port, configuredListener)) } lbLsCfgs := make(map[int32]*elbv2gw.ListenerConfiguration) @@ -554,8 +571,8 @@ func mapLoadBalancerListenerConfigsByPort(lbCfg elbv2gw.LoadBalancerConfiguratio return lbLsCfgs } -func generateListenerPortKey(listener gwv1.Listener) string { - return fmt.Sprintf("%s:%d", strings.ToLower(string(listener.Protocol)), listener.Port) +func generateListenerPortKey(port int32, listener gwListenerConfig) string { + return fmt.Sprintf("%s:%d", strings.ToLower(string(listener.protocol)), port) } func newListenerBuilder(loadBalancerType elbv2model.LoadBalancerType, tgBuilder targetGroupBuilder, tagHelper tagHelper, clusterName string, defaultSSLPolicy string, elbv2Client services.ELBV2, acmClient services.ACM, k8sClient client.Client, allowedCAARNs []string, secretsManager k8s.SecretsManager, logger logr.Logger) listenerBuilder { @@ -599,3 +616,19 @@ func getRoutingAction(config *elbv2gw.ListenerRuleConfiguration) *elbv2gw.Action } return nil } + +func mergeProtocols(storedProtocol, proposedProtocol elbv2model.Protocol) (elbv2model.Protocol, error) { + if storedProtocol == elbv2model.ProtocolTCP_UDP && (proposedProtocol == elbv2model.ProtocolTCP || proposedProtocol == elbv2model.ProtocolUDP) { + return elbv2model.ProtocolTCP_UDP, nil + } + + if storedProtocol == elbv2model.ProtocolTCP && proposedProtocol == elbv2model.ProtocolUDP { + return elbv2model.ProtocolTCP_UDP, nil + } + + if storedProtocol == elbv2model.ProtocolUDP && proposedProtocol == elbv2model.ProtocolTCP { + return elbv2model.ProtocolTCP_UDP, nil + } + + return elbv2model.ProtocolHTTP, errors.New("unsupported merge") +} diff --git a/pkg/gateway/model/model_build_listener_test.go b/pkg/gateway/model/model_build_listener_test.go index 689c0c6e8..0a6a3154c 100644 --- a/pkg/gateway/model/model_build_listener_test.go +++ b/pkg/gateway/model/model_build_listener_test.go @@ -33,7 +33,7 @@ func Test_mapGatewayListenerConfigsByPort(t *testing.T) { name string gateway *gwv1.Gateway routes map[int32][]routeutils.RouteDescriptor - want map[int32]*gwListenerConfig + want map[int32]gwListenerConfig wantErr bool }{ { @@ -49,7 +49,7 @@ func Test_mapGatewayListenerConfigsByPort(t *testing.T) { }, }, }, - want: map[int32]*gwListenerConfig{ + want: map[int32]gwListenerConfig{ 80: { protocol: elbv2model.ProtocolHTTP, hostnames: sets.New[string](), @@ -70,7 +70,7 @@ func Test_mapGatewayListenerConfigsByPort(t *testing.T) { }, }, }, - want: map[int32]*gwListenerConfig{ + want: map[int32]gwListenerConfig{ 443: { protocol: elbv2model.ProtocolTCP, hostnames: sets.New[string](), @@ -101,7 +101,7 @@ func Test_mapGatewayListenerConfigsByPort(t *testing.T) { }, }, }, - want: map[int32]*gwListenerConfig{ + want: map[int32]gwListenerConfig{ 80: { protocol: elbv2model.ProtocolHTTP, hostnames: sets.New[string](), @@ -137,7 +137,7 @@ func Test_mapGatewayListenerConfigsByPort(t *testing.T) { }, }, }, - want: map[int32]*gwListenerConfig{ + want: map[int32]gwListenerConfig{ 80: { protocol: elbv2model.ProtocolHTTP, hostnames: sets.New[string]("foo.example.com"), @@ -190,7 +190,7 @@ func Test_mapGatewayListenerConfigsByPort(t *testing.T) { }, }, }, - want: map[int32]*gwListenerConfig{ + want: map[int32]gwListenerConfig{ 80: { protocol: elbv2model.ProtocolHTTP, hostnames: sets.New[string]("foo.example.com", "bar.example.com"), @@ -218,7 +218,7 @@ func Test_mapGatewayListenerConfigsByPort(t *testing.T) { }, }, }, - want: map[int32]*gwListenerConfig{ + want: map[int32]gwListenerConfig{ 80: { protocol: elbv2model.ProtocolHTTP, hostnames: sets.New[string]("foo.example.com", "bar.example.com"), @@ -250,7 +250,7 @@ func Test_mapGatewayListenerConfigsByPort(t *testing.T) { }, }, }, - want: map[int32]*gwListenerConfig{ + want: map[int32]gwListenerConfig{ 80: { protocol: elbv2model.ProtocolHTTP, hostnames: sets.New[string]("foo.example.com", "r1.com", "r2.com", "r3.com", "r4.com", "r5.com", "r6.com"), @@ -287,7 +287,7 @@ func Test_mapGatewayListenerConfigsByPort(t *testing.T) { }, }, }, - want: map[int32]*gwListenerConfig{ + want: map[int32]gwListenerConfig{ 80: { protocol: elbv2model.ProtocolHTTP, hostnames: sets.New[string]("foo.example.com", "r1.com", "r2.com", "r3.com"), @@ -324,7 +324,7 @@ func Test_mapGatewayListenerConfigsByPort(t *testing.T) { }, }, }, - want: map[int32]*gwListenerConfig{ + want: map[int32]gwListenerConfig{ 80: { protocol: elbv2model.ProtocolHTTP, hostnames: sets.New[string]("foo.example.com", "r1.com", "r2.com", "r3.com"), @@ -332,6 +332,107 @@ func Test_mapGatewayListenerConfigsByPort(t *testing.T) { }, wantErr: false, }, + { + name: "listener valid merge", + gateway: &gwv1.Gateway{ + Spec: gwv1.GatewaySpec{ + Listeners: []gwv1.Listener{ + { + Name: "udp-1", + Port: 80, + Protocol: gwv1.UDPProtocolType, + }, + { + Name: "tcp-1", + Port: 80, + Protocol: gwv1.TCPProtocolType, + }, + { + Name: "tcp-2", + Port: 443, + Protocol: gwv1.TCPProtocolType, + }, + }, + }, + }, + routes: map[int32][]routeutils.RouteDescriptor{ + 80: { + &routeutils.MockRoute{}, + }, + 443: { + &routeutils.MockRoute{}, + }, + }, + want: map[int32]gwListenerConfig{ + 80: { + protocol: elbv2model.ProtocolTCP_UDP, + hostnames: sets.New[string](), + }, + 443: { + protocol: elbv2model.ProtocolTCP, + hostnames: sets.New[string](), + }, + }, + wantErr: false, + }, + { + name: "listener valid merge - multiple listeners", + gateway: &gwv1.Gateway{ + Spec: gwv1.GatewaySpec{ + Listeners: []gwv1.Listener{ + { + Name: "udp-1", + Port: 80, + Protocol: gwv1.UDPProtocolType, + }, + { + Name: "tcp-1", + Port: 80, + Protocol: gwv1.TCPProtocolType, + }, + { + Name: "tcp-2", + Port: 443, + Protocol: gwv1.TCPProtocolType, + }, + { + Name: "tcp-3", + Port: 80, + Protocol: gwv1.TCPProtocolType, + }, + { + Name: "tcp-4", + Port: 80, + Protocol: gwv1.TCPProtocolType, + }, + { + Name: "udp-2", + Port: 80, + Protocol: gwv1.UDPProtocolType, + }, + }, + }, + }, + routes: map[int32][]routeutils.RouteDescriptor{ + 80: { + &routeutils.MockRoute{}, + }, + 443: { + &routeutils.MockRoute{}, + }, + }, + want: map[int32]gwListenerConfig{ + 80: { + protocol: elbv2model.ProtocolTCP_UDP, + hostnames: sets.New[string](), + }, + 443: { + protocol: elbv2model.ProtocolTCP, + hostnames: sets.New[string](), + }, + }, + wantErr: false, + }, } for _, tt := range tests { @@ -474,17 +575,17 @@ func Test_mapLoadBalancerListenerConfigsByPort(t *testing.T) { tests := []struct { name string lbCfg elbv2gw.LoadBalancerConfiguration - listeners []gwv1.Listener + listeners map[int32]gwListenerConfig want map[int32]*elbv2gw.ListenerConfiguration }{ { name: "nil configuration", - listeners: []gwv1.Listener{}, + listeners: map[int32]gwListenerConfig{}, want: map[int32]*elbv2gw.ListenerConfiguration{}, }, { name: "nil listener configurations", - listeners: []gwv1.Listener{}, + listeners: map[int32]gwListenerConfig{}, lbCfg: elbv2gw.LoadBalancerConfiguration{ Spec: elbv2gw.LoadBalancerConfigurationSpec{ ListenerConfigurations: nil, @@ -494,7 +595,7 @@ func Test_mapLoadBalancerListenerConfigsByPort(t *testing.T) { }, { name: "empty listener configurations", - listeners: []gwv1.Listener{}, + listeners: map[int32]gwListenerConfig{}, lbCfg: elbv2gw.LoadBalancerConfiguration{ Spec: elbv2gw.LoadBalancerConfigurationSpec{ ListenerConfigurations: createListenerConfigs(), @@ -504,10 +605,9 @@ func Test_mapLoadBalancerListenerConfigsByPort(t *testing.T) { }, { name: "single HTTP listener", - listeners: []gwv1.Listener{ - { - Protocol: gwv1.HTTPProtocolType, - Port: 80, + listeners: map[int32]gwListenerConfig{ + 80: { + protocol: elbv2model.ProtocolHTTP, }, }, lbCfg: elbv2gw.LoadBalancerConfiguration{ @@ -523,18 +623,15 @@ func Test_mapLoadBalancerListenerConfigsByPort(t *testing.T) { }, { name: "multiple valid listeners", - listeners: []gwv1.Listener{ - { - Protocol: gwv1.HTTPProtocolType, - Port: 80, + listeners: map[int32]gwListenerConfig{ + 80: { + protocol: elbv2model.ProtocolHTTP, }, - { - Protocol: gwv1.HTTPSProtocolType, - Port: 443, + 443: { + protocol: elbv2model.ProtocolHTTPS, }, - { - Protocol: gwv1.HTTPProtocolType, - Port: 8080, + 8080: { + protocol: elbv2model.ProtocolHTTP, }, }, lbCfg: elbv2gw.LoadBalancerConfiguration{ @@ -560,18 +657,15 @@ func Test_mapLoadBalancerListenerConfigsByPort(t *testing.T) { }, { name: "conflicting listener protocols", - listeners: []gwv1.Listener{ - { - Protocol: gwv1.HTTPProtocolType, - Port: 80, + listeners: map[int32]gwListenerConfig{ + 80: { + protocol: elbv2model.ProtocolHTTP, }, - { - Protocol: gwv1.HTTPSProtocolType, - Port: 443, + 443: { + protocol: elbv2model.ProtocolHTTPS, }, - { - Protocol: gwv1.HTTPProtocolType, - Port: 8080, + 8080: { + protocol: elbv2model.ProtocolHTTP, }, }, lbCfg: elbv2gw.LoadBalancerConfiguration{ @@ -596,6 +690,24 @@ func Test_mapLoadBalancerListenerConfigsByPort(t *testing.T) { }, }, }, + { + name: "single TCP_UDP listener", + listeners: map[int32]gwListenerConfig{ + 80: { + protocol: elbv2model.ProtocolTCP_UDP, + }, + }, + lbCfg: elbv2gw.LoadBalancerConfiguration{ + Spec: elbv2gw.LoadBalancerConfigurationSpec{ + ListenerConfigurations: createListenerConfigs("TCP_UDP:80"), + }, + }, + want: map[int32]*elbv2gw.ListenerConfiguration{ + 80: { + ProtocolPort: "TCP_UDP:80", + }, + }, + }, } for _, tt := range tests { @@ -678,7 +790,7 @@ func TestBuildCertificates(t *testing.T) { name string gateway *gwv1.Gateway port int32 - gwLsCfg *gwListenerConfig + gwLsCfg gwListenerConfig lbLsCfg *elbv2gw.ListenerConfiguration setupMocks func(mockCertDiscovery *certs.MockCertDiscovery) want []elbv2model.Certificate @@ -698,7 +810,7 @@ func TestBuildCertificates(t *testing.T) { }, }, port: 443, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolHTTPS, hostnames: sets.New[string]("my-host-1", "my-host-2"), }, @@ -725,7 +837,7 @@ func TestBuildCertificates(t *testing.T) { }, }, port: 443, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolTLS, hostnames: sets.New[string]("my-host-1", "my-host-2"), }, @@ -758,7 +870,7 @@ func TestBuildCertificates(t *testing.T) { }, }, port: 443, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolHTTPS, hostnames: sets.New[string]("my-host-1", "my-host-2"), }, @@ -795,7 +907,7 @@ func TestBuildCertificates(t *testing.T) { }, }, port: 443, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolTLS, hostnames: sets.New[string]("example.com"), }, @@ -829,7 +941,7 @@ func TestBuildCertificates(t *testing.T) { }, }, port: 443, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolTLS, hostnames: sets.New[string]("example.com", "*.example.org"), }, @@ -866,7 +978,7 @@ func TestBuildCertificates(t *testing.T) { }, }, port: 443, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolHTTPS, hostnames: sets.New[string]("example.com"), }, @@ -895,7 +1007,7 @@ func TestBuildCertificates(t *testing.T) { }, }, port: 443, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolHTTP, hostnames: sets.New[string](), }, @@ -918,7 +1030,7 @@ func TestBuildCertificates(t *testing.T) { }, }, port: 443, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolHTTPS, hostnames: sets.New[string](), }, @@ -980,7 +1092,7 @@ func Test_buildMutualAuthenticationAttributes(t *testing.T) { tests := []struct { name string protocol elbv2model.Protocol - gwLsCfg *gwListenerConfig + gwLsCfg gwListenerConfig lbLsCfg *elbv2gw.ListenerConfiguration describeTrustStores describeTrustStoresCall want *elbv2model.MutualAuthenticationAttributes @@ -989,7 +1101,7 @@ func Test_buildMutualAuthenticationAttributes(t *testing.T) { { name: "non-secure protocol should return nil", protocol: elbv2model.ProtocolHTTP, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolHTTP, hostnames: sets.New[string]("example.com"), }, @@ -1000,7 +1112,7 @@ func Test_buildMutualAuthenticationAttributes(t *testing.T) { { name: "nil lbLsCfg should return nil", protocol: elbv2model.ProtocolHTTPS, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolHTTPS, hostnames: sets.New[string]("example.com"), }, @@ -1011,7 +1123,7 @@ func Test_buildMutualAuthenticationAttributes(t *testing.T) { { name: "nil mutualAuthentication should return off mode", protocol: elbv2model.ProtocolHTTPS, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolHTTPS, hostnames: sets.New[string]("example.com"), }, @@ -1024,7 +1136,7 @@ func Test_buildMutualAuthenticationAttributes(t *testing.T) { { name: "verify mode with truststore name should resolve ARN", protocol: elbv2model.ProtocolHTTPS, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolHTTPS, hostnames: sets.New[string]("example.com"), }, @@ -1052,7 +1164,7 @@ func Test_buildMutualAuthenticationAttributes(t *testing.T) { { name: "verify mode with truststore ARN should use ARN directly", protocol: elbv2model.ProtocolHTTPS, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolHTTPS, hostnames: sets.New[string]("example.com"), }, @@ -1073,7 +1185,7 @@ func Test_buildMutualAuthenticationAttributes(t *testing.T) { { name: "verify mode with all options", protocol: elbv2model.ProtocolHTTPS, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolHTTPS, hostnames: sets.New[string]("example.com"), }, @@ -1096,7 +1208,7 @@ func Test_buildMutualAuthenticationAttributes(t *testing.T) { { name: "verify mode with nil ignoreClientCertificateExpiry", protocol: elbv2model.ProtocolHTTPS, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolHTTPS, hostnames: sets.New[string]("example.com"), }, @@ -1119,7 +1231,7 @@ func Test_buildMutualAuthenticationAttributes(t *testing.T) { { name: "passthrough mode", protocol: elbv2model.ProtocolHTTPS, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolHTTPS, hostnames: sets.New[string]("example.com"), }, @@ -1139,7 +1251,7 @@ func Test_buildMutualAuthenticationAttributes(t *testing.T) { { name: "off mode", protocol: elbv2model.ProtocolHTTPS, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolHTTPS, hostnames: sets.New[string]("example.com"), }, @@ -1159,7 +1271,7 @@ func Test_buildMutualAuthenticationAttributes(t *testing.T) { { name: "error on truststore ARN resolution", protocol: elbv2model.ProtocolHTTPS, - gwLsCfg: &gwListenerConfig{ + gwLsCfg: gwListenerConfig{ protocol: elbv2model.ProtocolHTTPS, hostnames: sets.New[string]("example.com"), }, diff --git a/pkg/gateway/model/model_build_target_group.go b/pkg/gateway/model/model_build_target_group.go index e7363f791..fb51e6adb 100644 --- a/pkg/gateway/model/model_build_target_group.go +++ b/pkg/gateway/model/model_build_target_group.go @@ -5,15 +5,13 @@ import ( "crypto/sha256" "encoding/hex" "fmt" - "regexp" - "strconv" - awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "regexp" elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway" @@ -24,6 +22,7 @@ import ( elbv2modelk8s "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2/k8s" "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_utils" gwv1 "sigs.k8s.io/gateway-api/apis/v1" + "strconv" ) type buildTargetGroupOutput struct { @@ -33,7 +32,7 @@ type buildTargetGroupOutput struct { type targetGroupBuilder interface { buildTargetGroup(stack core.Stack, - gw *gwv1.Gateway, listenerPort int32, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) + gw *gwv1.Gateway, listenerPort int32, listenerProtocol elbv2model.Protocol, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) getLocalFrontendNlbData() map[string]*elbv2model.FrontendNlbTargetGroupState } @@ -109,10 +108,10 @@ func newTargetGroupBuilder(clusterName string, vpcId string, tagHelper tagHelper } func (builder *targetGroupBuilderImpl) buildTargetGroup(stack core.Stack, - gw *gwv1.Gateway, listenerPort int32, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) { + gw *gwv1.Gateway, listenerPort int32, listenerProtocol elbv2model.Protocol, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) { if backend.ServiceBackend != nil { - tg, err := builder.buildTargetGroupFromService(stack, gw, lbIPType, routeDescriptor, *backend.ServiceBackend) + tg, err := builder.buildTargetGroupFromService(stack, gw, listenerProtocol, lbIPType, routeDescriptor, *backend.ServiceBackend) if err != nil { return nil, err } @@ -120,7 +119,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroup(stack core.Stack, } if backend.GatewayBackend != nil { - tg, err := builder.buildTargetGroupFromGateway(stack, gw, listenerPort, lbIPType, routeDescriptor, *backend.GatewayBackend) + tg, err := builder.buildTargetGroupFromGateway(stack, gw, listenerPort, listenerProtocol, lbIPType, routeDescriptor, *backend.GatewayBackend) if err != nil { return nil, err } @@ -136,12 +135,14 @@ func (builder *targetGroupBuilderImpl) buildTargetGroup(stack core.Stack, } func (builder *targetGroupBuilderImpl) buildTargetGroupFromService(stack core.Stack, - gw *gwv1.Gateway, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backendConfig routeutils.ServiceBackendConfig) (*elbv2model.TargetGroup, error) { + gw *gwv1.Gateway, listenerProtocol elbv2model.Protocol, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backendConfig routeutils.ServiceBackendConfig) (*elbv2model.TargetGroup, error) { targetGroupProps := backendConfig.GetTargetGroupProps() - tgSpec, err := builder.buildTargetGroupSpec(gw, routeDescriptor, lbIPType, &backendConfig, targetGroupProps) + + tgSpec, err := builder.buildTargetGroupSpec(gw, routeDescriptor, listenerProtocol, lbIPType, &backendConfig, targetGroupProps) if err != nil { return nil, err } + tgResID := builder.buildTargetGroupResourceID(k8s.NamespacedName(gw), backendConfig.GetBackendNamespacedName(), routeDescriptor.GetRouteNamespacedName(), routeDescriptor.GetRouteKind(), backendConfig.GetIdentifierPort(), tgSpec.TargetControlPort) if tg, exists := builder.tgByResID[tgResID]; exists { return tg, nil @@ -166,14 +167,14 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupFromService(stack core.St } func (builder *targetGroupBuilderImpl) buildTargetGroupFromGateway(stack core.Stack, - gw *gwv1.Gateway, listenerPort int32, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backendConfig routeutils.GatewayBackendConfig) (*elbv2model.TargetGroup, error) { + gw *gwv1.Gateway, listenerPort int32, listenerProtocol elbv2model.Protocol, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backendConfig routeutils.GatewayBackendConfig) (*elbv2model.TargetGroup, error) { targetGroupProps := backendConfig.GetTargetGroupProps() tgResID := builder.buildTargetGroupResourceID(k8s.NamespacedName(gw), backendConfig.GetBackendNamespacedName(), routeDescriptor.GetRouteNamespacedName(), routeDescriptor.GetRouteKind(), backendConfig.GetIdentifierPort(), nil) if tg, exists := builder.tgByResID[tgResID]; exists { return tg, nil } - tgSpec, err := builder.buildTargetGroupSpec(gw, routeDescriptor, lbIPType, &backendConfig, targetGroupProps) + tgSpec, err := builder.buildTargetGroupSpec(gw, routeDescriptor, listenerProtocol, lbIPType, &backendConfig, targetGroupProps) if err != nil { return nil, err } @@ -259,9 +260,9 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(gw *gwv1.Gate }, nil } -func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, route routeutils.RouteDescriptor, lbIPType elbv2model.IPAddressType, backendConfig routeutils.TargetGroupConfigurator, targetGroupProps *elbv2gw.TargetGroupProps) (elbv2model.TargetGroupSpec, error) { +func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, route routeutils.RouteDescriptor, listenerProtocol elbv2model.Protocol, lbIPType elbv2model.IPAddressType, backendConfig routeutils.TargetGroupConfigurator, targetGroupProps *elbv2gw.TargetGroupProps) (elbv2model.TargetGroupSpec, error) { targetType := backendConfig.GetTargetType(builder.defaultTargetType) - tgProtocol, err := builder.buildTargetGroupProtocol(targetGroupProps, route) + tgProtocol, err := builder.buildTargetGroupProtocol(targetGroupProps, route, listenerProtocol) if err != nil { return elbv2model.TargetGroupSpec{}, err } @@ -300,7 +301,6 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, ro Port: awssdk.Int32(tgPort), Protocol: tgProtocol, ProtocolVersion: tgProtocolVersion, - TargetControlPort: targetControlPort, IPAddressType: ipAddressType, HealthCheckConfig: &healthCheckConfig, TargetGroupAttributes: builder.convertMapToAttributes(tgAttributesMap), @@ -334,7 +334,6 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupName(targetGroupProps *el if tgProtocolVersion != nil { _, _ = uuidHash.Write([]byte(*tgProtocolVersion)) } - if targetControlPort != nil { _, _ = uuidHash.Write([]byte(strconv.Itoa(int(*targetControlPort)))) } @@ -353,13 +352,13 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupIPAddressType(backendConf return addressType, nil } -func (builder *targetGroupBuilderImpl) buildTargetGroupProtocol(targetGroupProps *elbv2gw.TargetGroupProps, route routeutils.RouteDescriptor) (elbv2model.Protocol, error) { +func (builder *targetGroupBuilderImpl) buildTargetGroupProtocol(targetGroupProps *elbv2gw.TargetGroupProps, route routeutils.RouteDescriptor, listenerProtocol elbv2model.Protocol) (elbv2model.Protocol, error) { // TODO - Not convinced that this is good, maybe auto detect certs == HTTPS / TLS. if builder.loadBalancerType == elbv2model.LoadBalancerTypeApplication { return builder.buildL7TargetGroupProtocol(targetGroupProps, route) } - return builder.buildL4TargetGroupProtocol(targetGroupProps, route) + return builder.buildL4TargetGroupProtocol(targetGroupProps, route, listenerProtocol) } func (builder *targetGroupBuilderImpl) buildL7TargetGroupProtocol(targetGroupProps *elbv2gw.TargetGroupProps, route routeutils.RouteDescriptor) (elbv2model.Protocol, error) { @@ -376,7 +375,11 @@ func (builder *targetGroupBuilderImpl) buildL7TargetGroupProtocol(targetGroupPro } } -func (builder *targetGroupBuilderImpl) buildL4TargetGroupProtocol(targetGroupProps *elbv2gw.TargetGroupProps, route routeutils.RouteDescriptor) (elbv2model.Protocol, error) { +func (builder *targetGroupBuilderImpl) buildL4TargetGroupProtocol(targetGroupProps *elbv2gw.TargetGroupProps, route routeutils.RouteDescriptor, listenerProtocol elbv2model.Protocol) (elbv2model.Protocol, error) { + if listenerProtocol == elbv2model.ProtocolTCP_UDP { + return listenerProtocol, nil + } + if targetGroupProps == nil || targetGroupProps.Protocol == nil { return builder.inferTargetGroupProtocolFromRoute(route), nil } diff --git a/pkg/gateway/model/model_build_target_group_binding_network.go b/pkg/gateway/model/model_build_target_group_binding_network.go index a469e3479..154ead0ad 100644 --- a/pkg/gateway/model/model_build_target_group_binding_network.go +++ b/pkg/gateway/model/model_build_target_group_binding_network.go @@ -292,7 +292,7 @@ func (builder *targetGroupBindingNetworkBuilderImpl) buildPeersFromSourceRangeCI func (builder *targetGroupBindingNetworkBuilderImpl) buildHealthCheckSourceCIDRs(preserveClientIP bool, trafficSource, subnetCIDRs []string, tgPort, hcPort intstr.IntOrString, tgProtocol elbv2model.Protocol, defaultRangeUsed bool) []string { - if tgProtocol != elbv2model.ProtocolUDP && + if tgProtocol != elbv2model.ProtocolUDP && tgProtocol != elbv2model.ProtocolTCP_UDP && (hcPort.String() == shared_constants.HealthCheckPortTrafficPort || hcPort.IntValue() == tgPort.IntValue()) { if !preserveClientIP { return nil diff --git a/pkg/gateway/model/model_build_target_group_test.go b/pkg/gateway/model/model_build_target_group_test.go index d02dbc13a..01156765d 100644 --- a/pkg/gateway/model/model_build_target_group_test.go +++ b/pkg/gateway/model/model_build_target_group_test.go @@ -2,8 +2,6 @@ package model import ( "context" - "testing" - awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/pkg/errors" "github.com/stretchr/testify/assert" @@ -22,6 +20,7 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_utils" gwv1 "sigs.k8s.io/gateway-api/apis/v1" + "testing" ) func Test_buildTargetGroupSpec(t *testing.T) { @@ -318,7 +317,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { builder := newTargetGroupBuilder("my-cluster", "vpc-xxx", tagger, tc.lbType, &mockTargetGroupBindingNetworkingBuilder{}, gateway.NewTargetGroupConfigConstructor(), tc.defaultTargetType, nil) - out, err := builder.(*targetGroupBuilderImpl).buildTargetGroupSpec(tc.gateway, tc.route, elbv2model.IPAddressTypeIPV4, tc.backend, nil) + out, err := builder.(*targetGroupBuilderImpl).buildTargetGroupSpec(tc.gateway, tc.route, elbv2model.ProtocolHTTP, elbv2model.IPAddressTypeIPV4, tc.backend, nil) if tc.expectErr { assert.Error(t, err) return @@ -824,6 +823,7 @@ func Test_buildTargetGroupName(t *testing.T) { builder := targetGroupBuilderImpl{ clusterName: clusterName, } + var targetControlPort *int32 if tc.targetGroupProps != nil { targetControlPort = tc.targetGroupProps.TargetControlPort @@ -917,6 +917,7 @@ func Test_buildTargetGroupIPAddressType(t *testing.T) { func Test_buildTargetGroupProtocol(t *testing.T) { testCases := []struct { name string + listenerProtocol elbv2model.Protocol lbType elbv2model.LoadBalancerType targetGroupProps *elbv2gw.TargetGroupProps route routeutils.RouteDescriptor @@ -924,8 +925,9 @@ func Test_buildTargetGroupProtocol(t *testing.T) { expectErr bool }{ { - name: "alb - auto detect - http", - lbType: elbv2model.LoadBalancerTypeApplication, + name: "alb - auto detect - http", + listenerProtocol: elbv2model.ProtocolHTTPS, + lbType: elbv2model.LoadBalancerTypeApplication, route: &routeutils.MockRoute{ Kind: routeutils.HTTPRouteKind, Name: "r1", @@ -934,8 +936,9 @@ func Test_buildTargetGroupProtocol(t *testing.T) { expected: elbv2model.ProtocolHTTP, }, { - name: "alb - auto detect - grpc", - lbType: elbv2model.LoadBalancerTypeApplication, + name: "alb - auto detect - grpc", + listenerProtocol: elbv2model.ProtocolHTTPS, + lbType: elbv2model.LoadBalancerTypeApplication, route: &routeutils.MockRoute{ Kind: routeutils.GRPCRouteKind, Name: "r1", @@ -944,8 +947,9 @@ func Test_buildTargetGroupProtocol(t *testing.T) { expected: elbv2model.ProtocolHTTP, }, { - name: "alb - auto detect - tls", - lbType: elbv2model.LoadBalancerTypeApplication, + name: "alb - auto detect - tls", + listenerProtocol: elbv2model.ProtocolHTTPS, + lbType: elbv2model.LoadBalancerTypeApplication, route: &routeutils.MockRoute{ Kind: routeutils.TLSRouteKind, Name: "r1", @@ -954,8 +958,9 @@ func Test_buildTargetGroupProtocol(t *testing.T) { expected: elbv2model.ProtocolHTTPS, }, { - name: "nlb - auto detect - tcp", - lbType: elbv2model.LoadBalancerTypeNetwork, + name: "nlb - auto detect - tcp", + listenerProtocol: elbv2model.ProtocolTLS, + lbType: elbv2model.LoadBalancerTypeNetwork, route: &routeutils.MockRoute{ Kind: routeutils.TCPRouteKind, Name: "r1", @@ -964,8 +969,9 @@ func Test_buildTargetGroupProtocol(t *testing.T) { expected: elbv2model.ProtocolTCP, }, { - name: "alb - auto detect - udp", - lbType: elbv2model.LoadBalancerTypeNetwork, + name: "alb - auto detect - udp", + listenerProtocol: elbv2model.ProtocolUDP, + lbType: elbv2model.LoadBalancerTypeNetwork, route: &routeutils.MockRoute{ Kind: routeutils.UDPRouteKind, Name: "r1", @@ -974,8 +980,9 @@ func Test_buildTargetGroupProtocol(t *testing.T) { expected: elbv2model.ProtocolUDP, }, { - name: "nlb - auto detect - tls", - lbType: elbv2model.LoadBalancerTypeNetwork, + name: "nlb - auto detect - tls", + listenerProtocol: elbv2model.ProtocolTCP, + lbType: elbv2model.LoadBalancerTypeNetwork, route: &routeutils.MockRoute{ Kind: routeutils.TLSRouteKind, Name: "r1", @@ -984,8 +991,9 @@ func Test_buildTargetGroupProtocol(t *testing.T) { expected: elbv2model.ProtocolTLS, }, { - name: "alb - specified - http", - lbType: elbv2model.LoadBalancerTypeApplication, + name: "alb - specified - http", + listenerProtocol: elbv2model.ProtocolHTTP, + lbType: elbv2model.LoadBalancerTypeApplication, targetGroupProps: &elbv2gw.TargetGroupProps{ Protocol: protocolPtr(elbv2gw.ProtocolHTTP), }, @@ -997,8 +1005,9 @@ func Test_buildTargetGroupProtocol(t *testing.T) { expected: elbv2model.ProtocolHTTP, }, { - name: "alb - specified - https", - lbType: elbv2model.LoadBalancerTypeApplication, + name: "alb - specified - https", + listenerProtocol: elbv2model.ProtocolHTTPS, + lbType: elbv2model.LoadBalancerTypeApplication, targetGroupProps: &elbv2gw.TargetGroupProps{ Protocol: protocolPtr(elbv2gw.ProtocolHTTPS), }, @@ -1010,8 +1019,9 @@ func Test_buildTargetGroupProtocol(t *testing.T) { expected: elbv2model.ProtocolHTTPS, }, { - name: "alb - specified - invalid protocol", - lbType: elbv2model.LoadBalancerTypeApplication, + name: "alb - specified - invalid protocol", + listenerProtocol: elbv2model.ProtocolHTTPS, + lbType: elbv2model.LoadBalancerTypeApplication, targetGroupProps: &elbv2gw.TargetGroupProps{ Protocol: protocolPtr(elbv2gw.ProtocolTCP), }, @@ -1023,8 +1033,9 @@ func Test_buildTargetGroupProtocol(t *testing.T) { expectErr: true, }, { - name: "nlb - auto detect - tcp", - lbType: elbv2model.LoadBalancerTypeNetwork, + name: "nlb - auto detect - tcp", + listenerProtocol: elbv2model.ProtocolTCP, + lbType: elbv2model.LoadBalancerTypeNetwork, route: &routeutils.MockRoute{ Kind: routeutils.TCPRouteKind, Name: "r1", @@ -1033,18 +1044,20 @@ func Test_buildTargetGroupProtocol(t *testing.T) { expected: elbv2model.ProtocolTCP, }, { - name: "alb - auto detect - udp", - lbType: elbv2model.LoadBalancerTypeNetwork, + name: "nlb - auto detect - tcp_udp", + listenerProtocol: elbv2model.ProtocolTCP_UDP, + lbType: elbv2model.LoadBalancerTypeNetwork, route: &routeutils.MockRoute{ Kind: routeutils.UDPRouteKind, Name: "r1", Namespace: "ns", }, - expected: elbv2model.ProtocolUDP, + expected: elbv2model.ProtocolTCP_UDP, }, { - name: "nlb - auto detect - tls", - lbType: elbv2model.LoadBalancerTypeNetwork, + name: "nlb - auto detect - tls", + listenerProtocol: elbv2model.ProtocolTLS, + lbType: elbv2model.LoadBalancerTypeNetwork, route: &routeutils.MockRoute{ Kind: routeutils.TLSRouteKind, Name: "r1", @@ -1053,8 +1066,9 @@ func Test_buildTargetGroupProtocol(t *testing.T) { expected: elbv2model.ProtocolTLS, }, { - name: "nlb - specified - tcp protocol", - lbType: elbv2model.LoadBalancerTypeNetwork, + name: "nlb - specified - tcp protocol", + listenerProtocol: elbv2model.ProtocolTCP, + lbType: elbv2model.LoadBalancerTypeNetwork, targetGroupProps: &elbv2gw.TargetGroupProps{ Protocol: protocolPtr(elbv2gw.ProtocolTCP), }, @@ -1066,8 +1080,9 @@ func Test_buildTargetGroupProtocol(t *testing.T) { expected: elbv2model.ProtocolTCP, }, { - name: "nlb - specified - udp protocol", - lbType: elbv2model.LoadBalancerTypeNetwork, + name: "nlb - specified - udp protocol", + listenerProtocol: elbv2model.ProtocolUDP, + lbType: elbv2model.LoadBalancerTypeNetwork, targetGroupProps: &elbv2gw.TargetGroupProps{ Protocol: protocolPtr(elbv2gw.ProtocolUDP), }, @@ -1079,8 +1094,9 @@ func Test_buildTargetGroupProtocol(t *testing.T) { expected: elbv2model.ProtocolUDP, }, { - name: "nlb - specified - tcpudp protocol", - lbType: elbv2model.LoadBalancerTypeNetwork, + name: "nlb - specified - tcpudp protocol", + listenerProtocol: elbv2model.ProtocolTCP, + lbType: elbv2model.LoadBalancerTypeNetwork, targetGroupProps: &elbv2gw.TargetGroupProps{ Protocol: protocolPtr(elbv2gw.ProtocolTCP_UDP), }, @@ -1092,8 +1108,9 @@ func Test_buildTargetGroupProtocol(t *testing.T) { expected: elbv2model.ProtocolTCP_UDP, }, { - name: "nlb - specified - tls protocol", - lbType: elbv2model.LoadBalancerTypeNetwork, + name: "nlb - specified - tls protocol", + listenerProtocol: elbv2model.ProtocolTCP, + lbType: elbv2model.LoadBalancerTypeNetwork, targetGroupProps: &elbv2gw.TargetGroupProps{ Protocol: protocolPtr(elbv2gw.ProtocolTLS), }, @@ -1105,8 +1122,9 @@ func Test_buildTargetGroupProtocol(t *testing.T) { expected: elbv2model.ProtocolTLS, }, { - name: "nlb - specified - invalid protocol", - lbType: elbv2model.LoadBalancerTypeNetwork, + name: "nlb - specified - invalid protocol", + listenerProtocol: elbv2model.ProtocolTCP, + lbType: elbv2model.LoadBalancerTypeNetwork, targetGroupProps: &elbv2gw.TargetGroupProps{ Protocol: protocolPtr(elbv2gw.ProtocolHTTPS), }, @@ -1117,6 +1135,17 @@ func Test_buildTargetGroupProtocol(t *testing.T) { }, expectErr: true, }, + { + name: "nlb - tcp_udp listener", + listenerProtocol: elbv2model.ProtocolTCP_UDP, + lbType: elbv2model.LoadBalancerTypeNetwork, + route: &routeutils.MockRoute{ + Kind: routeutils.TCPRouteKind, + Name: "r1", + Namespace: "ns", + }, + expected: elbv2model.ProtocolTCP_UDP, + }, } for _, tc := range testCases { @@ -1124,7 +1153,7 @@ func Test_buildTargetGroupProtocol(t *testing.T) { builder := targetGroupBuilderImpl{ loadBalancerType: tc.lbType, } - res, err := builder.buildTargetGroupProtocol(tc.targetGroupProps, tc.route) + res, err := builder.buildTargetGroupProtocol(tc.targetGroupProps, tc.route, tc.listenerProtocol) if tc.expectErr { assert.Error(t, err) return @@ -1668,7 +1697,7 @@ func Test_buildTargetGroupTags(t *testing.T) { } } - tgSpec, err := builder.(*targetGroupBuilderImpl).buildTargetGroupSpec(gateway, route, elbv2model.IPAddressTypeIPV4, backend, tgProps) + tgSpec, err := builder.(*targetGroupBuilderImpl).buildTargetGroupSpec(gateway, route, elbv2model.ProtocolHTTP, elbv2model.IPAddressTypeIPV4, backend, tgProps) if tc.expectErr { assert.Error(t, err) @@ -1774,7 +1803,7 @@ func Test_buildTargetGroupFromGateway(t *testing.T) { impl.tgByResID[tgResID] = existingTG } - result, err := impl.buildTargetGroupFromGateway(stack, tc.gateway, tc.listenerPort, tc.lbIPType, tc.route, *tc.backendConfig) + result, err := impl.buildTargetGroupFromGateway(stack, tc.gateway, tc.listenerPort, elbv2model.ProtocolHTTP, tc.lbIPType, tc.route, *tc.backendConfig) assert.NoError(t, err) assert.NotNil(t, result) @@ -1798,125 +1827,6 @@ func Test_buildTargetGroupFromGateway(t *testing.T) { } } -func Test_buildTargetControlPort(t *testing.T) { - testCases := []struct { - name string - targetGroupProps *elbv2gw.TargetGroupProps - tgProtocol elbv2model.Protocol - expectedPort *int32 - tgType elbv2model.TargetType - }{ - { - name: "nil targetGroupProps", - targetGroupProps: nil, - tgProtocol: elbv2model.ProtocolHTTP, - tgType: elbv2model.TargetTypeIP, - expectedPort: nil, - }, - { - name: "nil TargetControlPort", - targetGroupProps: &elbv2gw.TargetGroupProps{ - TargetControlPort: nil, - }, - tgType: elbv2model.TargetTypeIP, - tgProtocol: elbv2model.ProtocolHTTP, - expectedPort: nil, - }, - { - name: "HTTP protocol with target control port", - targetGroupProps: &elbv2gw.TargetGroupProps{ - TargetControlPort: awssdk.Int32(3000), - }, - tgProtocol: elbv2model.ProtocolHTTP, - tgType: elbv2model.TargetTypeIP, - expectedPort: awssdk.Int32(3000), - }, - { - name: "HTTPS protocol with target control port", - targetGroupProps: &elbv2gw.TargetGroupProps{ - TargetControlPort: awssdk.Int32(3000), - }, - tgType: elbv2model.TargetTypeIP, - tgProtocol: elbv2model.ProtocolHTTPS, - expectedPort: awssdk.Int32(3000), - }, - { - name: "TCP protocol with target control port - should return nil", - targetGroupProps: &elbv2gw.TargetGroupProps{ - TargetControlPort: awssdk.Int32(3000), - }, - tgType: elbv2model.TargetTypeIP, - tgProtocol: elbv2model.ProtocolTCP, - expectedPort: nil, - }, - { - name: "UDP protocol with target control port - should return nil", - targetGroupProps: &elbv2gw.TargetGroupProps{ - TargetControlPort: awssdk.Int32(3000), - }, - tgProtocol: elbv2model.ProtocolUDP, - tgType: elbv2model.TargetTypeIP, - expectedPort: nil, - }, - { - name: "TLS protocol with target control port - should return nil", - targetGroupProps: &elbv2gw.TargetGroupProps{ - TargetControlPort: awssdk.Int32(3000), - }, - tgProtocol: elbv2model.ProtocolTLS, - tgType: elbv2model.TargetTypeIP, - expectedPort: nil, - }, - { - name: "Instance target type with target control port - should return nil", - targetGroupProps: &elbv2gw.TargetGroupProps{ - TargetControlPort: awssdk.Int32(3000), - }, - tgProtocol: elbv2model.ProtocolTLS, - tgType: elbv2model.TargetTypeInstance, - expectedPort: nil, - }, - { - name: "ALB target type with target control port - should return nil", - targetGroupProps: &elbv2gw.TargetGroupProps{ - TargetControlPort: awssdk.Int32(3000), - }, - tgProtocol: elbv2model.ProtocolTCP, - tgType: elbv2model.TargetTypeALB, - expectedPort: nil, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - builder := &targetGroupBuilderImpl{} - result, err := builder.buildTargetControlPort(tc.targetGroupProps, tc.tgProtocol, tc.tgType) - - // Check for error conditions - if tc.targetGroupProps != nil && tc.targetGroupProps.TargetControlPort != nil { - if tc.tgProtocol != elbv2model.ProtocolHTTP && tc.tgProtocol != elbv2model.ProtocolHTTPS { - assert.Error(t, err) - assert.Nil(t, result) - return - } - if tc.tgType == elbv2model.TargetTypeInstance || tc.tgType == elbv2model.TargetTypeALB { - assert.Error(t, err) - assert.Nil(t, result) - return - } - } - - assert.NoError(t, err) - if tc.expectedPort == nil { - assert.Nil(t, result) - } else { - assert.NotNil(t, result) - assert.Equal(t, *tc.expectedPort, *result) - } - }) - } -} - func protocolPtr(protocol elbv2gw.Protocol) *elbv2gw.Protocol { return &protocol } diff --git a/pkg/gateway/routeutils/backend_service.go b/pkg/gateway/routeutils/backend_service.go index 0dad940a2..9043c3e6a 100644 --- a/pkg/gateway/routeutils/backend_service.go +++ b/pkg/gateway/routeutils/backend_service.go @@ -173,9 +173,8 @@ func serviceLoader(ctx context.Context, k8sClient client.Client, routeIdentifier return nil, nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch svc object %+v", svcIdentifier)) } - // TODO -- This should be updated, to handle UDP and TCP on the same service port. - // Currently, it will just arbitrarily take one. - + // We just take 1, we don't care about the underlying protocol + // This works for singular protocols [TCP] or dual protocols [TCP_UDP]. var servicePort *corev1.ServicePort for _, svcPort := range svc.Spec.Ports { From 8ed19057a3b735ca4cdd38271b1f479c29689072 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Tue, 25 Nov 2025 09:46:29 -0800 Subject: [PATCH 2/3] documentation for tcp_udp --- docs/guide/gateway/l4gateway.md | 89 +++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/docs/guide/gateway/l4gateway.md b/docs/guide/gateway/l4gateway.md index 200bfc6d2..f1016920a 100644 --- a/docs/guide/gateway/l4gateway.md +++ b/docs/guide/gateway/l4gateway.md @@ -85,6 +85,95 @@ spec: * **L4 Listener Materialization:** The controller processes the `my-tcp-app-route` resource. Given that the `TCPRoute` validly references the `my-tcp-gateway` and its `tcp-app` listener, an **NLB Listener** is materialized on the provisioned NLB. This listener will be configured for `TCP` protocol on `port 8080`, as specified in the `Gateway`'s listener definition. A default forward action is subsequently configured on the NLB Listener, directing all incoming traffic on `port 8080` to the newly created Target Group for service `my-tcp-service` in `backendRefs` section of `my-tcp-app-route`. * **Target Group Creation:** An **AWS Target Group** is created for the Kubernetes Service `my-tcp-service` with default configuration. The cluster nodes are then registered as targets within this new Target Group. + +### Combined Protocols +AWS NLB supports combining TCP and UDP on the same listener; the protocol is called TCP_UDP. This powerful +paradigm allows the load balancer to serve different protocols for different applications on the same listener port. +The LBC implements this protocol merging capability. + +#### Combined protocol quirks + +AWS NLB assumes that in a combined protocol set up, +all targets are able to serve both protocols. To prevent configuration duplication, we follow this same pattern for constructing +the combined protocol listener. TCP_UDP listeners are able to attach routes of type TCP and UDP, each route attached +generates a TCP_UDP target group. + + +#### Combined protocol examples + +```yaml +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: my-tcp-udp-gateway + namespace: tcp-udp +spec: + gatewayClassName: aws-nlb-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: Same + name: tcp-app + port: 80 + protocol: TCP + - allowedRoutes: + namespaces: + from: Same + name: udp-app + port: 80 + protocol: UDP +--- +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: UDPRoute +metadata: + name: my-udp-app-route + namespace: tcp-udp +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: my-tcp-udp-gateway + sectionName: udp-app + rules: + - backendRefs: + - group: "" + kind: Service + name: udpechoserver + port: 8080 + weight: 1 +``` + +To customize the target group created, it's no different from a single protocol +```yaml +apiVersion: gateway.k8s.aws/v1beta1 +kind: TargetGroupConfiguration +metadata: + name: example-tg-config + namespace: tcp-udp +spec: + defaultConfiguration: + targetType: ip + targetReference: + group: "" + kind: Service + name: udpechoserver +``` + +To customize the listener: +```yaml +apiVersion: gateway.k8s.aws/v1beta1 +kind: LoadBalancerConfiguration +metadata: + name: nlb-lb-config + namespace: tcp-udp +spec: + listenerConfigurations: + - protocolPort: TCP_UDP:80 + listenerAttributes: + - key: tcp.idle_timeout.seconds + value: "60" +``` + ### L4 Gateway API Limitations for NLBs The LBC implementation of the Gateway API for L4 routes, which provisions NLB, introduces specific constraints to align with NLB capabilities. These limitations are enforced during the reconciliation process and are critical for successful L4 traffic management. From 4d6292d4a5ab79d5449d2363945d1863b22f729b Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Tue, 25 Nov 2025 16:43:21 -0800 Subject: [PATCH 3/3] e2e test for tcp_udp --- .../targetgroup_configuration_controller.go | 23 ++++- test/e2e/gateway/nlb_instance_target_test.go | 97 ++++++++++++++++++- test/e2e/gateway/nlb_ip_target_test.go | 92 +++++++++++++++++- test/e2e/gateway/nlb_test_helper.go | 39 +++++++- test/e2e/gateway/route_validator.go | 1 + .../gateway/shared_resource_definitions.go | 4 +- 6 files changed, 243 insertions(+), 13 deletions(-) diff --git a/controllers/gateway/targetgroup_configuration_controller.go b/controllers/gateway/targetgroup_configuration_controller.go index 5b486cda0..eeaa14f7f 100644 --- a/controllers/gateway/targetgroup_configuration_controller.go +++ b/controllers/gateway/targetgroup_configuration_controller.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" @@ -115,11 +116,25 @@ func (r *targetgroupConfigurationReconciler) handleDelete(tgConf *elbv2gw.Target Name: tgConf.Spec.TargetReference.Name, } - eligibleForRemoval := r.serviceReferenceCounter.IsEligibleForRemoval(svcReference, allGateways) + svc := &corev1.Service{} + err := r.k8sClient.Get(context.Background(), svcReference, svc) - // if the targetgroup configuration is still in use, we should not delete it - if !eligibleForRemoval { - return fmt.Errorf("targetgroup configuration [%+v] is still in use", k8s.NamespacedName(tgConf)) + referenceCheckNeeded := true + if err != nil { + notFoundErr := client.IgnoreNotFound(err) + if notFoundErr != nil { + return notFoundErr + } + referenceCheckNeeded = false + } + + if referenceCheckNeeded { + eligibleForRemoval := r.serviceReferenceCounter.IsEligibleForRemoval(svcReference, allGateways) + + // if the targetgroup configuration is still in use, we should not delete it + if !eligibleForRemoval { + return fmt.Errorf("targetgroup configuration [%+v] is still in use", k8s.NamespacedName(tgConf)) + } } return r.finalizerManager.RemoveFinalizers(context.Background(), tgConf, shared_constants.TargetGroupConfigurationFinalizer) } diff --git a/test/e2e/gateway/nlb_instance_target_test.go b/test/e2e/gateway/nlb_instance_target_test.go index b1d02fb5c..e7f9af265 100644 --- a/test/e2e/gateway/nlb_instance_target_test.go +++ b/test/e2e/gateway/nlb_instance_target_test.go @@ -3,10 +3,6 @@ package gateway import ( "context" "fmt" - "strconv" - "strings" - "time" - awssdk "github.com/aws/aws-sdk-go-v2/aws" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -14,6 +10,9 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/test/framework/http" "sigs.k8s.io/aws-load-balancer-controller/test/framework/utils" "sigs.k8s.io/aws-load-balancer-controller/test/framework/verifier" + "strconv" + "strings" + "time" ) var _ = Describe("test nlb gateway using instance targets reconciled by the aws load balancer controller", func() { @@ -34,8 +33,11 @@ var _ = Describe("test nlb gateway using instance targets reconciled by the aws }) AfterEach(func() { stack.Cleanup(ctx, tf) - auxiliaryStack.Cleanup(ctx, tf) + if auxiliaryStack != nil { + auxiliaryStack.Cleanup(ctx, tf) + } }) + Context(fmt.Sprintf("with NLB instance target configuration, using readiness gates %+v", false), func() { BeforeEach(func() {}) It("should provision internet-facing load balancer resources", func() { @@ -567,5 +569,90 @@ var _ = Describe("test nlb gateway using instance targets reconciled by the aws validateL4RouteStatusNotPermitted(tf, stack, hasTLS) }) }) + Context(fmt.Sprintf("with NLB instance target using TCP_UDP listener"), func() { + BeforeEach(func() {}) + It("should provision internet-facing load balancer resources", func() { + interf := elbv2gw.LoadBalancerSchemeInternetFacing + lbcSpec := elbv2gw.LoadBalancerConfigurationSpec{ + Scheme: &interf, + } + + instanceTargetType := elbv2gw.TargetTypeInstance + tgSpec := elbv2gw.TargetGroupConfigurationSpec{ + DefaultConfiguration: elbv2gw.TargetGroupProps{ + TargetType: &instanceTargetType, + }, + } + By("deploying stack", func() { + err := stack.DeployTCP_UDP(ctx, tf, lbcSpec, tgSpec, false) + Expect(err).NotTo(HaveOccurred()) + }) + + By("checking gateway status for lb dns name", func() { + dnsName = stack.GetLoadBalancerIngressHostName() + Expect(dnsName).ToNot(BeEmpty()) + }) + + By("querying AWS loadbalancer from the dns name", func() { + var err error + lbARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, dnsName) + Expect(err).NotTo(HaveOccurred()) + Expect(lbARN).ToNot(BeEmpty()) + }) + + By("verifying AWS loadbalancer resources", func() { + nodeList, err := stack.GetWorkerNodes(ctx, tf) + Expect(err).ToNot(HaveOccurred()) + expectedTargetGroups := []verifier.ExpectedTargetGroup{ + { + Protocol: "TCP_UDP", + Port: stack.nlbResourceStack.commonStack.svcs[0].Spec.Ports[0].NodePort, + NumTargets: len(nodeList), + TargetType: "instance", + TargetGroupHC: &verifier.TargetGroupHC{ + Protocol: "TCP", + Port: "traffic-port", + Interval: 15, + Timeout: 5, + HealthyThreshold: 3, + UnhealthyThreshold: 3, + }, + }, + } + + listenerPortMap := map[string]string{ + "80": "TCP_UDP", + } + + err = verifier.VerifyAWSLoadBalancerResources(ctx, tf, lbARN, verifier.LoadBalancerExpectation{ + Type: "network", + Scheme: "internet-facing", + Listeners: listenerPortMap, + TargetGroups: expectedTargetGroups, + }) + Expect(err).NotTo(HaveOccurred()) + }) + By("waiting for target group targets to be healthy", func() { + nodeList, err := stack.GetWorkerNodes(ctx, tf) + Expect(err).ToNot(HaveOccurred()) + err = verifier.WaitUntilTargetsAreHealthy(ctx, tf, lbARN, len(nodeList)) + Expect(err).NotTo(HaveOccurred()) + }) + By("waiting until DNS name is available", func() { + err := utils.WaitUntilDNSNameAvailable(ctx, dnsName) + Expect(err).NotTo(HaveOccurred()) + }) + By("sending http request to the lb", func() { + url := fmt.Sprintf("http://%v/any-path", dnsName) + err := tf.HTTPVerifier.VerifyURL(url, http.ResponseCodeMatches(200)) + Expect(err).NotTo(HaveOccurred()) + }) + By("sending udp request to the lb", func() { + endpoint := fmt.Sprintf("%v:80", dnsName) + err := tf.UDPVerifier.VerifyUDP(endpoint) + Expect(err).NotTo(HaveOccurred()) + }) + }) + }) }) }) diff --git a/test/e2e/gateway/nlb_ip_target_test.go b/test/e2e/gateway/nlb_ip_target_test.go index 00cbdeeaf..f7dad604a 100644 --- a/test/e2e/gateway/nlb_ip_target_test.go +++ b/test/e2e/gateway/nlb_ip_target_test.go @@ -32,9 +32,12 @@ var _ = Describe("test nlb gateway using ip targets reconciled by the aws load b }) AfterEach(func() { stack.Cleanup(ctx, tf) - auxiliaryStack.Cleanup(ctx, tf) + if auxiliaryStack != nil { + auxiliaryStack.Cleanup(ctx, tf) + } }) for _, readinessGateEnabled := range []bool{true, false} { + Context(fmt.Sprintf("with NLB ip target configuration, using readiness gates %+v", readinessGateEnabled), func() { It("should provision internet-facing load balancer resources", func() { interf := elbv2gw.LoadBalancerSchemeInternetFacing @@ -282,6 +285,7 @@ var _ = Describe("test nlb gateway using ip targets reconciled by the aws load b }) }) }) + Context(fmt.Sprintf("with NLB ip target configuration, using no SG, using readiness gates %+v", readinessGateEnabled), func() { It("should provision internet-facing load balancer resources", func() { interf := elbv2gw.LoadBalancerSchemeInternetFacing @@ -529,5 +533,91 @@ var _ = Describe("test nlb gateway using ip targets reconciled by the aws load b }) }) }) + + Context(fmt.Sprintf("with TCP_UDP listener, using readiness gates %+v", readinessGateEnabled), func() { + It("should provision internet-facing load balancer resources", func() { + interf := elbv2gw.LoadBalancerSchemeInternetFacing + lbcSpec := elbv2gw.LoadBalancerConfigurationSpec{ + Scheme: &interf, + } + + ipTargetType := elbv2gw.TargetTypeIP + tgSpec := elbv2gw.TargetGroupConfigurationSpec{ + DefaultConfiguration: elbv2gw.TargetGroupProps{ + TargetType: &ipTargetType, + }, + } + + By("deploying stack", func() { + err := stack.DeployTCP_UDP(ctx, tf, lbcSpec, tgSpec, false) + Expect(err).NotTo(HaveOccurred()) + }) + + By("checking gateway status for lb dns name", func() { + dnsName = stack.GetLoadBalancerIngressHostName() + Expect(dnsName).ToNot(BeEmpty()) + }) + + By("querying AWS loadbalancer from the dns name", func() { + var err error + lbARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, dnsName) + Expect(err).NotTo(HaveOccurred()) + Expect(lbARN).ToNot(BeEmpty()) + }) + + targetNumber := int(*stack.nlbResourceStack.commonStack.dps[0].Spec.Replicas) + + expectedTargetGroups := []verifier.ExpectedTargetGroup{ + { + Protocol: "TCP_UDP", + Port: 8080, + NumTargets: targetNumber, + TargetType: "ip", + TargetGroupHC: &verifier.TargetGroupHC{ + Protocol: "TCP", + Port: "traffic-port", + Interval: 15, + Timeout: 5, + HealthyThreshold: 3, + UnhealthyThreshold: 3, + }, + }, + } + + listenerPortMap := map[string]string{ + "80": "TCP_UDP", + } + + fmt.Println("------VALIDATE NOW!--------") + time.Sleep(10 * time.Minute) + By("verifying AWS loadbalancer resources", func() { + err := verifier.VerifyAWSLoadBalancerResources(ctx, tf, lbARN, verifier.LoadBalancerExpectation{ + Type: "network", + Scheme: "internet-facing", + Listeners: listenerPortMap, + TargetGroups: expectedTargetGroups, + }) + Expect(err).NotTo(HaveOccurred()) + }) + By("waiting for target group targets to be healthy", func() { + err := verifier.WaitUntilTargetsAreHealthy(ctx, tf, lbARN, targetNumber) + Expect(err).NotTo(HaveOccurred()) + }) + By("waiting until DNS name is available", func() { + err := utils.WaitUntilDNSNameAvailable(ctx, dnsName) + Expect(err).NotTo(HaveOccurred()) + }) + By("sending http request to the lb", func() { + url := fmt.Sprintf("http://%v/any-path", dnsName) + err := tf.HTTPVerifier.VerifyURL(url, http.ResponseCodeMatches(200)) + Expect(err).NotTo(HaveOccurred()) + }) + By("sending udp request to the lb", func() { + endpoint := fmt.Sprintf("%v:80", dnsName) + err := tf.UDPVerifier.VerifyUDP(endpoint) + Expect(err).NotTo(HaveOccurred()) + }) + }) + }) } }) diff --git a/test/e2e/gateway/nlb_test_helper.go b/test/e2e/gateway/nlb_test_helper.go index 7031db938..544ab25e4 100644 --- a/test/e2e/gateway/nlb_test_helper.go +++ b/test/e2e/gateway/nlb_test_helper.go @@ -85,13 +85,49 @@ func (s *NLBTestStack) Deploy(ctx context.Context, f *framework.Framework, auxil lbc := buildLoadBalancerConfig(lbConfSpec) tgcTCP := buildTargetGroupConfig(defaultTgConfigName, tgConfSpec, svcTCP) tgcUDP := buildTargetGroupConfig(udpDefaultTgConfigName, tgConfSpec, svcUDP) - udpr := buildUDPRoute() + udpr := buildUDPRoute("port8080") s.nlbResourceStack = newNLBResourceStack([]*appsv1.Deployment{dpTCP, dpUDP}, []*corev1.Service{svcTCP, svcUDP}, gwc, gw, lbc, []*elbv2gw.TargetGroupConfiguration{tgcTCP, tgcUDP}, tcprs, []*gwalpha2.UDPRoute{udpr}, nil, "nlb-gateway-e2e", readinessGateEnabled) return s.nlbResourceStack.Deploy(ctx, f) } +func (s *NLBTestStack) DeployTCP_UDP(ctx context.Context, f *framework.Framework, lbConfSpec elbv2gw.LoadBalancerConfigurationSpec, tgConfSpec elbv2gw.TargetGroupConfigurationSpec, readinessGateEnabled bool) error { + dpUDP := buildUDPDeploymentSpec() + svcUDP := buildUDPServiceSpec() + gwc := buildGatewayClassSpec("gateway.k8s.aws/nlb") + + if f.Options.IPFamily == framework.IPv6 { + v6 := elbv2gw.LoadBalancerIpAddressTypeDualstack + lbConfSpec.IpAddressType = &v6 + } + + listeners := []gwv1.Listener{ + { + Name: "port80tcp", + Port: 80, + Protocol: gwv1.TCPProtocolType, + }, + { + Name: "port80udp", + Port: 80, + Protocol: gwv1.UDPProtocolType, + }, + } + + tcprs := []*gwalpha2.TCPRoute{} + + gw := buildBasicGatewaySpec(gwc, listeners) + + lbc := buildLoadBalancerConfig(lbConfSpec) + tgcUDP := buildTargetGroupConfig(udpDefaultTgConfigName, tgConfSpec, svcUDP) + udpr := buildUDPRoute("port80udp") + + s.nlbResourceStack = newNLBResourceStack([]*appsv1.Deployment{dpUDP}, []*corev1.Service{svcUDP}, gwc, gw, lbc, []*elbv2gw.TargetGroupConfiguration{tgcUDP}, tcprs, []*gwalpha2.UDPRoute{udpr}, nil, "nlb-gateway-e2e", readinessGateEnabled) + + return s.nlbResourceStack.Deploy(ctx, f) +} + func (s *NLBTestStack) DeployFrontendNLB(ctx context.Context, albStack ALBTestStack, f *framework.Framework, lbConfSpec elbv2gw.LoadBalancerConfigurationSpec, hasTLS bool, readinessGateEnabled bool) error { gwc := buildGatewayClassSpec("gateway.k8s.aws/nlb") @@ -285,6 +321,7 @@ func validateL4RouteStatusNotPermitted(tf *framework.Framework, stack NLBTestSta }, }, } + validateRouteStatus(tf, stack.nlbResourceStack.tcprs, tcpRouteStatusConverter, tcpValidationInfo) validateRouteStatus(tf, stack.nlbResourceStack.udprs, udpRouteStatusConverter, udpValidationInfo) } diff --git a/test/e2e/gateway/route_validator.go b/test/e2e/gateway/route_validator.go index 542a83e25..3f8d90cab 100644 --- a/test/e2e/gateway/route_validator.go +++ b/test/e2e/gateway/route_validator.go @@ -41,6 +41,7 @@ func validateRouteStatus[R any](tf *framework.Framework, routes []R, routeStatus Expect(string(cond.Status)).To(Equal(listener.resolvedRefsStatus)) Expect(cond.Reason).To(Equal(listener.resolvedRefReason)) } else if cond.Type == string(gwv1.RouteConditionAccepted) { + fmt.Printf("%+v, %+v\n", listener, cond) Expect(string(cond.Status)).To(Equal(listener.acceptedStatus)) Expect(cond.Reason).To(Equal(listener.acceptedReason)) } else { diff --git a/test/e2e/gateway/shared_resource_definitions.go b/test/e2e/gateway/shared_resource_definitions.go index 87fa4b487..84ce5692d 100644 --- a/test/e2e/gateway/shared_resource_definitions.go +++ b/test/e2e/gateway/shared_resource_definitions.go @@ -348,7 +348,7 @@ func buildFENLBTCPRoute(albGatewayName, albNamespace string, port gwalpha2.PortN return tcpr } -func buildUDPRoute() *gwalpha2.UDPRoute { +func buildUDPRoute(sectionName string) *gwalpha2.UDPRoute { port := gwalpha2.PortNumber(8080) udpr := &gwalpha2.UDPRoute{ ObjectMeta: metav1.ObjectMeta{ @@ -359,7 +359,7 @@ func buildUDPRoute() *gwalpha2.UDPRoute { ParentRefs: []gwv1.ParentReference{ { Name: defaultName, - SectionName: (*gwv1.SectionName)(awssdk.String("port8080")), + SectionName: (*gwv1.SectionName)(awssdk.String(sectionName)), }, }, },