adds endpoint-security-group-tags flag

Author: carflo

main.go

@@ -108,7 +108,7 @@ func main() {
 	tgbResManager := targetgroupbinding.NewDefaultResourceManager(mgr.GetClient(), cloud.ELBV2(), cloud.EC2(),
 		podInfoRepo, sgManager, sgReconciler, vpcInfoProvider,
 		cloud.VpcID(), controllerCFG.ClusterName, controllerCFG.FeatureGates.Enabled(config.EndpointsFailOpen), controllerCFG.EnableEndpointSlices, controllerCFG.DisableRestrictedSGRules,
-		mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log)
+		controllerCFG.EndpointSGTags, mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log)
 	backendSGProvider := networking.NewBackendSGProvider(controllerCFG.ClusterName, controllerCFG.BackendSecurityGroup,
 		cloud.VpcID(), cloud.EC2(), mgr.GetClient(), controllerCFG.DefaultTags, ctrl.Log.WithName("backend-sg-provider"))
 	ingGroupReconciler := ingress.NewGroupReconciler(cloud, mgr.GetClient(), mgr.GetEventRecorderFor("ingress"),

pkg/config/controller_config.go

@@ -18,6 +18,7 @@ const (
 	flagDefaultTags                                  = "default-tags"
 	flagDefaultTargetType                            = "default-target-type"
 	flagExternalManagedTags                          = "external-managed-tags"
+	flagEndpointSGTags                               = "endpoint-security-group-tags"
 	flagServiceMaxConcurrentReconciles               = "service-max-concurrent-reconciles"
 	flagTargetGroupBindingMaxConcurrentReconciles    = "targetgroupbinding-max-concurrent-reconciles"
 	flagTargetGroupBindingMaxExponentialBackoffDelay = "targetgroupbinding-max-exponential-backoff-delay"
@@ -74,6 +75,9 @@ type ControllerConfig struct {
 	// List of Tag keys on AWS resources that will be managed externally.
 	ExternalManagedTags []string
 
+	// AWS Tags that will be used by the controller to find the worker node security group to add inbound rules from NLBs.
+	EndpointSGTags map[string]string
+
 	// Default SSL Policy that will be applied to all ingresses or services that do not have
 	// the SSL Policy annotation.
 	DefaultSSLPolicy string
@@ -128,7 +132,8 @@ func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) {
 		"Enable EndpointSlices for IP targets instead of Endpoints")
 	fs.BoolVar(&cfg.DisableRestrictedSGRules, flagDisableRestrictedSGRules, defaultDisableRestrictedSGRules,
 		"Disable the usage of restricted security group rules")
-
+	fs.StringToStringVar(&cfg.EndpointSGTags, flagEndpointSGTags, nil,
+		"AWS Tags that will be used by the controller to find the worker node security group to add inbound rules from NLBs")
 	cfg.FeatureGates.BindFlags(fs)
 	cfg.AWSConfig.BindFlags(fs)
 	cfg.RuntimeConfig.BindFlags(fs)

pkg/targetgroupbinding/networking_manager.go

@@ -45,7 +45,7 @@ type NetworkingManager interface {
 
 // NewDefaultNetworkingManager constructs defaultNetworkingManager.
 func NewDefaultNetworkingManager(k8sClient client.Client, podENIResolver networking.PodENIInfoResolver, nodeENIResolver networking.NodeENIInfoResolver,
-	sgManager networking.SecurityGroupManager, sgReconciler networking.SecurityGroupReconciler, vpcID string, clusterName string, logger logr.Logger, disabledRestrictedSGRulesFlag bool) *defaultNetworkingManager {
+	sgManager networking.SecurityGroupManager, sgReconciler networking.SecurityGroupReconciler, vpcID string, clusterName string, endpointSGTags map[string]string, logger logr.Logger, disabledRestrictedSGRulesFlag bool) *defaultNetworkingManager {
 
 	return &defaultNetworkingManager{
 		k8sClient:       k8sClient,
@@ -55,6 +55,7 @@ func NewDefaultNetworkingManager(k8sClient client.Client, podENIResolver network
 		sgReconciler:    sgReconciler,
 		vpcID:           vpcID,
 		clusterName:     clusterName,
+		endpointSGTags:  endpointSGTags,
 		logger:          logger,
 
 		mutex:                         sync.Mutex{},
@@ -74,6 +75,7 @@ type defaultNetworkingManager struct {
 	sgReconciler    networking.SecurityGroupReconciler
 	vpcID           string
 	clusterName     string
+	endpointSGTags  map[string]string
 	logger          logr.Logger
 
 	// mutex will serialize our TargetGroup's networking reconcile requests.
@@ -518,19 +520,36 @@ func (m *defaultNetworkingManager) resolveEndpointSGForENI(ctx context.Context,
 		return "", err
 	}
 	clusterResourceTagKey := fmt.Sprintf("kubernetes.io/cluster/%s", m.clusterName)
-	sgIDsWithClusterTag := sets.NewString()
+	sgIDsWithMatchingEndpointSGTags := sets.NewString()
 	for sgID, sgInfo := range sgInfoByID {
+		isMatch := true
 		if _, ok := sgInfo.Tags[clusterResourceTagKey]; ok {
-			sgIDsWithClusterTag.Insert(sgID)
+			for endpointSGTagKey, endpointSGTagValue := range m.endpointSGTags {
+				if sgInfo.Tags[endpointSGTagKey] != endpointSGTagValue {
+					isMatch = false
+					break
+				}
+			}
+		} else {
+			continue
+		}
+
+		if isMatch {
+			sgIDsWithMatchingEndpointSGTags.Insert(sgID)
 		}
 	}
-	if len(sgIDsWithClusterTag) != 1 {
+
+	if len(sgIDsWithMatchingEndpointSGTags) != 1 {
 		// user may provide incorrect `--cluster-name` at bootstrap or modify the tag key unexpectedly, it is hard to find out if no clusterName included in error message.
 		// having `clusterName` included in error message might be helpful for shorten the troubleshooting time spent.
-		return "", errors.Errorf("expect exactly one securityGroup tagged with %v for eni %v, got: %v (clusterName: %v)",
-			clusterResourceTagKey, eniInfo.NetworkInterfaceID, sgIDsWithClusterTag.List(), m.clusterName)
+		if len(m.endpointSGTags) == 0 {
+			return "", errors.Errorf("expect exactly one securityGroup tagged with %v for eni %v, got: %v (clusterName: %v)",
+				clusterResourceTagKey, eniInfo.NetworkInterfaceID, sgIDsWithMatchingEndpointSGTags.List(), m.clusterName)
+		}
+		return "", errors.Errorf("expect exactly one securityGroup tagged with %v and %v for eni %v, got: %v (clusterName: %v)",
+			clusterResourceTagKey, m.endpointSGTags, eniInfo.NetworkInterfaceID, sgIDsWithMatchingEndpointSGTags.List(), m.clusterName)
 	}
-	sgID, _ := sgIDsWithClusterTag.PopAny()
+	sgID, _ := sgIDsWithMatchingEndpointSGTags.PopAny()
 	return sgID, nil
 }
 

pkg/targetgroupbinding/resource_manager.go

@@ -42,6 +42,7 @@ func NewDefaultResourceManager(k8sClient client.Client, elbv2Client services.ELB
 	podInfoRepo k8s.PodInfoRepo, sgManager networking.SecurityGroupManager, sgReconciler networking.SecurityGroupReconciler,
 	vpcInfoProvider networking.VPCInfoProvider,
 	vpcID string, clusterName string, failOpenEnabled bool, endpointSliceEnabled bool, disabledRestrictedSGRulesFlag bool,
+	endpointSGTags map[string]string,
 	eventRecorder record.EventRecorder, logger logr.Logger) *defaultResourceManager {
 	targetsManager := NewCachedTargetsManager(elbv2Client, logger)
 	endpointResolver := backend.NewDefaultEndpointResolver(k8sClient, podInfoRepo, failOpenEnabled, endpointSliceEnabled, logger)
@@ -50,7 +51,7 @@ func NewDefaultResourceManager(k8sClient client.Client, elbv2Client services.ELB
 	podENIResolver := networking.NewDefaultPodENIInfoResolver(k8sClient, ec2Client, nodeInfoProvider, vpcID, logger)
 	nodeENIResolver := networking.NewDefaultNodeENIInfoResolver(nodeInfoProvider, logger)
 
-	networkingManager := NewDefaultNetworkingManager(k8sClient, podENIResolver, nodeENIResolver, sgManager, sgReconciler, vpcID, clusterName, logger, disabledRestrictedSGRulesFlag)
+	networkingManager := NewDefaultNetworkingManager(k8sClient, podENIResolver, nodeENIResolver, sgManager, sgReconciler, vpcID, clusterName, endpointSGTags, logger, disabledRestrictedSGRulesFlag)
 	return &defaultResourceManager{
 		k8sClient:         k8sClient,
 		targetsManager:    targetsManager,