refactor: move resource patching into the stack

feature: reduce resource waste by setting replicas to 1 for the new unused stack, that will not receive any traffic

Signed-off-by: Sandor Szücs <[email protected]>

Author: szuecs

pkg/core/stackset.go

@@ -52,22 +52,6 @@ func sanitizeServicePorts(service *zv1.StackServiceSpec) *zv1.StackServiceSpec {
 	return service
 }
 
-func patchForwardBackend(rg *zv1.RouteGroupSpec) {
-	rg.AdditionalBackends = []rgv1.RouteGroupBackend{
-		{
-			Name: forwardBackendName,
-			Type: rgv1.ForwardRouteGroupBackend,
-		},
-	}
-	for _, route := range rg.Routes {
-		route.Backends = []rgv1.RouteGroupBackendReference{
-			{
-				BackendName: forwardBackendName,
-			},
-		}
-	}
-}
-
 // NewStack returns an (optional) stack that should be created
 func (ssc *StackSetContainer) NewStack() (*StackContainer, string) {
 	_, forwardMigration := ssc.StackSet.ObjectMeta.Annotations[forwardBackendAnnotation]
@@ -89,9 +73,6 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) {
 
 		if ssc.StackSet.Spec.Ingress != nil {
 			spec.Ingress = ssc.StackSet.Spec.Ingress.DeepCopy()
-			if forwardMigration {
-				spec.Ingress.EmbeddedObjectMetaWithAnnotations.Annotations["zalando.org/skipper-backend"] = "forward"
-			}
 		}
 
 		if ssc.StackSet.Spec.ExternalIngress != nil {
@@ -100,9 +81,11 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) {
 
 		if ssc.StackSet.Spec.RouteGroup != nil {
 			spec.RouteGroup = ssc.StackSet.Spec.RouteGroup.DeepCopy()
-			if forwardMigration {
-				patchForwardBackend(spec.RouteGroup)
-			}
+		}
+
+		stackAnnotations := ssc.StackSet.Spec.StackTemplate.Annotations
+		if forwardMigration {
+			stackAnnotations[forwardBackendAnnotation] = forwardBackendName
 		}
 
 		return &StackContainer{
@@ -125,7 +108,7 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) {
 						ssc.StackSet.Labels,
 						map[string]string{StackVersionLabelKey: stackVersion},
 					),
-					Annotations: ssc.StackSet.Spec.StackTemplate.Annotations,
+					Annotations: stackAnnotations,
 				},
 				Spec: *spec,
 			},

pkg/core/types.go

@@ -419,18 +419,34 @@ func (sc *StackContainer) updateStackResources() error {
 }
 
 func (sc *StackContainer) updateFromResources() {
-	sc.stackReplicas = effectiveReplicas(sc.Stack.Spec.StackSpec.Replicas)
+	_, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]
+
+	if clusterMigration {
+		// we do not use these so reduce wasted resources!
+		sc.stackReplicas = 1
+	} else {
+		sc.stackReplicas = effectiveReplicas(sc.Stack.Spec.StackSpec.Replicas)
+	}
 
 	var deploymentUpdated, serviceUpdated, ingressUpdated, routeGroupUpdated, hpaUpdated bool
 	var ingressSegmentUpdated, routeGroupSegmentUpdated bool
 
 	// deployment
 	if sc.Resources.Deployment != nil {
 		deployment := sc.Resources.Deployment
-		sc.deploymentReplicas = effectiveReplicas(deployment.Spec.Replicas)
-		sc.createdReplicas = deployment.Status.Replicas
-		sc.readyReplicas = deployment.Status.ReadyReplicas
-		sc.updatedReplicas = deployment.Status.UpdatedReplicas
+
+		if clusterMigration {
+			// we do not use these so reduce wasted resources!
+			sc.deploymentReplicas = 1
+			sc.createdReplicas = 1
+			sc.readyReplicas = 1
+			sc.updatedReplicas = 1
+		} else {
+			sc.deploymentReplicas = effectiveReplicas(deployment.Spec.Replicas)
+			sc.createdReplicas = deployment.Status.Replicas
+			sc.readyReplicas = deployment.Status.ReadyReplicas
+			sc.updatedReplicas = deployment.Status.UpdatedReplicas
+		}
 		deploymentUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.Deployment.ObjectMeta) && deployment.Status.ObservedGeneration == deployment.Generation
 	}
 
@@ -442,13 +458,20 @@ func (sc *StackContainer) updateFromResources() {
 		// the per-stack ingress must either be present and up-to-date, or not present and not expected.
 		// the per-stack ingress is not expected if the stack has no hostnames matching the cluster domain.
 		if sc.Resources.Ingress != nil {
+			if clusterMigration {
+				sc.Resources.Ingress.Annotations["zalando.org/skipper-backend"] = "forward"
+			}
 			ingressUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.Ingress.ObjectMeta)
 		} else {
 			hostnames := sc.stackHostnames(sc.ingressSpec, false)
 			ingressUpdated = len(hostnames) == 0
 		}
-		ingressSegmentUpdated = sc.Resources.IngressSegment != nil &&
-			IsResourceUpToDate(sc.Stack, sc.Resources.IngressSegment.ObjectMeta)
+		if sc.Resources.IngressSegment != nil {
+			if clusterMigration {
+				sc.Resources.IngressSegment.Annotations["zalando.org/skipper-backend"] = "forward"
+			}
+			ingressSegmentUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.IngressSegment.ObjectMeta)
+		}
 	} else {
 		// ignore if ingress is not set
 		ingressUpdated = sc.Resources.Ingress == nil
@@ -460,24 +483,33 @@ func (sc *StackContainer) updateFromResources() {
 		// the per-stack route group must either be present and up-to-date, or not present and not expected.
 		// the per-stack route group is not expected if the stack has no hostnames matching the cluster domain.
 		if sc.Resources.RouteGroup != nil {
+			if clusterMigration {
+				patchForwardBackend(&sc.Resources.RouteGroup.Spec)
+			}
 			routeGroupUpdated = IsResourceUpToDate(sc.Stack, sc.Resources.RouteGroup.ObjectMeta)
 		} else {
 			hostnames := sc.stackHostnames(sc.routeGroupSpec, false)
 			routeGroupUpdated = len(hostnames) == 0
 		}
-		routeGroupSegmentUpdated = sc.Resources.RouteGroupSegment != nil &&
-			IsResourceUpToDate(
+
+		if sc.Resources.RouteGroupSegment != nil {
+			if clusterMigration {
+				patchForwardBackend(&sc.Resources.RouteGroupSegment.Spec)
+			}
+
+			routeGroupSegmentUpdated = IsResourceUpToDate(
 				sc.Stack,
 				sc.Resources.RouteGroupSegment.ObjectMeta,
 			)
+		}
 	} else {
 		// ignore if route group is not set
 		routeGroupUpdated = sc.Resources.RouteGroup == nil
 		routeGroupSegmentUpdated = sc.Resources.RouteGroupSegment == nil
 	}
 
-	// hpa
-	if sc.IsAutoscaled() {
+	// hpa not used if we migrate cluster to reduce wasted resources
+	if sc.IsAutoscaled() && !clusterMigration {
 		hpaUpdated = sc.Resources.HPA != nil && IsResourceUpToDate(sc.Stack, sc.Resources.HPA.ObjectMeta)
 	} else {
 		hpaUpdated = sc.Resources.HPA == nil
@@ -494,10 +526,27 @@ func (sc *StackContainer) updateFromResources() {
 
 	status := sc.Stack.Status
 	sc.noTrafficSince = unwrapTime(status.NoTrafficSince)
-	if status.Prescaling.Active {
+	// do not prescale on cluste rmigration to reduce wasted resources
+	if status.Prescaling.Active && !clusterMigration {
 		sc.prescalingActive = true
 		sc.prescalingReplicas = status.Prescaling.Replicas
 		sc.prescalingDesiredTrafficWeight = status.Prescaling.DesiredTrafficWeight
 		sc.prescalingLastTrafficIncrease = unwrapTime(status.Prescaling.LastTrafficIncrease)
 	}
 }
+
+func patchForwardBackend(rg *rgv1.RouteGroupSpec) {
+	rg.Backends = []rgv1.RouteGroupBackend{
+		{
+			Name: forwardBackendName,
+			Type: rgv1.ForwardRouteGroupBackend,
+		},
+	}
+	for _, route := range rg.Routes {
+		route.Backends = []rgv1.RouteGroupBackendReference{
+			{
+				BackendName: forwardBackendName,
+			},
+		}
+	}
+}