feature: zalando.org/forward-backend annotation support to enable migration to eks

docs/stack_crd.yaml

@@ -475,6 +475,22 @@ spec:
                     - type: integer
                     - type: string
                     x-kubernetes-int-or-string: true
+                  metadata:
+                    description: |-
+                      EmbeddedObjectMetaWithAnnotations defines the metadata which can be attached
+                      to a resource. It's a slimmed down version of metav1.ObjectMeta only
+                      containing annotations.
+                    properties:
+                      annotations:
+                        additionalProperties:
+                          type: string
+                        description: |-
+                          Annotations is an unstructured key value map stored with a resource that may be
+                          set by external tools to store and retrieve arbitrary metadata. They are not
+                          queryable and should be preserved when modifying objects.
+                          More info: http://kubernetes.io/docs/user-guide/annotations
+                        type: object
+                    type: object
                 required:
                 - backendPort
                 type: object

docs/stackset_crd.yaml

@@ -67,6 +67,22 @@ spec:
                     - type: integer
                     - type: string
                     x-kubernetes-int-or-string: true
+                  metadata:
+                    description: |-
+                      EmbeddedObjectMetaWithAnnotations defines the metadata which can be attached
+                      to a resource. It's a slimmed down version of metav1.ObjectMeta only
+                      containing annotations.
+                    properties:
+                      annotations:
+                        additionalProperties:
+                          type: string
+                        description: |-
+                          Annotations is an unstructured key value map stored with a resource that may be
+                          set by external tools to store and retrieve arbitrary metadata. They are not
+                          queryable and should be preserved when modifying objects.
+                          More info: http://kubernetes.io/docs/user-guide/annotations
+                        type: object
+                    type: object
                 required:
                 - backendPort
                 type: object
@@ -3589,12 +3605,6 @@ spec:
                                                 Must be set if and only if type is "Localhost".
                                               type: string
                                             type:
-                                              description: |-
-                                                type indicates which kind of AppArmor profile will be applied.
-                                                Valid options are:
-                                                  Localhost - a profile pre-loaded on the node.
-                                                  RuntimeDefault - the container runtime's default profile.
-                                                  Unconfined - no AppArmor enforcement.
                                               type: string
                                           required:
                                           - type

pkg/apis/zalando.org/v1/types.go

@@ -124,7 +124,8 @@ func (s *StackSetIngressSpec) GetAnnotations() map[string]string {
 // backendport for ingress managed outside of stackset.
 // +k8s:deepcopy-gen=true
 type StackSetExternalIngressSpec struct {
-	BackendPort intstr.IntOrString `json:"backendPort"`
+	EmbeddedObjectMetaWithAnnotations `json:"metadata,omitempty"`
+	BackendPort                       intstr.IntOrString `json:"backendPort"`
 }
 
 // RouteGroupSpec defines the specification for defining a RouteGroup attached

pkg/core/stack_resources.go

@@ -76,6 +76,23 @@ func objectMetaInjectLabels(objectMeta metav1.ObjectMeta, labels map[string]stri
 	return objectMeta
 }
 
+// patchForwardBackend rewrites a RouteGroupSpec to send all traffic to another cluster chosen by the operator of skipper
+func patchForwardBackend(rg *rgv1.RouteGroupSpec) {
+	rg.Backends = []rgv1.RouteGroupBackend{
+		{
+			Name: forwardBackendName,
+			Type: rgv1.ForwardRouteGroupBackend,
+		},
+	}
+	for _, route := range rg.Routes {
+		route.Backends = []rgv1.RouteGroupBackendReference{
+			{
+				BackendName: forwardBackendName,
+			},
+		}
+	}
+}
+
 func (sc *StackContainer) objectMeta(segment bool) metav1.ObjectMeta {
 	resourceLabels := mapCopy(sc.Stack.Labels)
 
@@ -169,6 +186,10 @@ func (sc *StackContainer) selector() map[string]string {
 	return limitLabels(sc.Stack.Labels, selectorLabels)
 }
 
+// GenerateDeployment generates a deployment as configured in the
+// stack.  On cluster migrations set by stackset annotation
+// "zalando.org/forward-backend", the deployment will be set to
+// replicas 1.
 func (sc *StackContainer) GenerateDeployment() *appsv1.Deployment {
 	stack := sc.Stack
 
@@ -224,9 +245,19 @@ func (sc *StackContainer) GenerateDeployment() *appsv1.Deployment {
 	if strategy != nil {
 		deployment.Spec.Strategy = *strategy
 	}
+
+	if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration {
+		i := int32(1)
+		deployment.Spec.Replicas = &i
+	}
+
 	return deployment
 }
 
+// GenerateHPA generates a hpa as configured in the
+// stack.  On cluster migrations set by stackset annotation
+// "zalando.org/forward-backend", the hpa will be set to
+// minReplicas = maxReplicass = 1.
 func (sc *StackContainer) GenerateHPA() (
 	*autoscaling.HorizontalPodAutoscaler,
 	error,
@@ -281,6 +312,12 @@ func (sc *StackContainer) GenerateHPA() (
 		result.Spec.MinReplicas = &pr
 	}
 
+	if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration {
+		i := int32(1)
+		result.Spec.MinReplicas = &i
+		result.Spec.MaxReplicas = i
+	}
+
 	return result, nil
 }
 
@@ -379,6 +416,10 @@ func (sc *StackContainer) GenerateIngressSegment() (
 	return res, nil
 }
 
+// generateIngress generates an ingress as configured in the stack.
+// On cluster migrations set by stackset annotation
+// "zalando.org/forward-backend", the annotation will be copied to the
+// ingress.
 func (sc *StackContainer) generateIngress(segment bool) (
 	*networking.Ingress,
 	error,
@@ -430,6 +471,9 @@ func (sc *StackContainer) generateIngress(segment bool) (
 			Rules: rules,
 		},
 	}
+	if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration {
+		result.Annotations["zalando.org/skipper-backend"] = "forward"
+	}
 
 	// insert annotations
 	result.Annotations = mergeLabels(
@@ -470,6 +514,10 @@ func (sc *StackContainer) GenerateRouteGroupSegment() (
 	return res, nil
 }
 
+// generateRouteGroup generates an RouteGroup as configured in the
+// stack.  On cluster migrations set by stackset annotation
+// "zalando.org/forward-backend", the RouteGroup will be patched by
+// patchForwardBackend() to execute the migration.
 func (sc *StackContainer) generateRouteGroup(segment bool) (
 	*rgv1.RouteGroup,
 	error,
@@ -529,6 +577,10 @@ func (sc *StackContainer) generateRouteGroup(segment bool) (
 		sc.routeGroupSpec.GetAnnotations(),
 	)
 
+	if _, clusterMigration := sc.Stack.Annotations[forwardBackendAnnotation]; clusterMigration {
+		patchForwardBackend(&result.Spec)
+	}
+
 	return result, nil
 }
 

pkg/core/stackset.go

@@ -17,6 +17,8 @@ const (
 	StackVersionLabelKey     = "stack-version"
 
 	ingressTrafficAuthoritativeAnnotation = "zalando.org/traffic-authoritative"
+	forwardBackendAnnotation              = "zalando.org/forward-backend"
+	forwardBackendName                    = "fwd"
 )
 
 var (
@@ -52,6 +54,7 @@ func sanitizeServicePorts(service *zv1.StackServiceSpec) *zv1.StackServiceSpec {
 
 // NewStack returns an (optional) stack that should be created
 func (ssc *StackSetContainer) NewStack() (*StackContainer, string) {
+	_, forwardMigration := ssc.StackSet.ObjectMeta.Annotations[forwardBackendAnnotation]
 	observedStackVersion := ssc.StackSet.Status.ObservedStackVersion
 	stackVersion := currentStackVersion(ssc.StackSet)
 	stackName := generateStackName(ssc.StackSet, stackVersion)
@@ -74,12 +77,20 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) {
 
 		if ssc.StackSet.Spec.ExternalIngress != nil {
 			spec.ExternalIngress = ssc.StackSet.Spec.ExternalIngress.DeepCopy()
+			if forwardMigration {
+				spec.ExternalIngress.Annotations[forwardBackendAnnotation] = forwardBackendName
+			}
 		}
 
 		if ssc.StackSet.Spec.RouteGroup != nil {
 			spec.RouteGroup = ssc.StackSet.Spec.RouteGroup.DeepCopy()
 		}
 
+		stackAnnotations := ssc.StackSet.Spec.StackTemplate.Annotations
+		if forwardMigration {
+			stackAnnotations[forwardBackendAnnotation] = forwardBackendName
+		}
+
 		return &StackContainer{
 			Stack: &zv1.Stack{
 				ObjectMeta: metav1.ObjectMeta{
@@ -100,7 +111,7 @@ func (ssc *StackSetContainer) NewStack() (*StackContainer, string) {
 						ssc.StackSet.Labels,
 						map[string]string{StackVersionLabelKey: stackVersion},
 					),
-					Annotations: ssc.StackSet.Spec.StackTemplate.Annotations,
+					Annotations: stackAnnotations,
 				},
 				Spec: *spec,
 			},