feat: add recreate-stateful-strategy, orphan, background, foreground(default) (#1286)

* feat: add recreate-stateful-strategy, orphan, background, foreground(default)

Signed-off-by: Husni Alhamdani <[email protected]>

* feat: add recreate-stateful-strategy, orphan, background, foreground(default)

Signed-off-by: Husni Alhamdani <[email protected]>

* typo

Signed-off-by: yangw <[email protected]>

* common function

Signed-off-by: yangw <[email protected]>

* docs

Signed-off-by: yangw <[email protected]>

---------

Signed-off-by: Husni Alhamdani <[email protected]>
Signed-off-by: yangw <[email protected]>
Co-authored-by: yangw <[email protected]>

Author: husnialhamdani

docs/content/en/docs/Advance Configuration/Upgrading/_index.md

@@ -170,3 +170,48 @@ spec:
 - If application is highly critical, in such scenarios it would make sense to create a new cluster and migrate the application pointing to it
 {{< /alert >}}
 
+## StatefulSet Recreation Strategy
+
+In some scenarios, you may need to recreate the StatefulSet completely, for example when you need to change immutable fields. Redis Operator provides an annotation `redis.opstreelabs.in/recreate-statefulset` that can be set to `true` to recreate the StatefulSet.
+
+Redis Operator supports specifying the deletion propagation strategy when recreating StatefulSets. This allows more control over how dependent resources like Pods are handled during the recreation process.
+
+### Available Strategies
+
+You can control the deletion behavior using the annotation `redis.opstreelabs.in/recreate-statefulset-strategy` with the following values:
+
+- **foreground** (default): The StatefulSet and its dependent objects (like Pods) are deleted synchronously. Kubernetes waits for all dependent objects to be deleted before finalizing the StatefulSet deletion.
+- **background**: The StatefulSet is deleted immediately, and dependent objects are deleted asynchronously in the background.
+- **orphan**: The StatefulSet is deleted but its dependent objects (Pods) are kept running. This provides the most control and minimizes downtime, but requires manual cleanup later.
+
+### Example Usage
+
+Here's how to configure Redis with a specific StatefulSet recreation strategy:
+
+```yaml
+apiVersion: redis.redis.opstreelabs.in/v1beta2
+kind: Redis
+metadata:
+  name: redis-standalone
+  annotations:
+    redis.opstreelabs.in/recreate-statefulset: "true"
+    redis.opstreelabs.in/recreate-statefulset-strategy: "orphan"
+spec:
+  kubernetesConfig:
+    image: "quay.io/opstree/redis:v7.0.5"
+    imagePullPolicy: "IfNotPresent"
+  # ... other configuration ...
+```
+
+### When to Use Different Strategies
+
+- **foreground**: When you want to ensure a complete, clean recreation of the Redis system and can tolerate downtime.
+- **background**: When you want to minimize the time the StatefulSet object itself exists in a "Terminating" state and don't need to wait for dependent objects to be deleted.
+- **orphan**: When you need to avoid downtime during StatefulSet recreation, especially in production environments. The existing Pods will continue to serve traffic while the new StatefulSet is being created.
+
+{{< alert color="warning" title="Warning" >}}
+Using the "orphan" strategy will keep the Pods running, but they won't be managed by the StatefulSet until the new one adopts them. If the Pods need to be updated, you may need to manually delete them so the new StatefulSet can create fresh Pods.
+{{< /alert >}}
+
+This feature is useful when upgrading Redis with changes to immutable StatefulSet fields, allowing administrators to control the recreation process according to their operational requirements.
+

docs/content/en/docs/CRD Reference/Redis API/_index.md

@@ -19,6 +19,23 @@ This page documents the Redis API Schema definitions for the redis API group.
 
 Package v1beta2 contains API Schema definitions for the redis v1beta2 API group
 
+### Annotations
+
+Redis Operator supports the following annotations that can be added to Redis, RedisCluster, RedisReplication, and RedisSentinel resources:
+
+| Annotation | Description | Default | Values |
+| --- | --- | --- | --- |
+| `redis.opstreelabs.in/recreate-statefulset` | Controls whether the StatefulSet should be recreated when changed | `false` | `"true"`, `"false"` |
+| `redis.opstreelabs.in/recreate-statefulset-strategy` | Controls how dependent resources are handled when the StatefulSet is recreated | `foreground` | `"foreground"`, `"background"`, `"orphan"` |
+
+#### Deletion Propagation Strategies
+
+When `redis.opstreelabs.in/recreate-statefulset` is set to `"true"`, you can control the deletion behavior using the `redis.opstreelabs.in/recreate-statefulset-strategy` annotation:
+
+- **foreground**: The StatefulSet and its dependent objects (like Pods) are deleted synchronously
+- **background**: The StatefulSet is deleted immediately, and dependent objects are deleted asynchronously
+- **orphan**: The StatefulSet is deleted but its dependent objects (Pods) are kept running
+
 ### Resource Types
 
 - [Redis](#redis)

pkg/k8sutils/const.go

@@ -1,7 +1,8 @@
 package k8sutils
 
 const (
-	AnnotationKeyRecreateStatefulset = "redis.opstreelabs.in/recreate-statefulset"
+	AnnotationKeyRecreateStatefulset         = "redis.opstreelabs.in/recreate-statefulset"
+	AnnotationKeyRecreateStatefulsetStrategy = "redis.opstreelabs.in/recreate-statefulset-strategy"
 )
 
 const (

pkg/k8sutils/redis-cluster.go

@@ -73,6 +73,7 @@ func generateRedisClusterParams(ctx context.Context, cr *redisv1beta2.RedisClust
 	}
 	if value, found := cr.ObjectMeta.GetAnnotations()[AnnotationKeyRecreateStatefulset]; found && value == "true" {
 		res.RecreateStatefulSet = true
+		res.RecreateStatefulsetStrategy = getDeletionPropagationStrategy(cr.ObjectMeta.GetAnnotations())
 	}
 	return res
 }

pkg/k8sutils/redis-replication.go

@@ -123,6 +123,7 @@ func generateRedisReplicationParams(cr *redisv1beta2.RedisReplication) statefulS
 	}
 	if value, found := cr.ObjectMeta.GetAnnotations()[AnnotationKeyRecreateStatefulset]; found && value == "true" {
 		res.RecreateStatefulSet = true
+		res.RecreateStatefulsetStrategy = getDeletionPropagationStrategy(cr.ObjectMeta.GetAnnotations())
 	}
 	return res
 }

pkg/k8sutils/redis-sentinel.go

@@ -119,6 +119,7 @@ func generateRedisSentinelParams(ctx context.Context, cr *redisv1beta2.RedisSent
 	}
 	if value, found := cr.ObjectMeta.GetAnnotations()[AnnotationKeyRecreateStatefulset]; found && value == "true" {
 		res.RecreateStatefulSet = true
+		res.RecreateStatefulsetStrategy = getDeletionPropagationStrategy(cr.ObjectMeta.GetAnnotations())
 	}
 	return res
 }

pkg/k8sutils/redis-standalone.go

@@ -131,6 +131,7 @@ func generateRedisStandaloneParams(cr *redisv1beta2.Redis) statefulSetParameters
 	}
 	if value, found := cr.ObjectMeta.GetAnnotations()[AnnotationKeyRecreateStatefulset]; found && value == "true" {
 		res.RecreateStatefulSet = true
+		res.RecreateStatefulsetStrategy = getDeletionPropagationStrategy(cr.ObjectMeta.GetAnnotations())
 	}
 	return res
 }

pkg/k8sutils/statefulset.go

@@ -110,6 +110,7 @@ type statefulSetParameters struct {
 	ServiceAccountName            *string
 	UpdateStrategy                appsv1.StatefulSetUpdateStrategy
 	RecreateStatefulSet           bool
+	RecreateStatefulsetStrategy   *metav1.DeletionPropagation
 	TerminationGracePeriodSeconds *int64
 	IgnoreAnnotations             []string
 	HostNetwork                   bool
@@ -174,11 +175,11 @@ func CreateOrUpdateStateFul(ctx context.Context, cl kubernetes.Interface, namesp
 		}
 		return err
 	}
-	return patchStatefulSet(ctx, storedStateful, statefulSetDef, namespace, params.RecreateStatefulSet, cl)
+	return patchStatefulSet(ctx, storedStateful, statefulSetDef, namespace, params.RecreateStatefulSet, params.RecreateStatefulsetStrategy, cl)
 }
 
 // patchStatefulSet patches the Redis StatefulSet by applying changes while maintaining atomicity.
-func patchStatefulSet(ctx context.Context, storedStateful, newStateful *appsv1.StatefulSet, namespace string, recreateStatefulSet bool, cl kubernetes.Interface) error {
+func patchStatefulSet(ctx context.Context, storedStateful, newStateful *appsv1.StatefulSet, namespace string, recreateStatefulSet bool, deletePropagation *metav1.DeletionPropagation, cl kubernetes.Interface) error {
 	// Sync system-managed fields to ensure atomic update.
 	syncManagedFields(storedStateful, newStateful)
 
@@ -223,7 +224,7 @@ func patchStatefulSet(ctx context.Context, storedStateful, newStateful *appsv1.S
 		return err
 	}
 
-	return updateStatefulSet(ctx, cl, namespace, newStateful, recreateStatefulSet)
+	return updateStatefulSet(ctx, cl, namespace, newStateful, recreateStatefulSet, deletePropagation)
 }
 
 // syncManagedFields syncs system-managed fields from the stored object to the new object.
@@ -830,7 +831,7 @@ func createStatefulSet(ctx context.Context, cl kubernetes.Interface, namespace s
 }
 
 // updateStatefulSet is a method to update statefulset in Kubernetes
-func updateStatefulSet(ctx context.Context, cl kubernetes.Interface, namespace string, stateful *appsv1.StatefulSet, recreateStateFulSet bool) error {
+func updateStatefulSet(ctx context.Context, cl kubernetes.Interface, namespace string, stateful *appsv1.StatefulSet, recreateStateFulSet bool, deletePropagation *metav1.DeletionPropagation) error {
 	_, err := cl.AppsV1().StatefulSets(namespace).Update(context.TODO(), stateful, metav1.UpdateOptions{})
 	if recreateStateFulSet {
 		sErr, ok := err.(*apierrors.StatusError)
@@ -840,8 +841,7 @@ func updateStatefulSet(ctx context.Context, cl kubernetes.Interface, namespace s
 				failMsg[messageCount] = cause.Message
 			}
 			log.FromContext(ctx).V(1).Info("recreating StatefulSet because the update operation wasn't possible", "reason", strings.Join(failMsg, ", "))
-			propagationPolicy := metav1.DeletePropagationForeground
-			if err := cl.AppsV1().StatefulSets(namespace).Delete(context.TODO(), stateful.GetName(), metav1.DeleteOptions{PropagationPolicy: &propagationPolicy}); err != nil { //nolint:gocritic
+			if err := cl.AppsV1().StatefulSets(namespace).Delete(context.TODO(), stateful.GetName(), metav1.DeleteOptions{PropagationPolicy: deletePropagation}); err != nil { //nolint:gocritic
 				return errors.Wrap(err, "failed to delete StatefulSet to avoid forbidden action")
 			}
 		}
@@ -874,3 +874,27 @@ func getSidecars(sidecars *[]redisv1beta2.Sidecar) []redisv1beta2.Sidecar {
 	}
 	return *sidecars
 }
+
+// getDeletionPropagationStrategy returns the deletion propagation strategy based on the annotation
+func getDeletionPropagationStrategy(annotations map[string]string) *metav1.DeletionPropagation {
+	if annotations == nil {
+		return nil
+	}
+
+	if strategy, exists := annotations[AnnotationKeyRecreateStatefulsetStrategy]; exists {
+		var propagation metav1.DeletionPropagation
+
+		switch strings.ToLower(strategy) {
+		case "orphan":
+			propagation = metav1.DeletePropagationOrphan
+		case "background":
+			propagation = metav1.DeletePropagationBackground
+		default:
+			propagation = metav1.DeletePropagationForeground
+		}
+
+		return &propagation
+	}
+
+	return nil
+}

pkg/k8sutils/statefulset_test.go

@@ -336,12 +336,13 @@ func Test_createStatefulSet(t *testing.T) {
 
 func TestUpdateStatefulSet(t *testing.T) {
 	tests := []struct {
-		name            string
-		existingStsSpec appsv1.StatefulSetSpec
-		updatedStsSpec  appsv1.StatefulSetSpec
-		recreateSts     bool
-		stsPresent      bool
-		expectErr       error
+		name              string
+		existingStsSpec   appsv1.StatefulSetSpec
+		updatedStsSpec    appsv1.StatefulSetSpec
+		recreateSts       bool
+		deletePropagation metav1.DeletionPropagation
+		stsPresent        bool
+		expectErr         error
 	}{
 		{
 			name: "Update StatefulSet without recreate in existing Statefulset",
@@ -439,7 +440,7 @@ func TestUpdateStatefulSet(t *testing.T) {
 			} else {
 				client = k8sClientFake.NewSimpleClientset()
 			}
-			err := updateStatefulSet(context.TODO(), client, updatedSts.GetNamespace(), &updatedSts, test.recreateSts)
+			err := updateStatefulSet(context.TODO(), client, updatedSts.GetNamespace(), &updatedSts, test.recreateSts, &test.deletePropagation)
 			if test.expectErr != nil {
 				assert.Error(err, "Expected Error while updating Statefulset")
 				assert.Equal(test.expectErr, err)