Use go-retry for UpdateTargetInstanceAfterPromotion

Co-authored-by: Trong Nguyen <trong.huu.nguyen@nav.no>

Author: mortenlj

internal/pkg/application/application.go

@@ -18,8 +18,6 @@ import (
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-const UpdateRetries = 3
-
 func ScaleApplication(ctx context.Context, cfg *config.Config, mgr *common_main.Manager, replicas int32) error {
 	mgr.Logger.Info("scaling application", "name", cfg.ApplicationName, "replicas", replicas)
 	scaleApplyConfiguration := autoscaling_v1.Scale{

internal/pkg/instance/instance.go

@@ -28,7 +28,6 @@ import (
 
 const (
 	dummyAppImage              = "europe-north1-docker.pkg.dev/nais-io/nais/images/kafka-debug:latest"
-	updateRetries              = 3
 	migrationAuthNetworkPrefix = "migrator:"
 )
 
@@ -291,8 +290,32 @@ func PrepareTargetInstance(ctx context.Context, target *resolved.Instance, mgr *
 }
 
 func UpdateTargetInstanceAfterPromotion(ctx context.Context, target *resolved.Instance, mgr *common_main.Manager) error {
-	err := updateTargetInstanceAfterPromotionWithRetries(ctx, target, mgr, updateRetries)
+	mgr.Logger.Info("updating target instance after promotion")
+
+	b := retry.NewConstant(1 * time.Second)
+	b = retry.WithMaxDuration(5*time.Minute, b)
+
+	err := retry.Do(ctx, b, func(ctx context.Context) error {
+		targetSqlInstance, err := mgr.SqlInstanceClient.Get(ctx, target.Name)
+		if err != nil {
+			return fmt.Errorf("failed to get target instance: %w", err)
+		}
+
+		targetSqlInstance.Spec.InstanceType = ptr.To("CLOUD_SQL_INSTANCE")
+		targetSqlInstance.Spec.MasterInstanceRef = nil
+
+		_, err = mgr.SqlInstanceClient.Update(ctx, targetSqlInstance)
+		if err != nil {
+			if k8s_errors.IsConflict(err) {
+				mgr.Logger.Warn("retrying update of target instance", "error", err)
+				return retry.RetryableError(err)
+			}
+			return err
+		}
+		return nil
+	})
 	if err != nil {
+		mgr.Logger.Error("failed to update target instance after promotion", "error", err)
 		return err
 	}
 
@@ -315,26 +338,6 @@ func UpdateTargetInstanceAfterPromotion(ctx context.Context, target *resolved.In
 	return nil
 }
 
-func updateTargetInstanceAfterPromotionWithRetries(ctx context.Context, target *resolved.Instance, mgr *common_main.Manager, retries int) error {
-	targetSqlInstance, err := mgr.SqlInstanceClient.Get(ctx, target.Name)
-	if err != nil {
-		return fmt.Errorf("failed to get target instance: %w", err)
-	}
-
-	targetSqlInstance.Spec.InstanceType = ptr.To("CLOUD_SQL_INSTANCE")
-	targetSqlInstance.Spec.MasterInstanceRef = nil
-
-	_, err = mgr.SqlInstanceClient.Update(ctx, targetSqlInstance)
-	if err != nil {
-		if k8s_errors.IsConflict(err) && retries > 0 {
-			mgr.Logger.Info("retrying update of target instance", "remaining_retries", retries)
-			return updateTargetInstanceAfterPromotionWithRetries(ctx, target, mgr, retries-1)
-		}
-		return err
-	}
-	return nil
-}
-
 func DeleteInstance(ctx context.Context, instanceName string, gcpProject *resolved.GcpProject, mgr *common_main.Manager) error {
 	instancesService := mgr.SqlAdminService.Instances