feat: enable force unlock and optimize failure handling in execute api (#1339)

Author: ffforest

pkg/server/handler/stack/utils.go

@@ -101,6 +101,7 @@ func requestHelper(r *http.Request) (context.Context, *httplog.Logger, *stackman
 	dryrunParam, _ := strconv.ParseBool(r.URL.Query().Get("dryrun"))
 	forceParam, _ := strconv.ParseBool(r.URL.Query().Get("force"))
 	noCacheParam, _ := strconv.ParseBool(r.URL.Query().Get("noCache"))
+	unlockParam, _ := strconv.ParseBool(r.URL.Query().Get("unlock"))
 	importResourcesParam, _ := strconv.ParseBool(r.URL.Query().Get("importResources"))
 	specIDParam := r.URL.Query().Get("specID")
 	// TODO: Should match automatically eventually???
@@ -123,6 +124,7 @@ func requestHelper(r *http.Request) (context.Context, *httplog.Logger, *stackman
 		SpecID:          specIDParam,
 		ImportResources: importResourcesParam,
 		NoCache:         noCacheParam,
+		Unlock:          unlockParam,
 	}
 	params := stackmanager.StackRequestParams{
 		StackID:       uint(id),

pkg/server/manager/stack/execute.go

@@ -132,7 +132,8 @@ func (m *StackManager) PreviewStack(ctx context.Context, params *StackRequestPar
 	// TODO: This is a temporary solution to prevent multiple requests from previewing the same stack and cause concurrency issues
 	// To override this, pass in force == true
 	if stackEntity.StackInOperation() && !params.ExecuteParams.Force {
-		return nil, ErrStackInOperation
+		err = ErrStackInOperation
+		return nil, err
 	}
 
 	// Set stack sync state to previewing
@@ -250,11 +251,12 @@ func (m *StackManager) ApplyStack(ctx context.Context, params *StackRequestParam
 	releaseCreated := false
 	// Ensure the state is updated properly
 	defer func() {
-		if !releaseCreated {
-			return
-		}
 		if err != nil {
 			stackEntity.SyncState = constant.StackStateApplyFailed
+			if !releaseCreated {
+				m.stackRepo.Update(ctx, stackEntity)
+				return
+			}
 			release.UpdateReleasePhase(rel, apiv1.ReleasePhaseFailed, relLock)
 			_ = release.UpdateApplyRelease(storage, rel, params.ExecuteParams.Dryrun, relLock)
 		} else {
@@ -274,7 +276,8 @@ func (m *StackManager) ApplyStack(ctx context.Context, params *StackRequestParam
 	// TODO: This is a temporary solution to prevent multiple requests from applying the same stack and cause concurrency issues
 	// To override this, pass in force == true
 	if stackEntity.StackInOperation() && !params.ExecuteParams.Force {
-		return ErrStackInOperation
+		err = ErrStackInOperation
+		return err
 	}
 	// Temporarily commented out
 	// if stackEntity.LastPreviewedRevision == "" || stackEntity.SyncState != constant.StackStatePreviewed {
@@ -301,13 +304,22 @@ func (m *StackManager) ApplyStack(ctx context.Context, params *StackRequestParam
 	if err != nil {
 		return err
 	}
+	// Allow force unlock of the release
+	if params.ExecuteParams.Unlock {
+		err = unlockRelease(ctx, storage)
+		if err != nil {
+			return err
+		}
+	}
+	// Get the latest state from the release
 	priorState, err := release.GetLatestState(storage)
 	if err != nil {
 		return err
 	}
 	if priorState == nil {
 		priorState = &apiv1.State{}
 	}
+	// Create new release
 	rel, err = release.NewApplyRelease(storage, project.Name, stackEntity.Name, ws.Name)
 	if err != nil {
 		return err
@@ -468,13 +480,14 @@ func (m *StackManager) DestroyStack(ctx context.Context, params *StackRequestPar
 	rel := &apiv1.Release{}
 	releaseCreated := false
 	defer func() {
-		if !releaseCreated {
-			return
-		}
 		if err != nil {
+			stackEntity.SyncState = constant.StackStateDestroyFailed
+			if !releaseCreated {
+				m.stackRepo.Update(ctx, stackEntity)
+				return
+			}
 			rel.Phase = apiv1.ReleasePhaseFailed
 			_ = release.UpdateDestroyRelease(storage, rel)
-			stackEntity.SyncState = constant.StackStateDestroyFailed
 		} else {
 			rel.Phase = apiv1.ReleasePhaseSucceeded
 			err = release.UpdateDestroyRelease(storage, rel)
@@ -490,7 +503,8 @@ func (m *StackManager) DestroyStack(ctx context.Context, params *StackRequestPar
 	// TODO: This is a temporary solution to prevent multiple requests from destroying the same stack and cause concurrency issues
 	// To override this, pass in force == true
 	if stackEntity.StackInOperation() && !params.ExecuteParams.Force {
-		return ErrStackInOperation
+		err = ErrStackInOperation
+		return err
 	}
 
 	// Set stack sync state to destroying
@@ -514,6 +528,14 @@ func (m *StackManager) DestroyStack(ctx context.Context, params *StackRequestPar
 	if err != nil {
 		return err
 	}
+	// Allow force unlock of the release
+	if params.ExecuteParams.Unlock {
+		err = unlockRelease(ctx, storage)
+		if err != nil {
+			return err
+		}
+	}
+	// Create destroy release
 	rel, err = release.CreateDestroyRelease(storage, project.Name, stack.Name, ws.Name)
 	if err != nil {
 		return

pkg/server/manager/stack/types.go

@@ -59,6 +59,7 @@ type StackExecuteParams struct {
 	Force           bool
 	ImportResources bool
 	NoCache         bool
+	Unlock          bool
 }
 
 type RunRequestParams struct {

pkg/server/manager/stack/util.go

@@ -22,6 +22,7 @@ import (
 	engineapi "kusionstack.io/kusion/pkg/engine/api"
 	sourceapi "kusionstack.io/kusion/pkg/engine/api/source"
 	"kusionstack.io/kusion/pkg/engine/operation/models"
+	"kusionstack.io/kusion/pkg/engine/release"
 	"kusionstack.io/kusion/pkg/engine/runtime/terraform/tfops"
 	workspacemanager "kusionstack.io/kusion/pkg/server/manager/workspace"
 	logutil "kusionstack.io/kusion/pkg/server/util/logging"
@@ -453,3 +454,30 @@ func logToAll(sysLogger *httplog.Logger, runLogger *httplog.Logger, level string
 		sysLogger.Error("unknown log level", "level", level)
 	}
 }
+
+func unlockRelease(ctx context.Context, storage release.Storage) error {
+	logger := logutil.GetLogger(ctx)
+	logger.Info("Getting workdir from stack source...")
+	// Get the latest release.
+	r, err := release.GetLatestRelease(storage)
+	if err != nil {
+		return err
+	}
+	if r == nil {
+		logger.Info("No release file found for given stack")
+		return nil
+	}
+
+	// Update the phase to 'failed', if it was not succeeded or failed.
+	if r.Phase != v1.ReleasePhaseSucceeded && r.Phase != v1.ReleasePhaseFailed {
+		r.Phase = v1.ReleasePhaseFailed
+		if err := storage.Update(r); err != nil {
+			return err
+		}
+		logger.Info("Successfully update release phase!")
+		return nil
+	} else {
+		logger.Info("No need to update the release phase, current phase: ", "phase", r.Phase)
+	}
+	return nil
+}