feat: added serverless timeout to request opeartion on mac pool service

Currently when a machine from the pool is requested it is responsabilitie from the requester to release the machine if something happens there is no way to get back the machine, this feature allows to set a serverless execution for the release operation based on a timeout

Signed-off-by: Adrian Riobo <[email protected]>

Author: adrianriobo

cmd/mapt/cmd/aws/services/mac-pool.go

@@ -204,6 +204,7 @@ func request() *cobra.Command {
 					PoolName:             viper.GetString(paramName),
 					Architecture:         viper.GetString(awsParams.MACArch),
 					OSVersion:            viper.GetString(awsParams.MACOSVersion),
+					Timeout:              viper.GetString(params.Timeout),
 					SetupGHActionsRunner: viper.IsSet(params.InstallGHActionsRunner)}); err != nil {
 				logging.Error(err)
 			}
@@ -216,6 +217,7 @@ func request() *cobra.Command {
 	flagSet.StringP(paramName, "", "", paramNameDesc)
 	flagSet.StringP(awsParams.MACArch, "", awsParams.MACArchDefault, awsParams.MACArchDesc)
 	flagSet.StringP(awsParams.MACOSVersion, "", awsParams.MACOSVersion, awsParams.MACOSVersionDefault)
+	flagSet.StringP(params.Timeout, "", "", params.TimeoutDesc)
 	flagSet.AddFlagSet(params.GetGHActionsFlagset())
 	params.AddCirrusFlags(flagSet)
 	c.PersistentFlags().AddFlagSet(flagSet)
@@ -235,6 +237,7 @@ func release() *cobra.Command {
 				&maptContext.ContextArgs{
 					Debug:      viper.IsSet(params.Debug),
 					DebugLevel: viper.GetUint(params.DebugLevel),
+					Serverless: viper.IsSet(params.Serverless),
 				},
 				viper.GetString(awsParams.MACDHID)); err != nil {
 				logging.Error(err)
@@ -244,6 +247,7 @@ func release() *cobra.Command {
 	}
 	flagSet := pflag.NewFlagSet(awsParams.MACReleaseCmd, pflag.ExitOnError)
 	flagSet.StringP(awsParams.MACDHID, "", "", awsParams.MACDHIDDesc)
+	flagSet.Bool(params.Serverless, false, params.ServerlessDesc)
 	c.PersistentFlags().AddFlagSet(flagSet)
 	err := c.MarkPersistentFlagRequired(awsParams.MACDHID)
 	if err != nil {

pkg/provider/aws/action/fedora/fedora.go

@@ -251,9 +251,14 @@ func (r *Request) deploy(ctx *pulumi.Context) error {
 	ctx.Export(fmt.Sprintf("%s-%s", r.Prefix, outputHost),
 		c.GetHostIP(!r.Airgap))
 	if len(r.Timeout) > 0 {
-		if err = serverless.CreateDestroyOperation(ctx,
-			r.region, r.Prefix, awsFedoraDedicatedID,
-			"fedora", r.Timeout); err != nil {
+		if err = serverless.OneTimeDelayedTask(ctx,
+			r.region, r.Prefix,
+			awsFedoraDedicatedID,
+			fmt.Sprintf("aws %s destroy --project-name %s --backed-url %s --serverless",
+				"fedora",
+				maptContext.ProjectName(),
+				maptContext.BackedURL()),
+			r.Timeout); err != nil {
 			return err
 		}
 	}

pkg/provider/aws/action/mac-pool/mac-pool.go

@@ -113,6 +113,7 @@ func Request(ctx *maptContext.ContextArgs, r *RequestMachineArgs) error {
 		Version:              *hi.OSVersion,
 		Architecture:         *hi.Arch,
 		SetupGHActionsRunner: r.SetupGHActionsRunner,
+		Timeout:              r.Timeout,
 	}
 
 	// TODO here we would change based on the integration-mode requested
@@ -121,6 +122,7 @@ func Request(ctx *maptContext.ContextArgs, r *RequestMachineArgs) error {
 	if err != nil {
 		return err
 	}
+
 	// We update the runID on the dedicated host
 	return tag.Update(maptContext.TagKeyRunID,
 		maptContext.RunID(),
@@ -348,6 +350,10 @@ func requestReleaserPolicy() (*string, error) {
 					"ec2:Describe*",
 					"ec2:ImportKeyPair",
 					"ec2:DeleteKeyPair",
+					"cloudformation:GetResource",
+					"scheduler:GetSchedule",
+					"cloudformation:DeleteResource",
+					"cloudformation:GetResourceRequestStatus",
 				},
 				"Resource": []string{
 					"*",

pkg/provider/aws/action/mac-pool/types.go

@@ -40,6 +40,8 @@ type RequestMachineArgs struct {
 	// This is sampler for type of integration / usage
 	// for the request machine
 	SetupGHActionsRunner bool
+	// If timeout is set a severless scheduled task will be created to self destroy the resources
+	Timeout string
 }
 
 type ReleaseMachineArgs struct {

pkg/provider/aws/action/rhel/rhel.go

@@ -248,9 +248,14 @@ func (r *Request) deploy(ctx *pulumi.Context) error {
 	ctx.Export(fmt.Sprintf("%s-%s", r.Prefix, outputHost),
 		c.GetHostIP(!r.Airgap))
 	if len(r.Timeout) > 0 {
-		if err = serverless.CreateDestroyOperation(ctx,
-			r.region, r.Prefix, awsRHELDedicatedID,
-			"rhel", r.Timeout); err != nil {
+		if err = serverless.OneTimeDelayedTask(ctx,
+			r.region, r.Prefix,
+			awsRHELDedicatedID,
+			fmt.Sprintf("aws %s destroy --project-name %s --backed-url %s --serverless",
+				"rhel",
+				maptContext.ProjectName(),
+				maptContext.BackedURL()),
+			r.Timeout); err != nil {
 			return err
 		}
 	}

pkg/provider/aws/action/windows/windows.go

@@ -282,9 +282,14 @@ func (r *Request) deploy(ctx *pulumi.Context) error {
 	ctx.Export(fmt.Sprintf("%s-%s", r.Prefix, outputHost),
 		c.GetHostIP(!r.Airgap))
 	if len(r.Timeout) > 0 {
-		if err = serverless.CreateDestroyOperation(ctx,
-			r.region, r.Prefix, awsWindowsDedicatedID,
-			"windows", r.Timeout); err != nil {
+		if err = serverless.OneTimeDelayedTask(ctx,
+			r.region, r.Prefix,
+			awsWindowsDedicatedID,
+			fmt.Sprintf("aws %s destroy --project-name %s --backed-url %s --serverless",
+				"windows",
+				maptContext.ProjectName(),
+				maptContext.BackedURL()),
+			r.Timeout); err != nil {
 			return err
 		}
 	}

pkg/provider/aws/modules/mac/machine/machine.go

@@ -16,6 +16,7 @@ import (
 	"github.com/redhat-developer/mapt/pkg/provider/aws/modules/mac"
 	macSetup "github.com/redhat-developer/mapt/pkg/provider/aws/modules/mac/machine/setup"
 	"github.com/redhat-developer/mapt/pkg/provider/aws/modules/network"
+	"github.com/redhat-developer/mapt/pkg/provider/aws/modules/serverless"
 	qEC2 "github.com/redhat-developer/mapt/pkg/provider/aws/services/ec2/compute"
 	"github.com/redhat-developer/mapt/pkg/provider/aws/services/ec2/keypair"
 	securityGroup "github.com/redhat-developer/mapt/pkg/provider/aws/services/ec2/security-group"
@@ -68,12 +69,12 @@ func ReplaceMachine(h *mac.HostInformation) error {
 	}
 	// Set a default request
 	r := &Request{
-		Prefix:             *h.Prefix,
-		Architecture:       *h.Arch,
-		Version:            *h.OSVersion,
-		lock:               false,
-		remoteTimeout:      releaseTimeout,
-		isRequestOperation: false,
+		Prefix:               *h.Prefix,
+		Architecture:         *h.Arch,
+		Version:              *h.OSVersion,
+		lock:                 false,
+		sshConnectionTimeout: releaseTimeout,
+		isRequestOperation:   false,
 	}
 	return r.manageMacMachine(h)
 }
@@ -85,7 +86,7 @@ func ReplaceMachine(h *mac.HostInformation) error {
 func (r *Request) ManageRequest(h *mac.HostInformation) error {
 	r.lock = true
 	r.isRequestOperation = true
-	r.remoteTimeout = requestTimeout
+	r.sshConnectionTimeout = requestTimeout
 	return r.manageMacMachine(h)
 }
 
@@ -241,6 +242,18 @@ func (r *Request) deployerMachine(ctx *pulumi.Context) error {
 		return err
 	}
 	ctx.Export(fmt.Sprintf("%s-%s", r.Prefix, outputLock), pulumi.Bool(r.lock))
+
+	// We offer serverless release so there should be a timeout and operation should be a request
+	if len(r.Timeout) > 0 && r.isRequestOperation {
+		if err = serverless.OneTimeDelayedTask(ctx,
+			*r.Region, r.Prefix, awsMacMachineID,
+			fmt.Sprintf("aws mac-pool release --dedicated-host-id %s --serverless",
+				*r.dedicatedHost.Host.HostId),
+			r.Timeout); err != nil {
+			return err
+		}
+	}
+
 	return machineLock(ctx,
 		resourcesUtil.GetResourceName(
 			r.Prefix, awsMacMachineID, "mac-lock"), r.lock,
@@ -367,7 +380,7 @@ func (r *Request) bootstrapscript(ctx *pulumi.Context,
 		r.isRequestOperation,
 		pulumi.String(r.currentPrivateKey),
 		mk.PrivateKeyOpenssh)
-	timeout := util.If(len(r.remoteTimeout) > 0, r.remoteTimeout, defaultTimeout)
+	timeout := util.If(len(r.sshConnectionTimeout) > 0, r.sshConnectionTimeout, defaultTimeout)
 	rc, err := remote.NewCommand(ctx,
 		resourcesUtil.GetResourceName(r.Prefix, awsMacMachineID, "bootstrap-cmd"),
 		&remote.CommandArgs{
@@ -431,7 +444,7 @@ func (r *Request) readiness(ctx *pulumi.Context,
 	mk *tls.PrivateKey,
 	b *bastion.Bastion,
 	dependecies []pulumi.Resource) (*remote.Command, error) {
-	timeout := util.If(len(r.remoteTimeout) > 0, r.remoteTimeout, defaultTimeout)
+	timeout := util.If(len(r.sshConnectionTimeout) > 0, r.sshConnectionTimeout, defaultTimeout)
 	return remote.NewCommand(ctx,
 		resourcesUtil.GetResourceName(r.Prefix, awsMacMachineID, "readiness-cmd"),
 		&remote.CommandArgs{

pkg/provider/aws/modules/mac/machine/setup/request.sh

@@ -9,14 +9,14 @@ set new_password "{{.NewPassword}}"
 
 spawn sudo dscl . -passwd /Users/{{.Username}}
 expect "New Password:"
-send "$new_password\r"
+send "\$new_password\r"
 expect "Permission denied. Please enter user's old password:"
-send "$old_password\r"
+send "\$old_password\r"
 expect eof
 EOF
 chmod +x change_password.exp
 ./change_password.exp
-rm change_password.exp
+# rm change_password.exp
 
 # Autologin
 sudo curl -o /tmp/kcpassword https://raw.githubusercontent.com/xfreebird/kcpassword/master/kcpassword

pkg/provider/aws/modules/mac/machine/types.go

@@ -17,16 +17,18 @@ type Request struct {
 	// setup as github actions runner
 	SetupGHActionsRunner bool
 	Airgap               bool
+	// If timeout is set a severless scheduled task will be created to self destroy the resources
+	Timeout string
 	// For airgap scenario there is an orchestation of
 	// a phase with connectivity on the machine (allowing bootstraping)
 	// a pahase with connectivyt off where the subnet for the target lost the nat gateway
 	airgapPhaseConnectivity network.Connectivity
 	// dh linkage
 	dedicatedHost *mac.HostInformation
 	// operation control params
-	isRequestOperation bool
-	lock               bool
-	remoteTimeout      string
+	isRequestOperation   bool
+	lock                 bool
+	sshConnectionTimeout string
 	// values used to increase security on request operations
 	// currentPrivateKey pulumi.StringPtrInput
 	// currentPassword   pulumi.StringPtrInput

pkg/provider/aws/modules/serverless/serverless.go

@@ -25,9 +25,9 @@ var (
 	ErrInvalidBackedURLForTimeout = fmt.Errorf("timeout can action can not be set due to backed url pointing to local file. Please use external storage or remote timeout option")
 )
 
-func CreateDestroyOperation(ctx *pulumi.Context,
+func OneTimeDelayedTask(ctx *pulumi.Context,
 	region, prefix, componentID string,
-	target string,
+	cmd string,
 	delay string) error {
 	if err := checkBackedURLForServerless(); err != nil {
 		return err
@@ -37,11 +37,9 @@ func CreateDestroyOperation(ctx *pulumi.Context,
 		return err
 	}
 	r := &serverlessRequestArgs{
-		region: region,
-		command: fmt.Sprintf("aws %s destroy --project-name %s --backed-url %s --serverless",
-			target,
-			maptContext.ProjectName(),
-			maptContext.BackedURL()),
+		region:             region,
+		command:            cmd,
+		scheduleType:       OneTime,
 		scheduleExpression: se,
 		prefix:             prefix,
 		componentID:        componentID,
@@ -110,7 +108,7 @@ func (a *serverlessRequestArgs) deploy(ctx *pulumi.Context) error {
 	if err != nil {
 		return err
 	}
-	sRole, err := createSchedulerRole(ctx,
+	sRoleArn, err := getSchedulerRole(ctx,
 		a.prefix,
 		a.componentID)
 	if err != nil {
@@ -143,7 +141,7 @@ func (a *serverlessRequestArgs) deploy(ctx *pulumi.Context) error {
 					},
 				},
 				Arn:     clusterArn,
-				RoleArn: sRole.Arn,
+				RoleArn: sRoleArn,
 			},
 			ScheduleExpression:         pulumi.String(*se),
 			ScheduleExpressionTimezone: pulumi.String(data.RegionTimezones[a.region]),
@@ -260,8 +258,24 @@ func createTaskRole(ctx *pulumi.Context, roleName, prefix, componentID string) (
 	return r, nil
 }
 
+// As part of the runtime for serverless invocation we need a fixed role for task execution the region as so if
+// it exists it will pick the role otherwise it will create and will not be deleted
+func getSchedulerRole(ctx *pulumi.Context, prefix, componentID string) (*pulumi.StringOutput, error) {
+	roleName := fmt.Sprintf("%s-%s", maptServerlessDefaultPrefix, "sch-role")
+	roleArn, err := data.GetRole(roleName)
+	if err != nil {
+		if role, err := createSchedulerRole(ctx, roleName, prefix, componentID); err != nil {
+			return nil, err
+		} else {
+			return &role.Arn, nil
+		}
+	}
+	rarn := pulumi.String(*roleArn).ToStringOutput()
+	return &rarn, nil
+}
+
 // https://docs.aws.amazon.com/scheduler/latest/UserGuide/setting-up.html#setting-up-execution-role
-func createSchedulerRole(ctx *pulumi.Context, prefix, componentID string) (*iam.Role, error) {
+func createSchedulerRole(ctx *pulumi.Context, roleName, prefix, componentID string) (*iam.Role, error) {
 	trustPolicyContent, err := json.Marshal(map[string]interface{}{
 		"Version": "2012-10-17",
 		"Statement": []map[string]interface{}{
@@ -279,12 +293,13 @@ func createSchedulerRole(ctx *pulumi.Context, prefix, componentID string) (*iam.
 	}
 	// Need to creeate policies and attach
 	r, err := iam.NewRole(ctx,
-		resourcesUtil.GetResourceName(prefix, componentID, "role-sche"),
+		resourcesUtil.GetResourceName(prefix, componentID, "sch-role"),
 		&iam.RoleArgs{
-			Name:             pulumi.String(fmt.Sprintf("mapt-sche-%s", maptContext.RunID())),
+			Name:             pulumi.String(roleName),
 			AssumeRolePolicy: pulumi.String(string(trustPolicyContent)),
 			Tags:             maptContext.ResourceTags(),
-		})
+		},
+		pulumi.RetainOnDelete(true))
 	if err != nil {
 		return nil, err
 	}
@@ -336,8 +351,8 @@ func generateOneTimeScheduleExpression(region, delay string) (string, error) {
 	}
 	// Add the duration to the current time
 	futureTime := currentTime.Add(duration)
-	se := scheduleExpressionByType(OneTime, futureTime.Format("2006-01-02T15:04:05"))
-	return *se, nil
+	se := futureTime.Format("2006-01-02T15:04:05")
+	return se, nil
 }
 
 func scheduleExpressionByType(st scheduleType, se string) *string {