chore: add retryable error handling for deploy helper

helpers/foundation-deployer/stages/apply.go

@@ -70,9 +70,12 @@ func DeployBootstrapStage(t testing.TB, s steps.Steps, tfvars GlobalTFVars, c Co
 
 	terraformDir := filepath.Join(c.FoundationPath, BootstrapStep)
 	options := &terraform.Options{
-		TerraformDir: terraformDir,
-		Logger:       c.Logger,
-		NoColor:      true,
+		TerraformDir:             terraformDir,
+		Logger:                   c.Logger,
+		NoColor:                  true,
+		RetryableTerraformErrors: testutils.RetryableTransientErrors,
+		MaxRetries:               MaxRetries,
+		TimeBetweenRetries:       TimeBetweenRetries,
 	}
 	// terraform deploy
 	err = applyLocal(t, options, "", c.PolicyPath, c.ValidatorProject)
@@ -177,9 +180,12 @@ func DeployBootstrapStage(t testing.TB, s steps.Steps, tfvars GlobalTFVars, c Co
 	// Init gcp-bootstrap terraform
 	err = s.RunStep("gcp-bootstrap.init-tf", func() error {
 		options := &terraform.Options{
-			TerraformDir: filepath.Join(gcpBootstrapPath, "envs", "shared"),
-			Logger:       c.Logger,
-			NoColor:      true,
+			TerraformDir:             filepath.Join(gcpBootstrapPath, "envs", "shared"),
+			Logger:                   c.Logger,
+			NoColor:                  true,
+			RetryableTerraformErrors: testutils.RetryableTransientErrors,
+			MaxRetries:               MaxRetries,
+			TimeBetweenRetries:       TimeBetweenRetries,
 		}
 		_, err := terraform.InitE(t, options)
 		return err
@@ -461,9 +467,12 @@ func deployStage(t testing.TB, sc StageConf, s steps.Steps, c CommonConf) error
 	for _, bu := range groupunit {
 		for _, localStep := range sc.LocalSteps {
 			buOptions := &terraform.Options{
-				TerraformDir: filepath.Join(filepath.Join(c.CheckoutPath, sc.Repo), bu, localStep),
-				Logger:       c.Logger,
-				NoColor:      true,
+				TerraformDir:             filepath.Join(filepath.Join(c.CheckoutPath, sc.Repo), bu, localStep),
+				Logger:                   c.Logger,
+				NoColor:                  true,
+				RetryableTerraformErrors: testutils.RetryableTransientErrors,
+				MaxRetries:               MaxRetries,
+				TimeBetweenRetries:       TimeBetweenRetries,
 			}
 
 			err := s.RunStep(fmt.Sprintf("%s.%s.apply-%s", sc.Stage, bu, localStep), func() error {

helpers/foundation-deployer/stages/data.go

@@ -19,6 +19,7 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"time"
 
 	"github.com/gruntwork-io/terratest/modules/logger"
 	"github.com/gruntwork-io/terratest/modules/terraform"
@@ -28,20 +29,22 @@ import (
 )
 
 const (
-	PoliciesRepo     = "gcp-policies"
-	BootstrapRepo    = "gcp-bootstrap"
-	OrgRepo          = "gcp-org"
-	EnvironmentsRepo = "gcp-environments"
-	NetworksRepo     = "gcp-networks"
-	ProjectsRepo     = "gcp-projects"
-	AppInfraRepo     = "bu1-example-app"
-	BootstrapStep    = "0-bootstrap"
-	OrgStep          = "1-org"
-	EnvironmentsStep = "2-environments"
-	HubAndSpokeStep  = "3-networks-hub-and-spoke"
-	SvpcStep         = "3-networks-svpc"
-	ProjectsStep     = "4-projects"
-	AppInfraStep     = "5-app-infra"
+	PoliciesRepo       = "gcp-policies"
+	BootstrapRepo      = "gcp-bootstrap"
+	OrgRepo            = "gcp-org"
+	EnvironmentsRepo   = "gcp-environments"
+	NetworksRepo       = "gcp-networks"
+	ProjectsRepo       = "gcp-projects"
+	AppInfraRepo       = "bu1-example-app"
+	BootstrapStep      = "0-bootstrap"
+	OrgStep            = "1-org"
+	EnvironmentsStep   = "2-environments"
+	HubAndSpokeStep    = "3-networks-hub-and-spoke"
+	SvpcStep           = "3-networks-svpc"
+	ProjectsStep       = "4-projects"
+	AppInfraStep       = "5-app-infra"
+	MaxRetries         = 2
+	TimeBetweenRetries = 2 * time.Minute
 )
 
 type CommonConf struct {

helpers/foundation-deployer/stages/destroy.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"time"
 
 	"github.com/gruntwork-io/terratest/modules/terraform"
 	"github.com/mitchellh/go-testing-interface"
@@ -61,9 +60,12 @@ func forceBackendMigration(t testing.TB, repo, groupUnit, env string, c CommonCo
 	}
 	if exist {
 		options := &terraform.Options{
-			TerraformDir: tfDir,
-			Logger:       c.Logger,
-			NoColor:      true,
+			TerraformDir:             tfDir,
+			Logger:                   c.Logger,
+			NoColor:                  true,
+			RetryableTerraformErrors: testutils.RetryableTransientErrors,
+			MaxRetries:               MaxRetries,
+			TimeBetweenRetries:       TimeBetweenRetries,
 		}
 		_, err := terraform.InitE(t, options)
 		if err != nil {
@@ -164,8 +166,8 @@ func destroyStage(t testing.TB, sc StageConf, s steps.Steps, c CommonConf) error
 					Logger:                   c.Logger,
 					NoColor:                  true,
 					RetryableTerraformErrors: testutils.RetryableTransientErrors,
-					MaxRetries:               2,
-					TimeBetweenRetries:       2 * time.Minute,
+					MaxRetries:               MaxRetries,
+					TimeBetweenRetries:       TimeBetweenRetries,
 				}
 				conf := utils.CloneCSR(t, sc.Repo, gcpPath, sc.CICDProject, c.Logger)
 				branch := e
@@ -198,8 +200,8 @@ func destroyStage(t testing.TB, sc StageConf, s steps.Steps, c CommonConf) error
 				Logger:                   c.Logger,
 				NoColor:                  true,
 				RetryableTerraformErrors: testutils.RetryableTransientErrors,
-				MaxRetries:               2,
-				TimeBetweenRetries:       2 * time.Minute,
+				MaxRetries:               MaxRetries,
+				TimeBetweenRetries:       TimeBetweenRetries,
 			}
 			conf := utils.CloneCSR(t, ProjectsRepo, gcpPath, sc.CICDProject, c.Logger)
 			err := conf.CheckoutBranch("production")

helpers/foundation-deployer/stages/vet.go

@@ -27,6 +27,8 @@ import (
 
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils"
+
+	"github.com/terraform-google-modules/terraform-example-foundation/test/integration/testutils"
 )
 
 // TerraformVet runs gcloud terraform vet on the plan of the provided terraform directory
@@ -37,10 +39,13 @@ func TerraformVet(t testing.TB, terraformDir, policyPath, project string) error
 	fmt.Println("")
 
 	options := &terraform.Options{
-		TerraformDir: terraformDir,
-		Logger:       logger.Discard,
-		NoColor:      true,
-		PlanFilePath: filepath.Join(os.TempDir(), "plan.tfplan"),
+		TerraformDir:             terraformDir,
+		Logger:                   logger.Discard,
+		NoColor:                  true,
+		PlanFilePath:             filepath.Join(os.TempDir(), "plan.tfplan"),
+		RetryableTerraformErrors: testutils.RetryableTransientErrors,
+		MaxRetries:               MaxRetries,
+		TimeBetweenRetries:       TimeBetweenRetries,
 	}
 	_, err := terraform.PlanE(t, options)
 	if err != nil {