diff --git a/.github/workflows/go-lint.yaml b/.github/workflows/go-lint.yaml index 12fa9e0e3..ba8ec5bae 100644 --- a/.github/workflows/go-lint.yaml +++ b/.github/workflows/go-lint.yaml @@ -16,7 +16,7 @@ name: go-lint on: pull_request: branches: - - master + - main paths: - ".github/workflows/go-lint.yaml" - "helpers/foundation-deployer/**" diff --git a/.github/workflows/go-test.yaml b/.github/workflows/go-test.yaml index e71eeab3b..c1d14fb53 100644 --- a/.github/workflows/go-test.yaml +++ b/.github/workflows/go-test.yaml @@ -17,7 +17,7 @@ name: go-test on: pull_request: branches: - - 'master' + - 'main' paths: - 'helpers/foundation-deployer/**' - '.github/workflows/go-test.yaml' diff --git a/helpers/foundation-deployer/gcp/gcp.go b/helpers/foundation-deployer/gcp/gcp.go index d0c47b282..eeba7cb78 100644 --- a/helpers/foundation-deployer/gcp/gcp.go +++ b/helpers/foundation-deployer/gcp/gcp.go @@ -47,6 +47,16 @@ func NewGCP() GCP { } } +// IsComponentInstalled checks if a given gcloud component is installed +func (g GCP) IsComponentInstalled(t testing.TB, componentID string) bool { + filter := fmt.Sprintf("\"id='%s'\"",componentID) + components := g.Runf(t, "components list --filter %s", filter).Array() + if len(components) == 0 { + return false + } + return components[0].Get("state.name").String() != "Not Installed" +} + // GetBuilds gets all Cloud Build builds form a project and region that satisfy the given filter. func (g GCP) GetBuilds(t testing.TB, projectID, region, filter string) map[string]string { var result = map[string]string{} @@ -116,12 +126,12 @@ func (g GCP) WaitBuildSuccess(t testing.TB, project, region, repo, commitSha, fa return err } if status != StatusSuccess { - return fmt.Errorf("%s\nSee:\nhttps://console.cloud.google.com/cloud-build/builds;region=%s/%s?project=%s\nfor details.\n", failureMsg, region, build, project) + return fmt.Errorf("%s\nSee:\nhttps://console.cloud.google.com/cloud-build/builds;region=%s/%s?project=%s\nfor details", failureMsg, region, build, project) } } else { status := g.GetLastBuildStatus(t, project, region, filter) if status != StatusSuccess { - return fmt.Errorf("%s\nSee:\nhttps://console.cloud.google.com/cloud-build/builds;region=%s/%s?project=%s\nfor details.\n", failureMsg, region, build, project) + return fmt.Errorf("%s\nSee:\nhttps://console.cloud.google.com/cloud-build/builds;region=%s/%s?project=%s\nfor details", failureMsg, region, build, project) } } return nil diff --git a/helpers/foundation-deployer/gcp/gcp_test.go b/helpers/foundation-deployer/gcp/gcp_test.go index d0c4ccace..58e63b335 100644 --- a/helpers/foundation-deployer/gcp/gcp_test.go +++ b/helpers/foundation-deployer/gcp/gcp_test.go @@ -25,6 +25,40 @@ import ( "github.com/tidwall/gjson" ) +func TestIsComponentInstalledFound(t *gotest.T) { + betaComponents, err := os.ReadFile(filepath.Join(".", "testdata", "beta_components_installed.json")) + assert.NoError(t, err) + gcp := GCP{ + Runf: func(t testing.TB, cmd string, args ...interface{}) gjson.Result { + return gjson.Result{ + Type: gjson.JSON, + Raw: string(betaComponents[:]), + } + }, + sleepTime: 1, + } + componentID := "beta" + result := gcp.IsComponentInstalled(t, componentID) + assert.True(t, result, "component '%s' should be installed", componentID) +} + +func TestIsComponentInstalledNotFound(t *gotest.T) { + betaComponents, err := os.ReadFile(filepath.Join(".", "testdata", "beta_components_not_installed.json")) + assert.NoError(t, err) + gcp := GCP{ + Runf: func(t testing.TB, cmd string, args ...interface{}) gjson.Result { + return gjson.Result{ + Type: gjson.JSON, + Raw: string(betaComponents[:]), + } + }, + sleepTime: 1, + } + componentID := "beta" + result := gcp.IsComponentInstalled(t, componentID) + assert.False(t, result, "component '%s' should not be installed", componentID) +} + func TestGetLastBuildStatus(t *gotest.T) { current, err := os.ReadFile(filepath.Join(".", "testdata", "success_build.json")) assert.NoError(t, err) @@ -101,7 +135,7 @@ func TestWaitBuildSuccess(t *gotest.T) { sleepTime: 1, } - err = gcp.WaitBuildSuccess(t, "prj-b-cicd-0123", "us-central1", "repo","", "failed_test_for_WaitBuildSuccess", 40) + err = gcp.WaitBuildSuccess(t, "prj-b-cicd-0123", "us-central1", "repo", "", "failed_test_for_WaitBuildSuccess", 40) assert.Error(t, err, "should have failed") assert.Contains(t, err.Error(), "failed_test_for_WaitBuildSuccess", "should have failed with custom info") assert.Equal(t, callCount, 3, "Runf must be called three times") @@ -133,7 +167,7 @@ func TestWaitBuildTimeout(t *gotest.T) { sleepTime: 1, } - err = gcp.WaitBuildSuccess(t, "prj-b-cicd-0123", "us-central1", "repo","", "failed_test_for_WaitBuildSuccess", 1) + err = gcp.WaitBuildSuccess(t, "prj-b-cicd-0123", "us-central1", "repo", "", "failed_test_for_WaitBuildSuccess", 1) assert.Error(t, err, "should have failed") assert.Contains(t, err.Error(), "timeout waiting for build '736f4689-2497-4382-afd0-b5f0f50eea5b' execution", "should have failed with timeout error") assert.Equal(t, callCount, 3, "Runf must be called three times") diff --git a/helpers/foundation-deployer/gcp/testdata/beta_components_installed.json b/helpers/foundation-deployer/gcp/testdata/beta_components_installed.json new file mode 100644 index 000000000..e1d20898a --- /dev/null +++ b/helpers/foundation-deployer/gcp/testdata/beta_components_installed.json @@ -0,0 +1,30 @@ +[ + { + "current_version_string": "2025.05.30", + "gdu_only": false, + "id": "beta", + "is_configuration": false, + "is_hidden": false, + "latest_version_string": "2025.08.29", + "name": "gcloud Beta Commands", + "platform": { + "architecture": { + "file_name": "x86_64", + "id": "x86_64", + "name": "x86_64" + }, + "operating_system": { + "clean_version": "6.6.87", + "file_name": "linux", + "id": "LINUX", + "name": "Linux", + "version": "6.6.87.2-microsoft-standard-WSL2" + } + }, + "platform_required": false, + "size": 797, + "state": { + "name": "Update Available" + } + } +] diff --git a/helpers/foundation-deployer/gcp/testdata/beta_components_not_installed.json b/helpers/foundation-deployer/gcp/testdata/beta_components_not_installed.json new file mode 100644 index 000000000..98b3f5711 --- /dev/null +++ b/helpers/foundation-deployer/gcp/testdata/beta_components_not_installed.json @@ -0,0 +1,30 @@ +[ + { + "current_version_string": "2025.05.30", + "gdu_only": false, + "id": "beta", + "is_configuration": false, + "is_hidden": false, + "latest_version_string": "2025.08.29", + "name": "gcloud Beta Commands", + "platform": { + "architecture": { + "file_name": "x86_64", + "id": "x86_64", + "name": "x86_64" + }, + "operating_system": { + "clean_version": "6.6.87", + "file_name": "linux", + "id": "LINUX", + "name": "Linux", + "version": "6.6.87.2-microsoft-standard-WSL2" + } + }, + "platform_required": false, + "size": 797, + "state": { + "name": "Not Installed" + } + } +] diff --git a/helpers/foundation-deployer/main.go b/helpers/foundation-deployer/main.go index 9d8d6ce1e..9d0df9d8a 100644 --- a/helpers/foundation-deployer/main.go +++ b/helpers/foundation-deployer/main.go @@ -34,7 +34,6 @@ import ( var ( validatorApis = []string{ - "securitycenter.googleapis.com", "accesscontextmanager.googleapis.com", } ) @@ -94,6 +93,14 @@ func main() { // init infra gotest.Init() t := &testing.RuntimeT{} + + // validate gcloud components + err = stages.ValidateComponents(t) + if err != nil { + fmt.Printf("# Failed validating gcloud components. Error: %s\n", err.Error()) + os.Exit(1) + } + conf := stages.CommonConf{ FoundationPath: globalTFVars.FoundationCodePath, CheckoutPath: globalTFVars.CodeCheckoutPath, @@ -108,6 +115,9 @@ func main() { conf.ValidatorProject = *globalTFVars.ValidatorProjectId var apis []string gcpConf := gcp.NewGCP() + if globalTFVars.EnableSccResourcesInTerraform != nil && *globalTFVars.EnableSccResourcesInTerraform { + validatorApis = append(validatorApis, "securitycenter.googleapis.com") + } for _, a := range validatorApis { if !gcpConf.IsApiEnabled(t, *globalTFVars.ValidatorProjectId, a) { apis = append(apis, a) diff --git a/helpers/foundation-deployer/stages/data.go b/helpers/foundation-deployer/stages/data.go index 70cec7fa6..37b25ecf0 100644 --- a/helpers/foundation-deployer/stages/data.go +++ b/helpers/foundation-deployer/stages/data.go @@ -320,11 +320,11 @@ func ReadGlobalTFVars(file string) (GlobalTFVars, error) { } _, err := os.Stat(file) if os.IsNotExist(err) { - return globalTfvars, fmt.Errorf("tfvars file '%s' does not exits\n", file) + return globalTfvars, fmt.Errorf("tfvars file '%s' does not exits", file) } err = utils.ReadTfvars(file, &globalTfvars) if err != nil { - return globalTfvars, fmt.Errorf("Failed to load tfvars file %s. Error: %s\n", file, err.Error()) + return globalTfvars, fmt.Errorf("failed to load tfvars file %s. Error: %s", file, err.Error()) } return globalTfvars, nil } diff --git a/helpers/foundation-deployer/stages/validate.go b/helpers/foundation-deployer/stages/validate.go index 89b9ea9e4..9ab20dbc1 100644 --- a/helpers/foundation-deployer/stages/validate.go +++ b/helpers/foundation-deployer/stages/validate.go @@ -33,11 +33,30 @@ const ( func ValidateDirectories(g GlobalTFVars) error { _, err := os.Stat(g.FoundationCodePath) if os.IsNotExist(err) { - return fmt.Errorf("Stopping execution, FoundationCodePath directory '%s' does not exits\n", g.FoundationCodePath) + return fmt.Errorf("stopping execution, FoundationCodePath directory '%s' does not exits", g.FoundationCodePath) } _, err = os.Stat(g.CodeCheckoutPath) if os.IsNotExist(err) { - return fmt.Errorf("Stopping execution, CodeCheckoutPath directory '%s' does not exits\n", g.CodeCheckoutPath) + return fmt.Errorf("stopping execution, CodeCheckoutPath directory '%s' does not exits", g.CodeCheckoutPath) + } + return nil +} + +// ValidateComponents checks if gcloud Beta Components and Terraform Tools are installed +func ValidateComponents(t testing.TB) error { + gcpConf := gcp.NewGCP() + components := []string{ + "beta", + "terraform-tools", + } + missing := []string{} + for _, c := range components { + if !gcpConf.IsComponentInstalled(t, c) { + missing = append(missing, fmt.Sprintf("'%s' not installed", c)) + } + } + if len(missing) > 0 { + return fmt.Errorf("missing Google Cloud SDK component:%v", missing) } return nil } diff --git a/helpers/foundation-deployer/stages/vet.go b/helpers/foundation-deployer/stages/vet.go index 4ee52e61a..09bffe395 100644 --- a/helpers/foundation-deployer/stages/vet.go +++ b/helpers/foundation-deployer/stages/vet.go @@ -51,22 +51,35 @@ func TerraformVet(t testing.TB, terraformDir, policyPath, project string) error return err } jsonFile, err := utils.WriteTmpFileWithExtension(jsonPlan, "json") - defer os.Remove(jsonFile) - defer os.Remove(options.PlanFilePath) + + defer func() { + err := os.Remove(jsonFile) + if err != nil { + fmt.Fprintf(os.Stderr, "Error removing file: %s\n", err) + } + }() + + defer func() { + err := os.Remove(options.PlanFilePath) + if err != nil { + fmt.Fprintf(os.Stderr, "Error removing file: %s\n", err) + } + }() + if err != nil { return err } command := fmt.Sprintf("beta terraform vet %s --policy-library=%s --project=%s --quiet", jsonFile, policyPath, project) result, err := gcloud.RunCmdE(t, command) - if err != nil && !(strings.Contains(err.Error(), "Validating resources") && strings.Contains(err.Error(), "done")) { + if err != nil && (!strings.Contains(err.Error(), "Validating resources") || !strings.Contains(err.Error(), "done")) { return err } if !gjson.Valid(result) { - return fmt.Errorf("Error parsing output, invalid json: %s", result) + return fmt.Errorf("error parsing output, invalid json: %s", result) } if len(gjson.Parse(result).Array()) > 0 { - return fmt.Errorf("Policy violations found: %s", result) + return fmt.Errorf("policy violations found: %s", result) } fmt.Println("") fmt.Println("# The configuration passed tf vet.") diff --git a/helpers/foundation-deployer/utils/files.go b/helpers/foundation-deployer/utils/files.go index d7e61d026..b7eab50b9 100644 --- a/helpers/foundation-deployer/utils/files.go +++ b/helpers/foundation-deployer/utils/files.go @@ -74,7 +74,7 @@ func ReplaceStringInFile(filename, old, new string) error { if err != nil { return err } - return os.WriteFile(filename, bytes.Replace(f, []byte(old), []byte(new), -1), 0644) + return os.WriteFile(filename, bytes.ReplaceAll(f, []byte(old), []byte(new)), 0644) } // FindFiles find files with the given filename under the directory skipping terraform temp dir. diff --git a/helpers/foundation-deployer/utils/logger.go b/helpers/foundation-deployer/utils/logger.go index 45f0a242e..286bbc5d0 100644 --- a/helpers/foundation-deployer/utils/logger.go +++ b/helpers/foundation-deployer/utils/logger.go @@ -32,7 +32,10 @@ func NewCustomLogger() CustomLogger { } } func (c CustomLogger) Logf(t grunttest.TestingT, format string, args ...interface{}) { - fmt.Fprintln(os.Stdout, fmt.Sprintf(c.baseFmt, fmt.Sprintf(format, args...))) + _, err := fmt.Fprintln(os.Stdout, fmt.Sprintf(c.baseFmt, fmt.Sprintf(format, args...))) + if err != nil { + fmt.Fprintf(os.Stderr, "Error writing log: %s\n", err) + } } func GetLogger(quiet bool) *logger.Logger {