add support to run specific (multiple) tests and envs through command line -t flag and change default time from 2s to 15s

Author: kataras

Diff for: .gitignore

@@ -8,3 +8,4 @@ tests.dev.yml
 dev
 client_secret.json
 binaries
+_code/rest/request.json

Diff for: benchmark.go

@@ -33,7 +33,7 @@ type (
 		NumberOfConnections uint64            `yaml:"NumberOfConnections"` // defaults to 125.
 		NumberOfRequests    uint64            `yaml:"NumberOfRequests"`
 		Duration            time.Duration     `yaml:"Duration"`
-		Timeout             time.Duration     `yaml:"Timeout"` // defaults to 2s
+		Timeout             time.Duration     `yaml:"Timeout"` // defaults to 15s
 		Headers             map[string]string `yaml:"Headers"`
 		Method              string            `yaml:"Method"`
 		URL                 string            `yaml:"URL"`
@@ -114,9 +114,9 @@ func (t *Test) buildArgs() (args []string) {
 		t.NumberOfConnections = 125
 	}
 
-	// default timeout to 2 seconds (this can be omitted, as it's the bombardier's default).
+	// default timeout to 15 seconds (this can be omitted, as it's the bombardier's default).
 	if t.Timeout == 0 {
-		t.Timeout = 2 * time.Second
+		t.Timeout = 15 * time.Second
 	}
 
 	// if not number of requests to fire defined and not a test duration,
@@ -373,6 +373,10 @@ func runBenchmark(t *Test, env *TestEnv) (err error) {
 }
 
 func benchmark(t *Test) error {
+	if len(t.Envs) == 0 {
+		return nil
+	}
+
 	for _, env := range t.Envs {
 		if !env.CanBenchmark() {
 			continue

Diff for: main.go

@@ -23,7 +23,6 @@ const (
 var (
 	waitRunDur         = flag.Duration("wait-run", 3*time.Second, "wait time between tests")
 	testsFile          = flag.String("i", "./tests.yml", "yaml file path contains the tests to run")
-	specificTest       = flag.String("t", "", "run only a specific test by its name")
 	outputDir          = flag.String("o", "./", "directory to save generaged Markdown and CSV files")
 	enableREADMEOutput = flag.Bool("readme", false, "to generate a README.md file near the RESULTS.md")
 	spreadsheetID      = flag.String("g-spreadsheet", "", "Google Spreadsheet ID to send results")
@@ -35,6 +34,8 @@ var (
 
 // server-benchmarks --wait-run=3s -i ./tests.dev.yml -o ./dev -g-spreadsheet $GoogleSpreadsheetID -g-secret client_secret.json
 func main() {
+	var specificTests stringSlice
+	flag.Var(&specificTests, "t", "run only specific tests by their names") // support multiple -t flags.
 	flag.Parse()
 
 	if _, err := os.Stat("/.dockerenv"); err == nil || os.IsExist(err) {
@@ -44,11 +45,7 @@ func main() {
 	tests, err := readTests(*testsFile)
 	catch(err)
 
-	if specificTestName := strings.ToLower(*specificTest); specificTestName != "" {
-		tests = slices.DeleteFunc(tests, func(t *Test) bool {
-			return strings.ToLower(t.Name) != specificTestName
-		})
-	}
+	tests = filterTests(specificTests, tests...)
 
 	// TESTS
 	for _, t := range tests {
@@ -95,6 +92,39 @@ func readTests(filename string) ([]*Test, error) {
 	return tests, nil
 }
 
+func filterTests(specificTests stringSlice, tests ...*Test) []*Test {
+	testsAndEnvsToKeep := make(map[string][]string, len(specificTests))
+	for _, specificTest := range specificTests {
+		if specificTestName := strings.ToLower(specificTest); specificTestName != "" {
+			specificTestEnvName := ""
+			if dotParts := strings.Split(specificTestName, "."); len(dotParts) > 1 {
+				specificTestName = strings.Join(dotParts[0:len(dotParts)-1], ".") // name all except last dot.
+				specificTestEnvName = dotParts[len(dotParts)-1]
+			}
+			testsAndEnvsToKeep[specificTestName] = append(testsAndEnvsToKeep[specificTestName], specificTestEnvName)
+		}
+	}
+
+	// Delete all tests and envs that are not in the specificTests list.
+	return slices.DeleteFunc(tests, func(t *Test) bool {
+		// keep only the specific test.
+		testName := strings.ToLower(t.Name)
+		if specificTestEnvNames, ok := testsAndEnvsToKeep[testName]; ok {
+			// keep only the specific test's environment.
+			// E.g. -t rest.iris -t rest.iris-private
+			if len(specificTestEnvNames) > 0 {
+				t.Envs = slices.DeleteFunc(t.Envs, func(e *TestEnv) bool {
+					return !slices.Contains(specificTestEnvNames, strings.ToLower(e.GetName()))
+				})
+			}
+
+			return false
+		}
+
+		return true
+	})
+}
+
 func cleanup() {
 	for i, f := range downloadedFiles {
 		if err := os.Remove(f); err != nil && !errors.Is(err, os.ErrNotExist) {

Diff for: slice_flag.go

@@ -0,0 +1,25 @@
+package main
+
+import "strings"
+
+type stringSlice []string
+
+func (s *stringSlice) String() string {
+	return strings.Join(*s, ", ")
+}
+
+func (s *stringSlice) Set(value string) error {
+	*s = append(*s, value)
+	return nil
+}
+
+// type sliceFlag[T any] struct {
+// 	slice []T
+// }
+// func (s *sliceFlag[T]) String() string {
+// 	return fmt.Sprintf("%v", s.slice)
+// }
+// func (s *sliceFlag[T]) Set(value string) error {
+// 	s.slice = append(s.slice, *(*T)(unsafe.Pointer(&value)))
+// 	return nil
+// }

Diff for: tests.yml

@@ -11,7 +11,7 @@
   # or
   # Duration: 5s
   #
-  # Timeout: 2s
+  # Timeout: 15s
   # Method: POST
   # BodyFile: ./request_payload.json
   # Headers: