Add profile flag to commands to allow running profiles at the same time (#1368)

Author: mrodm

cmd/benchmark.go

@@ -13,6 +13,7 @@ import (
 	"github.com/dustin/go-humanize"
 
 	"github.com/elastic/elastic-package/internal/corpusgenerator"
+	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/kibana"
 
 	"github.com/spf13/cobra"
@@ -57,6 +58,8 @@ func setupBenchmarkCommand() *cobraext.Command {
 		Long:  benchLongDescription,
 	}
 
+	cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar))
+
 	pipelineCmd := getPipelineCommand()
 	cmd.AddCommand(pipelineCmd)
 
@@ -248,6 +251,11 @@ func systemCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("locating package root failed: %w", err)
 	}
 
+	profile, err := cobraext.GetProfileFlag(cmd)
+	if err != nil {
+		return nil
+	}
+
 	signal.Enable()
 
 	esClient, err := elasticsearch.NewClient()
@@ -272,6 +280,7 @@ func systemCommandAction(cmd *cobra.Command, args []string) error {
 		system.WithPackageRootPath(packageRootPath),
 		system.WithESAPI(esClient.API),
 		system.WithKibanaClient(kc),
+		system.WithProfile(profile),
 	)
 	runner := system.NewSystemBenchmark(opts)
 

cmd/service.go

@@ -12,6 +12,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/elastic/elastic-package/internal/cobraext"
+	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/packages"
 	"github.com/elastic/elastic-package/internal/service"
 )
@@ -35,6 +36,7 @@ func setupServiceCommand() *cobraext.Command {
 		Long:  serviceLongDescription,
 	}
 	cmd.AddCommand(upCommand)
+	cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar))
 
 	return cobraext.NewCommand(cmd, cobraext.ContextPackage)
 }
@@ -58,8 +60,14 @@ func upCommandAction(cmd *cobra.Command, args []string) error {
 
 	variantFlag, _ := cmd.Flags().GetString(cobraext.VariantFlagName)
 
+	profile, err := cobraext.GetProfileFlag(cmd)
+	if err != nil {
+		return nil
+	}
+
 	_, serviceName := filepath.Split(packageRoot)
 	err = service.BootUp(service.Options{
+		Profile:            profile,
 		ServiceName:        serviceName,
 		PackageRootPath:    packageRoot,
 		DataStreamRootPath: dataStreamPath,

cmd/testrunner.go

@@ -226,14 +226,14 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command
 		var results []testrunner.TestResult
 		for _, folder := range testFolders {
 			r, err := testrunner.Run(testType, testrunner.TestOptions{
+				Profile:            profile,
 				TestFolder:         folder,
 				PackageRootPath:    packageRootPath,
 				GenerateTestResult: generateTestResult,
 				API:                esClient.API,
 				DeferCleanup:       deferCleanup,
 				ServiceVariant:     variantFlag,
 				WithCoverage:       testCoverage,
-				Profile:            profile,
 			})
 
 			results = append(results, r...)

internal/benchrunner/runners/system/options.go

@@ -9,10 +9,12 @@ import (
 
 	"github.com/elastic/elastic-package/internal/elasticsearch"
 	"github.com/elastic/elastic-package/internal/kibana"
+	"github.com/elastic/elastic-package/internal/profile"
 )
 
 // Options contains benchmark runner options.
 type Options struct {
+	Profile          *profile.Profile
 	ESAPI            *elasticsearch.API
 	KibanaClient     *kibana.Client
 	DeferCleanup     time.Duration
@@ -72,3 +74,9 @@ func WithDataReindexing(b bool) OptionFunc {
 		opts.ReindexData = b
 	}
 }
+
+func WithProfile(p *profile.Profile) OptionFunc {
+	return func(opts *Options) {
+		opts.Profile = p
+	}
+}

internal/benchrunner/runners/system/runner.go

@@ -230,6 +230,7 @@ func (r *runner) run() (report reporters.Reportable, err error) {
 		// Setup service.
 		logger.Debug("setting up service...")
 		serviceDeployer, err := servicedeployer.Factory(servicedeployer.FactoryOptions{
+			Profile:  r.options.Profile,
 			RootPath: r.options.PackageRootPath,
 		})
 

internal/benchrunner/runners/system/servicedeployer/compose.go

@@ -15,12 +15,14 @@ import (
 	"github.com/elastic/elastic-package/internal/docker"
 	"github.com/elastic/elastic-package/internal/files"
 	"github.com/elastic/elastic-package/internal/logger"
+	"github.com/elastic/elastic-package/internal/profile"
 	"github.com/elastic/elastic-package/internal/stack"
 )
 
 // DockerComposeServiceDeployer knows how to deploy a service defined via
 // a Docker Compose file.
 type DockerComposeServiceDeployer struct {
+	profile  *profile.Profile
 	ymlPaths []string
 }
 
@@ -32,8 +34,9 @@ type dockerComposeDeployedService struct {
 }
 
 // NewDockerComposeServiceDeployer returns a new instance of a DockerComposeServiceDeployer.
-func NewDockerComposeServiceDeployer(ymlPaths []string) (*DockerComposeServiceDeployer, error) {
+func NewDockerComposeServiceDeployer(profile *profile.Profile, ymlPaths []string) (*DockerComposeServiceDeployer, error) {
 	return &DockerComposeServiceDeployer{
+		profile:  profile,
 		ymlPaths: ymlPaths,
 	}, nil
 }
@@ -53,7 +56,7 @@ func (d *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedSer
 	}
 
 	// Verify the Elastic stack network
-	err = stack.EnsureStackNetworkUp()
+	err = stack.EnsureStackNetworkUp(d.profile)
 	if err != nil {
 		return nil, fmt.Errorf("elastic stack network is not ready: %w", err)
 	}
@@ -85,7 +88,7 @@ func (d *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedSer
 	outCtxt.Hostname = p.ContainerName(serviceName)
 
 	// Connect service network with stack network (for the purpose of metrics collection)
-	err = docker.ConnectToNetwork(p.ContainerName(serviceName), stack.Network())
+	err = docker.ConnectToNetwork(p.ContainerName(serviceName), stack.Network(d.profile))
 	if err != nil {
 		return nil, fmt.Errorf("can't attach service container to the stack network: %w", err)
 	}

internal/benchrunner/runners/system/servicedeployer/factory.go

@@ -9,12 +9,16 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+
+	"github.com/elastic/elastic-package/internal/profile"
 )
 
 const devDeployDir = "_dev/benchmark/system/deploy"
 
 // FactoryOptions defines options used to create an instance of a service deployer.
 type FactoryOptions struct {
+	Profile *profile.Profile
+
 	RootPath string
 }
 
@@ -37,7 +41,7 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) {
 	case "docker":
 		dockerComposeYMLPath := filepath.Join(serviceDeployerPath, "docker-compose.yml")
 		if _, err := os.Stat(dockerComposeYMLPath); err == nil {
-			return NewDockerComposeServiceDeployer([]string{dockerComposeYMLPath})
+			return NewDockerComposeServiceDeployer(options.Profile, []string{dockerComposeYMLPath})
 		}
 	}
 	return nil, fmt.Errorf("unsupported service deployer (name: %s)", serviceDeployerName)

internal/kind/kind.go

@@ -10,6 +10,7 @@ import (
 	"github.com/elastic/elastic-package/internal/docker"
 	"github.com/elastic/elastic-package/internal/kubectl"
 	"github.com/elastic/elastic-package/internal/logger"
+	"github.com/elastic/elastic-package/internal/profile"
 	"github.com/elastic/elastic-package/internal/stack"
 )
 
@@ -33,13 +34,13 @@ func VerifyContext() error {
 }
 
 // ConnectToElasticStackNetwork function ensures that the control plane node is connected to the Elastic stack network.
-func ConnectToElasticStackNetwork() error {
+func ConnectToElasticStackNetwork(profile *profile.Profile) error {
 	containerID, err := controlPlaneContainerID()
 	if err != nil {
 		return fmt.Errorf("can't find kind-control plane node: %w", err)
 	}
 
-	stackNetwork := stack.Network()
+	stackNetwork := stack.Network(profile)
 	logger.Debugf("check network connectivity between service container %s (ID: %s) and the stack network %s", ControlPlaneContainerName, containerID, stackNetwork)
 
 	networkDescriptions, err := docker.InspectNetwork(stackNetwork)

internal/service/boot.go

@@ -11,6 +11,7 @@ import (
 	"syscall"
 
 	"github.com/elastic/elastic-package/internal/logger"
+	"github.com/elastic/elastic-package/internal/profile"
 
 	"github.com/elastic/elastic-package/internal/configuration/locations"
 	"github.com/elastic/elastic-package/internal/testrunner/runners/system"
@@ -19,6 +20,8 @@ import (
 
 // Options define the details of the service which should be booted up.
 type Options struct {
+	Profile *profile.Profile
+
 	ServiceName        string
 	PackageRootPath    string
 	DataStreamRootPath string
@@ -30,6 +33,7 @@ type Options struct {
 func BootUp(options Options) error {
 	logger.Debugf("Create new instance of the service deployer")
 	serviceDeployer, err := servicedeployer.Factory(servicedeployer.FactoryOptions{
+		Profile:            options.Profile,
 		PackageRootPath:    options.DataStreamRootPath,
 		DataStreamRootPath: options.DataStreamRootPath,
 		Variant:            options.Variant,

internal/stack/boot.go

@@ -13,11 +13,20 @@ import (
 	"github.com/elastic/elastic-package/internal/builder"
 	"github.com/elastic/elastic-package/internal/configuration/locations"
 	"github.com/elastic/elastic-package/internal/files"
+	"github.com/elastic/elastic-package/internal/profile"
 )
 
-// DockerComposeProjectName is the name of the Docker Compose project used to boot up
+// baseComposeProjectName is the base name of the Docker Compose project used to boot up
 // Elastic Stack containers.
-const DockerComposeProjectName = "elastic-package-stack"
+const baseComposeProjectName = "elastic-package-stack"
+
+// DockerComposeProjectName returns the docker compose project name for a given profile.
+func DockerComposeProjectName(profile *profile.Profile) string {
+	if profile.ProfileName == "default" {
+		return baseComposeProjectName
+	}
+	return baseComposeProjectName + "-" + profile.ProfileName
+}
 
 // BootUp function boots up the Elastic stack.
 func BootUp(options Options) error {
@@ -81,7 +90,7 @@ func BootUp(options Options) error {
 		// to fail too.
 		// As a workaround, try to give another chance to docker-compose if only
 		// elastic-agent failed.
-		if onlyElasticAgentFailed() {
+		if onlyElasticAgentFailed(options) {
 			fmt.Println("Elastic Agent failed to start, trying again.")
 			err = dockerComposeUp(options)
 		}
@@ -98,8 +107,8 @@ func BootUp(options Options) error {
 	return nil
 }
 
-func onlyElasticAgentFailed() bool {
-	status, err := Status()
+func onlyElasticAgentFailed(options Options) bool {
+	status, err := Status(options)
 	if err != nil {
 		fmt.Printf("Failed to check status of the stack after failure: %v\n", err)
 		return false

internal/stack/compose.go

@@ -52,7 +52,7 @@ func (eb *envBuilder) build() []string {
 }
 
 func dockerComposeBuild(options Options) error {
-	c, err := compose.NewProject(DockerComposeProjectName, options.Profile.Path(profileStackPath, SnapshotFile))
+	c, err := compose.NewProject(DockerComposeProjectName(options.Profile), options.Profile.Path(profileStackPath, SnapshotFile))
 	if err != nil {
 		return fmt.Errorf("could not create docker compose project: %w", err)
 	}
@@ -78,7 +78,7 @@ func dockerComposeBuild(options Options) error {
 }
 
 func dockerComposePull(options Options) error {
-	c, err := compose.NewProject(DockerComposeProjectName, options.Profile.Path(profileStackPath, SnapshotFile))
+	c, err := compose.NewProject(DockerComposeProjectName(options.Profile), options.Profile.Path(profileStackPath, SnapshotFile))
 	if err != nil {
 		return fmt.Errorf("could not create docker compose project: %w", err)
 	}
@@ -104,7 +104,7 @@ func dockerComposePull(options Options) error {
 }
 
 func dockerComposeUp(options Options) error {
-	c, err := compose.NewProject(DockerComposeProjectName, options.Profile.Path(profileStackPath, SnapshotFile))
+	c, err := compose.NewProject(DockerComposeProjectName(options.Profile), options.Profile.Path(profileStackPath, SnapshotFile))
 	if err != nil {
 		return fmt.Errorf("could not create docker compose project: %w", err)
 	}
@@ -136,7 +136,7 @@ func dockerComposeUp(options Options) error {
 }
 
 func dockerComposeDown(options Options) error {
-	c, err := compose.NewProject(DockerComposeProjectName, options.Profile.Path(profileStackPath, SnapshotFile))
+	c, err := compose.NewProject(DockerComposeProjectName(options.Profile), options.Profile.Path(profileStackPath, SnapshotFile))
 	if err != nil {
 		return fmt.Errorf("could not create docker compose project: %w", err)
 	}
@@ -182,10 +182,10 @@ func withIsReadyServices(services []string) []string {
 	return allServices
 }
 
-func dockerComposeStatus() ([]ServiceStatus, error) {
+func dockerComposeStatus(options Options) ([]ServiceStatus, error) {
 	var services []ServiceStatus
 	// query directly to docker to avoid load environment variables (e.g. STACK_VERSION_VARIANT) and profiles
-	containerIDs, err := docker.ContainerIDsWithLabel(projectLabelDockerCompose, DockerComposeProjectName)
+	containerIDs, err := docker.ContainerIDsWithLabel(projectLabelDockerCompose, DockerComposeProjectName(options.Profile))
 	if err != nil {
 		return nil, err
 	}

internal/stack/dump.go

@@ -60,7 +60,7 @@ func dumpStackLogs(options DumpOptions) error {
 			writeLogFiles(logsPath, serviceName, content)
 		}
 
-		err = copyDockerInternalLogs(serviceName, logsPath)
+		err = copyDockerInternalLogs(serviceName, logsPath, options.Profile)
 		if err != nil {
 			logger.Errorf("can't copy internal logs (service: %s): %v", serviceName, err)
 		}

internal/stack/logs.go

@@ -22,7 +22,7 @@ func dockerComposeLogs(serviceName string, profile *profile.Profile) ([]byte, er
 
 	snapshotFile := profile.Path(profileStackPath, SnapshotFile)
 
-	p, err := compose.NewProject(DockerComposeProjectName, snapshotFile)
+	p, err := compose.NewProject(DockerComposeProjectName(profile), snapshotFile)
 	if err != nil {
 		return nil, fmt.Errorf("could not create docker compose project: %w", err)
 	}
@@ -43,14 +43,14 @@ func dockerComposeLogs(serviceName string, profile *profile.Profile) ([]byte, er
 	return out, nil
 }
 
-func copyDockerInternalLogs(serviceName, outputPath string) error {
+func copyDockerInternalLogs(serviceName, outputPath string, profile *profile.Profile) error {
 	switch serviceName {
 	case elasticAgentService, fleetServerService:
 	default:
 		return nil // we need to pull internal logs only from Elastic-Agent and Fleets Server container
 	}
 
-	p, err := compose.NewProject(DockerComposeProjectName)
+	p, err := compose.NewProject(DockerComposeProjectName(profile))
 	if err != nil {
 		return fmt.Errorf("could not create docker compose project: %w", err)
 	}

internal/stack/network.go

@@ -8,18 +8,19 @@ import (
 	"fmt"
 
 	"github.com/elastic/elastic-package/internal/docker"
+	"github.com/elastic/elastic-package/internal/profile"
 )
 
 // EnsureStackNetworkUp function verifies if stack network is up and running.
-func EnsureStackNetworkUp() error {
-	_, err := docker.InspectNetwork(Network())
+func EnsureStackNetworkUp(profile *profile.Profile) error {
+	_, err := docker.InspectNetwork(Network(profile))
 	if err != nil {
 		return fmt.Errorf("network not available: %w", err)
 	}
 	return nil
 }
 
 // Network function returns the stack network name.
-func Network() string {
-	return fmt.Sprintf("%s_default", DockerComposeProjectName)
+func Network(profile *profile.Profile) string {
+	return fmt.Sprintf("%s_default", DockerComposeProjectName(profile))
 }

internal/stack/providers.go

@@ -75,5 +75,5 @@ func (*composeProvider) Dump(options DumpOptions) (string, error) {
 }
 
 func (*composeProvider) Status(options Options) ([]ServiceStatus, error) {
-	return Status()
+	return Status(options)
 }