diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go index 85fcc75416c..7f2d6723c60 100644 --- a/api/v1alpha1/envoygateway_types.go +++ b/api/v1alpha1/envoygateway_types.go @@ -430,7 +430,25 @@ type EnvoyGatewayInfrastructureProvider struct { // EnvoyGatewayHostInfrastructureProvider defines configuration for the Host Infrastructure provider. type EnvoyGatewayHostInfrastructureProvider struct { - // TODO: Add config as use cases are better understood. + // ConfigHome is the directory for configuration files. + // Defaults to ~/.config/envoy-gateway + // +optional + ConfigHome *string `json:"configHome,omitempty"` + + // DataHome is the directory for persistent data (Envoy binaries). + // Defaults to ~/.local/share/envoy-gateway + // +optional + DataHome *string `json:"dataHome,omitempty"` + + // StateHome is the directory for persistent state (logs). + // Defaults to ~/.local/state/envoy-gateway + // +optional + StateHome *string `json:"stateHome,omitempty"` + + // RuntimeDir is the directory for ephemeral runtime files. + // Defaults to /tmp/envoy-gateway-${UID} + // +optional + RuntimeDir *string `json:"runtimeDir,omitempty"` } // RateLimit defines the configuration associated with the Rate Limit Service diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index abd8cc941fc..15647e88127 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1947,6 +1947,26 @@ func (in *EnvoyGatewayFileResourceProvider) DeepCopy() *EnvoyGatewayFileResource // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EnvoyGatewayHostInfrastructureProvider) DeepCopyInto(out *EnvoyGatewayHostInfrastructureProvider) { *out = *in + if in.ConfigHome != nil { + in, out := &in.ConfigHome, &out.ConfigHome + *out = new(string) + **out = **in + } + if in.DataHome != nil { + in, out := &in.DataHome, &out.DataHome + *out = new(string) + **out = **in + } + if in.StateHome != nil { + in, out := &in.StateHome, &out.StateHome + *out = new(string) + **out = **in + } + if in.RuntimeDir != nil { + in, out := &in.RuntimeDir, &out.RuntimeDir + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvoyGatewayHostInfrastructureProvider. @@ -1965,7 +1985,7 @@ func (in *EnvoyGatewayInfrastructureProvider) DeepCopyInto(out *EnvoyGatewayInfr if in.Host != nil { in, out := &in.Host, &out.Host *out = new(EnvoyGatewayHostInfrastructureProvider) - **out = **in + (*in).DeepCopyInto(*out) } } diff --git a/examples/standalone/envoy-gateway.yaml b/examples/standalone/envoy-gateway.yaml index 54454b867c2..a3434a676d4 100644 --- a/examples/standalone/envoy-gateway.yaml +++ b/examples/standalone/envoy-gateway.yaml @@ -11,6 +11,15 @@ provider: paths: ["/tmp/envoy-gateway-test"] infrastructure: type: Host + # Optional: Configure XDG-compliant directory paths under host: + # If not specified, uses XDG Base Directory defaults: + # - configHome: ~/.config/envoy-gateway + # - dataHome: ~/.local/share/envoy-gateway + # - stateHome: ~/.local/state/envoy-gateway + # - runtimeDir: /tmp/envoy-gateway-${UID} + # Example custom configuration: + # host: + # dataHome: /custom/data/path host: {} logging: level: diff --git a/internal/cmd/certgen.go b/internal/cmd/certgen.go index 13c35791ba6..16e266e120c 100644 --- a/internal/cmd/certgen.go +++ b/internal/cmd/certgen.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "path" + "path/filepath" "github.com/spf13/cobra" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" @@ -20,9 +21,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" clicfg "sigs.k8s.io/controller-runtime/pkg/client/config" + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/crypto" "github.com/envoyproxy/gateway/internal/envoygateway" "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/infrastructure/host" "github.com/envoyproxy/gateway/internal/provider/kubernetes" "github.com/envoyproxy/gateway/internal/utils/file" ) @@ -32,26 +35,29 @@ var overwriteControlPlaneCerts bool var disableTopologyInjector bool -// TODO: make this path configurable or use server config directly. const ( - defaultLocalCertPath = "/tmp/envoy-gateway/certs" topologyWebhookNamePrefix = "envoy-gateway-topology-injector" ) // GetCertGenCommand returns the certGen cobra command to be executed. func GetCertGenCommand() *cobra.Command { - var local bool + var ( + local bool + dataHome string + ) cmd := &cobra.Command{ Use: "certgen", Short: "Generate Control Plane Certificates", RunE: func(cmd *cobra.Command, args []string) error { - return certGen(cmd.Context(), cmd.OutOrStdout(), local) + return certGen(cmd.Context(), cmd.OutOrStdout(), local, dataHome) }, } cmd.PersistentFlags().BoolVarP(&local, "local", "l", false, "Generate all the certificates locally.") + cmd.PersistentFlags().StringVar(&dataHome, "data-home", "", + "Directory for certificates (defaults to ~/.local/share/envoy-gateway)") cmd.PersistentFlags().BoolVarP(&overwriteControlPlaneCerts, "overwrite", "o", false, "Updates the secrets containing the control plane certs.") cmd.PersistentFlags().BoolVar(&disableTopologyInjector, "disable-topology-injector", false, @@ -60,7 +66,7 @@ func GetCertGenCommand() *cobra.Command { } // certGen generates control plane certificates. -func certGen(ctx context.Context, logOut io.Writer, local bool) error { +func certGen(ctx context.Context, logOut io.Writer, local bool, dataHome string) error { cfg, err := config.New(logOut, io.Discard) if err != nil { return err @@ -86,8 +92,21 @@ func certGen(ctx context.Context, logOut io.Writer, local bool) error { return fmt.Errorf("failed to patch webhook: %w", err) } } else { - log.Info("generated certificates", "path", defaultLocalCertPath) - if err = outputCertsForLocal(defaultLocalCertPath, certs); err != nil { + // Use provided dataHome or default + hostCfg := &egv1a1.EnvoyGatewayHostInfrastructureProvider{} + if dataHome != "" { + hostCfg.DataHome = &dataHome + } + + paths, err := host.GetPaths(hostCfg) + if err != nil { + return fmt.Errorf("failed to determine paths: %w", err) + } + + certPath := filepath.Join(paths.DataHome, "certs") + log.Info("generated certificates", "path", certPath) + + if err = outputCertsForLocal(certPath, certs); err != nil { return fmt.Errorf("failed to output certificates locally: %w", err) } } diff --git a/internal/gatewayapi/runner/runner.go b/internal/gatewayapi/runner/runner.go index c78131160bb..7fad17a928d 100644 --- a/internal/gatewayapi/runner/runner.go +++ b/internal/gatewayapi/runner/runner.go @@ -12,6 +12,7 @@ import ( "fmt" "os" "path" + "path/filepath" "github.com/docker/docker/pkg/fileutils" "github.com/telepresenceio/watchable" @@ -29,6 +30,7 @@ import ( extension "github.com/envoyproxy/gateway/internal/extension/types" "github.com/envoyproxy/gateway/internal/gatewayapi" "github.com/envoyproxy/gateway/internal/gatewayapi/resource" + "github.com/envoyproxy/gateway/internal/infrastructure/host" "github.com/envoyproxy/gateway/internal/message" "github.com/envoyproxy/gateway/internal/utils" "github.com/envoyproxy/gateway/internal/wasm" @@ -40,15 +42,8 @@ const ( serveTLSKeyFilepath = "/certs/tls.key" serveTLSCaFilepath = "/certs/ca.crt" - // TODO: Make these path configurable. - // Default certificates path for envoy-gateway with Host infrastructure provider. - localTLSCertFilepath = "/tmp/envoy-gateway/certs/envoy-gateway/tls.crt" - localTLSKeyFilepath = "/tmp/envoy-gateway/certs/envoy-gateway/tls.key" - localTLSCaFilepath = "/tmp/envoy-gateway/certs/envoy-gateway/ca.crt" - hmacSecretName = "envoy-oidc-hmac" // nolint: gosec hmacSecretKey = "hmac-secret" - hmacSecretPath = "/tmp/envoy-gateway/certs/envoy-oidc-hmac/hmac-secret" // nolint: gosec ) type Config struct { @@ -375,12 +370,31 @@ func (r *Runner) loadTLSConfig(ctx context.Context) (tlsConfig *tls.Config, salt } case r.EnvoyGateway.Provider.IsRunningOnHost(): - salt, err = os.ReadFile(hmacSecretPath) + // Get config + var hostCfg *egv1a1.EnvoyGatewayHostInfrastructureProvider + if p := r.EnvoyGateway.Provider; p != nil && p.Custom != nil && + p.Custom.Infrastructure != nil && p.Custom.Infrastructure.Host != nil { + hostCfg = p.Custom.Infrastructure.Host + } + + paths, err := host.GetPaths(hostCfg) + if err != nil { + return nil, nil, fmt.Errorf("failed to determine paths: %w", err) + } + + // Read HMAC secret + hmacPath := filepath.Join(paths.CertDir("envoy-oidc-hmac"), "hmac-secret") + salt, err = os.ReadFile(hmacPath) if err != nil { return nil, nil, fmt.Errorf("failed to get hmac secret: %w", err) } - tlsConfig, err = crypto.LoadTLSConfig(localTLSCertFilepath, localTLSKeyFilepath, localTLSCaFilepath) + certDir := paths.CertDir("envoy-gateway") + certPath := filepath.Join(certDir, "tls.crt") + keyPath := filepath.Join(certDir, "tls.key") + caPath := filepath.Join(certDir, "ca.crt") + + tlsConfig, err = crypto.LoadTLSConfig(certPath, keyPath, caPath) if err != nil { return nil, nil, fmt.Errorf("failed to create tls config: %w", err) } diff --git a/internal/gatewayapi/runner/runner_test.go b/internal/gatewayapi/runner/runner_test.go index 929548aa5ac..fc1093a431a 100644 --- a/internal/gatewayapi/runner/runner_test.go +++ b/internal/gatewayapi/runner/runner_test.go @@ -7,7 +7,9 @@ package runner import ( "context" + "crypto/tls" "os" + "path/filepath" "reflect" "testing" "time" @@ -18,6 +20,7 @@ import ( gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/envoyproxy/gateway/internal/crypto" "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/extension/registry" "github.com/envoyproxy/gateway/internal/ir" @@ -235,3 +238,66 @@ func TestDeleteAllKeys(t *testing.T) { require.Empty(t, r.keyCache.SecurityPolicyStatus) require.Empty(t, r.keyCache.EnvoyExtensionPolicyStatus) } + +func TestLoadTLSConfig_HostMode(t *testing.T) { + // Create temporary directory structure for certs using t.TempDir() + configHome := t.TempDir() + certsDir := filepath.Join(configHome, "certs", "envoy-gateway") + hmacDir := filepath.Join(configHome, "certs", "envoy-oidc-hmac") + require.NoError(t, os.MkdirAll(certsDir, 0o750)) + require.NoError(t, os.MkdirAll(hmacDir, 0o750)) + + // Create test certificates using internal/crypto package + cfg, err := config.New(os.Stdout, os.Stderr) + require.NoError(t, err) + + // Generate certificates with default provider (crypto.GenerateCerts only supports Kubernetes) + certs, err := crypto.GenerateCerts(cfg) + require.NoError(t, err) + + // Write certificates to temp directory + caFile := filepath.Join(certsDir, "ca.crt") + certFile := filepath.Join(certsDir, "tls.crt") + keyFile := filepath.Join(certsDir, "tls.key") + hmacFile := filepath.Join(hmacDir, "hmac-secret") + + require.NoError(t, os.WriteFile(caFile, certs.CACertificate, 0o600)) + require.NoError(t, os.WriteFile(certFile, certs.EnvoyGatewayCertificate, 0o600)) + require.NoError(t, os.WriteFile(keyFile, certs.EnvoyGatewayPrivateKey, 0o600)) + require.NoError(t, os.WriteFile(hmacFile, certs.OIDCHMACSecret, 0o600)) + + // Configure host mode with custom configHome (certs are stored in configHome) + // MUST be set BEFORE creating Runner since Config{Server: *cfg} makes a copy + cfg.EnvoyGateway.Provider = &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: &egv1a1.EnvoyGatewayCustomProvider{ + Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{ + Type: egv1a1.InfrastructureProviderTypeHost, + Host: &egv1a1.EnvoyGatewayHostInfrastructureProvider{ + ConfigHome: &configHome, + }, + }, + }, + } + + r := &Runner{ + Config: Config{ + Server: *cfg, + }, + } + + // Test loadTLSConfig with host mode + tlsConfig, salt, err := r.loadTLSConfig(context.Background()) + require.NoError(t, err) + require.NotNil(t, tlsConfig) + require.NotNil(t, salt) + + // Verify HMAC secret was loaded + require.Equal(t, certs.OIDCHMACSecret, salt) + + // Verify TLS config properties + // crypto.LoadTLSConfig uses GetConfigForClient callback to load certs on demand + require.NotNil(t, tlsConfig.GetConfigForClient) + require.Equal(t, tls.RequireAndVerifyClientCert, tlsConfig.ClientAuth) + require.Equal(t, uint16(tls.VersionTLS13), tlsConfig.MinVersion) +} diff --git a/internal/globalratelimit/runner/runner.go b/internal/globalratelimit/runner/runner.go index 477b346db48..3cd4a33ea84 100644 --- a/internal/globalratelimit/runner/runner.go +++ b/internal/globalratelimit/runner/runner.go @@ -11,6 +11,7 @@ import ( "fmt" "math" "net" + "path/filepath" "strconv" discoveryv3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" @@ -25,6 +26,7 @@ import ( egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/crypto" "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/infrastructure/host" "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/ratelimit" "github.com/envoyproxy/gateway/internal/ir" "github.com/envoyproxy/gateway/internal/message" @@ -43,12 +45,6 @@ const ( rateLimitTLSKeyFilepath = "/certs/tls.key" // rateLimitTLSCACertFilepath is the ratelimit ca cert file. rateLimitTLSCACertFilepath = "/certs/ca.crt" - - // TODO: Make these path configurable. - // Default certificates path for envoy-gateway with Host infrastructure provider. - localTLSCertFilepath = "/tmp/envoy-gateway/certs/envoy-gateway/tls.crt" - localTLSKeyFilepath = "/tmp/envoy-gateway/certs/envoy-gateway/tls.key" - localTLSCaFilepath = "/tmp/envoy-gateway/certs/envoy-gateway/ca.crt" ) type Config struct { @@ -216,21 +212,37 @@ func (r *Runner) addNewSnapshot(ctx context.Context, resource types.XdsResources } func (r *Runner) loadTLSConfig() (tlsConfig *tls.Config, err error) { + var certPath, keyPath, caPath string + switch { case r.EnvoyGateway.Provider.IsRunningOnKubernetes(): - tlsConfig, err = crypto.LoadTLSConfig(rateLimitTLSCertFilepath, rateLimitTLSKeyFilepath, rateLimitTLSCACertFilepath) - if err != nil { - return nil, fmt.Errorf("failed to create tls config: %w", err) + certPath = rateLimitTLSCertFilepath + keyPath = rateLimitTLSKeyFilepath + caPath = rateLimitTLSCACertFilepath + case r.EnvoyGateway.Provider.IsRunningOnHost(): + // Get configuration from provider + var hostCfg *egv1a1.EnvoyGatewayHostInfrastructureProvider + if p := r.EnvoyGateway.Provider; p != nil && p.Custom != nil && + p.Custom.Infrastructure != nil && p.Custom.Infrastructure.Host != nil { + hostCfg = p.Custom.Infrastructure.Host } - case r.EnvoyGateway.Provider.IsRunningOnHost(): - tlsConfig, err = crypto.LoadTLSConfig(localTLSCertFilepath, localTLSKeyFilepath, localTLSCaFilepath) + paths, err := host.GetPaths(hostCfg) if err != nil { - return nil, fmt.Errorf("failed to create tls config: %w", err) + return nil, fmt.Errorf("failed to determine paths: %w", err) } + certDir := paths.CertDir("envoy-gateway") + certPath = filepath.Join(certDir, "tls.crt") + keyPath = filepath.Join(certDir, "tls.key") + caPath = filepath.Join(certDir, "ca.crt") default: return nil, fmt.Errorf("no valid tls certificates") } + + tlsConfig, err = crypto.LoadTLSConfig(certPath, keyPath, caPath) + if err != nil { + return nil, fmt.Errorf("failed to create tls config: %w", err) + } return } diff --git a/internal/globalratelimit/runner/runner_test.go b/internal/globalratelimit/runner/runner_test.go index 9b5ae49bd1f..932131fc70f 100644 --- a/internal/globalratelimit/runner/runner_test.go +++ b/internal/globalratelimit/runner/runner_test.go @@ -7,8 +7,10 @@ package runner import ( "context" + "crypto/tls" "fmt" "os" + "path/filepath" "testing" "time" @@ -22,6 +24,7 @@ import ( "github.com/stretchr/testify/require" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/envoyproxy/gateway/internal/crypto" "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/ratelimit" "github.com/envoyproxy/gateway/internal/ir" @@ -246,3 +249,58 @@ func Test_subscribeAndTranslate(t *testing.T) { }) } } + +func TestLoadTLSConfig_HostMode(t *testing.T) { + // Create temporary directory structure for certs using t.TempDir() + configHome := t.TempDir() + certsDir := filepath.Join(configHome, "certs", "envoy-gateway") + require.NoError(t, os.MkdirAll(certsDir, 0o750)) + + // Create test certificates using internal/crypto package + cfg, err := config.New(os.Stdout, os.Stderr) + require.NoError(t, err) + + // Generate certificates with default provider (crypto.GenerateCerts only supports Kubernetes) + certs, err := crypto.GenerateCerts(cfg) + require.NoError(t, err) + + // Write certificates to temp directory + caFile := filepath.Join(certsDir, "ca.crt") + certFile := filepath.Join(certsDir, "tls.crt") + keyFile := filepath.Join(certsDir, "tls.key") + + require.NoError(t, os.WriteFile(caFile, certs.CACertificate, 0o600)) + require.NoError(t, os.WriteFile(certFile, certs.EnvoyGatewayCertificate, 0o600)) + require.NoError(t, os.WriteFile(keyFile, certs.EnvoyGatewayPrivateKey, 0o600)) + + // Configure host mode with custom configHome (certs are stored in configHome) + // MUST be set BEFORE creating Runner since Config{Server: *cfg} makes a copy + cfg.EnvoyGateway.Provider = &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: &egv1a1.EnvoyGatewayCustomProvider{ + Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{ + Type: egv1a1.InfrastructureProviderTypeHost, + Host: &egv1a1.EnvoyGatewayHostInfrastructureProvider{ + ConfigHome: &configHome, + }, + }, + }, + } + + r := &Runner{ + Config: Config{ + Server: *cfg, + }, + } + + // Test loadTLSConfig with host mode + tlsConfig, err := r.loadTLSConfig() + require.NoError(t, err) + require.NotNil(t, tlsConfig) + + // Verify TLS config properties + // crypto.LoadTLSConfig uses GetConfigForClient callback to load certs on demand + require.NotNil(t, tlsConfig.GetConfigForClient) + require.Equal(t, tls.RequireAndVerifyClientCert, tlsConfig.ClientAuth) + require.Equal(t, uint16(tls.VersionTLS13), tlsConfig.MinVersion) +} diff --git a/internal/infrastructure/host/infra.go b/internal/infrastructure/host/infra.go index 27c782cdc3f..c586f2c53df 100644 --- a/internal/infrastructure/host/infra.go +++ b/internal/infrastructure/host/infra.go @@ -20,10 +20,6 @@ import ( ) const ( - // TODO: Make these path configurable. - defaultHomeDir = "/tmp/envoy-gateway" - defaultLocalCertPathDir = "/tmp/envoy-gateway/certs/envoy" - // XdsTLSCertFilename is the fully qualified name of the file containing Envoy's // xDS server TLS certificate. XdsTLSCertFilename = "tls.crt" @@ -38,8 +34,9 @@ const ( // Infra manages the creation and deletion of host process // based on Infra IR resources. type Infra struct { - HomeDir string - Logger logging.Logger + // Paths contains the XDG-compliant directory paths. + Paths *Paths + Logger logging.Logger // EnvoyGateway is the configuration used to startup Envoy Gateway. EnvoyGateway *egv1a1.EnvoyGateway @@ -47,7 +44,7 @@ type Infra struct { // proxyContextMap store the context of each running proxy by its name for lifecycle management. proxyContextMap map[string]*proxyContext - // TODO: remove this field once it supports the configurable homeDir + // sdsConfigPath is the path to SDS configuration files. sdsConfigPath string // defaultEnvoyImage is the default Envoy image to use if no Envoy version is set. @@ -60,27 +57,41 @@ type Infra struct { } func NewInfra(runnerCtx context.Context, cfg *config.Server, logger logging.Logger) (*Infra, error) { - // Ensure the home directory exist. - if err := os.MkdirAll(defaultHomeDir, 0o750); err != nil { - return nil, fmt.Errorf("failed to create dir: %w", err) + // Get configuration from provider + var hostCfg *egv1a1.EnvoyGatewayHostInfrastructureProvider + if p := cfg.EnvoyGateway.Provider; p != nil && p.Custom != nil && + p.Custom.Infrastructure != nil && p.Custom.Infrastructure.Host != nil { + hostCfg = p.Custom.Infrastructure.Host + } + + // Get paths using helper + paths, err := GetPaths(hostCfg) + if err != nil { + return nil, fmt.Errorf("failed to determine paths: %w", err) + } + + // Ensure the data directory exists + if err := os.MkdirAll(paths.DataHome, 0o750); err != nil { + return nil, fmt.Errorf("failed to create data directory: %w", err) } - // Check local certificates dir exist. - if _, err := os.Lstat(defaultLocalCertPathDir); err != nil { - return nil, fmt.Errorf("failed to stat dir: %w", err) + // Check local certificates dir exist + certPath := paths.CertDir("envoy") + if _, err := os.Lstat(certPath); err != nil { + return nil, fmt.Errorf("failed to stat cert dir: %w", err) } - // Ensure the sds config exist. - if err := createSdsConfig(defaultLocalCertPathDir); err != nil { + // Ensure the sds config exist + if err := createSdsConfig(certPath); err != nil { return nil, fmt.Errorf("failed to create sds config: %w", err) } infra := &Infra{ - HomeDir: defaultHomeDir, + Paths: paths, Logger: logger, EnvoyGateway: cfg.EnvoyGateway, proxyContextMap: make(map[string]*proxyContext), - sdsConfigPath: defaultLocalCertPathDir, + sdsConfigPath: certPath, defaultEnvoyImage: egv1a1.DefaultEnvoyProxyImage, Stdout: cfg.Stdout, Stderr: cfg.Stderr, diff --git a/internal/infrastructure/host/paths.go b/internal/infrastructure/host/paths.go new file mode 100644 index 00000000000..bd64cc3bb99 --- /dev/null +++ b/internal/infrastructure/host/paths.go @@ -0,0 +1,71 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package host + +import ( + "fmt" + "os" + "os/user" + "path/filepath" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" +) + +// Paths contains all XDG-style directory paths for host infrastructure. +type Paths struct { + ConfigHome string + DataHome string + StateHome string + RuntimeDir string +} + +// GetPaths returns directory paths from config or XDG defaults. +// This follows the same pattern as func-e's ConfigHome(), DataHome(), StateHome(), RuntimeDir() +// but uses configuration fields instead of RunOptions. +func GetPaths(cfg *egv1a1.EnvoyGatewayHostInfrastructureProvider) (*Paths, error) { + u, err := user.Current() + if err != nil { + return nil, fmt.Errorf("failed to get current user: %w", err) + } + + paths := &Paths{} + + // ConfigHome + if cfg != nil && cfg.ConfigHome != nil { + paths.ConfigHome = *cfg.ConfigHome + } else { + paths.ConfigHome = filepath.Join(u.HomeDir, ".config", "envoy-gateway") + } + + // DataHome (Envoy binaries, shared application data) + if cfg != nil && cfg.DataHome != nil { + paths.DataHome = *cfg.DataHome + } else { + paths.DataHome = filepath.Join(u.HomeDir, ".local", "share", "envoy-gateway") + } + + // StateHome (logs, persistent state) + if cfg != nil && cfg.StateHome != nil { + paths.StateHome = *cfg.StateHome + } else { + paths.StateHome = filepath.Join(u.HomeDir, ".local", "state", "envoy-gateway") + } + + // RuntimeDir (ephemeral files) + if cfg != nil && cfg.RuntimeDir != nil { + paths.RuntimeDir = *cfg.RuntimeDir + } else { + // Use UID for multi-user safety, like func-e does + paths.RuntimeDir = filepath.Join(os.TempDir(), fmt.Sprintf("envoy-gateway-%s", u.Uid)) + } + + return paths, nil +} + +// CertDir returns the certificate directory path (under ConfigHome). +func (p *Paths) CertDir(component string) string { + return filepath.Join(p.ConfigHome, "certs", component) +} diff --git a/internal/infrastructure/host/paths_test.go b/internal/infrastructure/host/paths_test.go new file mode 100644 index 00000000000..31fbf7f7e7f --- /dev/null +++ b/internal/infrastructure/host/paths_test.go @@ -0,0 +1,119 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package host + +import ( + "os" + "os/user" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" +) + +func TestGetPaths_Defaults(t *testing.T) { + // Test with nil config - should return XDG defaults + paths, err := GetPaths(nil) + require.NoError(t, err) + require.NotNil(t, paths) + + u, _ := user.Current() + + // Verify XDG defaults to home and temp dirs + require.Equal(t, filepath.Join(u.HomeDir, ".config", "envoy-gateway"), paths.ConfigHome) + require.Equal(t, filepath.Join(u.HomeDir, ".local", "share", "envoy-gateway"), paths.DataHome) + require.Equal(t, filepath.Join(u.HomeDir, ".local", "state", "envoy-gateway"), paths.StateHome) + require.Equal(t, filepath.Join(os.TempDir(), "envoy-gateway-"+u.Uid), paths.RuntimeDir) +} + +func TestGetPaths_CustomConfig(t *testing.T) { + // Test with custom configuration + customConfigHome := "/custom/config" + customDataHome := "/custom/data" + customStateHome := "/custom/state" + customRuntimeDir := "/custom/runtime" + + cfg := &egv1a1.EnvoyGatewayHostInfrastructureProvider{ + ConfigHome: &customConfigHome, + DataHome: &customDataHome, + StateHome: &customStateHome, + RuntimeDir: &customRuntimeDir, + } + + paths, err := GetPaths(cfg) + require.NoError(t, err) + require.NotNil(t, paths) + + // Verify custom paths are used + require.Equal(t, customConfigHome, paths.ConfigHome) + require.Equal(t, customDataHome, paths.DataHome) + require.Equal(t, customStateHome, paths.StateHome) + require.Equal(t, customRuntimeDir, paths.RuntimeDir) +} + +func TestGetPaths_PartialConfig(t *testing.T) { + // Test with only some fields configured + customDataHome := "/custom/data" + + cfg := &egv1a1.EnvoyGatewayHostInfrastructureProvider{ + DataHome: &customDataHome, + } + + paths, err := GetPaths(cfg) + require.NoError(t, err) + require.NotNil(t, paths) + + u, _ := user.Current() + + // Verify custom dataHome but defaults for others + require.Equal(t, filepath.Join(u.HomeDir, ".config", "envoy-gateway"), paths.ConfigHome) + require.Equal(t, customDataHome, paths.DataHome) + require.Equal(t, filepath.Join(u.HomeDir, ".local", "state", "envoy-gateway"), paths.StateHome) + require.Equal(t, filepath.Join(os.TempDir(), "envoy-gateway-"+u.Uid), paths.RuntimeDir) +} + +func TestPaths_CertDir(t *testing.T) { + // Test CertDir helper - certs are stored under ConfigHome + paths := &Paths{ + ConfigHome: "/test/config", + } + + certDir := paths.CertDir("envoy") + require.Equal(t, filepath.Join("/test/config", "certs", "envoy"), certDir) + + certDir = paths.CertDir("envoy-gateway") + require.Equal(t, filepath.Join("/test/config", "certs", "envoy-gateway"), certDir) +} + +func TestGetPaths_RuntimeDirUID(t *testing.T) { + // Verify UID is included in runtime directory + paths, err := GetPaths(nil) + require.NoError(t, err) + + u, _ := user.Current() + expectedPrefix := filepath.Join(os.TempDir(), "envoy-gateway-"+u.Uid) + + require.Equal(t, expectedPrefix, paths.RuntimeDir) +} + +func TestGetPaths_EmptyConfig(t *testing.T) { + // Test with empty (but non-nil) config - should still use defaults + cfg := &egv1a1.EnvoyGatewayHostInfrastructureProvider{} + + paths, err := GetPaths(cfg) + require.NoError(t, err) + require.NotNil(t, paths) + + u, _ := user.Current() + + // Should use defaults when all fields are nil + require.Equal(t, filepath.Join(u.HomeDir, ".config", "envoy-gateway"), paths.ConfigHome) + require.Equal(t, filepath.Join(u.HomeDir, ".local", "share", "envoy-gateway"), paths.DataHome) + require.Equal(t, filepath.Join(u.HomeDir, ".local", "state", "envoy-gateway"), paths.StateHome) + require.Equal(t, filepath.Join(os.TempDir(), "envoy-gateway-"+u.Uid), paths.RuntimeDir) +} diff --git a/internal/infrastructure/host/proxy_infra.go b/internal/infrastructure/host/proxy_infra.go index b730573373e..033478971e8 100644 --- a/internal/infrastructure/host/proxy_infra.go +++ b/internal/infrastructure/host/proxy_infra.go @@ -99,7 +99,10 @@ func (i *Infra) runEnvoy(ctx context.Context, envoyVersion, name string, args [] exit <- struct{}{} }() err := func_e.Run(pCtx, args, - api.HomeDir(i.HomeDir), + api.ConfigHome(i.Paths.ConfigHome), + api.DataHome(i.Paths.DataHome), + api.StateHome(i.Paths.StateHome), + api.RuntimeDir(i.Paths.RuntimeDir), api.Out(i.Stdout), api.EnvoyOut(i.Stdout), api.EnvoyErr(i.Stderr), diff --git a/internal/infrastructure/host/proxy_infra_test.go b/internal/infrastructure/host/proxy_infra_test.go index aa275fac6c3..7ddebc5f882 100644 --- a/internal/infrastructure/host/proxy_infra_test.go +++ b/internal/infrastructure/host/proxy_infra_test.go @@ -6,15 +6,13 @@ package host import ( - "bytes" "io" + "os" "path" "testing" "time" "github.com/stretchr/testify/require" - func_e "github.com/tetratelabs/func-e" - "github.com/tetratelabs/func-e/api" "k8s.io/utils/ptr" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" @@ -46,8 +44,14 @@ func newMockInfra(t *testing.T, cfg *config.Server) *Infra { err = createSdsConfig(proxyDir) require.NoError(t, err) + paths := &Paths{ + ConfigHome: homeDir, + DataHome: homeDir, + StateHome: homeDir, + RuntimeDir: homeDir, + } infra := &Infra{ - HomeDir: homeDir, + Paths: paths, Logger: logging.DefaultLogger(io.Discard, egv1a1.LogLevelInfo), EnvoyGateway: cfg.EnvoyGateway, proxyContextMap: make(map[string]*proxyContext), @@ -63,7 +67,6 @@ func TestInfraCreateProxy(t *testing.T) { require.NoError(t, err) infra := newMockInfra(t, cfg) - // TODO: add more tests once it supports configurable homeDir and runDir. testCases := []struct { name string expect bool @@ -95,37 +98,6 @@ func TestInfraCreateProxy(t *testing.T) { } } -func TestInfra_runEnvoy_stopEnvoy(t *testing.T) { - tmpdir := t.TempDir() - // Ensures that all the required binaries are available. - err := func_e.Run(t.Context(), []string{"--version"}, api.HomeDir(tmpdir)) - require.NoError(t, err) - - stdout := &bytes.Buffer{} - stderr := &bytes.Buffer{} - i := &Infra{ - proxyContextMap: make(map[string]*proxyContext), - HomeDir: tmpdir, - Logger: logging.DefaultLogger(stdout, egv1a1.LogLevelInfo), - Stdout: stdout, - Stderr: stderr, - } - // Ensures that run -> stop will successfully stop the envoy and we can - // run it again without any issues. - for range 5 { - args := []string{ - "--config-yaml", - "admin: {address: {socket_address: {address: '127.0.0.1', port_value: 9901}}}", - } - i.runEnvoy(t.Context(), "", "test", args) - require.Len(t, i.proxyContextMap, 1) - i.stopEnvoy("test") - require.Empty(t, i.proxyContextMap) - // If the cleanup didn't work, the error due to "address already in use" will be tried to be written to the nil logger, - // which will panic. - } -} - func TestExtractSemver(t *testing.T) { tests := []struct { image string @@ -154,46 +126,97 @@ func TestExtractSemver(t *testing.T) { } } -// TestInfra_runEnvoy_OutputRedirection verifies that Envoy output goes to configured writers, not os.Stdout/Stderr. -func TestInfra_runEnvoy_OutputRedirection(t *testing.T) { - tmpdir := t.TempDir() - // Ensures that all the required binaries are available. - err := func_e.Run(t.Context(), []string{"--version"}, api.HomeDir(tmpdir)) - require.NoError(t, err) +// TestInfra_runEnvoy verifies Envoy process lifecycle, output redirection, and XDG directory usage. +func TestInfra_runEnvoy(t *testing.T) { + // Create separate XDG directories + baseDir := t.TempDir() + configHome := path.Join(baseDir, "config") + dataHome := path.Join(baseDir, "data") + stateHome := path.Join(baseDir, "state") + runtimeDir := path.Join(baseDir, "runtime") // Create separate buffers for stdout and stderr buffers := utils.DumpLogsOnFail(t, "stdout", "stderr") stdout := buffers[0] stderr := buffers[1] + paths := &Paths{ + ConfigHome: configHome, + DataHome: dataHome, + StateHome: stateHome, + RuntimeDir: runtimeDir, + } i := &Infra{ proxyContextMap: make(map[string]*proxyContext), - HomeDir: tmpdir, + Paths: paths, Logger: logging.DefaultLogger(stdout, egv1a1.LogLevelInfo), Stdout: stdout, Stderr: stderr, } - // Run envoy with an invalid config to force it to write to stderr and exit quickly + // Run envoy once to let func-e set up all XDG directories args := []string{ "--config-yaml", - "invalid: yaml: syntax", + "admin: {address: {socket_address: {address: '127.0.0.1', port_value: 9901}}}", } - i.runEnvoy(t.Context(), "", "test", args) require.Len(t, i.proxyContextMap, 1) - // Wait a bit for envoy to fail + // Wait for func-e to create all XDG directories require.Eventually(t, func() bool { - return stderr.Len() > 0 || stdout.Len() > 0 - }, 5*time.Second, 100*time.Millisecond, "expected output to be written to buffers") + _, err := os.Stat(path.Join(configHome, "envoy-version")) + return err == nil + }, 5*time.Second, 100*time.Millisecond, "envoy-version file should be created in configHome") i.stopEnvoy("test") require.Empty(t, i.proxyContextMap) - // Verify that output was captured in buffers (either stdout or stderr should have content) - totalOutput := stdout.Len() + stderr.Len() - require.Positive(t, totalOutput, "expected some output to be captured in stdout or stderr buffers") + t.Run("xdg_directory_state", func(t *testing.T) { + // Verify XDG directories were created at configured paths by func-e + // This proves the Paths configuration was properly propagated to func-e API + + // ConfigHome must exist with envoy-version file + require.DirExists(t, configHome, "configHome should exist at configured path") + require.FileExists(t, path.Join(configHome, "envoy-version"), "envoy-version file should exist in configHome") + + // DataHome must exist with envoy-versions subdirectory for downloaded binaries + require.DirExists(t, dataHome, "dataHome should exist at configured path") + require.DirExists(t, path.Join(dataHome, "envoy-versions"), "envoy-versions dir should exist under dataHome") + + // StateHome must exist with envoy-runs subdirectory for per-run logs + require.DirExists(t, stateHome, "stateHome should exist at configured path") + require.DirExists(t, path.Join(stateHome, "envoy-runs"), "envoy-runs dir should exist under stateHome") + + // RuntimeDir must exist - func-e creates runID subdirectories with admin-address.txt + require.DirExists(t, runtimeDir, "runtimeDir should exist at configured path") + + // Verify each XDG directory is separate (not the same path) + require.NotEqual(t, configHome, dataHome, "configHome and dataHome must be different") + require.NotEqual(t, dataHome, stateHome, "dataHome and stateHome must be different") + require.NotEqual(t, stateHome, runtimeDir, "stateHome and runtimeDir must be different") + }) + + t.Run("output_redirection", func(t *testing.T) { + // Verify output was captured in buffers (not os.Stdout/Stderr) + totalOutput := stdout.Len() + stderr.Len() + require.Positive(t, totalOutput, "expected some output to be captured in stdout or stderr buffers") + }) + + t.Run("stop_start_cycle", func(t *testing.T) { + // Ensures that run -> stop cycle works multiple times without issues + for range 5 { + args := []string{ + "--config-yaml", + "admin: {address: {socket_address: {address: '127.0.0.1', port_value: 9901}}}", + } + i.runEnvoy(t.Context(), "", "test", args) + require.Len(t, i.proxyContextMap, 1) + i.stopEnvoy("test") + require.Empty(t, i.proxyContextMap) + // If the cleanup didn't work, the error due to "address already in use" will be + // tried to be written to the nil logger, which will panic. + } + }) } func TestGetEnvoyVersion(t *testing.T) { diff --git a/internal/xds/runner/runner.go b/internal/xds/runner/runner.go index 93430568dd8..e010da2439a 100644 --- a/internal/xds/runner/runner.go +++ b/internal/xds/runner/runner.go @@ -11,6 +11,7 @@ import ( "fmt" "math/rand" "net" + "path/filepath" "strconv" "time" @@ -32,6 +33,7 @@ import ( "github.com/envoyproxy/gateway/internal/crypto" "github.com/envoyproxy/gateway/internal/envoygateway/config" extension "github.com/envoyproxy/gateway/internal/extension/types" + "github.com/envoyproxy/gateway/internal/infrastructure/host" "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/ratelimit" "github.com/envoyproxy/gateway/internal/ir" "github.com/envoyproxy/gateway/internal/message" @@ -56,11 +58,6 @@ const ( // xDS server trusted CA certificate. xdsTLSCaFilepath = "/certs/ca.crt" - // TODO: Make these path configurable. - // Default certificates path for envoy-gateway with Host infrastructure provider. - localTLSCertFilepath = "/tmp/envoy-gateway/certs/envoy-gateway/tls.crt" - localTLSKeyFilepath = "/tmp/envoy-gateway/certs/envoy-gateway/tls.key" - localTLSCaFilepath = "/tmp/envoy-gateway/certs/envoy-gateway/ca.crt" // defaultKubernetesIssuer is the default issuer URL for Kubernetes. // This is used for validating Service Account JWT tokens. defaultKubernetesIssuer = "https://kubernetes.default.svc.cluster.local" @@ -349,9 +346,22 @@ func (r *Runner) loadTLSConfig() (tlsConfig *tls.Config, err error) { keyPath = xdsTLSKeyFilepath caPath = xdsTLSCaFilepath case r.EnvoyGateway.Provider.IsRunningOnHost(): - certPath = localTLSCertFilepath - keyPath = localTLSKeyFilepath - caPath = localTLSCaFilepath + // Get config + var hostCfg *egv1a1.EnvoyGatewayHostInfrastructureProvider + if p := r.EnvoyGateway.Provider; p != nil && p.Custom != nil && + p.Custom.Infrastructure != nil && p.Custom.Infrastructure.Host != nil { + hostCfg = p.Custom.Infrastructure.Host + } + + paths, err := host.GetPaths(hostCfg) + if err != nil { + return nil, fmt.Errorf("failed to determine paths: %w", err) + } + + certDir := paths.CertDir("envoy-gateway") + certPath = filepath.Join(certDir, "tls.crt") + keyPath = filepath.Join(certDir, "tls.key") + caPath = filepath.Join(certDir, "ca.crt") default: return nil, fmt.Errorf("no valid tls certificates") } diff --git a/internal/xds/runner/runner_test.go b/internal/xds/runner/runner_test.go index 687641c0775..2100919faed 100644 --- a/internal/xds/runner/runner_test.go +++ b/internal/xds/runner/runner_test.go @@ -543,3 +543,64 @@ func TestGetRandomMaxConnectionAge(t *testing.T) { // Verify we got different values (randomness check) assert.Len(t, counts, len(maxConnectionAgeValues), "Should see all possible values") } + +func TestLoadTLSConfig_HostMode(t *testing.T) { + // Create temporary directory structure for certs using t.TempDir() + configHome := t.TempDir() + certsDir := filepath.Join(configHome, "certs", "envoy-gateway") + err := os.MkdirAll(certsDir, 0o750) + require.NoError(t, err) + + // Create test certificates + trustedCACert := certyaml.Certificate{ + Subject: "cn=test-ca", + } + serverCert := certyaml.Certificate{ + Subject: "cn=test-server", + SubjectAltNames: []string{"DNS:localhost"}, + Issuer: &trustedCACert, + } + + caFile := filepath.Join(certsDir, "ca.crt") + certFile := filepath.Join(certsDir, "tls.crt") + keyFile := filepath.Join(certsDir, "tls.key") + + err = trustedCACert.WritePEM(caFile, keyFile) + require.NoError(t, err) + err = serverCert.WritePEM(certFile, keyFile) + require.NoError(t, err) + + // Configure host mode with custom configHome (certs are stored in configHome) + // MUST be set BEFORE creating Runner since Config{Server: *cfg} makes a copy + cfg, err := config.New(os.Stdout, os.Stderr) + require.NoError(t, err) + + cfg.EnvoyGateway.Provider = &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: &egv1a1.EnvoyGatewayCustomProvider{ + Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{ + Type: egv1a1.InfrastructureProviderTypeHost, + Host: &egv1a1.EnvoyGatewayHostInfrastructureProvider{ + ConfigHome: &configHome, + }, + }, + }, + } + + r := &Runner{ + Config: Config{ + Server: *cfg, + }, + } + + // Test loadTLSConfig with host mode + tlsConfig, err := r.loadTLSConfig() + require.NoError(t, err) + require.NotNil(t, tlsConfig) + + // Verify TLS config properties + // crypto.LoadTLSConfig uses GetConfigForClient callback to load certs on demand + require.NotNil(t, tlsConfig.GetConfigForClient) + require.Equal(t, tls.RequireAndVerifyClientCert, tlsConfig.ClientAuth) + require.Equal(t, uint16(tls.VersionTLS13), tlsConfig.MinVersion) +} diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 16fdbd57b52..4ab53f3fb5c 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -1353,6 +1353,12 @@ EnvoyGatewayHostInfrastructureProvider defines configuration for the Host Infras _Appears in:_ - [EnvoyGatewayInfrastructureProvider](#envoygatewayinfrastructureprovider) +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `configHome` | _string_ | false | | ConfigHome is the directory for configuration files.
Defaults to ~/.config/envoy-gateway | +| `dataHome` | _string_ | false | | DataHome is the directory for persistent data (Envoy binaries).
Defaults to ~/.local/share/envoy-gateway | +| `stateHome` | _string_ | false | | StateHome is the directory for persistent state (logs).
Defaults to ~/.local/state/envoy-gateway | +| `runtimeDir` | _string_ | false | | RuntimeDir is the directory for ephemeral runtime files.
Defaults to /tmp/envoy-gateway-$\{UID\} | #### EnvoyGatewayInfrastructureProvider diff --git a/site/content/en/latest/tasks/operations/standalone-deployment-mode.md b/site/content/en/latest/tasks/operations/standalone-deployment-mode.md index 1adbe723ce8..7615eed03d3 100644 --- a/site/content/en/latest/tasks/operations/standalone-deployment-mode.md +++ b/site/content/en/latest/tasks/operations/standalone-deployment-mode.md @@ -49,6 +49,13 @@ ensure the Envoy Gateway works properly. envoy-gateway certgen --local ``` +By default, certificates are stored in `~/.config/envoy-gateway/certs/`. You can customize this +location using the `--config-home` flag (certs will be in a `certs/` subdirectory): + +```shell +envoy-gateway certgen --local --config-home /custom/config/path +``` + ### Start Envoy Gateway Start Envoy Gateway by the following command: @@ -73,7 +80,15 @@ provider: paths: ["/tmp/envoy-gateway-test"] infrastructure: type: Host - host: {} + host: + # Optional: Configure XDG-compliant directory paths + # If not specified, uses XDG Base Directory defaults: + # - configHome: ~/.config/envoy-gateway + # - dataHome: ~/.local/share/envoy-gateway + # - stateHome: ~/.local/state/envoy-gateway + # - runtimeDir: /tmp/envoy-gateway-${UID} + # Example custom configuration: + # dataHome: /custom/data/path logging: level: default: info @@ -156,7 +171,7 @@ All runners in Envoy Gateway are using TLS connection, so create these TLS certi ensure the Envoy Gateway works properly. ```shell -docker run --rm --volume /tmp/envoy-gateway-test:/tmp/envoy-gateway envoyproxy/gateway:{{< helm-version >}} certgen --local +docker run --rm --volume /tmp/envoy-gateway-test:/tmp/envoy-gateway envoyproxy/gateway:{{< helm-version >}} certgen --local --data-home /tmp/envoy-gateway ``` ### Start Envoy Gateway @@ -177,7 +192,12 @@ provider: paths: ["/tmp/envoy-gateway/config"] infrastructure: type: Host - host: {} + host: + # Configure configHome and dataHome to use the mounted volume + # configHome: for certificates + # dataHome: for Envoy binaries + configHome: /tmp/envoy-gateway + dataHome: /tmp/envoy-gateway logging: level: default: info