feat: make host mode directory paths configurable (#7225)

This adds configuration fields to EnvoyGatewayHostInfrastructureProvider to allow users to specify custom paths for configuration, data, state, and runtime directories, following XDG Base Directory Specification conventions while maintaining backward compatibility.

The implementation introduces four configurable directory paths:

- configHome: certificates and configuration files (default: ~/.config/envoy-gateway)
- dataHome: Envoy binaries, sharable across configs (default: ~/.local/share/envoy-gateway)
- stateHome: logs and persistent state (default: ~/.local/state/envoy-gateway)
- runtimeDir: ephemeral runtime files (default: /tmp/envoy-gateway-${UID})

Certificates are stored under configHome to ensure isolation between different configurations when running multiple EnvoyGateway instances in parallel, preventing certificate conflicts.

These paths are propagated to func-e which creates subdirectories as needed:
- dataHome/envoy-versions/ for Envoy binaries
- stateHome/envoy-runs/{runID}/ for per-run logs
- runtimeDir/{runID}/ for per-run runtime files

The changes include path resolution helpers, updated runners to use configurable paths, comprehensive test coverage, and updated documentation.

Signed-off-by: Adrian Cole <[email protected]>

Author: codefromthecrypt

api/v1alpha1/envoygateway_types.go

@@ -430,7 +430,25 @@ type EnvoyGatewayInfrastructureProvider struct {
 
 // EnvoyGatewayHostInfrastructureProvider defines configuration for the Host Infrastructure provider.
 type EnvoyGatewayHostInfrastructureProvider struct {
-	// TODO: Add config as use cases are better understood.
+	// ConfigHome is the directory for configuration files.
+	// Defaults to ~/.config/envoy-gateway
+	// +optional
+	ConfigHome *string `json:"configHome,omitempty"`
+
+	// DataHome is the directory for persistent data (Envoy binaries).
+	// Defaults to ~/.local/share/envoy-gateway
+	// +optional
+	DataHome *string `json:"dataHome,omitempty"`
+
+	// StateHome is the directory for persistent state (logs).
+	// Defaults to ~/.local/state/envoy-gateway
+	// +optional
+	StateHome *string `json:"stateHome,omitempty"`
+
+	// RuntimeDir is the directory for ephemeral runtime files.
+	// Defaults to /tmp/envoy-gateway-${UID}
+	// +optional
+	RuntimeDir *string `json:"runtimeDir,omitempty"`
 }
 
 // RateLimit defines the configuration associated with the Rate Limit Service

api/v1alpha1/zz_generated.deepcopy.go

@@ -11,6 +11,15 @@ provider:
         paths: ["/tmp/envoy-gateway-test"]
     infrastructure:
       type: Host
+      # Optional: Configure XDG-compliant directory paths under host:
+      # If not specified, uses XDG Base Directory defaults:
+      # - configHome: ~/.config/envoy-gateway
+      # - dataHome: ~/.local/share/envoy-gateway
+      # - stateHome: ~/.local/state/envoy-gateway
+      # - runtimeDir: /tmp/envoy-gateway-${UID}
+      # Example custom configuration:
+      # host:
+      #   dataHome: /custom/data/path
       host: {}
 logging:
   level:

examples/standalone/envoy-gateway.yaml

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"io"
 	"path"
+	"path/filepath"
 
 	"github.com/spf13/cobra"
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
@@ -20,9 +21,11 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	clicfg "sigs.k8s.io/controller-runtime/pkg/client/config"
 
+	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
 	"github.com/envoyproxy/gateway/internal/crypto"
 	"github.com/envoyproxy/gateway/internal/envoygateway"
 	"github.com/envoyproxy/gateway/internal/envoygateway/config"
+	"github.com/envoyproxy/gateway/internal/infrastructure/host"
 	"github.com/envoyproxy/gateway/internal/provider/kubernetes"
 	"github.com/envoyproxy/gateway/internal/utils/file"
 )
@@ -32,26 +35,29 @@ var overwriteControlPlaneCerts bool
 
 var disableTopologyInjector bool
 
-// TODO: make this path configurable or use server config directly.
 const (
-	defaultLocalCertPath      = "/tmp/envoy-gateway/certs"
 	topologyWebhookNamePrefix = "envoy-gateway-topology-injector"
 )
 
 // GetCertGenCommand returns the certGen cobra command to be executed.
 func GetCertGenCommand() *cobra.Command {
-	var local bool
+	var (
+		local    bool
+		dataHome string
+	)
 
 	cmd := &cobra.Command{
 		Use:   "certgen",
 		Short: "Generate Control Plane Certificates",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return certGen(cmd.Context(), cmd.OutOrStdout(), local)
+			return certGen(cmd.Context(), cmd.OutOrStdout(), local, dataHome)
 		},
 	}
 
 	cmd.PersistentFlags().BoolVarP(&local, "local", "l", false,
 		"Generate all the certificates locally.")
+	cmd.PersistentFlags().StringVar(&dataHome, "data-home", "",
+		"Directory for certificates (defaults to ~/.local/share/envoy-gateway)")
 	cmd.PersistentFlags().BoolVarP(&overwriteControlPlaneCerts, "overwrite", "o", false,
 		"Updates the secrets containing the control plane certs.")
 	cmd.PersistentFlags().BoolVar(&disableTopologyInjector, "disable-topology-injector", false,
@@ -60,7 +66,7 @@ func GetCertGenCommand() *cobra.Command {
 }
 
 // certGen generates control plane certificates.
-func certGen(ctx context.Context, logOut io.Writer, local bool) error {
+func certGen(ctx context.Context, logOut io.Writer, local bool, dataHome string) error {
 	cfg, err := config.New(logOut, io.Discard)
 	if err != nil {
 		return err
@@ -86,8 +92,21 @@ func certGen(ctx context.Context, logOut io.Writer, local bool) error {
 			return fmt.Errorf("failed to patch webhook: %w", err)
 		}
 	} else {
-		log.Info("generated certificates", "path", defaultLocalCertPath)
-		if err = outputCertsForLocal(defaultLocalCertPath, certs); err != nil {
+		// Use provided dataHome or default
+		hostCfg := &egv1a1.EnvoyGatewayHostInfrastructureProvider{}
+		if dataHome != "" {
+			hostCfg.DataHome = &dataHome
+		}
+
+		paths, err := host.GetPaths(hostCfg)
+		if err != nil {
+			return fmt.Errorf("failed to determine paths: %w", err)
+		}
+
+		certPath := filepath.Join(paths.DataHome, "certs")
+		log.Info("generated certificates", "path", certPath)
+
+		if err = outputCertsForLocal(certPath, certs); err != nil {
 			return fmt.Errorf("failed to output certificates locally: %w", err)
 		}
 	}

internal/cmd/certgen.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"path/filepath"
 
 	"github.com/docker/docker/pkg/fileutils"
 	"github.com/telepresenceio/watchable"
@@ -29,6 +30,7 @@ import (
 	extension "github.com/envoyproxy/gateway/internal/extension/types"
 	"github.com/envoyproxy/gateway/internal/gatewayapi"
 	"github.com/envoyproxy/gateway/internal/gatewayapi/resource"
+	"github.com/envoyproxy/gateway/internal/infrastructure/host"
 	"github.com/envoyproxy/gateway/internal/message"
 	"github.com/envoyproxy/gateway/internal/utils"
 	"github.com/envoyproxy/gateway/internal/wasm"
@@ -40,15 +42,8 @@ const (
 	serveTLSKeyFilepath  = "/certs/tls.key"
 	serveTLSCaFilepath   = "/certs/ca.crt"
 
-	// TODO: Make these path configurable.
-	// Default certificates path for envoy-gateway with Host infrastructure provider.
-	localTLSCertFilepath = "/tmp/envoy-gateway/certs/envoy-gateway/tls.crt"
-	localTLSKeyFilepath  = "/tmp/envoy-gateway/certs/envoy-gateway/tls.key"
-	localTLSCaFilepath   = "/tmp/envoy-gateway/certs/envoy-gateway/ca.crt"
-
 	hmacSecretName = "envoy-oidc-hmac" // nolint: gosec
 	hmacSecretKey  = "hmac-secret"
-	hmacSecretPath = "/tmp/envoy-gateway/certs/envoy-oidc-hmac/hmac-secret" // nolint: gosec
 )
 
 type Config struct {
@@ -375,12 +370,31 @@ func (r *Runner) loadTLSConfig(ctx context.Context) (tlsConfig *tls.Config, salt
 		}
 
 	case r.EnvoyGateway.Provider.IsRunningOnHost():
-		salt, err = os.ReadFile(hmacSecretPath)
+		// Get config
+		var hostCfg *egv1a1.EnvoyGatewayHostInfrastructureProvider
+		if p := r.EnvoyGateway.Provider; p != nil && p.Custom != nil &&
+			p.Custom.Infrastructure != nil && p.Custom.Infrastructure.Host != nil {
+			hostCfg = p.Custom.Infrastructure.Host
+		}
+
+		paths, err := host.GetPaths(hostCfg)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to determine paths: %w", err)
+		}
+
+		// Read HMAC secret
+		hmacPath := filepath.Join(paths.CertDir("envoy-oidc-hmac"), "hmac-secret")
+		salt, err = os.ReadFile(hmacPath)
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to get hmac secret: %w", err)
 		}
 
-		tlsConfig, err = crypto.LoadTLSConfig(localTLSCertFilepath, localTLSKeyFilepath, localTLSCaFilepath)
+		certDir := paths.CertDir("envoy-gateway")
+		certPath := filepath.Join(certDir, "tls.crt")
+		keyPath := filepath.Join(certDir, "tls.key")
+		caPath := filepath.Join(certDir, "ca.crt")
+
+		tlsConfig, err = crypto.LoadTLSConfig(certPath, keyPath, caPath)
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to create tls config: %w", err)
 		}

internal/gatewayapi/runner/runner.go

@@ -7,7 +7,9 @@ package runner
 
 import (
 	"context"
+	"crypto/tls"
 	"os"
+	"path/filepath"
 	"reflect"
 	"testing"
 	"time"
@@ -18,6 +20,7 @@ import (
 	gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 
 	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
+	"github.com/envoyproxy/gateway/internal/crypto"
 	"github.com/envoyproxy/gateway/internal/envoygateway/config"
 	"github.com/envoyproxy/gateway/internal/extension/registry"
 	"github.com/envoyproxy/gateway/internal/ir"
@@ -235,3 +238,66 @@ func TestDeleteAllKeys(t *testing.T) {
 	require.Empty(t, r.keyCache.SecurityPolicyStatus)
 	require.Empty(t, r.keyCache.EnvoyExtensionPolicyStatus)
 }
+
+func TestLoadTLSConfig_HostMode(t *testing.T) {
+	// Create temporary directory structure for certs using t.TempDir()
+	configHome := t.TempDir()
+	certsDir := filepath.Join(configHome, "certs", "envoy-gateway")
+	hmacDir := filepath.Join(configHome, "certs", "envoy-oidc-hmac")
+	require.NoError(t, os.MkdirAll(certsDir, 0o750))
+	require.NoError(t, os.MkdirAll(hmacDir, 0o750))
+
+	// Create test certificates using internal/crypto package
+	cfg, err := config.New(os.Stdout, os.Stderr)
+	require.NoError(t, err)
+
+	// Generate certificates with default provider (crypto.GenerateCerts only supports Kubernetes)
+	certs, err := crypto.GenerateCerts(cfg)
+	require.NoError(t, err)
+
+	// Write certificates to temp directory
+	caFile := filepath.Join(certsDir, "ca.crt")
+	certFile := filepath.Join(certsDir, "tls.crt")
+	keyFile := filepath.Join(certsDir, "tls.key")
+	hmacFile := filepath.Join(hmacDir, "hmac-secret")
+
+	require.NoError(t, os.WriteFile(caFile, certs.CACertificate, 0o600))
+	require.NoError(t, os.WriteFile(certFile, certs.EnvoyGatewayCertificate, 0o600))
+	require.NoError(t, os.WriteFile(keyFile, certs.EnvoyGatewayPrivateKey, 0o600))
+	require.NoError(t, os.WriteFile(hmacFile, certs.OIDCHMACSecret, 0o600))
+
+	// Configure host mode with custom configHome (certs are stored in configHome)
+	// MUST be set BEFORE creating Runner since Config{Server: *cfg} makes a copy
+	cfg.EnvoyGateway.Provider = &egv1a1.EnvoyGatewayProvider{
+		Type: egv1a1.ProviderTypeCustom,
+		Custom: &egv1a1.EnvoyGatewayCustomProvider{
+			Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{
+				Type: egv1a1.InfrastructureProviderTypeHost,
+				Host: &egv1a1.EnvoyGatewayHostInfrastructureProvider{
+					ConfigHome: &configHome,
+				},
+			},
+		},
+	}
+
+	r := &Runner{
+		Config: Config{
+			Server: *cfg,
+		},
+	}
+
+	// Test loadTLSConfig with host mode
+	tlsConfig, salt, err := r.loadTLSConfig(context.Background())
+	require.NoError(t, err)
+	require.NotNil(t, tlsConfig)
+	require.NotNil(t, salt)
+
+	// Verify HMAC secret was loaded
+	require.Equal(t, certs.OIDCHMACSecret, salt)
+
+	// Verify TLS config properties
+	// crypto.LoadTLSConfig uses GetConfigForClient callback to load certs on demand
+	require.NotNil(t, tlsConfig.GetConfigForClient)
+	require.Equal(t, tls.RequireAndVerifyClientCert, tlsConfig.ClientAuth)
+	require.Equal(t, uint16(tls.VersionTLS13), tlsConfig.MinVersion)
+}

internal/gatewayapi/runner/runner_test.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"math"
 	"net"
+	"path/filepath"
 	"strconv"
 
 	discoveryv3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
@@ -25,6 +26,7 @@ import (
 	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
 	"github.com/envoyproxy/gateway/internal/crypto"
 	"github.com/envoyproxy/gateway/internal/envoygateway/config"
+	"github.com/envoyproxy/gateway/internal/infrastructure/host"
 	"github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/ratelimit"
 	"github.com/envoyproxy/gateway/internal/ir"
 	"github.com/envoyproxy/gateway/internal/message"
@@ -43,12 +45,6 @@ const (
 	rateLimitTLSKeyFilepath = "/certs/tls.key"
 	// rateLimitTLSCACertFilepath is the ratelimit ca cert file.
 	rateLimitTLSCACertFilepath = "/certs/ca.crt"
-
-	// TODO: Make these path configurable.
-	// Default certificates path for envoy-gateway with Host infrastructure provider.
-	localTLSCertFilepath = "/tmp/envoy-gateway/certs/envoy-gateway/tls.crt"
-	localTLSKeyFilepath  = "/tmp/envoy-gateway/certs/envoy-gateway/tls.key"
-	localTLSCaFilepath   = "/tmp/envoy-gateway/certs/envoy-gateway/ca.crt"
 )
 
 type Config struct {
@@ -216,21 +212,37 @@ func (r *Runner) addNewSnapshot(ctx context.Context, resource types.XdsResources
 }
 
 func (r *Runner) loadTLSConfig() (tlsConfig *tls.Config, err error) {
+	var certPath, keyPath, caPath string
+
 	switch {
 	case r.EnvoyGateway.Provider.IsRunningOnKubernetes():
-		tlsConfig, err = crypto.LoadTLSConfig(rateLimitTLSCertFilepath, rateLimitTLSKeyFilepath, rateLimitTLSCACertFilepath)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create tls config: %w", err)
+		certPath = rateLimitTLSCertFilepath
+		keyPath = rateLimitTLSKeyFilepath
+		caPath = rateLimitTLSCACertFilepath
+	case r.EnvoyGateway.Provider.IsRunningOnHost():
+		// Get configuration from provider
+		var hostCfg *egv1a1.EnvoyGatewayHostInfrastructureProvider
+		if p := r.EnvoyGateway.Provider; p != nil && p.Custom != nil &&
+			p.Custom.Infrastructure != nil && p.Custom.Infrastructure.Host != nil {
+			hostCfg = p.Custom.Infrastructure.Host
 		}
 
-	case r.EnvoyGateway.Provider.IsRunningOnHost():
-		tlsConfig, err = crypto.LoadTLSConfig(localTLSCertFilepath, localTLSKeyFilepath, localTLSCaFilepath)
+		paths, err := host.GetPaths(hostCfg)
 		if err != nil {
-			return nil, fmt.Errorf("failed to create tls config: %w", err)
+			return nil, fmt.Errorf("failed to determine paths: %w", err)
 		}
 
+		certDir := paths.CertDir("envoy-gateway")
+		certPath = filepath.Join(certDir, "tls.crt")
+		keyPath = filepath.Join(certDir, "tls.key")
+		caPath = filepath.Join(certDir, "ca.crt")
 	default:
 		return nil, fmt.Errorf("no valid tls certificates")
 	}
+
+	tlsConfig, err = crypto.LoadTLSConfig(certPath, keyPath, caPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create tls config: %w", err)
+	}
 	return
 }