|
| 1 | +package ha |
| 2 | + |
| 3 | +import ( |
| 4 | + "crypto/tls" |
| 5 | + "crypto/x509" |
| 6 | + "database/sql" |
| 7 | + "errors" |
| 8 | + "fmt" |
| 9 | + "net/http" |
| 10 | + "net/url" |
| 11 | + "os" |
| 12 | + "strings" |
| 13 | + "time" |
| 14 | + |
| 15 | + clientv3 "go.etcd.io/etcd/client/v3" |
| 16 | + |
| 17 | + "github.com/block/proto-fleet/server/internal/infrastructure/db" |
| 18 | + "github.com/block/proto-fleet/server/internal/runtimejobs" |
| 19 | +) |
| 20 | + |
| 21 | +// Config keeps HA opt-in so existing single-instance deployments remain |
| 22 | +// standalone and do not need etcd or Patroni credentials. |
| 23 | +type Config struct { |
| 24 | + Enabled bool `help:"Enable active/passive Fleet runtime ownership." default:"false" env:"ENABLED"` |
| 25 | + ClusterPath string `help:"Patroni DCS cluster path." default:"/service/proto-fleet" env:"CLUSTER_PATH"` |
| 26 | + EtcdEndpoints []string `help:"Comma-separated HTTPS etcd endpoints." env:"ETCD_ENDPOINTS" sep:","` |
| 27 | + EtcdUsername string `help:"Read-only etcd observer username." default:"fleet-observer" env:"ETCD_USERNAME"` |
| 28 | + EtcdPasswordFile string `help:"Path to the etcd observer password file." default:"/etc/proto-fleet/ha/fleet-etcd-password" env:"ETCD_PASSWORD_FILE" type:"path"` |
| 29 | + ServiceCAFile string `help:"Path to the CA that signs etcd and Patroni service certificates." default:"/etc/proto-fleet/ha/service-ca.crt" env:"SERVICE_CA_FILE" type:"path"` |
| 30 | + LeaseDuration time.Duration `help:"Fleet active lease duration." default:"10s" env:"LEASE_DURATION"` |
| 31 | + RenewInterval time.Duration `help:"Fleet active lease renewal interval." default:"3s" env:"RENEW_INTERVAL"` |
| 32 | + RetryInterval time.Duration `help:"Passive ownership retry interval." default:"1s" env:"RETRY_INTERVAL"` |
| 33 | + DialTimeout time.Duration `help:"etcd connection timeout." default:"5s" env:"DIAL_TIMEOUT"` |
| 34 | +} |
| 35 | + |
| 36 | +// NewConfiguredRuntime creates a standalone runtime unless HA is explicitly |
| 37 | +// enabled. In HA mode, cleanup closes the etcd client. |
| 38 | +func NewConfiguredRuntime( |
| 39 | + config Config, |
| 40 | + conn *sql.DB, |
| 41 | + group *runtimejobs.Group, |
| 42 | + healthy func() bool, |
| 43 | +) (*Runtime, func() error, error) { |
| 44 | + if !config.Enabled { |
| 45 | + runtime, err := NewStandaloneRuntime(group, healthy) |
| 46 | + return runtime, func() error { return nil }, err |
| 47 | + } |
| 48 | + if err := config.Validate(); err != nil { |
| 49 | + return nil, nil, err |
| 50 | + } |
| 51 | + |
| 52 | + password, err := readRuntimeSecret(config.EtcdPasswordFile) |
| 53 | + if err != nil { |
| 54 | + return nil, nil, fmt.Errorf("read HA etcd password: %w", err) |
| 55 | + } |
| 56 | + tlsConfig, err := loadServiceTLS(config.ServiceCAFile) |
| 57 | + if err != nil { |
| 58 | + return nil, nil, err |
| 59 | + } |
| 60 | + etcd, err := NewEtcdClient(clientv3.Config{ |
| 61 | + Endpoints: config.EtcdEndpoints, |
| 62 | + Username: config.EtcdUsername, |
| 63 | + Password: password, |
| 64 | + TLS: tlsConfig.Clone(), |
| 65 | + DialTimeout: config.DialTimeout, |
| 66 | + }) |
| 67 | + if err != nil { |
| 68 | + return nil, nil, fmt.Errorf("create HA etcd client: %w", err) |
| 69 | + } |
| 70 | + |
| 71 | + transport, ok := http.DefaultTransport.(*http.Transport) |
| 72 | + if !ok { |
| 73 | + _ = etcd.Close() |
| 74 | + return nil, nil, errors.New("default HTTP transport is not configurable for HA") |
| 75 | + } |
| 76 | + patroniTransport := transport.Clone() |
| 77 | + patroniTransport.TLSClientConfig = tlsConfig.Clone() |
| 78 | + cleanup := func() error { |
| 79 | + patroniTransport.CloseIdleConnections() |
| 80 | + return etcd.Close() |
| 81 | + } |
| 82 | + patroni := NewPatroniHTTPClient(&http.Client{ |
| 83 | + Transport: patroniTransport, |
| 84 | + Timeout: defaultHAHTTPTimeout, |
| 85 | + }) |
| 86 | + queries := db.NewFailoverResettingQuerier(db.NewRetryDB(conn)) |
| 87 | + observer, err := NewObserver(config.ClusterPath, etcd, queries, patroni) |
| 88 | + if err != nil { |
| 89 | + _ = cleanup() |
| 90 | + return nil, nil, err |
| 91 | + } |
| 92 | + coordinator, err := NewCoordinator(observer, NewLeaseStore(queries), CoordinatorConfig{ |
| 93 | + LeaseDuration: config.LeaseDuration, |
| 94 | + RenewInterval: config.RenewInterval, |
| 95 | + RetryInterval: config.RetryInterval, |
| 96 | + }) |
| 97 | + if err != nil { |
| 98 | + _ = cleanup() |
| 99 | + return nil, nil, err |
| 100 | + } |
| 101 | + runtime, err := NewRuntime(coordinator, group, healthy) |
| 102 | + if err != nil { |
| 103 | + _ = cleanup() |
| 104 | + return nil, nil, err |
| 105 | + } |
| 106 | + return runtime, cleanup, nil |
| 107 | +} |
| 108 | + |
| 109 | +func (config Config) Validate() error { |
| 110 | + if !config.Enabled { |
| 111 | + return nil |
| 112 | + } |
| 113 | + if config.ClusterPath == "" || len(config.EtcdEndpoints) == 0 || config.EtcdUsername == "" || |
| 114 | + config.EtcdPasswordFile == "" || config.ServiceCAFile == "" || config.DialTimeout <= 0 { |
| 115 | + return errors.New("enabled HA requires cluster path, etcd endpoints and credentials, service CA, and a positive dial timeout") |
| 116 | + } |
| 117 | + for _, endpoint := range config.EtcdEndpoints { |
| 118 | + parsed, err := url.Parse(endpoint) |
| 119 | + if err != nil || parsed.Scheme != "https" || parsed.Host == "" { |
| 120 | + return fmt.Errorf("HA etcd endpoint must be an HTTPS URL: %q", endpoint) |
| 121 | + } |
| 122 | + } |
| 123 | + return nil |
| 124 | +} |
| 125 | + |
| 126 | +func readRuntimeSecret(path string) (string, error) { |
| 127 | + contents, err := os.ReadFile(path) |
| 128 | + if err != nil { |
| 129 | + return "", fmt.Errorf("read secret file: %w", err) |
| 130 | + } |
| 131 | + secret := strings.TrimSpace(string(contents)) |
| 132 | + if secret == "" { |
| 133 | + return "", errors.New("secret file is empty") |
| 134 | + } |
| 135 | + return secret, nil |
| 136 | +} |
| 137 | + |
| 138 | +func loadServiceTLS(path string) (*tls.Config, error) { |
| 139 | + contents, err := os.ReadFile(path) |
| 140 | + if err != nil { |
| 141 | + return nil, fmt.Errorf("read HA service CA: %w", err) |
| 142 | + } |
| 143 | + roots := x509.NewCertPool() |
| 144 | + if !roots.AppendCertsFromPEM(contents) { |
| 145 | + return nil, errors.New("HA service CA contains no certificates") |
| 146 | + } |
| 147 | + return &tls.Config{ |
| 148 | + MinVersion: tls.VersionTLS13, |
| 149 | + RootCAs: roots, |
| 150 | + }, nil |
| 151 | +} |
0 commit comments