diff --git a/README.md b/README.md index b095941..3b6a4cc 100644 --- a/README.md +++ b/README.md @@ -324,9 +324,30 @@ $ lndinit -v init-wallet \ --init-rpc.server=localhost:10009 \ --init-rpc.tls-cert-path=$HOME/.lnd/tls.cert \ --init-rpc.watch-only \ - --init-rpc.accounts-file=/tmp/accounts.json + --init-rpc.accounts-file=/tmp/accounts.json \ + --init-rpc.watch-only-birthday=2024-07-01 ``` +**NOTE**: The accounts JSON file only contains the account xpubs, not the +birthday of the master key they were derived from. If +`--init-rpc.watch-only-birthday` isn't specified, `lnd` has to assume the aezeed +epoch (`2017-08-24`) as the wallet's birthday and rescans the chain from there, +which on mainnet means walking hundreds of thousands of blocks and can take +multiple hours. Pointing the flag at the actual birthday of the seed on the +remote signer avoids that. The value can be given as a Unix timestamp in +seconds, as an RFC3339 timestamp (`2024-07-01T00:00:00Z`) or as a plain +`YYYY-MM-DD` date, which is interpreted as midnight UTC. When in doubt, pick a +date slightly _before_ the seed was created, since a birthday that is too late +makes `lnd` skip the blocks the wallet's funds are in. + +A wallet that already has history needs one more flag, +`--init-rpc.recovery-window`, which sets the address look-ahead `lnd` uses to +scan for keys that have been used. It defaults to zero, meaning no addresses are +recovered, which is the right answer for a brand new node. When re-creating a +wallet that has been in use, set it the way `lncli createwatchonly` does (it +prompts with a default of `2500`), otherwise the rescan starts at the right +height but never derives far enough to find the wallet's addresses. + #### 5. Store the wallet password in a file Because we now only have the wallet password as a value in a k8s secret, we need @@ -389,7 +410,9 @@ exist). This can make it hard to follow exactly what is happening when debugging the initialization. To assist with debugging, the following two flags can be used: -- `--verbose (-v)`: Log debug information to `stderr`. +- `--verbose (-v)`: Log progress information to `stderr`, equivalent to + `--debuglevel=info`. Use `--debuglevel=debug` or `--debuglevel=trace` for more + detail. - `--error-on-existing (-e)`: Exit with a non-zero return code (128) if the result of an operation already exists. See example below. diff --git a/cmd_init_wallet.go b/cmd_init_wallet.go index 8da9f54..5a53e5a 100644 --- a/cmd_init_wallet.go +++ b/cmd_init_wallet.go @@ -6,6 +6,7 @@ import ( "math" "os" "path/filepath" + "strconv" "strings" "time" @@ -31,6 +32,24 @@ const ( typeFile = "file" typeRpc = "rpc" + + // birthdayDateFormat is the calendar date notation a watch-only + // wallet's birthday can be expressed in, next to a raw Unix timestamp + // and a full RFC3339 timestamp. + birthdayDateFormat = "2006-01-02" + + // maxBirthdayDrift is how far into the future a watch-only wallet's + // birthday is allowed to be. We tolerate some slack to account for + // clock skew between the machine that produced the timestamp and the + // one running lndinit. Anything beyond that is rejected, since a + // birthday far enough in the future makes lnd start scanning at the + // chain tip, where it would miss any funds the wallet already holds. + // + // The window is a day rather than something tighter because btcwallet + // rewinds the birthday by 48 hours before it looks for the birthday + // block, so a value inside this window still starts the scan at least a + // day before the wallet could have been created. + maxBirthdayDrift = 24 * time.Hour ) var ( @@ -58,10 +77,12 @@ type initTypeFile struct { } type initTypeRpc struct { - Server string `long:"server" description:"The host:port of the RPC server to connect to"` - TLSCertPath string `long:"tls-cert-path" description:"The full path to the RPC server's TLS certificate"` - WatchOnly bool `long:"watch-only" description:"Don't require a seed to be set, initialize the wallet as watch-only; requires the accounts-file flag to be specified"` - AccountsFile string `long:"accounts-file" description:"The JSON file that contains all accounts xpubs for initializing a watch-only wallet"` + Server string `long:"server" description:"The host:port of the RPC server to connect to"` + TLSCertPath string `long:"tls-cert-path" description:"The full path to the RPC server's TLS certificate"` + WatchOnly bool `long:"watch-only" description:"Don't require a seed to be set, initialize the wallet as watch-only; requires the accounts-file flag to be specified"` + AccountsFile string `long:"accounts-file" description:"The JSON file that contains all accounts xpubs for initializing a watch-only wallet"` + WatchOnlyBirthday string `long:"watch-only-birthday" description:"The birthday of the watch-only wallet's master key, either as a Unix timestamp in seconds, an RFC3339 timestamp or a YYYY-MM-DD date; if unset, lnd assumes the aezeed epoch (2017-08-24) and rescans the chain from there, which can take hours; requires the watch-only flag to be specified"` + RecoveryWindow int32 `long:"recovery-window" description:"The address look-ahead used to scan for used keys when the wallet being initialized already has history; a value of zero, the default, means no addresses are recovered, which is what a brand new wallet wants"` } type initWalletCommand struct { @@ -102,6 +123,27 @@ func (x *initWalletCommand) Register(parser *flags.Parser) error { } func (x *initWalletCommand) Execute(_ []string) error { + // A birthday can only be given for a watch-only wallet created through + // RPC. Any other wallet is created from a seed that carries its birthday + // with it, so silently ignoring the flag there would be misleading. + birthdayApplies := x.InitType == typeRpc && x.InitRpc.WatchOnly + if x.InitRpc.WatchOnlyBirthday != "" && !birthdayApplies { + return fmt.Errorf("--init-rpc.watch-only-birthday can only " + + "be used in combination with --init-type=rpc and " + + "--init-rpc.watch-only") + } + + // The recovery window applies to any wallet created through RPC, but + // there's nothing sensible we could do with a negative look-ahead. + if x.InitRpc.RecoveryWindow < 0 { + return fmt.Errorf("invalid recovery window %d, must not be "+ + "negative", x.InitRpc.RecoveryWindow) + } + if x.InitRpc.RecoveryWindow != 0 && x.InitType != typeRpc { + return fmt.Errorf("--init-rpc.recovery-window can only be " + + "used in combination with --init-type=rpc") + } + // Do we require a seed? We don't if we do an RPC based, watch-only // initialization. requireSeed := (x.InitType == typeFile) || @@ -157,6 +199,32 @@ func (x *initWalletCommand) Execute(_ []string) error { // seed to be present but instead want to read an accounts JSON // file that contains all the wallet's xpubs. if x.InitRpc.WatchOnly { + // The accounts JSON file doesn't carry the birthday of + // the master key the accounts were derived from, so the + // operator has to tell us what it is. Without it lnd + // rescans the chain from the aezeed epoch, which on + // mainnet means walking hundreds of thousands of + // blocks. + birthday, err := parseWatchOnlyBirthday( + x.InitRpc.WatchOnlyBirthday, x.Network, + ) + if err != nil { + return err + } + + if birthday == 0 { + logger.Warn("No wallet birthday specified, " + + "lnd will rescan the chain from the " + + "aezeed epoch (2017-08-24) which can " + + "take multiple hours; use " + + "--init-rpc.watch-only-birthday to " + + "start the rescan at the wallet's " + + "actual birthday instead") + } else { + logger.Infof("Using wallet birthday %s", + formatBirthday(birthday)) + } + // For initializing a watch-only wallet we need the // accounts JSON file. logger.Info("Reading accounts from file") @@ -186,14 +254,20 @@ func (x *initWalletCommand) Execute(_ []string) error { } watchOnly = &lnrpc.WatchOnly{ - MasterKeyBirthdayTimestamp: 0, + MasterKeyBirthdayTimestamp: birthday, Accounts: rpcAccounts, } } + if x.InitRpc.RecoveryWindow != 0 { + logger.Infof("Using address look-ahead of %d for "+ + "recovery", x.InitRpc.RecoveryWindow) + } + return createWalletRpc( seedWords, seedPassPhrase, walletPassword, x.InitRpc.Server, x.InitRpc.TLSCertPath, watchOnly, + x.InitRpc.RecoveryWindow, ) default: @@ -397,7 +471,8 @@ func validateWallet(walletDir string, walletPassword []byte, } func createWalletRpc(seedWords []string, seedPassword, walletPassword, - rpcServer, tlsPath string, watchOnly *lnrpc.WatchOnly) error { + rpcServer, tlsPath string, watchOnly *lnrpc.WatchOnly, + recoveryWindow int32) error { // Since this will potentially run for a while (we need to wait for // compaction), make sure we catch any interrupt signals. @@ -433,6 +508,7 @@ func createWalletRpc(seedWords []string, seedPassword, walletPassword, AezeedPassphrase: []byte(seedPassword), WalletPassword: []byte(walletPassword), WatchOnly: watchOnly, + RecoveryWindow: recoveryWindow, }) return err } @@ -456,6 +532,72 @@ func checkSeed(seed, seedPassPhrase string) (*aezeed.CipherSeed, error) { return cipherSeed, nil } +// parseWatchOnlyBirthday turns the operator provided birthday of a watch-only +// wallet's master key into a Unix timestamp in seconds, as expected by lnd's +// InitWallet RPC. An empty value means the birthday is unknown and results in a +// zero timestamp, which makes lnd fall back to its own default. +func parseWatchOnlyBirthday(birthday, network string) (uint64, error) { + if birthday == "" { + return 0, nil + } + + birthdayTime, err := parseTimestampOrDate(birthday) + if err != nil { + return 0, fmt.Errorf("error parsing wallet birthday: %v", err) + } + + // A birthday from before the chain itself existed can only be a + // mistake, and would make lnd rescan all the way from the genesis + // block. + netParams, err := getNetworkParams(network) + if err != nil { + return 0, err + } + genesisTime := netParams.GenesisBlock.Header.Timestamp + if birthdayTime.Before(genesisTime) { + return 0, fmt.Errorf("invalid wallet birthday %s, is before "+ + "the %s genesis block time %s", + birthdayTime.Format(time.RFC3339), network, + genesisTime.UTC().Format(time.RFC3339)) + } + + if birthdayTime.After(time.Now().Add(maxBirthdayDrift)) { + return 0, fmt.Errorf("invalid wallet birthday %s, is more "+ + "than %v in the future (are the units seconds?)", + birthdayTime.Format(time.RFC3339), maxBirthdayDrift) + } + + return uint64(birthdayTime.Unix()), nil +} + +// parseTimestampOrDate parses a point in time that is either given as a Unix +// timestamp in seconds, as an RFC3339 timestamp or as a plain calendar date. +// Dates without a time of day are interpreted as midnight UTC. +func parseTimestampOrDate(value string) (time.Time, error) { + // A bare number is a Unix timestamp in seconds. + seconds, err := strconv.ParseInt(value, 10, 64) + if err == nil { + return time.Unix(seconds, 0).UTC(), nil + } + + for _, layout := range []string{time.RFC3339, birthdayDateFormat} { + parsed, err := time.Parse(layout, value) + if err == nil { + return parsed.UTC(), nil + } + } + + return time.Time{}, fmt.Errorf("value %q is neither a Unix timestamp "+ + "in seconds, an RFC3339 timestamp nor a %s date", value, + birthdayDateFormat) +} + +// formatBirthday renders a Unix timestamp in seconds as a human readable UTC +// timestamp for logging. +func formatBirthday(birthday uint64) string { + return time.Unix(int64(birthday), 0).UTC().Format(time.RFC3339) +} + func getNetworkParams(network string) (*chaincfg.Params, error) { switch strings.ToLower(network) { case "mainnet": diff --git a/cmd_init_wallet_test.go b/cmd_init_wallet_test.go index 9797fbb..a50f716 100644 --- a/cmd_init_wallet_test.go +++ b/cmd_init_wallet_test.go @@ -1,8 +1,10 @@ package main import ( + "fmt" "os" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -27,6 +29,235 @@ func TestReadInput(t *testing.T) { require.Equal(t, "p4ssw0rd", walletPassword) } +// TestParseWatchOnlyBirthday makes sure the birthday of a watch-only wallet can +// be given in any of the supported notations and that nonsensical values are +// rejected instead of silently turning into a full chain rescan. +func TestParseWatchOnlyBirthday(t *testing.T) { + t.Parallel() + + // A couple of reference points, expressed both as a timestamp and as + // the string we expect the parser to accept. + const ( + // mainnetGenesis is the timestamp of the mainnet genesis block. + mainnetGenesis = 1231006505 + + // regtestGenesis is the timestamp of the regtest genesis block. + regtestGenesis = 1296688602 + ) + + now := time.Now() + + testCases := []struct { + name string + birthday string + network string + expected uint64 + expectedErr string + }{{ + name: "empty means unknown", + birthday: "", + network: "mainnet", + expected: 0, + }, { + name: "unix timestamp in seconds", + birthday: "1719792000", + network: "mainnet", + expected: 1719792000, + }, { + name: "rfc3339 timestamp", + birthday: "2024-07-01T00:00:00Z", + network: "mainnet", + expected: 1719792000, + }, { + name: "rfc3339 timestamp with offset", + birthday: "2024-07-01T02:00:00+02:00", + network: "mainnet", + expected: 1719792000, + }, { + name: "calendar date is midnight utc", + birthday: "2024-07-01", + network: "mainnet", + expected: 1719792000, + }, { + name: "the genesis block itself", + birthday: fmt.Sprintf("%d", mainnetGenesis), + network: "mainnet", + expected: mainnetGenesis, + }, { + name: "now is fine", + birthday: fmt.Sprintf("%d", now.Unix()), + network: "mainnet", + expected: uint64(now.Unix()), + }, { + name: "slight clock skew is tolerated", + birthday: fmt.Sprintf("%d", now.Add(time.Hour).Unix()), + network: "mainnet", + expected: uint64(now.Add(time.Hour).Unix()), + }, { + name: "zero is before genesis", + birthday: "0", + network: "mainnet", + expectedErr: "before the mainnet genesis block time", + }, { + name: "negative timestamp", + birthday: "-1719792000", + network: "mainnet", + expectedErr: "before the mainnet genesis block time", + }, { + name: "date before genesis", + birthday: "2008-10-31", + network: "mainnet", + expectedErr: "before the mainnet genesis block time", + }, { + name: "milliseconds instead of seconds", + birthday: "1719792000000", + network: "mainnet", + expectedErr: "in the future", + }, { + name: "far future date", + birthday: now.AddDate(1, 0, 0).Format(time.RFC3339), + network: "mainnet", + expectedErr: "in the future", + }, { + name: "not a timestamp or date", + birthday: "yesterday", + network: "mainnet", + expectedErr: "is neither a Unix timestamp", + }, { + name: "date with slashes", + birthday: "2024/07/01", + network: "mainnet", + expectedErr: "is neither a Unix timestamp", + }, { + name: "unknown network", + birthday: "2024-07-01", + network: "fakenet", + expectedErr: "unknown network", + }, { + // The genesis block of each network has its own timestamp, so + // the lower bound has to follow the network. + name: "regtest genesis", + birthday: fmt.Sprintf("%d", regtestGenesis), + network: "regtest", + expected: regtestGenesis, + }, { + name: "mainnet date before regtest genesis", + birthday: "2010-07-01", + network: "regtest", + expectedErr: "before the regtest genesis block time", + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + birthday, err := parseWatchOnlyBirthday( + tc.birthday, tc.network, + ) + + if tc.expectedErr != "" { + require.ErrorContains(t, err, tc.expectedErr) + require.Zero(t, birthday) + + return + } + + require.NoError(t, err) + require.Equal(t, tc.expected, birthday) + }) + } +} + +// TestWatchOnlyBirthdayRequiresWatchOnly makes sure we don't silently ignore a +// birthday that was given for a wallet that isn't initialized as a watch-only +// one through RPC, which is the only combination that can act on it. +func TestWatchOnlyBirthdayRequiresWatchOnly(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + initType string + watchOnly bool + }{{ + name: "rpc without watch-only", + initType: typeRpc, + watchOnly: false, + }, { + name: "file init", + initType: typeFile, + watchOnly: false, + }, { + // The file based init doesn't look at any of the RPC flags, so + // the watch-only flag being set alongside it doesn't make the + // birthday reachable either. + name: "file init with the watch-only flag set", + initType: typeFile, + watchOnly: true, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + cmd := newInitWalletCommand() + cmd.InitType = tc.initType + cmd.InitRpc.WatchOnly = tc.watchOnly + cmd.InitRpc.WatchOnlyBirthday = "2024-07-01" + + err := cmd.Execute(nil) + require.ErrorContains( + t, err, "can only be used in combination with", + ) + }) + } +} + +// TestRecoveryWindowValidation makes sure a recovery window that can't be acted +// on is rejected instead of being dropped on the floor, the same way the +// birthday is. +func TestRecoveryWindowValidation(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + initType string + recoveryWindow int32 + expectedErr string + }{{ + name: "negative window", + initType: typeRpc, + recoveryWindow: -1, + expectedErr: "must not be negative", + }, { + name: "file init ignores the window", + initType: typeFile, + recoveryWindow: 2500, + expectedErr: "can only be used in combination with", + }, { + // A zero window is the default and means "don't recover", so it + // stays valid no matter which init type is used. This gets far + // enough to fail on the missing seed file instead, which we + // assert on through the sentinel that main maps to exit code + // EXIT_CODE_INPUT_MISSING rather than through the wrapping + // message. + name: "zero window is always fine", + initType: typeFile, + recoveryWindow: 0, + expectedErr: errInputMissing, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + cmd := newInitWalletCommand() + cmd.InitType = tc.initType + cmd.InitRpc.RecoveryWindow = tc.recoveryWindow + + err := cmd.Execute(nil) + require.ErrorContains(t, err, tc.expectedErr) + }) + } +} + func writeToTempFile(t *testing.T, data []byte) string { tempFileName, err := os.CreateTemp("", "*.txt") require.NoError(t, err) diff --git a/k8s.go b/k8s.go index 8264e33..5890cbe 100644 --- a/k8s.go +++ b/k8s.go @@ -243,11 +243,8 @@ func updateSecretValueK8s(client *kubernetes.Clientset, secret *api.Secret, "%v", opts.Name, opts.Namespace, err) } - jsonSecret, _ := asJSON(jsonK8sObject{ - TypeMeta: updatedSecret.TypeMeta, - ObjectMeta: updatedSecret.ObjectMeta, - }) - logger.Infof("Updated secret: %s", jsonSecret) + logger.Infof("Updated secret %s in namespace %s", + updatedSecret.Name, opts.Namespace) return nil } @@ -291,11 +288,8 @@ func createSecretK8s(client *kubernetes.Clientset, opts *k8sObjectOptions, "%v", opts.Name, opts.Namespace, err) } - jsonSecret, _ := asJSON(jsonK8sObject{ - TypeMeta: updatedSecret.TypeMeta, - ObjectMeta: updatedSecret.ObjectMeta, - }) - logger.Infof("Created secret: %s", jsonSecret) + logger.Infof("Created secret %s in namespace %s", + updatedSecret.Name, opts.Namespace) return nil } diff --git a/main.go b/main.go index c4c2cf3..66a2412 100644 --- a/main.go +++ b/main.go @@ -44,8 +44,10 @@ func main() { // just the global options, we do a pre-parsing without any commands // registered yet. We ignore any errors as that'll be handled later. _, _ = flags.NewParser(globalOpts, flags.IgnoreUnknown).Parse() - if globalOpts.Verbose { - globalOpts.DebugLevel = "error" + // An explicit --debuglevel always wins, -v only picks the level when + // none was given. + if globalOpts.Verbose && globalOpts.DebugLevel == "" { + globalOpts.DebugLevel = "info" } logger.Infof("Version %s commit=%s, debuglevel=%s", Version(), Commit,