vault: add base64 option for binary secret values

KV v2 stores entries as JSON strings, so a value that is not valid
UTF-8, such as a raw macaroon, has its invalid bytes replaced with the
Unicode replacement character on write and cannot be recovered. Text
values like a seed, a wallet password, or a PEM certificate are
unaffected, but binary values are silently corrupted. The Kubernetes
backend already avoids this with its base64 option; the Vault path had
no equivalent.

Add a base64 option to the Vault secret flags that encodes the value on
write and decodes it on read, wired into the store-secret, load-secret
and init-wallet commands so a caller can store binary values safely. It
is off by default, so text values are still stored verbatim.

Author: calvinrzachman

cmd_init_wallet.go

@@ -62,6 +62,7 @@ type secretSourceVault struct {
 	SeedKeyName           string `long:"seed-key-name" description:"The name of the entry within the secret that contains the seed"`
 	SeedPassphraseKeyName string `long:"seed-passphrase-key-name" description:"The name of the entry within the secret that contains the seed passphrase"`
 	WalletPasswordKeyName string `long:"wallet-password-key-name" description:"The name of the entry within the secret that contains the wallet password"`
+	Base64                bool   `long:"base64" description:"Decode entries as base64 when reading; must match how the secrets were stored"`
 }
 
 func (s *secretSourceVault) options(keyName string) *vaultSecretOptions {
@@ -73,6 +74,7 @@ func (s *secretSourceVault) options(keyName string) *vaultSecretOptions {
 		KVMount:       s.KVMount,
 		SecretPath:    s.SecretPath,
 		SecretKeyName: keyName,
+		Base64:        s.Base64,
 	}
 }
 

vault.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"strings"
@@ -38,6 +39,7 @@ type vaultSecretOptions struct {
 	KVMount       string `long:"kv-mount" description:"The mount path of the KV v2 secrets engine the secret lives in"`
 	SecretPath    string `long:"secret-path" description:"The path of the secret within the KV v2 engine, excluding the engine mount and the 'data/' segment (e.g. 'lnd/mynode/wallet')"`
 	SecretKeyName string `long:"secret-key-name" description:"The name of the key/entry within the secret"`
+	Base64        bool   `long:"base64" description:"Encode as base64 when storing and decode as base64 when reading; required for binary values since KV v2 stores entries as JSON strings"`
 }
 
 // jsonVaultObject is the subset of Vault response metadata that we surface when
@@ -84,6 +86,13 @@ func saveVault(content string, opts *vaultSecretOptions, overwrite bool) error {
 		return err
 	}
 
+	// Optionally base64-encode the value so binary content survives KV v2's
+	// JSON string storage, which would otherwise replace invalid UTF-8 bytes
+	// with the Unicode replacement character.
+	if opts.Base64 {
+		content = base64.StdEncoding.EncodeToString([]byte(content))
+	}
+
 	ctx := context.Background()
 	kv := client.KVv2(kvMount(opts))
 
@@ -196,6 +205,18 @@ func readVault(opts *vaultSecretOptions) (string, *jsonVaultObject, error) {
 			"is empty", opts.SecretPath, opts.SecretKeyName)
 	}
 
+	// Undo the optional base64 encoding applied on write to recover the
+	// original (possibly binary) value.
+	if opts.Base64 {
+		decoded, err := base64.StdEncoding.DecodeString(content)
+		if err != nil {
+			return "", nil, fmt.Errorf("failed to base64 decode "+
+				"entry %s of secret %s: %v", opts.SecretKeyName,
+				opts.SecretPath, err)
+		}
+		content = string(decoded)
+	}
+
 	return content, newJSONVaultObject(secret), nil
 }
 

vault_test.go

@@ -292,3 +292,27 @@ func TestVaultCheckAndSet(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, "seed-a", content)
 }
+
+// TestVaultBase64 verifies that a binary value is corrupted by KV v2's JSON
+// string storage when stored as-is, but round-trips intact when the base64
+// option is set.
+func TestVaultBase64(t *testing.T) {
+	addr, _ := newTestVault(t)
+	binary := string([]byte{0x01, 0xff, 0xfe, 0x80, 0x02})
+
+	// Stored as-is, the invalid UTF-8 bytes are replaced during JSON
+	// encoding, so the value does not survive the round trip.
+	plain := testVaultOptions(t, addr, "lnd/test/plain", "admin.macaroon")
+	require.NoError(t, saveVault(binary, plain, false))
+	got, _, err := readVault(plain)
+	require.NoError(t, err)
+	require.NotEqual(t, binary, got)
+
+	// With base64 the same value round-trips intact.
+	encoded := testVaultOptions(t, addr, "lnd/test/encoded", "admin.macaroon")
+	encoded.Base64 = true
+	require.NoError(t, saveVault(binary, encoded, false))
+	got, _, err = readVault(encoded)
+	require.NoError(t, err)
+	require.Equal(t, binary, got)
+}