Merge bitcoin#23093: Add ability to flush keypool and always flush when upgrading non-HD to HD

6531599 test: Add check that newkeypool flushes change addresses too (Samuel Dobson)
84fa19c Add release notes for keypool flush changes (Samuel Dobson)
f9603ee Add test for flushing keypool with newkeypool (Samuel Dobson)
6f6f7bb Make legacy wallet upgrades from non-HD to HD always flush the keypool (Samuel Dobson)
2434b10 Fix outdated keypool size default (Samuel Dobson)
22cc797 Add newkeypool RPC to flush the keypool (Samuel Dobson)

Pull request description:

  This PR makes two main changes:
  1) Adds a new RPC `newkeypool` which will entirely flush and refill the keypool.
  2) When upgradewallet is called on old, non-HD wallets upgrading them to HD, we now always flush the keypool and generate a new one, to immediately start using the HD generated keys.

  This PR is motivated by a number of users with old, pre-compressed-key wallets upgrading them and being confused about why they still can't generate p2sh-segwit or bech32 addresses -- this is due to uncompressed keys remaining in the keypool post-upgrade and being illegal in these newer address formats. There is currently no easy way to flush the keypool other than to call `getnewaddress` a hundred/thousand times or an ugly hack of using a `sethdseed` call.

ACKs for top commit:
  laanwj:
    re-ACK 6531599
  meshcollider:
    Added new commit 6531599 to avoid invalidating previous ACKs.
  instagibbs:
    ACK bitcoin@6531599

Tree-SHA512: 50c79c5d42dd27ab0ecdbfdc4071fdaa1b2dbb2f9195ed325b007106ff19226419ce57fe5b1539c0c24101b12f5e034bbcfb7bbb0451b766cb1071295383d774

Author: laanwj

doc/release-notes-23093.md

@@ -0,0 +1,11 @@
+Notable changes
+===============
+
+Updated RPCs
+------------
+
+- `upgradewallet` will now automatically flush the keypool if upgrading
+from a non-HD wallet to an HD wallet, to immediately start using the
+newly-generated HD keys.
+- a new RPC `newkeypool` has been added, which will flush (entirely
+clear and refill) the keypool.

src/wallet/rpcwallet.cpp

@@ -1854,7 +1854,7 @@ static RPCHelpMan keypoolrefill()
                 "\nFills the keypool."+
         HELP_REQUIRING_PASSPHRASE,
                 {
-                    {"newsize", RPCArg::Type::NUM, RPCArg::Default{100}, "The new keypool size"},
+                    {"newsize", RPCArg::Type::NUM, RPCArg::DefaultHint{strprintf("%u, or as set by -keypool", DEFAULT_KEYPOOL_SIZE)}, "The new keypool size"},
                 },
                 RPCResult{RPCResult::Type::NONE, "", ""},
                 RPCExamples{
@@ -1893,6 +1893,33 @@ static RPCHelpMan keypoolrefill()
 }
 
 
+static RPCHelpMan newkeypool()
+{
+    return RPCHelpMan{"newkeypool",
+                "\nEntirely clears and refills the keypool."+
+            HELP_REQUIRING_PASSPHRASE,
+                {},
+                RPCResult{RPCResult::Type::NONE, "", ""},
+                RPCExamples{
+            HelpExampleCli("newkeypool", "")
+            + HelpExampleRpc("newkeypool", "")
+                },
+        [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue
+{
+    std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request);
+    if (!pwallet) return NullUniValue;
+
+    LOCK(pwallet->cs_wallet);
+
+    LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet, true);
+    spk_man.NewKeyPool();
+
+    return NullUniValue;
+},
+    };
+}
+
+
 static RPCHelpMan walletpassphrase()
 {
     return RPCHelpMan{"walletpassphrase",
@@ -4875,6 +4902,7 @@ static const CRPCCommand commands[] =
     { "wallet",             &listwallets,                    },
     { "wallet",             &loadwallet,                     },
     { "wallet",             &lockunspent,                    },
+    { "wallet",             &newkeypool,                     },
     { "wallet",             &removeprunedfunds,              },
     { "wallet",             &rescanblockchain,               },
     { "wallet",             &send,                           },

src/wallet/scriptpubkeyman.cpp

@@ -489,7 +489,7 @@ bool LegacyScriptPubKeyMan::Upgrade(int prev_version, int new_version, bilingual
     }
     // Regenerate the keypool if upgraded to HD
     if (hd_upgrade) {
-        if (!TopUp()) {
+        if (!NewKeyPool()) {
             error = _("Unable to generate keys");
             return false;
         }

test/functional/wallet_keypool.py

@@ -138,6 +138,20 @@ def run_test(self):
             assert_equal(wi['keypoolsize_hd_internal'], 100)
             assert_equal(wi['keypoolsize'], 100)
 
+        if not self.options.descriptors:
+            # Check that newkeypool entirely flushes the keypool
+            start_keypath = nodes[0].getaddressinfo(nodes[0].getnewaddress())['hdkeypath']
+            start_change_keypath = nodes[0].getaddressinfo(nodes[0].getrawchangeaddress())['hdkeypath']
+            # flush keypool and get new addresses
+            nodes[0].newkeypool()
+            end_keypath = nodes[0].getaddressinfo(nodes[0].getnewaddress())['hdkeypath']
+            end_change_keypath = nodes[0].getaddressinfo(nodes[0].getrawchangeaddress())['hdkeypath']
+            # The new keypath index should be 100 more than the old one
+            new_index = int(start_keypath.rsplit('/',  1)[1][:-1]) + 100
+            new_change_index = int(start_change_keypath.rsplit('/',  1)[1][:-1]) + 100
+            assert_equal(end_keypath, "m/0'/0'/" + str(new_index) + "'")
+            assert_equal(end_change_keypath, "m/0'/1'/" + str(new_change_index) + "'")
+
         # create a blank wallet
         nodes[0].createwallet(wallet_name='w2', blank=True, disable_private_keys=True)
         w2 = nodes[0].get_wallet_rpc('w2')

test/functional/wallet_upgradewallet.py

@@ -234,18 +234,13 @@ def copy_split_hd():
         assert_equal(1, hd_chain_version)
         seed_id = bytearray(seed_id)
         seed_id.reverse()
-        old_kvs = new_kvs
-        # First 2 keys should still be non-HD
-        for i in range(0, 2):
-            info = wallet.getaddressinfo(wallet.getnewaddress())
-            assert 'hdkeypath' not in info
-            assert 'hdseedid' not in info
-        # Next key should be HD
+
+        # New keys (including change) should be HD (the two old keys have been flushed)
         info = wallet.getaddressinfo(wallet.getnewaddress())
         assert_equal(seed_id.hex(), info['hdseedid'])
         assert_equal('m/0\'/0\'/0\'', info['hdkeypath'])
         prev_seed_id = info['hdseedid']
-        # Change key should be the same keypool
+        # Change key should be HD and from the same keypool
         info = wallet.getaddressinfo(wallet.getrawchangeaddress())
         assert_equal(prev_seed_id, info['hdseedid'])
         assert_equal('m/0\'/0\'/1\'', info['hdkeypath'])
@@ -291,14 +286,7 @@ def copy_split_hd():
         hd_chain_version, external_counter, seed_id, internal_counter = struct.unpack('<iI20sI', hd_chain)
         assert_equal(2, hd_chain_version)
         assert_equal(2, internal_counter)
-        # Drain the keypool by fetching one external key and one change key. Should still be the same keypool
-        info = wallet.getaddressinfo(wallet.getnewaddress())
-        assert 'hdseedid' not in info
-        assert 'hdkeypath' not in info
-        info = wallet.getaddressinfo(wallet.getrawchangeaddress())
-        assert 'hdseedid' not in info
-        assert 'hdkeypath' not in info
-        # The next addresses are HD and should be on different HD chains
+        # The next addresses are HD and should be on different HD chains (the one remaining key in each pool should have been flushed)
         info = wallet.getaddressinfo(wallet.getnewaddress())
         ext_id = info['hdseedid']
         assert_equal('m/0\'/0\'/0\'', info['hdkeypath'])