Merge bitcoin#20267: Disable and fix tests for when BDB is not compiled

49797c3 tests: Disable bdb dump test when no bdb (Andrew Chow)
1194cf9 Fix wallet_send.py wallet setup to work with descriptors (Andrew Chow)
fbaea7b Require legacy wallet for wallet_upgradewallet.py (Andrew Chow)
b1b679e Explicitly mark legacy wallet tests as such (Andrew Chow)
09514e1 Setup wallets for interface_zmq.py (Andrew Chow)
4d03ef9 Use MiniWallet in rpc_net.py (Andrew Chow)
4de2382 Setup wallets for interface_bitcoin_cli.py (Andrew Chow)
7c71c62 Setup wallets with descriptors for feature_notifications (Andrew Chow)
1f1bef8 Have feature_filelock.py test both bdb and sqlite, depending on compiled (Andrew Chow)
c77975a Disable upgrades tests that require BDB if BDB is not compiled (Andrew Chow)
1f20cac Disable wallet_descriptor.py bdb format check if BDB is not compiled (Andrew Chow)
3641597 tests: Don't make any wallets unless wallet is required (Andrew Chow)
b9b88f5 Skip legacy wallet reliant tests if BDB is not compiled (Andrew Chow)
6f36242 tests: Set descriptors default based on compilation (Andrew Chow)

Pull request description:

  This PR fixes tests for when BDB is not compiled. Tests which rely on or test legacy wallet behavior are disabled and skipped when BDB is not compiled. For the components of some tests that are for legacy wallet things, those parts of the tests are skipped.

  For the majority of tests, changes are made so that they can be run with either legacy wallets or descriptor wallets without materially effecting the test. Most tests only need the wallet for balance and transactions, so the type of wallet is not an important part of those tests. Additionally, some tests are wallet agnostic and modified to instead use the test framework's MiniWallet.

ACKs for top commit:
  laanwj:
    ACK 49797c3
  ryanofsky:
    Code review ACK 49797c3. Only change since last review is dropping last commit. Previous review w/ suggestions for future followup is bitcoin#20267 (review)

Tree-SHA512: 69659f8a81fb437ecbca962f4082c12835282dbf1fba7d9952f727a49e01981d749af9b09feda1c8ca737516c7d7a08ef17e782795df3fa69892d5021b41c1ed

Author: laanwj

test/functional/feature_backwards_compatibility.py

@@ -354,73 +354,75 @@ def run_test(self):
         hdkeypath = v17_info["hdkeypath"]
         pubkey = v17_info["pubkey"]
 
-        # Copy the 0.16 wallet to the last Bitcoin Core version and open it:
-        shutil.copyfile(
-            os.path.join(node_v16_wallets_dir, "wallets/u1_v16"),
-            os.path.join(node_master_wallets_dir, "u1_v16")
-        )
-        load_res = node_master.loadwallet("u1_v16")
-        # Make sure this wallet opens without warnings. See https://github.com/bitcoin/bitcoin/pull/19054
-        assert_equal(load_res['warning'], '')
-        wallet = node_master.get_wallet_rpc("u1_v16")
-        info = wallet.getaddressinfo(v16_addr)
-        descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + v16_pubkey + ")"
-        assert_equal(info["desc"], descsum_create(descriptor))
-
-        # Now copy that same wallet back to 0.16 to make sure no automatic upgrade breaks it
-        os.remove(os.path.join(node_v16_wallets_dir, "wallets/u1_v16"))
-        shutil.copyfile(
-            os.path.join(node_master_wallets_dir, "u1_v16"),
-            os.path.join(node_v16_wallets_dir, "wallets/u1_v16")
-        )
-        self.start_node(-1, extra_args=["-wallet=u1_v16"])
-        wallet = node_v16.get_wallet_rpc("u1_v16")
-        info = wallet.validateaddress(v16_addr)
-        assert_equal(info, v16_info)
-
-        # Copy the 0.17 wallet to the last Bitcoin Core version and open it:
-        node_v17.unloadwallet("u1_v17")
-        shutil.copytree(
-            os.path.join(node_v17_wallets_dir, "u1_v17"),
-            os.path.join(node_master_wallets_dir, "u1_v17")
-        )
-        node_master.loadwallet("u1_v17")
-        wallet = node_master.get_wallet_rpc("u1_v17")
-        info = wallet.getaddressinfo(address)
-        descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + pubkey + ")"
-        assert_equal(info["desc"], descsum_create(descriptor))
-
-        # Now copy that same wallet back to 0.17 to make sure no automatic upgrade breaks it
-        node_master.unloadwallet("u1_v17")
-        shutil.rmtree(os.path.join(node_v17_wallets_dir, "u1_v17"))
-        shutil.copytree(
-            os.path.join(node_master_wallets_dir, "u1_v17"),
-            os.path.join(node_v17_wallets_dir, "u1_v17")
-        )
-        node_v17.loadwallet("u1_v17")
-        wallet = node_v17.get_wallet_rpc("u1_v17")
-        info = wallet.getaddressinfo(address)
-        assert_equal(info, v17_info)
-
-        # Copy the 0.19 wallet to the last Bitcoin Core version and open it:
-        shutil.copytree(
-            os.path.join(node_v19_wallets_dir, "w1_v19"),
-            os.path.join(node_master_wallets_dir, "w1_v19")
-        )
-        node_master.loadwallet("w1_v19")
-        wallet = node_master.get_wallet_rpc("w1_v19")
-        assert wallet.getaddressinfo(address_18075)["solvable"]
+        if self.is_bdb_compiled():
+            # Old wallets are BDB and will only work if BDB is compiled
+            # Copy the 0.16 wallet to the last Bitcoin Core version and open it:
+            shutil.copyfile(
+                os.path.join(node_v16_wallets_dir, "wallets/u1_v16"),
+                os.path.join(node_master_wallets_dir, "u1_v16")
+            )
+            load_res = node_master.loadwallet("u1_v16")
+            # Make sure this wallet opens without warnings. See https://github.com/bitcoin/bitcoin/pull/19054
+            assert_equal(load_res['warning'], '')
+            wallet = node_master.get_wallet_rpc("u1_v16")
+            info = wallet.getaddressinfo(v16_addr)
+            descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + v16_pubkey + ")"
+            assert_equal(info["desc"], descsum_create(descriptor))
+
+            # Now copy that same wallet back to 0.16 to make sure no automatic upgrade breaks it
+            os.remove(os.path.join(node_v16_wallets_dir, "wallets/u1_v16"))
+            shutil.copyfile(
+                os.path.join(node_master_wallets_dir, "u1_v16"),
+                os.path.join(node_v16_wallets_dir, "wallets/u1_v16")
+            )
+            self.start_node(-1, extra_args=["-wallet=u1_v16"])
+            wallet = node_v16.get_wallet_rpc("u1_v16")
+            info = wallet.validateaddress(v16_addr)
+            assert_equal(info, v16_info)
 
-        # Now copy that same wallet back to 0.19 to make sure no automatic upgrade breaks it
-        node_master.unloadwallet("w1_v19")
-        shutil.rmtree(os.path.join(node_v19_wallets_dir, "w1_v19"))
-        shutil.copytree(
-            os.path.join(node_master_wallets_dir, "w1_v19"),
-            os.path.join(node_v19_wallets_dir, "w1_v19")
-        )
-        node_v19.loadwallet("w1_v19")
-        wallet = node_v19.get_wallet_rpc("w1_v19")
-        assert wallet.getaddressinfo(address_18075)["solvable"]
+            # Copy the 0.17 wallet to the last Bitcoin Core version and open it:
+            node_v17.unloadwallet("u1_v17")
+            shutil.copytree(
+                os.path.join(node_v17_wallets_dir, "u1_v17"),
+                os.path.join(node_master_wallets_dir, "u1_v17")
+            )
+            node_master.loadwallet("u1_v17")
+            wallet = node_master.get_wallet_rpc("u1_v17")
+            info = wallet.getaddressinfo(address)
+            descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + pubkey + ")"
+            assert_equal(info["desc"], descsum_create(descriptor))
+
+            # Now copy that same wallet back to 0.17 to make sure no automatic upgrade breaks it
+            node_master.unloadwallet("u1_v17")
+            shutil.rmtree(os.path.join(node_v17_wallets_dir, "u1_v17"))
+            shutil.copytree(
+                os.path.join(node_master_wallets_dir, "u1_v17"),
+                os.path.join(node_v17_wallets_dir, "u1_v17")
+            )
+            node_v17.loadwallet("u1_v17")
+            wallet = node_v17.get_wallet_rpc("u1_v17")
+            info = wallet.getaddressinfo(address)
+            assert_equal(info, v17_info)
+
+            # Copy the 0.19 wallet to the last Bitcoin Core version and open it:
+            shutil.copytree(
+                os.path.join(node_v19_wallets_dir, "w1_v19"),
+                os.path.join(node_master_wallets_dir, "w1_v19")
+            )
+            node_master.loadwallet("w1_v19")
+            wallet = node_master.get_wallet_rpc("w1_v19")
+            assert wallet.getaddressinfo(address_18075)["solvable"]
+
+            # Now copy that same wallet back to 0.19 to make sure no automatic upgrade breaks it
+            node_master.unloadwallet("w1_v19")
+            shutil.rmtree(os.path.join(node_v19_wallets_dir, "w1_v19"))
+            shutil.copytree(
+                os.path.join(node_master_wallets_dir, "w1_v19"),
+                os.path.join(node_v19_wallets_dir, "w1_v19")
+            )
+            node_v19.loadwallet("w1_v19")
+            wallet = node_v19.get_wallet_rpc("w1_v19")
+            assert wallet.getaddressinfo(address_18075)["solvable"]
 
 if __name__ == '__main__':
     BackwardsCompatibilityTest().main()

test/functional/feature_filelock.py

@@ -4,6 +4,8 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Check that it's not possible to start a second bitcoind instance using the same datadir or wallet."""
 import os
+import random
+import string
 
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.test_node import ErrorMatch
@@ -27,11 +29,21 @@ def run_test(self):
         self.nodes[1].assert_start_raises_init_error(extra_args=['-datadir={}'.format(self.nodes[0].datadir), '-noserver'], expected_msg=expected_msg)
 
         if self.is_wallet_compiled():
-            self.nodes[0].createwallet(self.default_wallet_name)
-            wallet_dir = os.path.join(datadir, 'wallets')
-            self.log.info("Check that we can't start a second bitcoind instance using the same wallet")
-            expected_msg = "Error: Error initializing wallet database environment"
-            self.nodes[1].assert_start_raises_init_error(extra_args=['-walletdir={}'.format(wallet_dir), '-wallet=' + self.default_wallet_name, '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX)
+            def check_wallet_filelock(descriptors):
+                wallet_name = ''.join([random.choice(string.ascii_lowercase) for _ in range(6)])
+                self.nodes[0].createwallet(wallet_name=wallet_name, descriptors=descriptors)
+                wallet_dir = os.path.join(datadir, 'wallets')
+                self.log.info("Check that we can't start a second bitcoind instance using the same wallet")
+                if descriptors:
+                    expected_msg = "Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?"
+                else:
+                    expected_msg = "Error: Error initializing wallet database environment"
+                self.nodes[1].assert_start_raises_init_error(extra_args=['-walletdir={}'.format(wallet_dir), '-wallet=' + wallet_name, '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX)
+
+            if self.is_bdb_compiled():
+                check_wallet_filelock(False)
+            if self.is_sqlite_compiled():
+                check_wallet_filelock(True)
 
 if __name__ == '__main__':
     FilelockTest().main()

test/functional/feature_notifications.py

@@ -5,11 +5,11 @@
 """Test the -alertnotify, -blocknotify and -walletnotify options."""
 import os
 
-from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE, keyhash_to_p2pkh
+from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE
+from test_framework.descriptors import descsum_create
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_equal,
-    hex_str_to_bytes,
 )
 
 # Linux allow all characters other than \x00
@@ -49,6 +49,31 @@ def setup_network(self):
         super().setup_network()
 
     def run_test(self):
+        if self.is_wallet_compiled():
+            # Setup the descriptors to be imported to the wallet
+            seed = "cTdGmKFWpbvpKQ7ejrdzqYT2hhjyb3GPHnLAK7wdi5Em67YLwSm9"
+            xpriv = "tprv8ZgxMBicQKsPfHCsTwkiM1KT56RXbGGTqvc2hgqzycpwbHqqpcajQeMRZoBD35kW4RtyCemu6j34Ku5DEspmgjKdt2qe4SvRch5Kk8B8A2v"
+            desc_imports = [{
+                "desc": descsum_create("wpkh(" + xpriv + "/0/*)"),
+                "timestamp": 0,
+                "active": True,
+                "keypool": True,
+            },{
+                "desc": descsum_create("wpkh(" + xpriv + "/1/*)"),
+                "timestamp": 0,
+                "active": True,
+                "keypool": True,
+                "internal": True,
+            }]
+            # Make the wallets and import the descriptors
+            # Ensures that node 0 and node 1 share the same wallet for the conflicting transaction tests below.
+            for i, name in enumerate(self.wallet_names):
+                self.nodes[i].createwallet(wallet_name=name, descriptors=self.options.descriptors, blank=True, load_on_startup=True)
+                if self.options.descriptors:
+                    self.nodes[i].importdescriptors(desc_imports)
+                else:
+                    self.nodes[i].sethdseed(True, seed)
+
         self.log.info("test -blocknotify")
         block_count = 10
         blocks = self.nodes[1].generatetoaddress(block_count, self.nodes[1].getnewaddress() if self.is_wallet_compiled() else ADDRESS_BCRT1_UNSPENDABLE)
@@ -84,11 +109,10 @@ def run_test(self):
             for tx_file in os.listdir(self.walletnotify_dir):
                 os.remove(os.path.join(self.walletnotify_dir, tx_file))
 
-            # Conflicting transactions tests. Give node 0 same wallet seed as
-            # node 1, generate spends from node 0, and check notifications
+            # Conflicting transactions tests.
+            # Generate spends from node 0, and check notifications
             # triggered by node 1
             self.log.info("test -walletnotify with conflicting transactions")
-            self.nodes[0].sethdseed(seed=self.nodes[1].dumpprivkey(keyhash_to_p2pkh(hex_str_to_bytes(self.nodes[1].getwalletinfo()['hdseedid'])[::-1])))
             self.nodes[0].rescanblockchain()
             self.nodes[0].generatetoaddress(100, ADDRESS_BCRT1_UNSPENDABLE)
             self.sync_blocks()

test/functional/interface_bitcoin_cli.py

@@ -29,6 +29,8 @@ class TestBitcoinCli(BitcoinTestFramework):
     def set_test_params(self):
         self.setup_clean_chain = True
         self.num_nodes = 1
+        if self.is_wallet_compiled():
+            self.requires_wallet = True
 
     def skip_test_if_missing_module(self):
         self.skip_if_no_cli()

test/functional/interface_zmq.py

@@ -62,6 +62,8 @@ def receive_sequence(self):
 class ZMQTest (BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 2
+        if self.is_wallet_compiled():
+            self.requires_wallet = True
 
     def skip_test_if_missing_module(self):
         self.skip_if_no_py3_zmq()

test/functional/rpc_net.py

@@ -25,6 +25,7 @@
     assert_raises_rpc_error,
     p2p_port,
 )
+from test_framework.wallet import MiniWallet
 
 
 def assert_net_servicesnames(servicesflag, servicenames):
@@ -48,6 +49,9 @@ def set_test_params(self):
         self.supports_cli = False
 
     def run_test(self):
+        # We need miniwallet to make a transaction
+        self.wallet = MiniWallet(self.nodes[0])
+        self.wallet.generate(1)
         # Get out of IBD for the minfeefilter and getpeerinfo tests.
         self.nodes[0].generate(101)
 
@@ -74,8 +78,7 @@ def test_connection_count(self):
     def test_getpeerinfo(self):
         self.log.info("Test getpeerinfo")
         # Create a few getpeerinfo last_block/last_transaction values.
-        if self.is_wallet_compiled():
-            self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1)
+        self.wallet.send_self_transfer(from_node=self.nodes[0]) # Make a transaction so we can see it in the getpeerinfo results
         self.nodes[1].generate(1)
         self.sync_all()
         time_now = int(time.time())

test/functional/test_framework/test_framework.py

@@ -108,6 +108,9 @@ def __init__(self):
         # skipped. If list is truncated, wallet creation is skipped and keys
         # are not imported.
         self.wallet_names = None
+        # By default the wallet is not required. Set to true by skip_if_no_wallet().
+        # When False, we ignore wallet_names regardless of what it is.
+        self.requires_wallet = False
         self.set_test_params()
         assert self.wallet_names is None or len(self.wallet_names) <= self.num_nodes
         if self.options.timeout_factor == 0 :
@@ -184,15 +187,30 @@ def parse_args(self):
         parser.add_argument('--timeout-factor', dest="timeout_factor", type=float, default=1.0, help='adjust test timeouts by a factor. Setting it to 0 disables all timeouts')
 
         group = parser.add_mutually_exclusive_group()
-        group.add_argument("--descriptors", default=False, action="store_true",
+        group.add_argument("--descriptors", action='store_const', const=True,
                             help="Run test using a descriptor wallet", dest='descriptors')
-        group.add_argument("--legacy-wallet", default=False, action="store_false",
+        group.add_argument("--legacy-wallet", action='store_const', const=False,
                             help="Run test using legacy wallets", dest='descriptors')
 
         self.add_options(parser)
         self.options = parser.parse_args()
         self.options.previous_releases_path = previous_releases_path
 
+        config = configparser.ConfigParser()
+        config.read_file(open(self.options.configfile))
+        self.config = config
+
+        if self.options.descriptors is None:
+            # Prefer BDB unless it isn't available
+            if self.is_bdb_compiled():
+                self.options.descriptors = False
+            elif self.is_sqlite_compiled():
+                self.options.descriptors = True
+            else:
+                # If neither are compiled, tests requiring a wallet will be skipped and the value of self.options.descriptors won't matter
+                # It still needs to exist and be None in order for tests to work however.
+                self.options.descriptors = None
+
     def setup(self):
         """Call this method to start up the test framework object with options set."""
 
@@ -202,9 +220,8 @@ def setup(self):
 
         self.options.cachedir = os.path.abspath(self.options.cachedir)
 
-        config = configparser.ConfigParser()
-        config.read_file(open(self.options.configfile))
-        self.config = config
+        config = self.config
+
         fname_bitcoind = os.path.join(
             config["environment"]["BUILDDIR"],
             "src",
@@ -377,7 +394,7 @@ def setup_nodes(self):
             extra_args = self.extra_args
         self.add_nodes(self.num_nodes, extra_args)
         self.start_nodes()
-        if self.is_wallet_compiled():
+        if self.requires_wallet:
             self.import_deterministic_coinbase_privkeys()
         if not self.setup_clean_chain:
             for n in self.nodes:
@@ -769,10 +786,13 @@ def skip_if_no_bitcoind_zmq(self):
 
     def skip_if_no_wallet(self):
         """Skip the running test if wallet has not been compiled."""
+        self.requires_wallet = True
         if not self.is_wallet_compiled():
             raise SkipTest("wallet has not been compiled.")
         if self.options.descriptors:
             self.skip_if_no_sqlite()
+        else:
+            self.skip_if_no_bdb()
 
     def skip_if_no_sqlite(self):
         """Skip the running test if sqlite has not been compiled."""