HD wallet (#1405)

* HD wallet

Minimal set of changes (no refactoring) backported from Bitcoin upstream to make HD wallets work in Dash 0.12.1.x+

* minimal bip44 (hardcoded account and change)

* minimal bip39

Additional cmd-line options for new wallet:
-mnemonic
-mnemonicpassphrase

* Do not recreate HD wallet on encryption

Adjusted keypool.py test

* Do not store any private keys for hd wallet besides the master one

Derive all keys on the fly.
Original idea/implementation - btc PR9298, backported and improved

* actually use bip39

* pbkdf2 test

* backport wallet-hd.py test

* Allow specifying hd seed, add dumphdseed rpc, fix bugs

- -hdseed cmd-line param to specify HD seed on wallet creation
- dumphdseed rpc to dump HD seed
- allow seed of any size
- fix dumpwallet rpc bug (wasn't decrypting HD seed)
- print HD seed and extended public masterkey on dumpwallet

* top up keypool on HD wallet encryption

* split HD chain: external/internal

* add missing cs_wallet lock in init.cpp

* fix `const char *` issues (use strings)

* default mnemonic passphrase is an empty string in all cases

* store mnemonic/mnemonicpassphrase

replace dumphdseed with dumphdinfo

* Add fCrypted flag to CHDChain

* prepare internal structures for multiple HD accounts

(plus some code cleanup)

* use secure allocator for storing sensitive HD data

* use secure strings for mnemonic(passphrase)

* small fix in GenerateNewHDChain

* use 24 words for mnemonic by default

* make sure mnemonic passphrase provided by user does not exceed 256 symbols

* more usage of secure allocators and memory_cleanse

* code cleanup

* rename: CSecureVector -> SecureVector

* add missing include

* fix warning in rpcdump.cpp

* refactor mnemonic_check (also fix a bug)

* move bip39 functions to CMnemonic

* Few fixes for CMnemonic:
- use `SecureVector` for data, bits, seed
- `Check` should return bool

* init vectors with desired size where possible

Author: UdjinM6

qa/pull-tester/rpc-tests.py

@@ -90,6 +90,7 @@
 testScripts = [
     'bip68-112-113-p2p.py',
     'wallet.py',
+    'wallet-hd.py',
     'listtransactions.py',
     'receivedby.py',
     'mempool_resurrect_test.py',

qa/rpc-tests/fundrawtransaction.py

@@ -459,6 +459,7 @@ def run_test(self):
 
         # drain the keypool
         self.nodes[1].getnewaddress()
+        self.nodes[1].getrawchangeaddress()
         inputs = []
         outputs = {self.nodes[0].getnewaddress():1.1}
         rawTx = self.nodes[1].createrawtransaction(inputs, outputs)
@@ -472,6 +473,7 @@ def run_test(self):
 
         #refill the keypool
         self.nodes[1].walletpassphrase("test", 100)
+        self.nodes[1].keypoolrefill(2) #need to refill the keypool to get an internal change address
         self.nodes[1].walletlock()
 
         try:

qa/rpc-tests/keypool.py

@@ -14,36 +14,64 @@ class KeyPoolTest(BitcoinTestFramework):
 
     def run_test(self):
         nodes = self.nodes
+        addr_before_encrypting = nodes[0].getnewaddress()
+        addr_before_encrypting_data = nodes[0].validateaddress(addr_before_encrypting)
+        wallet_info_old = nodes[0].getwalletinfo()
+        assert(addr_before_encrypting_data['hdchainid'] == wallet_info_old['hdchainid'])
+
         # Encrypt wallet and wait to terminate
         nodes[0].encryptwallet('test')
         bitcoind_processes[0].wait()
         # Restart node 0
         nodes[0] = start_node(0, self.options.tmpdir)
         # Keep creating keys
         addr = nodes[0].getnewaddress()
+        addr_data = nodes[0].validateaddress(addr)
+        wallet_info = nodes[0].getwalletinfo()
+        assert(addr_before_encrypting_data['hdchainid'] == wallet_info['hdchainid'])
+        assert(addr_data['hdchainid'] == wallet_info['hdchainid'])
+
         try:
             addr = nodes[0].getnewaddress()
             raise AssertionError('Keypool should be exhausted after one address')
         except JSONRPCException as e:
             assert(e.error['code']==-12)
 
-        # put three new keys in the keypool
+        # put six (plus 2) new keys in the keypool (100% external-, +100% internal-keys, 1 in min)
         nodes[0].walletpassphrase('test', 12000)
-        nodes[0].keypoolrefill(3)
+        nodes[0].keypoolrefill(6)
         nodes[0].walletlock()
+        wi = nodes[0].getwalletinfo()
+        assert_equal(wi['keypoolsize_hd_internal'], 6)
+        assert_equal(wi['keypoolsize'], 6)
+
+        # drain the internal keys
+        nodes[0].getrawchangeaddress()
+        nodes[0].getrawchangeaddress()
+        nodes[0].getrawchangeaddress()
+        nodes[0].getrawchangeaddress()
+        nodes[0].getrawchangeaddress()
+        nodes[0].getrawchangeaddress()
+        # the next one should fail
+        try:
+            nodes[0].getrawchangeaddress()
+            raise AssertionError('Keypool should be exhausted after six addresses')
+        except JSONRPCException as e:
+            assert(e.error['code']==-12)
 
-        # drain the keys
         addr = set()
-        addr.add(nodes[0].getrawchangeaddress())
-        addr.add(nodes[0].getrawchangeaddress())
-        addr.add(nodes[0].getrawchangeaddress())
-        addr.add(nodes[0].getrawchangeaddress())
-        # assert that four unique addresses were returned
-        assert(len(addr) == 4)
+        # drain the external keys
+        addr.add(nodes[0].getnewaddress())
+        addr.add(nodes[0].getnewaddress())
+        addr.add(nodes[0].getnewaddress())
+        addr.add(nodes[0].getnewaddress())
+        addr.add(nodes[0].getnewaddress())
+        addr.add(nodes[0].getnewaddress())
+        assert(len(addr) == 6)
         # the next one should fail
         try:
-            addr = nodes[0].getrawchangeaddress()
-            raise AssertionError('Keypool should be exhausted after three addresses')
+            addr = nodes[0].getnewaddress()
+            raise AssertionError('Keypool should be exhausted after six addresses')
         except JSONRPCException as e:
             assert(e.error['code']==-12)
 
@@ -58,13 +86,18 @@ def run_test(self):
         nodes[0].generate(1)
         nodes[0].generate(1)
         nodes[0].generate(1)
-        nodes[0].generate(1)
         try:
             nodes[0].generate(1)
             raise AssertionError('Keypool should be exhausted after three addesses')
         except JSONRPCException as e:
             assert(e.error['code']==-12)
 
+        nodes[0].walletpassphrase('test', 100)
+        nodes[0].keypoolrefill(100)
+        wi = nodes[0].getwalletinfo()
+        assert_equal(wi['keypoolsize_hd_internal'], 100)
+        assert_equal(wi['keypoolsize'], 100)
+
     def setup_chain(self):
         print("Initializing test directory "+self.options.tmpdir)
         initialize_chain(self.options.tmpdir)

qa/rpc-tests/wallet-hd.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python2
+# coding=utf-8
+# ^^^^^^^^^^^^ TODO remove when supporting only Python3
+# Copyright (c) 2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test Hierarchical Deterministic wallet function."""
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import *
+
+class WalletHDTest(BitcoinTestFramework):
+
+    def setup_chain(self):
+        print("Initializing test directory "+self.options.tmpdir)
+        initialize_chain_clean(self.options.tmpdir, 2)
+
+    def setup_network(self):
+        self.nodes = start_nodes(2, self.options.tmpdir, [['-usehd=0'], ['-usehd=1', '-keypool=0']])
+        self.is_network_split = False
+        connect_nodes_bi(self.nodes, 0, 1)
+        self.is_network_split=False
+        self.sync_all()
+
+    def run_test (self):
+        tmpdir = self.options.tmpdir
+
+        # Make sure can't switch off usehd after wallet creation
+        stop_node(self.nodes[1],1)
+        try:
+            start_node(1, self.options.tmpdir, ['-usehd=0'])
+            raise AssertionError("Must not allow to turn off HD on an already existing HD wallet")
+        except Exception as e:
+            assert("dashd exited with status 1 during initialization" in str(e))
+        # assert_start_raises_init_error(1, self.options.tmpdir, ['-usehd=0'], 'already existing HD wallet')
+        # self.nodes[1] = start_node(1, self.options.tmpdir, self.node_args[1])
+        self.nodes[1] = start_node(1, self.options.tmpdir, ['-usehd=1', '-keypool=0'])
+        connect_nodes_bi(self.nodes, 0, 1)
+
+        # Make sure we use hd, keep chainid
+        chainid = self.nodes[1].getwalletinfo()['hdchainid']
+        assert_equal(len(chainid), 64)
+
+        # create an internal key
+        change_addr = self.nodes[1].getrawchangeaddress()
+        change_addrV= self.nodes[1].validateaddress(change_addr);
+        assert_equal(change_addrV["hdkeypath"], "m/44'/1'/0'/1/0") #first internal child key
+
+        # Import a non-HD private key in the HD wallet
+        non_hd_add = self.nodes[0].getnewaddress()
+        self.nodes[1].importprivkey(self.nodes[0].dumpprivkey(non_hd_add))
+
+        # This should be enough to keep the master key and the non-HD key
+        self.nodes[1].backupwallet(tmpdir + "/hd.bak")
+        #self.nodes[1].dumpwallet(tmpdir + "/hd.dump")
+
+        # Derive some HD addresses and remember the last
+        # Also send funds to each add
+        self.nodes[0].generate(101)
+        hd_add = None
+        num_hd_adds = 300
+        for i in range(num_hd_adds):
+            hd_add = self.nodes[1].getnewaddress()
+            hd_info = self.nodes[1].validateaddress(hd_add)
+            assert_equal(hd_info["hdkeypath"], "m/44'/1'/0'/0/"+str(i+1))
+            assert_equal(hd_info["hdchainid"], chainid)
+            self.nodes[0].sendtoaddress(hd_add, 1)
+            self.nodes[0].generate(1)
+        self.nodes[0].sendtoaddress(non_hd_add, 1)
+        self.nodes[0].generate(1)
+
+        # create an internal key (again)
+        change_addr = self.nodes[1].getrawchangeaddress()
+        change_addrV= self.nodes[1].validateaddress(change_addr);
+        assert_equal(change_addrV["hdkeypath"], "m/44'/1'/0'/1/1") #second internal child key
+
+        self.sync_all()
+        assert_equal(self.nodes[1].getbalance(), num_hd_adds + 1)
+
+        print("Restore backup ...")
+        stop_node(self.nodes[1],1)
+        os.remove(self.options.tmpdir + "/node1/regtest/wallet.dat")
+        shutil.copyfile(tmpdir + "/hd.bak", tmpdir + "/node1/regtest/wallet.dat")
+        self.nodes[1] = start_node(1, self.options.tmpdir, ['-usehd=1', '-keypool=0'])
+        #connect_nodes_bi(self.nodes, 0, 1)
+
+        # Assert that derivation is deterministic
+        hd_add_2 = None
+        for _ in range(num_hd_adds):
+            hd_add_2 = self.nodes[1].getnewaddress()
+            hd_info_2 = self.nodes[1].validateaddress(hd_add_2)
+            assert_equal(hd_info_2["hdkeypath"], "m/44'/1'/0'/0/"+str(_+1))
+            assert_equal(hd_info_2["hdchainid"], chainid)
+        assert_equal(hd_add, hd_add_2)
+
+        # Needs rescan
+        stop_node(self.nodes[1],1)
+        self.nodes[1] = start_node(1, self.options.tmpdir, ['-usehd=1', '-keypool=0', '-rescan'])
+        #connect_nodes_bi(self.nodes, 0, 1)
+        assert_equal(self.nodes[1].getbalance(), num_hd_adds + 1)
+
+        # send a tx and make sure its using the internal chain for the changeoutput
+        txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1)
+        outs = self.nodes[1].decoderawtransaction(self.nodes[1].gettransaction(txid)['hex'])['vout'];
+        keypath = ""
+        for out in outs:
+            if out['value'] != 1:
+                keypath = self.nodes[1].validateaddress(out['scriptPubKey']['addresses'][0])['hdkeypath']
+
+        assert_equal(keypath[0:13], "m/44'/1'/0'/1")
+
+if __name__ == '__main__':
+    WalletHDTest().main ()

src/Makefile.am

@@ -73,6 +73,8 @@ BITCOIN_CORE_H = \
   amount.h \
   arith_uint256.h \
   base58.h \
+  bip39.h \
+  bip39_english.h \
   bloom.h \
   cachemap.h \
   cachemultimap.h \
@@ -108,6 +110,7 @@ BITCOIN_CORE_H = \
   governance-votedb.h \
   flat-database.h \
   hash.h \
+  hdchain.h \
   httprpc.h \
   httpserver.h \
   init.h \
@@ -260,7 +263,6 @@ libbitcoin_wallet_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES)
 libbitcoin_wallet_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 libbitcoin_wallet_a_SOURCES = \
   activemasternode.cpp \
-  privatesend-client.cpp \
   dsnotificationinterface.cpp \
   instantx.cpp \
   masternode.cpp \
@@ -269,6 +271,7 @@ libbitcoin_wallet_a_SOURCES = \
   masternodeconfig.cpp \
   masternodeman.cpp \
   keepass.cpp \
+  privatesend-client.cpp \
   wallet/crypter.cpp \
   wallet/db.cpp \
   wallet/rpcdump.cpp \
@@ -329,13 +332,15 @@ libbitcoin_common_a_SOURCES = \
   amount.cpp \
   arith_uint256.cpp \
   base58.cpp \
+  bip39.cpp \
   chainparams.cpp \
   coins.cpp \
   compressor.cpp \
   consensus/merkle.cpp \
   core_read.cpp \
   core_write.cpp \
   hash.cpp \
+  hdchain.cpp \
   key.cpp \
   keystore.cpp \
   netbase.cpp \

src/Makefile.qt.include

@@ -194,6 +194,8 @@ RES_ICONS = \
   qt/res/icons/drkblue/eye_minus.png \
   qt/res/icons/drkblue/eye_plus.png \
   qt/res/icons/drkblue/filesave.png \
+  qt/res/icons/drkblue/hd_disabled.png \
+  qt/res/icons/drkblue/hd_enabled.png \
   qt/res/icons/drkblue/history.png \
   qt/res/icons/drkblue/key.png \
   qt/res/icons/drkblue/lock_closed.png \
@@ -242,6 +244,8 @@ RES_ICONS = \
   qt/res/icons/crownium/eye_minus.png \
   qt/res/icons/crownium/eye_plus.png \
   qt/res/icons/crownium/filesave.png \
+  qt/res/icons/crownium/hd_disabled.png \
+  qt/res/icons/crownium/hd_enabled.png \
   qt/res/icons/crownium/history.png \
   qt/res/icons/crownium/key.png \
   qt/res/icons/crownium/lock_closed.png \
@@ -290,6 +294,8 @@ RES_ICONS = \
   qt/res/icons/light/eye_minus.png \
   qt/res/icons/light/eye_plus.png \
   qt/res/icons/light/filesave.png \
+  qt/res/icons/light/hd_disabled.png \
+  qt/res/icons/light/hd_enabled.png \
   qt/res/icons/light/history.png \
   qt/res/icons/light/key.png \
   qt/res/icons/light/lock_closed.png \
@@ -338,6 +344,8 @@ RES_ICONS = \
   qt/res/icons/trad/eye_minus.png \
   qt/res/icons/trad/eye_plus.png \
   qt/res/icons/trad/filesave.png \
+  qt/res/icons/trad/hd_disabled.png \
+  qt/res/icons/trad/hd_enabled.png \
   qt/res/icons/trad/history.png \
   qt/res/icons/trad/key.png \
   qt/res/icons/trad/lock_closed.png \

src/Makefile.test.include

@@ -24,6 +24,7 @@ JSON_TEST_FILES = \
   test/data/base58_keys_valid.json \
   test/data/base58_encode_decode.json \
   test/data/base58_keys_invalid.json \
+  test/data/bip39_vectors.json \
   test/data/tx_invalid.json \
   test/data/tx_valid.json \
   test/data/sighash.json
@@ -42,6 +43,7 @@ BITCOIN_TESTS =\
   test/base58_tests.cpp \
   test/base64_tests.cpp \
   test/bip32_tests.cpp \
+  test/bip39_tests.cpp \
   test/bloom_tests.cpp \
   test/bswap_tests.cpp \
   test/cachemap_tests.cpp \