From bcac7a31e0a0223d2a538fd6c9564a0efd62f9f4 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Thu, 1 Sep 2022 15:33:34 -0400 Subject: [PATCH 01/17] Merge bitcoin/bitcoin#19602: wallet: Migrate legacy wallets to descriptor wallets 53e7ed075c49f853cc845afc7b2f058cabad0cb0 doc: Release notes and other docs for migration (Andrew Chow) 9c44bfe244f35f08ba576d8b979a90dcd68d2c77 Test migratewallet (Andrew Chow) 0b26e7cdf2659fd8b54d21fd2bd749f9f3e87af8 descriptors: addr() and raw() should return false for ToPrivateString (Andrew Chow) 31764c3f872f4f01b48d50585f86e97c41554954 Add migratewallet RPC (Andrew Chow) 0bf7b38bff422e7413bcd3dc0abe2568dd918ddc Implement MigrateLegacyToDescriptor (Andrew Chow) e7b16f925ae5b117e8b74ce814b63e19b19b50f4 Implement MigrateToSQLite (Andrew Chow) 5b62f095e790a0d4e2a70ece89465b64fc68358a wallet: Refactor SetupDescSPKMs to take CExtKey (Andrew Chow) 22401f17e026ead4bc3fe96967eec56a719a4f75 Implement LegacyScriptPubKeyMan::DeleteRecords (Andrew Chow) 35f428fae68ad974abdce0fa905148f620a9443c Implement LegacyScriptPubKeyMan::MigrateToDescriptor (Andrew Chow) ea1ab390e4dac128e3a37d4884528c3f4128ed83 scriptpubkeyman: Implement GetScriptPubKeys in Legacy (Andrew Chow) e664af29760527e75cd7e290be5f102b6d29ebee Apply label to all scriptPubKeys of imported combo() (Andrew Chow) Pull request description: This PR adds a new `migratewallet` RPC which migrates a legacy wallet to a descriptor wallet. Migrated wallets will need a new backup. If a wallet has watchonly stuff in it, a new watchonly descriptor wallet will be created containing those watchonly things. The related transactions, labels, and descriptors for those watchonly things will be removed from the original wallet. Migrated wallets will not have any of the legacy things be available for fetching from `getnewaddress` or `getrawchangeaddress`. Wallets that have private keys enabled will have newly generated descriptors. Wallets with private keys disabled will not have any active `ScriptPubKeyMan`s. For the basic HD wallet case of just generated keys, in addition to the standard descriptor wallet descriptors using the master key derived from the pre-existing hd seed, the migration will also create 3 descriptors for each HD chain in: a ranged combo external, a ranged combo internal, and a single key combo for the seed (the seed is a valid key that we can receive coins at!). The migrated wallet will then have newly generated descriptors as the active `ScriptPubKeyMan`s. This is equivalent to creating a new descriptor wallet and importing the 3 descriptors for each HD chain. For wallets containing non-HD keys, each key will have its own combo descriptor. There are also tests. ACKs for top commit: Sjors: tACK 53e7ed075c49f853cc845afc7b2f058cabad0cb0 w0xlt: reACK https://github.com/bitcoin/bitcoin/commit/53e7ed075c49f853cc845afc7b2f058cabad0cb0 Tree-SHA512: c0c003694ca2e17064922d08e8464278d314e970efb7df874b4fe04ec5d124c7206409ca701c65c099d17779ab2136ae63f1da2a9dba39b45f6d62cf93b5c60a --- doc/managing-wallets.md | 21 ++ doc/release-notes-19602.md | 9 + src/script/descriptor.cpp | 4 +- src/wallet/rpc/wallet.cpp | 54 +++ src/wallet/scriptpubkeyman.cpp | 330 +++++++++++++++++- src/wallet/scriptpubkeyman.h | 18 +- src/wallet/test/ismine_tests.cpp | 20 ++ src/wallet/wallet.cpp | 515 +++++++++++++++++++++++++++- src/wallet/wallet.h | 27 +- src/wallet/walletdb.cpp | 40 +++ src/wallet/walletdb.h | 6 + src/wallet/walletutil.h | 17 + test/functional/test_runner.py | 1 + test/functional/tool_wallet.py | 10 +- test/functional/wallet_migration.py | 420 +++++++++++++++++++++++ 15 files changed, 1456 insertions(+), 36 deletions(-) create mode 100644 doc/release-notes-19602.md create mode 100755 test/functional/wallet_migration.py diff --git a/doc/managing-wallets.md b/doc/managing-wallets.md index 584892fba64a..ca89186edd2f 100644 --- a/doc/managing-wallets.md +++ b/doc/managing-wallets.md @@ -121,3 +121,24 @@ $ dash-cli -rpcwallet="restored-wallet" getwalletinfo ``` The restored wallet can also be loaded in the GUI via `File` ->`Open wallet`. + +## Migrating Legacy Wallets to Descriptor Wallets + +Legacy wallets (traditional non-descriptor wallets) can be migrated to become Descriptor wallets +through the use of the `migratewallet` RPC. Migrated wallets will have all of their addresses and private keys added to +a newly created Descriptor wallet that has the same name as the original wallet. Because Descriptor +wallets do not support having private keys and watch-only scripts, there may be up to two +additional wallets created after migration. In addition to a descriptor wallet of the same name, +there may also be a wallet named `_watchonly` and `_solvables`. `_watchonly` +contains all of the watchonly scripts. `_solvables` contains any scripts which the wallet +knows but is not watching the corresponding P2SH scripts. + +Given that there is an extremely large number of possible configurations for the scripts that +Legacy wallets can know about, be watching for, and be able to sign for, `migratewallet` only +makes a best effort attempt to capture all of these things into Descriptor wallets. There may be +unforeseen configurations which result in some scripts being excluded. If a migration fails +unexpectedly or otherwise misses any scripts, please create an issue on GitHub. A backup of the +original wallet can be found in the wallet directory with the name `-.legacy.bak`. + +The backup can be restored using the `restorewallet` command as discussed in the +[Restoring the Wallet From a Backup](#16-restoring-the-wallet-from-a-backup) section diff --git a/doc/release-notes-19602.md b/doc/release-notes-19602.md new file mode 100644 index 000000000000..fc0a68a4a1a8 --- /dev/null +++ b/doc/release-notes-19602.md @@ -0,0 +1,9 @@ +Wallet +====== + +Migrating Legacy Wallets to Descriptor Wallets +--------------------------------------------- + +An experimental RPC `migratewallet` has been added to migrate Legacy (non-descriptor) wallets to +Descriptor wallets. More information about the migration process is available in the +[documentation](https://github.com/dashpay/dash/blob/master/doc/managing-wallets.md#migrating-legacy-wallets-to-descriptor-wallets). diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 49d14db91562..cf5a392faa44 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -587,7 +587,7 @@ class DescriptorImpl : public Descriptor return AddChecksum(ret); } - bool ToPrivateString(const SigningProvider& arg, std::string& out) const override final + bool ToPrivateString(const SigningProvider& arg, std::string& out) const override { bool ret = ToStringHelper(&arg, out, StringType::PRIVATE); out = AddChecksum(out); @@ -677,6 +677,7 @@ class AddressDescriptor final : public DescriptorImpl } } bool IsSingleType() const final { return true; } + bool ToPrivateString(const SigningProvider& arg, std::string& out) const final { return false; } }; /** A parsed raw(H) descriptor. */ @@ -702,6 +703,7 @@ class RawDescriptor final : public DescriptorImpl } } bool IsSingleType() const final { return true; } + bool ToPrivateString(const SigningProvider& arg, std::string& out) const final { return false; } }; /** A parsed pk(P) descriptor. */ diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index cd9740c8ab1f..8a8156a7ec72 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -1103,6 +1103,59 @@ RPCHelpMan simulaterawtransaction() }; } +static RPCHelpMan migratewallet() +{ + return RPCHelpMan{"migratewallet", + "EXPERIMENTAL warning: This call may not work as expected and may be changed in future releases\n" + "\nMigrate the wallet to a descriptor wallet.\n" + "A new wallet backup will need to be made.\n" + "\nThe migration process will create a backup of the wallet before migrating. This backup\n" + "file will be named -.legacy.bak and can be found in the directory\n" + "for this wallet. In the event of an incorrect migration, the backup can be restored using restorewallet." + + HELP_REQUIRING_PASSPHRASE, + {}, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "wallet_name", "The name of the primary migrated wallet"}, + {RPCResult::Type::STR, "watchonly_name", /*optional=*/true, "The name of the migrated wallet containing the watchonly scripts"}, + {RPCResult::Type::STR, "solvables_name", /*optional=*/true, "The name of the migrated wallet containing solvable but not watched scripts"}, + {RPCResult::Type::STR, "backup_path", "The location of the backup of the original wallet"}, + } + }, + RPCExamples{ + HelpExampleCli("migratewallet", "") + + HelpExampleRpc("migratewallet", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + std::shared_ptr wallet = GetWalletForJSONRPCRequest(request); + if (!wallet) return NullUniValue; + + EnsureWalletIsUnlocked(*wallet); + + WalletContext& context = EnsureWalletContext(request.context); + + util::Result res = MigrateLegacyToDescriptor(std::move(wallet), context); + if (!res) { + throw JSONRPCError(RPC_WALLET_ERROR, util::ErrorString(res).original); + } + + UniValue r{UniValue::VOBJ}; + r.pushKV("wallet_name", res->wallet_name); + if (res->watchonly_wallet) { + r.pushKV("watchonly_name", res->watchonly_wallet->GetName()); + } + if (res->solvables_wallet) { + r.pushKV("solvables_name", res->solvables_wallet->GetName()); + } + r.pushKV("backup_path", fs::PathToString(res->backup_path)); + + return r; + }, + }; +} + // addresses RPCHelpMan getaddressinfo(); RPCHelpMan getnewaddress(); @@ -1223,6 +1276,7 @@ Span GetWalletRPCCommands() {"wallet", &listwallets}, {"wallet", &loadwallet}, {"wallet", &lockunspent}, + {"wallet", &migratewallet}, {"wallet", &newkeypool}, {"wallet", &removeprunedfunds}, {"wallet", &rescanblockchain}, diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 221e3826722c..f3a752910ada 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -1235,9 +1235,10 @@ bool LegacyScriptPubKeyMan::GetKeyOrigin(const CKeyID& keyID, KeyOriginInfo& inf { LOCK(cs_KeyStore); auto it = mapKeyMetadata.find(keyID); - if (it != mapKeyMetadata.end()) { - meta = it->second; + if (it == mapKeyMetadata.end()) { + return false; } + meta = it->second; } if (meta.has_key_origin) { std::copy(meta.key_origin.fingerprint, meta.key_origin.fingerprint + 4, info.fingerprint); @@ -1820,6 +1821,319 @@ bool LegacyScriptPubKeyMan::GetHDChain(CHDChain& hdChainRet) const return !m_hd_chain.IsNull(); } +std::unordered_set LegacyScriptPubKeyMan::GetScriptPubKeys() const +{ + LOCK(cs_KeyStore); + std::unordered_set spks; + + // All keys have at least P2PK and P2PKH + for (const auto& key_pair : mapKeys) { + const CPubKey& pub = key_pair.second.GetPubKey(); + spks.insert(GetScriptForRawPubKey(pub)); + spks.insert(GetScriptForDestination(PKHash(pub))); + } + for (const auto& key_pair : mapCryptedKeys) { + const CPubKey& pub = key_pair.second.first; + spks.insert(GetScriptForRawPubKey(pub)); + spks.insert(GetScriptForDestination(PKHash(pub))); + } + // Dash: HD keys are stored in mapHdPubKeys, not in mapKeys/mapCryptedKeys + // Only P2PKH is used for HD keys in Dash (BIP44) + for (const auto& key_pair : mapHdPubKeys) { + const CPubKey& pub = key_pair.second.extPubKey.pubkey; + spks.insert(GetScriptForDestination(PKHash(pub))); + } + + // For every script in mapScript, only the ISMINE_SPENDABLE ones are being tracked. + // The watchonly ones will be in setWatchOnly which we deal with later + // For all keys, if they have segwit scripts, those scripts will end up in mapScripts + for (const auto& script_pair : mapScripts) { + const CScript& script = script_pair.second; + if (IsMine(script) == ISMINE_SPENDABLE) { + // Add ScriptHash for scripts that are not already P2SH + if (!script.IsPayToScriptHash()) { + spks.insert(GetScriptForDestination(ScriptHash(script))); + } + } else { + // Multisigs are special. They don't show up as ISMINE_SPENDABLE unless they are in a P2SH + // So check the P2SH of a multisig to see if we should insert it + std::vector> sols; + TxoutType type = Solver(script, sols); + if (type == TxoutType::MULTISIG) { + CScript ms_spk = GetScriptForDestination(ScriptHash(script)); + if (IsMine(ms_spk) != ISMINE_NO) { + spks.insert(ms_spk); + } + } + } + } + + // All watchonly scripts are raw + spks.insert(setWatchOnly.begin(), setWatchOnly.end()); + + return spks; +} + +std::optional LegacyScriptPubKeyMan::MigrateToDescriptor() +{ + LOCK(cs_KeyStore); + if (m_storage.IsLocked(false)) { + return std::nullopt; + } + + MigrationData out; + + std::unordered_set spks{GetScriptPubKeys()}; + + // Get all key ids + std::set keyids; + for (const auto& key_pair : mapKeys) { + keyids.insert(key_pair.first); + } + for (const auto& key_pair : mapCryptedKeys) { + keyids.insert(key_pair.first); + } + + // Get key metadata and figure out which keys don't have a seed + // Note that we do not ignore the seeds themselves because they are considered IsMine! + // In Dash, HD keys are tracked via mapHdPubKeys, not via metadata fields + for (auto keyid_it = keyids.begin(); keyid_it != keyids.end();) { + const CKeyID& keyid = *keyid_it; + if (mapHdPubKeys.count(keyid) > 0) { + // This key belongs to the HD chain, will be handled below + keyid_it = keyids.erase(keyid_it); + continue; + } + keyid_it++; + } + + // keyids is now all non-HD keys. Each key will have its own combo descriptor + for (const CKeyID& keyid : keyids) { + CKey key; + if (!GetKey(keyid, key)) { + assert(false); + } + + // Get birthdate from key meta + uint64_t creation_time = 0; + const auto& it = mapKeyMetadata.find(keyid); + if (it != mapKeyMetadata.end()) { + creation_time = it->second.nCreateTime; + } + + // Get the key origin + // Maybe this doesn't matter because floating keys here shouldn't have origins + KeyOriginInfo info; + bool has_info = GetKeyOrigin(keyid, info); + std::string origin_str = has_info ? "[" + HexStr(info.fingerprint) + FormatHDKeypath(info.path) + "]" : ""; + + // Construct the combo descriptor + std::string desc_str = "combo(" + origin_str + HexStr(key.GetPubKey()) + ")"; + FlatSigningProvider keys; + std::string error; + std::unique_ptr desc = Parse(desc_str, keys, error, false); + WalletDescriptor w_desc(std::move(desc), creation_time, 0, 0, 0); + + // Make the DescriptorScriptPubKeyMan and get the scriptPubKeys + auto desc_spk_man = std::unique_ptr(new DescriptorScriptPubKeyMan(m_storage, w_desc)); + desc_spk_man->AddDescriptorKey(key, key.GetPubKey()); + desc_spk_man->TopUp(); + auto desc_spks = desc_spk_man->GetScriptPubKeys(); + + // Remove the scriptPubKeys from our current set + for (const CScript& spk : desc_spks) { + size_t erased = spks.erase(spk); + assert(erased == 1); + assert(IsMine(spk) == ISMINE_SPENDABLE); + } + + out.desc_spkms.push_back(std::move(desc_spk_man)); + } + + // Handle HD keys: build one inactive combo() descriptor per BIP44 chain + // (external + internal). This mirrors Bitcoin Core's MigrateToDescriptor — + // combo descriptors emit P2PK + P2PKH + P2SH-P2PKH for every derived index, + // so the migrated wallet's script-centric IsMine recognizes every script + // form the legacy keyid-centric IsMine matched via HaveKey(). The active + // address-providing pkh() descriptors are still created later by + // SetupDescriptorScriptPubKeyMans() in ApplyMigrationData; the combos sit + // alongside as non-active history coverage. + if (!m_hd_chain.IsNull()) { + // Decrypt the HD chain if the wallet is encrypted + CHDChain hdChainDecrypted; + if (m_hd_chain.IsCrypted()) { + if (!m_storage.WithEncryptionKey([&](const CKeyingMaterial& encryption_key) { + return DecryptHDChain(encryption_key, hdChainDecrypted); + })) { + throw std::runtime_error(std::string(__func__) + ": DecryptHDChain failed"); + } + if (hdChainDecrypted.GetID() != hdChainDecrypted.GetSeedHash()) { + throw std::runtime_error(std::string(__func__) + ": Wrong HD chain!"); + } + } else { + hdChainDecrypted = m_hd_chain; + } + + CHDAccount acc; + if (!hdChainDecrypted.GetAccount(0, acc)) { + throw std::runtime_error(std::string(__func__) + ": GetAccount(0) failed"); + } + + // Derive the master key from the seed and extract the mnemonic for both + // the migration combo descriptors below and the active descriptors that + // ApplyMigrationData will create via SetupDescriptorScriptPubKeyMans. + CExtKey master_key; + master_key.SetSeed(MakeByteSpan(hdChainDecrypted.GetSeed())); + out.master_key = master_key; + SecureString ssMnemonic, ssMnemonicPassphrase; + if (hdChainDecrypted.GetMnemonic(ssMnemonic, ssMnemonicPassphrase)) { + out.mnemonic = ssMnemonic; + out.mnemonic_passphrase = ssMnemonicPassphrase; + } + + // Build per-chain inactive combo() descriptors with range_end set to the + // legacy chain counter so TopUp() populates every historical index. + const std::string xpub_str = EncodeExtPubKey(master_key.Neuter()); + for (int i = 0; i < 2; ++i) { + const uint32_t chain_counter = (i == 1) ? acc.nInternalChainCounter : acc.nExternalChainCounter; + const std::string desc_str = strprintf("combo(%s/%d'/%d'/0'/%d/*)", + xpub_str, BIP32_PURPOSE_STANDARD, Params().ExtCoinType(), i); + FlatSigningProvider keys; + std::string error; + std::unique_ptr desc = Parse(desc_str, keys, error, false); + if (!desc) { + throw std::runtime_error(std::string(__func__) + ": failed to parse migration combo descriptor: " + error); + } + WalletDescriptor w_desc(std::move(desc), 0, 0, chain_counter, 0); + + auto desc_spk_man = std::unique_ptr(new DescriptorScriptPubKeyMan(m_storage, w_desc)); + desc_spk_man->AddDescriptorKey(master_key.key, master_key.key.GetPubKey(), out.mnemonic, out.mnemonic_passphrase); + desc_spk_man->TopUp(); + auto desc_spks = desc_spk_man->GetScriptPubKeys(); + + // Erase every script form combo just produced from the tracking set. + // Legacy GetScriptPubKeys only enumerates P2PKH for HD keys, so the + // P2PK and P2SH-P2PKH forms are not in `spks` — only erase what's there. + for (const CScript& spk : desc_spks) { + spks.erase(spk); + } + + out.desc_spkms.push_back(std::move(desc_spk_man)); + } + } + + // Handle the rest of the scriptPubKeys which must be imports and may not have all info + for (auto it = spks.begin(); it != spks.end();) { + const CScript& spk = *it; + + // Get birthdate from script meta + uint64_t creation_time = 0; + const auto& mit = m_script_metadata.find(CScriptID(spk)); + if (mit != m_script_metadata.end()) { + creation_time = mit->second.nCreateTime; + } + + // InferDescriptor as that will get us all the solving info if it is there + std::unique_ptr desc = InferDescriptor(spk, *GetSolvingProvider(spk)); + // Get the private keys for this descriptor + std::vector scripts; + FlatSigningProvider keys; + if (!desc->Expand(0, DUMMY_SIGNING_PROVIDER, scripts, keys)) { + assert(false); + } + std::set privkeyids; + for (const auto& key_orig_pair : keys.origins) { + privkeyids.insert(key_orig_pair.first); + } + + std::vector desc_spks; + + // Make the descriptor string with private keys + std::string desc_str; + bool watchonly = !desc->ToPrivateString(*this, desc_str); + if (watchonly && !m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + out.watch_descs.push_back({desc->ToString(), creation_time}); + + // Get the scriptPubKeys without writing this to the wallet + FlatSigningProvider provider; + desc->Expand(0, provider, desc_spks, provider); + } else { + // Make the DescriptorScriptPubKeyMan and get the scriptPubKeys + WalletDescriptor w_desc(std::move(desc), creation_time, 0, 0, 0); + auto desc_spk_man = std::unique_ptr(new DescriptorScriptPubKeyMan(m_storage, w_desc)); + for (const auto& keyid : privkeyids) { + CKey key; + if (!GetKey(keyid, key)) { + continue; + } + desc_spk_man->AddDescriptorKey(key, key.GetPubKey()); + } + desc_spk_man->TopUp(); + auto desc_spks_set = desc_spk_man->GetScriptPubKeys(); + desc_spks.insert(desc_spks.end(), desc_spks_set.begin(), desc_spks_set.end()); + + out.desc_spkms.push_back(std::move(desc_spk_man)); + } + + // Remove the scriptPubKeys from our current set + for (const CScript& desc_spk : desc_spks) { + auto del_it = spks.find(desc_spk); + assert(del_it != spks.end()); + assert(IsMine(desc_spk) != ISMINE_NO); + it = spks.erase(del_it); + } + } + + // Multisigs are special. They don't show up as ISMINE_SPENDABLE unless they are in a P2SH + // So we have to check if any of our scripts are a multisig and if so, add the P2SH + for (const auto& script_pair : mapScripts) { + const CScript script = script_pair.second; + + // Get birthdate from script meta + uint64_t creation_time = 0; + const auto& it = m_script_metadata.find(CScriptID(script)); + if (it != m_script_metadata.end()) { + creation_time = it->second.nCreateTime; + } + + std::vector> sols; + TxoutType type = Solver(script, sols); + if (type == TxoutType::MULTISIG) { + CScript sh_spk = GetScriptForDestination(ScriptHash(script)); + + // We only want the multisigs that we have not already seen, i.e. they are not watchonly and not spendable + // For P2SH, a multisig is not ISMINE_NO when: + // * All keys are in the wallet + // * The multisig itself is watch only + // * The P2SH is watch only + // For P2SH-P2WSH, if the script is in the wallet, then it will have the same conditions as P2SH. + // For P2WSH, a multisig is not ISMINE_NO when, other than the P2SH conditions: + // * The P2WSH script is in the wallet and it is being watched + std::vector> keys(sols.begin() + 1, sols.begin() + sols.size() - 1); + if (HaveWatchOnly(sh_spk) || HaveWatchOnly(script) || HaveKeys(keys, *this)) { + // The above emulates IsMine for these 3 scriptPubKeys, so double check that by running IsMine + assert(IsMine(sh_spk) != ISMINE_NO); + continue; + } + assert(IsMine(sh_spk) == ISMINE_NO); + + std::unique_ptr sh_desc = InferDescriptor(sh_spk, *GetSolvingProvider(sh_spk)); + out.solvable_descs.push_back({sh_desc->ToString(), creation_time}); + } + } + + // Make sure that we have accounted for all scriptPubKeys + assert(spks.size() == 0); + return out; +} + +bool LegacyScriptPubKeyMan::DeleteRecords() +{ + LOCK(cs_KeyStore); + WalletBatch batch(m_storage.GetDatabase()); + return batch.EraseRecords(DBKeys::LEGACY_TYPES); +} + util::Result DescriptorScriptPubKeyMan::GetNewDestination() { // Returns true if this descriptor supports getting new addresses. Conditions where we may be unable to fetch them (e.g. locked) are caught later @@ -2072,11 +2386,11 @@ std::vector DescriptorScriptPubKeyMan::MarkUnusedAddresses(Wa return result; } -void DescriptorScriptPubKeyMan::AddDescriptorKey(const CKey& key, const CPubKey &pubkey) +void DescriptorScriptPubKeyMan::AddDescriptorKey(const CKey& key, const CPubKey &pubkey, const SecureString& mnemonic, const SecureString& mnemonic_passphrase) { LOCK(cs_desc_man); WalletBatch batch(m_storage.GetDatabase()); - if (!AddDescriptorKeyWithDB(batch, key, pubkey, "", "")) { + if (!AddDescriptorKeyWithDB(batch, key, pubkey, mnemonic, mnemonic_passphrase)) { throw std::runtime_error(std::string(__func__) + ": writing descriptor private key failed"); } } @@ -2513,19 +2827,19 @@ WalletDescriptor DescriptorScriptPubKeyMan::GetWalletDescriptor() const return m_wallet_descriptor; } -std::vector DescriptorScriptPubKeyMan::GetScriptPubKeys() const +std::unordered_set DescriptorScriptPubKeyMan::GetScriptPubKeys() const { return GetScriptPubKeys(0); } -std::vector DescriptorScriptPubKeyMan::GetScriptPubKeys(int32_t minimum_index) const +std::unordered_set DescriptorScriptPubKeyMan::GetScriptPubKeys(int32_t minimum_index) const { LOCK(cs_desc_man); - std::vector script_pub_keys; + std::unordered_set script_pub_keys; script_pub_keys.reserve(m_map_script_pub_keys.size()); for (auto const& [script_pub_key, index] : m_map_script_pub_keys) { - if (index >= minimum_index) script_pub_keys.push_back(script_pub_key); + if (index >= minimum_index) script_pub_keys.insert(script_pub_key); } return script_pub_keys; } diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 060e0cb2720e..1facd55bd452 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -239,6 +239,9 @@ class ScriptPubKeyMan virtual uint256 GetID() const { return uint256(); } + /** Returns a set of all the scriptPubKeys that this ScriptPubKeyMan watches */ + virtual std::unordered_set GetScriptPubKeys() const { return {}; }; + /** Prepends the wallet name in logging output to ease debugging in multi-wallet use cases */ template void WalletLogPrintf(std::string fmt, Params... parameters) const { @@ -252,6 +255,8 @@ class ScriptPubKeyMan boost::signals2::signal NotifyCanGetAddressesChanged; }; +class DescriptorScriptPubKeyMan; + class LegacyScriptPubKeyMan : public ScriptPubKeyMan, public FillableSigningProvider { private: @@ -507,6 +512,13 @@ class LegacyScriptPubKeyMan : public ScriptPubKeyMan, public FillableSigningProv const std::map& GetAllReserveKeys() const { return m_pool_key_to_index; } std::set GetKeys() const override; + std::unordered_set GetScriptPubKeys() const override; + + /** Get the DescriptorScriptPubKeyMans (with private keys) that have the same scriptPubKeys as this LegacyScriptPubKeyMan. + * Does not modify this ScriptPubKeyMan. */ + std::optional MigrateToDescriptor(); + /** Delete all the records ofthis LegacyScriptPubKeyMan from disk*/ + bool DeleteRecords(); }; /** Wraps a LegacyScriptPubKeyMan so that it can be returned in a new unique_ptr. Does not provide privkeys */ @@ -636,12 +648,12 @@ class DescriptorScriptPubKeyMan : public ScriptPubKeyMan bool HasWalletDescriptor(const WalletDescriptor& desc) const; void UpdateWalletDescriptor(WalletDescriptor& descriptor); bool CanUpdateToWalletDescriptor(const WalletDescriptor& descriptor, std::string& error); - void AddDescriptorKey(const CKey& key, const CPubKey &pubkey); + void AddDescriptorKey(const CKey& key, const CPubKey &pubkey, const SecureString& mnemonic = "", const SecureString& mnemonic_passphrase = ""); void WriteDescriptor(); WalletDescriptor GetWalletDescriptor() const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); - std::vector GetScriptPubKeys() const; - std::vector GetScriptPubKeys(int32_t minimum_index) const; + std::unordered_set GetScriptPubKeys() const override; + std::unordered_set GetScriptPubKeys(int32_t minimum_index) const; int32_t GetEndRange() const; bool GetDescriptorString(std::string& out, const bool priv) const; diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index 27a16f116296..a9a54b575aea 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -44,11 +44,13 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Keystore does not have key result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); // Keystore has key BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 1); } // P2PK uncompressed @@ -61,11 +63,13 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Keystore does not have key result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); // Keystore has key BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 1); } // P2PKH compressed @@ -78,11 +82,13 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Keystore does not have key result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); // Keystore has key BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 1); } // P2PKH uncompressed @@ -95,11 +101,13 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Keystore does not have key result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); // Keystore has key BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 1); } // P2SH @@ -114,16 +122,19 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Keystore does not have redeemScript or key result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); // Keystore has redeemScript but no key BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(redeemScript)); result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); // Keystore has redeemScript and key BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 1); } // (P2PKH inside) P2SH inside P2SH (invalid) @@ -142,6 +153,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); } // scriptPubKey multisig @@ -155,24 +167,28 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Keystore does not have any keys result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); // Keystore has 1/2 keys BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); // Keystore has 2/2 keys BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); // Keystore has 2/2 keys and the script BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); } // P2SH multisig @@ -189,11 +205,13 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Keystore has no redeemScript result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); // Keystore has redeemScript BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(redeemScript)); result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 1); } // OP_RETURN @@ -208,6 +226,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); } // Nonstandard @@ -222,6 +241,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); } } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 773d1dd6cc7d..e01dce4b099f 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -4003,24 +4003,10 @@ void CWallet::LoadDescriptorScriptPubKeyMan(uint256 id, WalletDescriptor& desc) } } -void CWallet::SetupDescriptorScriptPubKeyMans(const SecureString& mnemonic_arg, const SecureString mnemonic_passphrase) +void CWallet::SetupDescriptorScriptPubKeyMans(const CExtKey& master_key, const SecureString& mnemonic, const SecureString mnemonic_passphrase) { AssertLockHeld(cs_wallet); - if (!IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)) { - // Make a seed - // TODO: remove duplicated code with CHDChain::SetMnemonic - const SecureString mnemonic = mnemonic_arg.empty() ? CMnemonic::Generate(m_args.GetIntArg("-mnemonicbits", CHDChain::DEFAULT_MNEMONIC_BITS)) : mnemonic_arg; - if (!CMnemonic::Check(mnemonic)) { - throw std::runtime_error(std::string(__func__) + ": invalid mnemonic: `" + std::string(mnemonic) + "`"); - } - SecureVector seed_key; - CMnemonic::ToSeed(mnemonic, mnemonic_passphrase, seed_key); - - // Get the extended key - CExtKey master_key; - master_key.SetSeed(MakeByteSpan(seed_key)); - for (auto type : {PathDerivationType::BIP44_External, PathDerivationType::BIP44_Internal, PathDerivationType::DIP0009_CoinJoin}) { { // OUTPUT_TYPE is only one: LEGACY auto spk_manager = std::unique_ptr(new DescriptorScriptPubKeyMan(*this)); @@ -4040,6 +4026,27 @@ void CWallet::SetupDescriptorScriptPubKeyMans(const SecureString& mnemonic_arg, } } } +} + +void CWallet::SetupDescriptorScriptPubKeyMans(const SecureString& mnemonic_arg, const SecureString mnemonic_passphrase) +{ + AssertLockHeld(cs_wallet); + + if (!IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)) { + // Make a seed + // TODO: remove duplicated code with CHDChain::SetMnemonic + const SecureString mnemonic = mnemonic_arg.empty() ? CMnemonic::Generate(m_args.GetIntArg("-mnemonicbits", CHDChain::DEFAULT_MNEMONIC_BITS)) : mnemonic_arg; + if (!CMnemonic::Check(mnemonic)) { + throw std::runtime_error(std::string(__func__) + ": invalid mnemonic: `" + std::string(mnemonic) + "`"); + } + SecureVector seed_key; + CMnemonic::ToSeed(mnemonic, mnemonic_passphrase, seed_key); + + // Get the extended key + CExtKey master_key; + master_key.SetSeed(MakeByteSpan(seed_key)); + + SetupDescriptorScriptPubKeyMans(master_key, mnemonic, mnemonic_passphrase); } else { ExternalSigner signer = ExternalSignerScriptPubKeyMan::GetExternalSigner(); @@ -4211,9 +4218,13 @@ ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const Flat return nullptr; } - CTxDestination dest; - if (!internal && ExtractDestination(script_pub_keys.at(0), dest)) { - SetAddressBook(dest, label, "receive"); + if (!internal) { + for (const auto& script : script_pub_keys) { + CTxDestination dest; + if (ExtractDestination(script, dest)) { + SetAddressBook(dest, label, "receive"); + } + } } } @@ -4222,4 +4233,472 @@ ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const Flat return spk_man; } + +bool CWallet::MigrateToSQLite(bilingual_str& error) +{ + AssertLockHeld(cs_wallet); + + WalletLogPrintf("Migrating wallet storage database from BerkeleyDB to SQLite.\n"); + + if (m_database->Format() == "sqlite") { + error = _("Error: This wallet already uses SQLite"); + return false; + } + + // Get all of the records for DB type migration + std::unique_ptr batch = m_database->MakeBatch(); + std::vector> records; + if (!batch->StartCursor()) { + error = _("Error: Unable to begin reading all records in the database"); + return false; + } + bool complete = false; + while (true) { + CDataStream ss_key(SER_DISK, CLIENT_VERSION); + CDataStream ss_value(SER_DISK, CLIENT_VERSION); + bool ret = batch->ReadAtCursor(ss_key, ss_value, complete); + if (!ret) { + break; + } + SerializeData key(ss_key.begin(), ss_key.end()); + SerializeData value(ss_value.begin(), ss_value.end()); + records.emplace_back(key, value); + } + batch->CloseCursor(); + batch.reset(); + if (!complete) { + error = _("Error: Unable to read all records in the database"); + return false; + } + + // Close this database and delete the file + fs::path db_path = fs::PathFromString(m_database->Filename()); + fs::path db_dir = db_path.parent_path(); + m_database->Close(); + fs::remove(db_path); + + // Make new DB + DatabaseOptions opts; + opts.require_create = true; + opts.require_format = DatabaseFormat::SQLITE; + DatabaseStatus db_status; + std::unique_ptr new_db = MakeDatabase(db_dir, opts, db_status, error); + assert(new_db); // This is to prevent doing anything further with this wallet. The original file was deleted, but a backup exists. + m_database.reset(); + m_database = std::move(new_db); + + // Write existing records into the new DB + batch = m_database->MakeBatch(); + bool began = batch->TxnBegin(); + assert(began); // This is a critical error, the new db could not be written to. The original db exists as a backup, but we should not continue execution. + for (const auto& [key, value] : records) { + CDataStream ss_key(key, SER_DISK, CLIENT_VERSION); + CDataStream ss_value(value, SER_DISK, CLIENT_VERSION); + if (!batch->Write(ss_key, ss_value)) { + batch->TxnAbort(); + m_database->Close(); + fs::remove(m_database->Filename()); + assert(false); // This is a critical error, the new db could not be written to. The original db exists as a backup, but we should not continue execution. + } + } + bool committed = batch->TxnCommit(); + assert(committed); // This is a critical error, the new db could not be written to. The original db exists as a backup, but we should not continue execution. + return true; +} + +std::optional CWallet::GetDescriptorsForLegacy(bilingual_str& error) const +{ + AssertLockHeld(cs_wallet); + + LegacyScriptPubKeyMan* legacy_spkm = GetLegacyScriptPubKeyMan(); + if (!legacy_spkm) { + error = _("Error: This wallet is already a descriptor wallet"); + return std::nullopt; + } + + std::optional res = legacy_spkm->MigrateToDescriptor(); + if (res == std::nullopt) { + error = _("Error: Unable to produce descriptors for this legacy wallet. Make sure the wallet is unlocked first"); + return std::nullopt; + } + return res; +} + +bool CWallet::ApplyMigrationData(MigrationData& data, bilingual_str& error) +{ + AssertLockHeld(cs_wallet); + + LegacyScriptPubKeyMan* legacy_spkm = GetLegacyScriptPubKeyMan(); + if (!legacy_spkm) { + error = _("Error: This wallet is already a descriptor wallet"); + return false; + } + + for (auto& desc_spkm : data.desc_spkms) { + if (m_spk_managers.count(desc_spkm->GetID()) > 0) { + error = _("Error: Duplicate descriptors created during migration. Your wallet may be corrupted."); + return false; + } + m_spk_managers[desc_spkm->GetID()] = std::move(desc_spkm); + } + + // Remove the LegacyScriptPubKeyMan from disk + if (!legacy_spkm->DeleteRecords()) { + return false; + } + + // Remove the LegacyScriptPubKeyMan from memory + m_spk_managers.erase(legacy_spkm->GetID()); + m_external_spk_managers = nullptr; + m_internal_spk_managers = nullptr; + + // Setup new descriptors + SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + if (!IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + // Use the existing master key if we have it + if (data.master_key.key.IsValid()) { + SetupDescriptorScriptPubKeyMans(data.master_key, data.mnemonic, data.mnemonic_passphrase); + } else { + // Setup with a new seed if we don't. + SetupDescriptorScriptPubKeyMans("", ""); + } + } + + // Check if the transactions in the wallet are still ours. Either they belong here, or they belong in the watchonly wallet. + // We need to go through these in the tx insertion order so that lookups to spends works. + std::vector txids_to_delete; + for (const auto& [_pos, wtx] : wtxOrdered) { + if (!IsMine(*wtx->tx) && !IsFromMe(*wtx->tx)) { + // Check it is the watchonly wallet's + // solvable_wallet doesn't need to be checked because transactions for those scripts weren't being watched for + if (data.watchonly_wallet) { + LOCK(data.watchonly_wallet->cs_wallet); + if (data.watchonly_wallet->IsMine(*wtx->tx) || data.watchonly_wallet->IsFromMe(*wtx->tx)) { + // Add to watchonly wallet + if (!data.watchonly_wallet->AddToWallet(wtx->tx, wtx->m_state)) { + error = _("Error: Could not add watchonly tx to watchonly wallet"); + return false; + } + // Mark as to remove from this wallet + txids_to_delete.push_back(wtx->GetHash()); + continue; + } + } + // Both not ours and not in the watchonly wallet + error = strprintf(_("Error: Transaction %s in wallet cannot be identified to belong to migrated wallets"), wtx->GetHash().GetHex()); + return false; + } + } + // Do the removes + if (txids_to_delete.size() > 0) { + std::vector deleted_txids; + if (ZapSelectTx(txids_to_delete, deleted_txids) != DBErrors::LOAD_OK) { + error = _("Error: Could not delete watchonly transactions"); + return false; + } + if (deleted_txids != txids_to_delete) { + error = _("Error: Not all watchonly txs could be deleted"); + return false; + } + // Tell the GUI of each tx + for (const uint256& txid : deleted_txids) { + NotifyTransactionChanged(txid, CT_UPDATED); + } + } + + // Check the address book data in the same way we did for transactions + std::vector dests_to_delete; + for (const auto& addr_pair : m_address_book) { + // Labels applied to receiving addresses should go based on IsMine + if (addr_pair.second.purpose == "receive") { + if (!IsMine(addr_pair.first)) { + // Check the address book data is the watchonly wallet's + if (data.watchonly_wallet) { + LOCK(data.watchonly_wallet->cs_wallet); + if (data.watchonly_wallet->IsMine(addr_pair.first)) { + // Add to the watchonly. Preserve the labels, purpose, and change-ness + std::string label = addr_pair.second.GetLabel(); + std::string purpose = addr_pair.second.purpose; + if (!purpose.empty()) { + data.watchonly_wallet->m_address_book[addr_pair.first].purpose = purpose; + } + if (!addr_pair.second.IsChange()) { + data.watchonly_wallet->m_address_book[addr_pair.first].SetLabel(label); + } + dests_to_delete.push_back(addr_pair.first); + continue; + } + } + if (data.solvable_wallet) { + LOCK(data.solvable_wallet->cs_wallet); + if (data.solvable_wallet->IsMine(addr_pair.first)) { + // Add to the solvable. Preserve the labels, purpose, and change-ness + std::string label = addr_pair.second.GetLabel(); + std::string purpose = addr_pair.second.purpose; + if (!purpose.empty()) { + data.solvable_wallet->m_address_book[addr_pair.first].purpose = purpose; + } + if (!addr_pair.second.IsChange()) { + data.solvable_wallet->m_address_book[addr_pair.first].SetLabel(label); + } + dests_to_delete.push_back(addr_pair.first); + continue; + } + } + // Not ours, not in watchonly wallet, and not in solvable + error = _("Error: Address book data in wallet cannot be identified to belong to migrated wallets"); + return false; + } + } else { + // Labels for everything else (send) should be cloned to all + if (data.watchonly_wallet) { + LOCK(data.watchonly_wallet->cs_wallet); + // Add to the watchonly. Preserve the labels, purpose, and change-ness + std::string label = addr_pair.second.GetLabel(); + std::string purpose = addr_pair.second.purpose; + if (!purpose.empty()) { + data.watchonly_wallet->m_address_book[addr_pair.first].purpose = purpose; + } + if (!addr_pair.second.IsChange()) { + data.watchonly_wallet->m_address_book[addr_pair.first].SetLabel(label); + } + continue; + } + if (data.solvable_wallet) { + LOCK(data.solvable_wallet->cs_wallet); + // Add to the solvable. Preserve the labels, purpose, and change-ness + std::string label = addr_pair.second.GetLabel(); + std::string purpose = addr_pair.second.purpose; + if (!purpose.empty()) { + data.solvable_wallet->m_address_book[addr_pair.first].purpose = purpose; + } + if (!addr_pair.second.IsChange()) { + data.solvable_wallet->m_address_book[addr_pair.first].SetLabel(label); + } + continue; + } + } + } + // Remove the things to delete + if (dests_to_delete.size() > 0) { + for (const auto& dest : dests_to_delete) { + if (!DelAddressBook(dest)) { + error = _("Error: Unable to remove watchonly address book data"); + return false; + } + } + } + + // Connect the SPKM signals + ConnectScriptPubKeyManNotifiers(); + NotifyCanGetAddressesChanged(); + + WalletLogPrintf("Wallet migration complete.\n"); + + return true; +} + +bool DoMigration(CWallet& wallet, WalletContext& context, bilingual_str& error, MigrationResult& res) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +{ + AssertLockHeld(wallet.cs_wallet); + + // Get all of the descriptors from the legacy wallet + std::optional data = wallet.GetDescriptorsForLegacy(error); + if (data == std::nullopt) return false; + + // Create the watchonly and solvable wallets if necessary + if (data->watch_descs.size() > 0 || data->solvable_descs.size() > 0) { + DatabaseOptions options; + options.require_existing = false; + options.require_create = true; + + // Make the wallets + options.create_flags = WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET | WALLET_FLAG_DESCRIPTORS; + if (wallet.IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE)) { + options.create_flags |= WALLET_FLAG_AVOID_REUSE; + } + if (wallet.IsWalletFlagSet(WALLET_FLAG_KEY_ORIGIN_METADATA)) { + options.create_flags |= WALLET_FLAG_KEY_ORIGIN_METADATA; + } + if (data->watch_descs.size() > 0) { + wallet.WalletLogPrintf("Making a new watchonly wallet containing the watched scripts\n"); + + DatabaseStatus status; + std::vector warnings; + std::string wallet_name = wallet.GetName() + "_watchonly"; + data->watchonly_wallet = CreateWallet(context, wallet_name, std::nullopt, options, status, error, warnings); + if (status != DatabaseStatus::SUCCESS) { + error = _("Error: Failed to create new watchonly wallet"); + return false; + } + res.watchonly_wallet = data->watchonly_wallet; + LOCK(data->watchonly_wallet->cs_wallet); + + // Parse the descriptors and add them to the new wallet + for (const auto& [desc_str, creation_time] : data->watch_descs) { + // Parse the descriptor + FlatSigningProvider keys; + std::string parse_err; + std::unique_ptr desc = Parse(desc_str, keys, parse_err, /* require_checksum */ true); + assert(desc); // It shouldn't be possible to have the LegacyScriptPubKeyMan make an invalid descriptor + assert(!desc->IsRange()); // It shouldn't be possible to have LegacyScriptPubKeyMan make a ranged watchonly descriptor + + // Add to the wallet + WalletDescriptor w_desc(std::move(desc), creation_time, 0, 0, 0); + data->watchonly_wallet->AddWalletDescriptor(w_desc, keys, "", false); + } + + // Add the wallet to settings + UpdateWalletSetting(*context.chain, wallet_name, /*load_on_startup=*/true, warnings); + } + if (data->solvable_descs.size() > 0) { + wallet.WalletLogPrintf("Making a new watchonly wallet containing the unwatched solvable scripts\n"); + + DatabaseStatus status; + std::vector warnings; + std::string wallet_name = wallet.GetName() + "_solvables"; + data->solvable_wallet = CreateWallet(context, wallet_name, std::nullopt, options, status, error, warnings); + if (status != DatabaseStatus::SUCCESS) { + error = _("Error: Failed to create new watchonly wallet"); + return false; + } + res.solvables_wallet = data->solvable_wallet; + LOCK(data->solvable_wallet->cs_wallet); + + // Parse the descriptors and add them to the new wallet + for (const auto& [desc_str, creation_time] : data->solvable_descs) { + // Parse the descriptor + FlatSigningProvider keys; + std::string parse_err; + std::unique_ptr desc = Parse(desc_str, keys, parse_err, /* require_checksum */ true); + assert(desc); // It shouldn't be possible to have the LegacyScriptPubKeyMan make an invalid descriptor + assert(!desc->IsRange()); // It shouldn't be possible to have LegacyScriptPubKeyMan make a ranged watchonly descriptor + + // Add to the wallet + WalletDescriptor w_desc(std::move(desc), creation_time, 0, 0, 0); + data->solvable_wallet->AddWalletDescriptor(w_desc, keys, "", false); + } + + // Add the wallet to settings + UpdateWalletSetting(*context.chain, wallet_name, /*load_on_startup=*/true, warnings); + } + } + + // Add the descriptors to wallet, remove LegacyScriptPubKeyMan, and cleanup txs and address book data + if (!wallet.ApplyMigrationData(*data, error)) { + return false; + } + return true; +} + +util::Result MigrateLegacyToDescriptor(std::shared_ptr&& wallet, WalletContext& context) +{ + MigrationResult res; + bilingual_str error; + std::vector warnings; + + // Make a backup of the DB + std::string wallet_name = wallet->GetName(); + fs::path this_wallet_dir = fs::absolute(fs::PathFromString(wallet->GetDatabase().Filename())).parent_path(); + fs::path backup_filename = fs::PathFromString(strprintf("%s-%d.legacy.bak", wallet_name, GetTime())); + fs::path backup_path = this_wallet_dir / backup_filename; + if (!wallet->BackupWallet(fs::PathToString(backup_path))) { + return util::Error{_("Error: Unable to make a backup of your wallet")}; + } + res.backup_path = backup_path; + + // Unload the wallet so that nothing else tries to use it while we're changing it + if (!RemoveWallet(context, wallet, /*load_on_start=*/std::nullopt, warnings)) { + return util::Error{_("Unable to unload the wallet before migrating")}; + } + UnloadWallet(std::move(wallet)); + + // Load the wallet but only in the context of this function. + // No signals should be connected nor should anything else be aware of this wallet + WalletContext empty_context; + empty_context.args = context.args; + DatabaseOptions options; + options.require_existing = true; + DatabaseStatus status; + std::unique_ptr database = MakeWalletDatabase(wallet_name, options, status, error); + if (!database) { + return util::Error{Untranslated("Wallet file verification failed.") + Untranslated(" ") + error}; + } + + std::shared_ptr local_wallet = CWallet::Create(empty_context, wallet_name, std::move(database), options.create_flags, error, warnings); + if (!local_wallet) { + return util::Error{Untranslated("Wallet loading failed.") + Untranslated(" ") + error}; + } + + bool success = false; + { + LOCK(local_wallet->cs_wallet); + + // First change to using SQLite + if (!local_wallet->MigrateToSQLite(error)) return util::Error{error}; + + // Do the migration, and cleanup if it fails + success = DoMigration(*local_wallet, context, error, res); + } + + if (success) { + // Migration successful, unload the wallet locally, then reload it. + assert(local_wallet.use_count() == 1); + local_wallet.reset(); + LoadWallet(context, wallet_name, /*load_on_start=*/std::nullopt, options, status, error, warnings); + res.wallet_name = wallet_name; + } else { + // Migration failed, cleanup + // Copy the backup to the actual wallet dir + fs::path temp_backup_location = fsbridge::AbsPathJoin(GetWalletDir(), backup_filename); + fs::copy_file(backup_path, temp_backup_location, fs::copy_options::none); + + // Remember this wallet's walletdir to remove after unloading + std::vector wallet_dirs; + wallet_dirs.push_back(fs::PathFromString(local_wallet->GetDatabase().Filename()).parent_path()); + + // Unload the wallet locally + assert(local_wallet.use_count() == 1); + local_wallet.reset(); + + // Make list of wallets to cleanup + std::vector> created_wallets; + created_wallets.push_back(std::move(res.watchonly_wallet)); + created_wallets.push_back(std::move(res.solvables_wallet)); + + // Get the directories to remove after unloading + for (std::shared_ptr& w : created_wallets) { + wallet_dirs.push_back(fs::PathFromString(w->GetDatabase().Filename()).parent_path()); + } + + // Unload the wallets + for (std::shared_ptr& w : created_wallets) { + if (!RemoveWallet(context, w, /*load_on_start=*/false)) { + error += _("\nUnable to cleanup failed migration"); + return util::Error{error}; + } + UnloadWallet(std::move(w)); + } + + // Delete the wallet directories + for (fs::path& dir : wallet_dirs) { + fs::remove_all(dir); + } + + // Restore the backup + DatabaseStatus status; + std::vector warnings; + if (!RestoreWallet(context, temp_backup_location, wallet_name, /*load_on_start=*/std::nullopt, status, error, warnings)) { + error += _("\nUnable to restore backup of wallet."); + return util::Error{error}; + } + + // Move the backup to the wallet dir + fs::copy_file(temp_backup_location, backup_path, fs::copy_options::none); + fs::remove(temp_backup_location); + + return util::Error{error}; + } + return res; +} } // namespace wallet diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 0544840047ee..bab315999180 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -381,7 +381,7 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati std::string m_name; /** Internal database handle. */ - std::unique_ptr const m_database; + std::unique_ptr m_database; /** * The following is used to keep track of how far behind the wallet is @@ -1069,6 +1069,7 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati void DeactivateScriptPubKeyMan(uint256 id, bool internal); //! Create new DescriptorScriptPubKeyMans and add them to the wallet + void SetupDescriptorScriptPubKeyMans(const CExtKey& master_key, const SecureString& mnemonic, const SecureString mnemonic_passphrase) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void SetupDescriptorScriptPubKeyMans(const SecureString& mnemonic, const SecureString mnemonic_passphrase) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Return the DescriptorScriptPubKeyMan for a WalletDescriptor if it is already in the wallet @@ -1081,6 +1082,20 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati //! Add a descriptor to the wallet, return a ScriptPubKeyMan & associated output type ScriptPubKeyMan* AddWalletDescriptor(WalletDescriptor& desc, const FlatSigningProvider& signing_provider, const std::string& label, bool internal) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + /** Move all records from the BDB database to a new SQLite database for storage. + * The original BDB file will be deleted and replaced with a new SQLite file. + * A backup is not created. + * May crash if something unexpected happens in the filesystem. + */ + bool MigrateToSQLite(bilingual_str& error) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + //! Get all of the descriptors from a legacy wallet + std::optional GetDescriptorsForLegacy(bilingual_str& error) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + //! Adds the ScriptPubKeyMans given in MigrationData to this wallet, removes LegacyScriptPubKeyMan, + //! and where needed, moves tx and address book entries to watchonly_wallet or solvable_wallet + bool ApplyMigrationData(MigrationData& data, bilingual_str& error) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); }; /** @@ -1141,6 +1156,16 @@ bool RemoveWalletSetting(interfaces::Chain& chain, const std::string& wallet_nam bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, const CCoinControl* coin_control = nullptr); bool FillInputToWeight(CTxIn& txin, int64_t target_weight); + +struct MigrationResult { + std::string wallet_name; + std::shared_ptr watchonly_wallet; + std::shared_ptr solvables_wallet; + fs::path backup_path; +}; + +//! Do all steps to migrate a legacy wallet to a descriptor wallet +util::Result MigrateLegacyToDescriptor(std::shared_ptr&& wallet, WalletContext& context); } // namespace wallet #endif // BITCOIN_WALLET_WALLET_H diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index f408f9815b6e..1065b785f310 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -67,6 +67,7 @@ const std::string WALLETDESCRIPTORCKEY{"walletdescriptorckey"}; const std::string WALLETDESCRIPTORKEY{"walletdescriptorkey"}; const std::string WATCHMETA{"watchmeta"}; const std::string WATCHS{"watchs"}; +const std::unordered_set LEGACY_TYPES{CRYPTED_KEY, CRYPTED_HDCHAIN, CSCRIPT, DEFAULTKEY, HDCHAIN, HDPUBKEY, KEYMETA, KEY, OLD_KEY, POOL, PRIVATESEND_SALT, WATCHMETA, WATCHS}; } // namespace DBKeys // @@ -1147,6 +1148,45 @@ bool WalletBatch::WriteWalletFlags(const uint64_t flags) return WriteIC(DBKeys::FLAGS, flags); } +bool WalletBatch::EraseRecords(const std::unordered_set& types) +{ + // Get cursor + if (!m_batch->StartCursor()) + { + return false; + } + + // Iterate the DB and look for any records that have the type prefixes + while (true) + { + // Read next record + CDataStream key(SER_DISK, CLIENT_VERSION); + CDataStream value(SER_DISK, CLIENT_VERSION); + bool complete; + bool ret = m_batch->ReadAtCursor(key, value, complete); + if (complete) { + break; + } + else if (!ret) + { + m_batch->CloseCursor(); + return false; + } + + // Make a copy of key to avoid data being deleted by the following read of the type + Span key_data = MakeUCharSpan(key); + + std::string type; + key >> type; + + if (types.count(type) > 0) { + m_batch->Erase(key_data); + } + } + m_batch->CloseCursor(); + return true; +} + bool WalletBatch::TxnBegin() { return m_batch->TxnBegin(); diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 44aaa0145a30..b6af05f8901f 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -93,6 +93,9 @@ extern const std::string WALLETDESCRIPTORCKEY; extern const std::string WALLETDESCRIPTORKEY; extern const std::string WATCHMETA; extern const std::string WATCHS; + +// Keys in this set pertain only to the legacy wallet (LegacyScriptPubKeyMan) and are removed during migration from legacy to descriptors. +extern const std::unordered_set LEGACY_TYPES; } // namespace DBKeys class CKeyMetadata @@ -243,6 +246,9 @@ class WalletBatch bool WriteHDChain(const CHDChain& chain); bool WriteHDPubKey(const CHDPubKey& hdPubKey, const CKeyMetadata& keyMeta); + //! Delete records of the given types + bool EraseRecords(const std::unordered_set& types); + bool WriteWalletFlags(const uint64_t flags); //! Begin a new transaction bool TxnBegin(); diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index 42a22cd78cdf..c72bdee5fb60 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -7,6 +7,7 @@ #include #include