From 8aab17217f013b6eee999f7fd5ef16585ac5a68b Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Thu, 3 Sep 2020 12:51:28 -0400 Subject: [PATCH 1/3] wallet: Deep Clean walletDB deepClean(): - deletes all mapping of blocks, outpoints and tx hashes to wallet ids - deletes ENTIRE txdb (all history, all balance data, for all wallets) - keeps all account metadata (account name, type, address depth, etc) - keeps all name->wallet ID mapping (although this may be unnecessary) - keeps all address-hash path mapping (addresses from all accounts) wallet HTTP /deepclean: - requires admin token, like /rescan - requires setting parameter `I_HAVE_BACKED_UP_MY_WALLET` to `true` --- lib/wallet/http.js | 20 +++++++++++++ lib/wallet/walletdb.js | 68 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/lib/wallet/http.js b/lib/wallet/http.js index 907201e97..64241894d 100644 --- a/lib/wallet/http.js +++ b/lib/wallet/http.js @@ -200,6 +200,26 @@ class HTTP extends Server { await this.wdb.rescan(height); }); + // Deep Clean + this.post('/deepclean', async (req, res) => { + if (!req.admin) { + res.json(403); + return; + } + + const valid = Validator.fromRequest(req); + const disclaimer = valid.bool('I_HAVE_BACKED_UP_MY_WALLET', false); + + enforce( + disclaimer, + 'Deep Clean requires I_HAVE_BACKED_UP_MY_WALLET=true' + ); + + res.json(200, { success: true }); + + await this.wdb.deepClean(); + }); + // Resend this.post('/resend', async (req, res) => { if (!req.admin) { diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index e61e836f5..9cda2d421 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -503,6 +503,74 @@ class WalletDB extends EventEmitter { } } + /** + * Deep Clean: + * Keep all keys, account data, wallet maps (name and path). + * Dump all TX history and balance state. + * A rescan will be required but is not initiated automatically. + * @returns {Promise} + */ + + async deepClean() { + const unlock1 = await this.txLock.lock(); + const unlock2 = await this.writeLock.lock(); + const unlock3 = await this.readLock.lock(); + try { + return await this._deepClean(); + } finally { + unlock3(); + unlock2(); + unlock1(); + } + } + + /** + * Deep Clean (without locks): + * Keep all keys, account data, wallet maps (name and path). + * Dump all TX history and balance state. + * A rescan will be required but is not initiated automatically. + * @returns {Promise} + */ + + async _deepClean() { + this.logger.warning('Initiating Deep Clean...'); + + const b = this.db.batch(); + const removeRange = (opt) => { + return this.db.iterator(opt).each(key => b.del(key)); + }; + + this.logger.warning('Clearing block map, tx map and outpoint map...'); + // b[height] -> block->wid map + await removeRange({ + gte: layout.b.min(), + lte: layout.b.max() + }); + + // o[hash][index] -> outpoint->wid map + await removeRange({ + gte: layout.o.min(), + lte: layout.o.max() + }); + + // T[hash] -> tx->wid map + await removeRange({ + gte: layout.T.min(), + lte: layout.T.max() + }); + + this.logger.warning('Clearing all tx history for all wallets...'); + // t[wid]* -> txdb + await removeRange({ + gte: layout.t.min(), + lte: layout.t.max() + }); + + await b.write(); + + this.logger.warning('Deep Clean complete. A rescan is now required.'); + } + /** * Force a rescan. * @param {Number} height From 9e19110cfa58321cc284c36683e056bb22244b61 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Mon, 4 Jan 2021 11:28:41 -0500 Subject: [PATCH 2/3] test: deep clean --- lib/wallet/walletdb.js | 29 +++- test/wallet-deepclean-test.js | 258 ++++++++++++++++++++++++++++++++++ 2 files changed, 282 insertions(+), 5 deletions(-) create mode 100644 test/wallet-deepclean-test.js diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 9cda2d421..c5a4cc1ba 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -559,13 +559,32 @@ class WalletDB extends EventEmitter { lte: layout.T.max() }); - this.logger.warning('Clearing all tx history for all wallets...'); - // t[wid]* -> txdb - await removeRange({ - gte: layout.t.min(), - lte: layout.t.max() + const wids = await this.db.keys({ + gte: layout.W.min(), + lte: layout.W.max(), + parse: key => layout.W.decode(key)[0] }); + for (const wid of wids) { + const wallet = await this.get(wid); + this.logger.warning( + 'Clearing all tx history for wallet: %s (%d)', + wallet.id, wid + ); + + // remove all txdb data *except* blinds ('v') + const key = 'v'.charCodeAt(); + const prefix = layout.t.encode(wid); + await removeRange({ + gte: Buffer.concat([prefix, Buffer.alloc(1)]), + lt: Buffer.concat([prefix, Buffer.from([key])]) + }); + await removeRange({ + gt: Buffer.concat([prefix, Buffer.from([key + 1])]), + lte: Buffer.concat([prefix, Buffer.from([0xff])]) + }); + } + await b.write(); this.logger.warning('Deep Clean complete. A rescan is now required.'); diff --git a/test/wallet-deepclean-test.js b/test/wallet-deepclean-test.js new file mode 100644 index 000000000..3b890d5e7 --- /dev/null +++ b/test/wallet-deepclean-test.js @@ -0,0 +1,258 @@ +/* eslint-env mocha */ +/* eslint prefer-arrow-callback: "off" */ + +'use strict'; + +const assert = require('bsert'); +const Network = require('../lib/protocol/network'); +const FullNode = require('../lib/node/fullnode'); +const Address = require('../lib/primitives/address'); +const Resource = require('../lib/dns/resource'); + +const network = Network.get('regtest'); + +const node = new FullNode({ + memory: true, + network: 'regtest', + plugins: [require('../lib/wallet/plugin')] +}); + +const {wdb} = node.require('walletdb'); + +const nullBalance = { + account: -1, + tx: 0, + coin: 0, + unconfirmed: 0, + confirmed: 0, + lockedUnconfirmed: 0, + lockedConfirmed: 0 +}; + +let alice, aliceReceive, aliceAcct0; +let aliceAcct0Info, aliceNames, aliceBalance, aliceHistory; +let bob, bobReceive, bobAcct0; +let bobAcct0Info, bobNames, bobBalance, bobHistory; + +const aliceBlinds = []; +const bobBlinds = []; + +async function mineBlocks(n, addr) { + addr = addr ? addr : new Address().toString('regtest'); + for (let i = 0; i < n; i++) { + const block = await node.miner.mineBlock(null, addr); + await node.chain.add(block); + } +} + +describe('Wallet Deep Clean', function() { + this.timeout(20000); + + before(async () => { + await node.open(); + await node.connect(); + node.startSync(); + + alice = await wdb.create(); + aliceAcct0 = await alice.getAccount(0); + aliceReceive = await aliceAcct0.receiveAddress(); + + bob = await wdb.create(); + bobAcct0 = await bob.getAccount(0); + bobReceive = await bobAcct0.receiveAddress(); + }); + + after(async () => { + await node.close(); + }); + + it('should fund wallets', async () => { + // Also mines enough blocks to roll out all names + await mineBlocks(30, aliceReceive); + await mineBlocks(30, bobReceive); + }); + + it('should open 10 auctions and REGISTER names', async () => { + for (let i = 0; i < 10; i++) { + const w = i < 5 ? alice : bob; + const name = i < 5 ? `alice${i}` : `bob${i}`; + const array = i < 5 ? aliceBlinds : bobBlinds; + + await w.sendOpen(name, false, {account: 0}); + await mineBlocks(network.names.treeInterval + 2); + + // Send two bids so there is a winner/loser and name gets a value + const bid1 = await w.sendBid(name, 100000 + i, 200000 + i, {account: 0}); + const bid2 = await w.sendBid(name, 100000 - i, 200000 - i, {account: 0}); + saveBlindValue(bid1, array); + saveBlindValue(bid2, array); + await mineBlocks(network.names.biddingPeriod); + + await w.sendReveal(name, {account: 0}); + await mineBlocks(network.names.revealPeriod); + + const res = Resource.Resource.fromJSON({ + records: [ + { + type: 'TXT', + txt: [name] + } + ] + }); + + await w.sendUpdate(name, res, {account: 0}); + await w.sendRedeem(name, {account: 0}); + await mineBlocks(network.names.treeInterval); + } + }); + + it('should TRANSFER and FINALIZE some names', async () => { + const bobReceiveName = await bobAcct0.receiveAddress(); + await alice.sendTransfer('alice0', bobReceiveName); + await mineBlocks(network.names.transferLockup + 1); + + await alice.sendFinalize('alice0'); + await mineBlocks(10); + + const aliceReceiveName = await aliceAcct0.receiveAddress(); + await bob.sendTransfer('bob9', aliceReceiveName); + await mineBlocks(network.names.transferLockup + 1); + + await bob.sendFinalize('bob9'); + await mineBlocks(10); + }); + + it('should send 20 normal transactions', async () => { + for (let i = 0; i < 20; i++) { + const send = i < 5 ? alice : bob; + const rec = i < 5 ? bobAcct0 : aliceAcct0; + const address = rec.receiveAddress(); + const value = 1212 + (i * 10); + + await send.send({ + outputs: [ + { + address, + value + } + ] + }); + await mineBlocks(1); + } + }); + + it('should save wallet data', async () => { + aliceBalance = await alice.getBalance(); + aliceNames = await alice.getNames(); + aliceHistory = await alice.getHistory(); + aliceAcct0Info = await alice.getAccount(0); + + bobBalance = await bob.getBalance(); + bobNames = await bob.getNames(); + bobHistory = await bob.getHistory(); + bobAcct0Info = await bob.getAccount(0); + }); + + it('should DEEP CLEAN', async () => { + await wdb.deepClean(); + }); + + it('should have erased wallet data', async () => { + const aliceBalance2 = await alice.getBalance(); + const aliceNames2 = await alice.getNames(); + const aliceHistory2 = await alice.getHistory(); + const aliceAcct0Info2 = await alice.getAccount(0); + + const bobBalance2 = await bob.getBalance(); + const bobNames2 = await bob.getNames(); + const bobHistory2 = await bob.getHistory(); + const bobAcct0Info2 = await bob.getAccount(0); + + // Account metadata is fine + assert.deepStrictEqual(aliceAcct0Info, aliceAcct0Info2); + assert.deepStrictEqual(bobAcct0Info, bobAcct0Info2); + + // Blind values are fine + for (const blind of aliceBlinds) { + assert(await alice.getBlind(blind)); + } + for (const blind of bobBlinds) { + assert(await bob.getBlind(blind)); + } + + // Everything else is wiped + assert.deepStrictEqual(aliceBalance2.getJSON(), nullBalance); + assert.deepStrictEqual(bobBalance2.getJSON(), nullBalance); + compareNames(aliceNames2, []); + compareHistories(aliceHistory2, []); + compareNames(bobNames2, []); + compareHistories(bobHistory2, []); + }); + + it('should rescan wallets', async () => { + await wdb.rescan(0); + }); + + it('should have recovered wallet data', async () => { + const aliceBalance2 = await alice.getBalance(); + const aliceNames2 = await alice.getNames(); + const aliceHistory2 = await alice.getHistory(); + const aliceAcct0Info2 = await alice.getAccount(0); + + const bobBalance2 = await bob.getBalance(); + const bobNames2 = await bob.getNames(); + const bobHistory2 = await bob.getHistory(); + const bobAcct0Info2 = await bob.getAccount(0); + + assert.deepStrictEqual(aliceBalance, aliceBalance2); + assert.deepStrictEqual(aliceAcct0Info, aliceAcct0Info2); + compareNames(aliceNames, aliceNames2); + compareHistories(aliceHistory, aliceHistory2); + + assert.deepStrictEqual(bobBalance, bobBalance2); + assert.deepStrictEqual(bobAcct0Info, bobAcct0Info2); + compareNames(bobNames, bobNames2); + compareHistories(bobHistory, bobHistory2); + }); +}); + +function compareHistories(a, b) { + for (let i = 0; i < a.length; i ++) { + const objA = a[i]; + const objB = b[i]; + + for (const prop of Object.keys(objA)) { + // Wall-clock time that TX was inserted, ignore after rescan + if (prop === 'mtime') + continue; + + assert.deepStrictEqual(objA[prop], objB[prop]); + } + } +} + +function compareNames(a, b) { + for (let i = 0; i < a.length; i ++) { + const objA = a[i]; + const objB = b[i]; + + for (const prop of Object.keys(objA)) { + // Highest bid and current data are not transmitted in the FINALIZE + // so they are unknown to the wallet until the chain is rescanned. + if (prop === 'highest' || prop === 'data') + continue; + + assert.deepStrictEqual(objA[prop], objB[prop]); + } + } +} + +function saveBlindValue(tx, array) { + for (const output of tx.outputs) { + const cov = output.covenant; + if (!cov.isBid()) + continue; + + array.push(cov.getHash(3)); + } +} From 2fd89c6be22abd770ec947ee4d3a3d44295960fe Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Fri, 11 Sep 2020 15:11:57 -0400 Subject: [PATCH 3/3] pkg: CHANGELOG --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95626fbdb..fe3fc933e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## unreleased + ### Node changes - `FullNode` now parses option `--min-weight=` (`min-weight: ` in @@ -11,6 +12,17 @@ the miner will add transactions up to the minimum weight that would normally be ignored for being "free" (paying a fee below policy limit). The default value is raised from `0` to `5000` (a 1-in, 2-out BID transaction has a weight of about `889`). +### Wallet API changes + +- Adds new wallet HTTP endpoint `/deepclean` that requires a parameter +`I_HAVE_BACKED_UP_MY_WALLET=true`. This action wipes out balance and transaction +history in the wallet DB but retains key hashes and name maps. It should be used +only if the wallet state has been corrupted by issues like the +[reserved name registration bug](https://github.com/handshake-org/hsd/issues/454) +or the +[locked coins balance after FINALIZE bug](https://github.com/handshake-org/hsd/pull/464). +After the corrupt data has been cleared, **a walletDB rescan is required**. + ### Wallet changes - Fixes a bug that ignored the effect of sending or receiving a FINALIZE on a