[WIP] Explore wallet addresses using an embedded bwt electrum server

.env-sample

@@ -34,7 +34,7 @@
 #BTCEXP_BITCOIND_RPC_TIMEOUT=5000
 
 # Select optional "address API" to display address tx lists and balances
-# Options: electrumx, blockchain.com, blockchair.com, blockcypher.com
+# Options: electrumx, bwt, blockchain.com, blockchair.com, blockcypher.com
 # If electrumx set, the BTCEXP_ELECTRUMX_SERVERS variable must also be
 # set.
 # Default: none
@@ -112,4 +112,16 @@
 
 # Show tools list in a sub-nav at top of screen
 # Default: true
-BTCEXP_UI_SHOW_TOOLS_SUBHEADER=true
+#BTCEXP_UI_SHOW_TOOLS_SUBHEADER=true
+
+# Options for bwt
+# See https://github.com/shesek/bwt for more info
+# BTCEXP_BWT_XPUBS="xpub1;xpub2"
+# BTCEXP_BWT_BARE_XPUBS="xpub1;xpub2"
+# BTCEXP_BWT_DESCRIPTORS="wpkh(xpub1/*);pkh(xpub2/0/*)"
+# BTCEXP_BWT_BITCOIND_WALLET=<name>
+# BTCEXP_BWT_RESCAN_SINCE=<yyyy-mm-dd / "now">
+# BTCEXP_BWT_VERBOSE=[0-4]
+# BTCEXP_BWT_GAP_LIMIT=20
+# BTCEXP_BWT_INITIAL_IMPORT_SIZE=20
+# BTCEXP_BWT_POLL_INTERVAL=5

README.md

@@ -98,6 +98,20 @@ After successfull access with the token a cookie is used for authentication, so
 To improve user experience you can set `BTCEXP_SSO_LOGIN_REDIRECT_URL` to the URL of your SSO provider.
 This causes the users to be redirected to login page if not logged in.
 
+#### Wallet addresses history
+
+Showing the history of your wallet addresses is supported via [bwt](https://github.com/sheske/bwt).
+To enable this feature, install bwt with `npm install -g libbwt` then start btc-rpc-explorer
+with `--address-api bwt` and one or more `--descriptors`s or `--xpub`s.
+
+The wallet addresses will be derived and imported into Bitcoin Core. You can specify which wallet to use with `--bwt-bitcoind-wallet <name>`.
+
+Scanning the full blockchain history for transactions involving your addresses may take up to 20-30 minutes.
+To speed this up, you may specify a date to start rescanning from with `--bwt-rescan-since <yyyy-mm-dd>`, or
+use `now` to watch for new transactions only.
+
+For example: `--address-api bwt --bwt-rescan-since 2020-01-01 --descriptor 'wpkh(xpub69..oP/0/*)' --xpub xpub66..zQ`
+
 ## Run via Docker
 
 1. `docker build -t btc-rpc-explorer .`

app.js

@@ -45,6 +45,7 @@ var request = require("request");
 var qrcode = require("qrcode");
 var addressApi = require("./app/api/addressApi.js");
 var electrumAddressApi = require("./app/api/electrumAddressApi.js");
+var bwtAddressApi = require("./app/api/bwtAddressApi.js");
 var coreApi = require("./app/api/coreApi.js");
 var auth = require('./app/auth.js');
 var sso = require('./app/sso.js');
@@ -276,6 +277,11 @@ function onRpcConnectionVerified(getnetworkinfo, getblockchaininfo) {
 	// 1d / 7d volume
 	refreshNetworkVolumes();
 	setInterval(refreshNetworkVolumes, 30 * 60 * 1000);
+
+  if (config.addressApi == "bwt") {
+    bwtAddressApi.setup(config, global.activeBlockchain)
+      .catch(function(err) { utils.logError("bwt error", err) });
+  }
 }
 
 function refreshUtxoSetSummary() {

app/api/addressApi.js

@@ -10,7 +10,7 @@ var blockchairAddressApi = require("./blockchairAddressApi.js");
 var blockcypherAddressApi = require("./blockcypherAddressApi.js");
 
 function getSupportedAddressApis() {
-	return ["blockchain.com", "blockchair.com", "blockcypher.com", "electrumx"];
+	return ["blockchain.com", "blockchair.com", "blockcypher.com", "electrumx", "bwt"];
 }
 
 function getCurrentAddressApiFeatureSupport() {
@@ -35,7 +35,7 @@ function getCurrentAddressApiFeatureSupport() {
 			sortAsc: false
 		};
 
-	} else if (config.addressApi == "electrumx") {
+	} else if (config.addressApi == "electrumx" || config.addressApi == "bwt") {
 		return {
 			pageNumbers: true,
 			sortDesc: true,
@@ -57,7 +57,7 @@ function getAddressDetails(address, scriptPubkey, sort, limit, offset) {
 		} else if (config.addressApi == "blockcypher.com") {
 			promises.push(blockcypherAddressApi.getAddressDetails(address, scriptPubkey, sort, limit, offset));
 
-		} else if (config.addressApi == "electrumx") {
+		} else if (config.addressApi == "electrumx" || config.addressApi == "bwt") {
 			promises.push(electrumAddressApi.getAddressDetails(address, scriptPubkey, sort, limit, offset));
 
 		} else {

app/api/bwtAddressApi.js

@@ -0,0 +1,67 @@
+var debug = require("debug");
+var electrumAddressApi = require("./electrumAddressApi.js");
+
+async function setup(config, activeBlockchain) {
+  try { var { BwtDaemon } = require('libbwt'); }
+  catch (_) { throw new Error('The bwt backend requires installing the "libbwt" package'); }
+
+  var network = { main: 'bitcoin', test: 'testnet' }[activeBlockchain] || activeBlockchain;
+  var rpcCred = config.credentials.rpc;
+  var rpcUrl = `http://${rpcCred.host}:${rpcCred.port}`;
+
+  var bwt = await BwtDaemon({
+    network,
+    bitcoind_url: rpcUrl,
+    verbose: +debug.enabled('bwt'),
+    progress: reportProgress,
+    electrum: true,
+
+    ...getEnvOptions(),
+    ...getAuthOptions(rpcCred),
+    ...getDescsXpubsOptions(),
+  }).start();
+
+  var [ host, port ] = bwt.electrum_addr.split(':');
+  config.electrumXServers = [{ host, port, protocol: 'tcp' }];
+
+  await electrumAddressApi.connectToServers();
+}
+
+function reportProgress(type, progress, { eta }) {
+  if (type == 'scan') console.log('[bwt] Rescan in progress... (%f%%, eta %d minutes)', (progress*100).toFixed(1), (eta/60).toFixed(1))
+}
+
+function getEnvOptions() {
+  return [ 'bitcoind_wallet', 'rescan_since', 'verbose', 'gap_limit', 'initial_import_size', 'poll_interval' ]
+    .reduce((O, name, idx) => {
+      var envVal = process.env[`BTCEXP_BWT_${name.toUpperCase()}`];
+      if (envVal != null) O[name] = idx <= 1 ? envVal : +envVal;
+      return O;
+    }, {});
+}
+
+function getAuthOptions(rpcCred) {
+  return rpcCred.username
+    ? { bitcoind_auth: `${rpcCred.username}:${rpcCred.password}` }
+    : { bitcoind_cookie: rpcCred.cookie }
+}
+
+function getDescsXpubsOptions() {
+  var options = { descriptors: [], xpubs: [], bare_xpubs: [] };
+
+  var argTypes = { '--descriptor': options.descriptors, '--xpub': options.xpubs, '--bare-xpub': options.bare_xpubs };
+  process.argv.slice(0, -1).forEach((arg, i) => {
+    var tList = argTypes[arg];
+    if (tList) tList.push(process.argv[i+1]);
+  });
+
+  var envTypes = { 'DESCRIPTORS': options.descriptors, 'XPUBS': options.xpubs, 'BARE_XPUBS': options.bare_xpubs };
+  Object.entries(envTypes).forEach(([ envName, tList ]) => {
+    var envVal = process.env[`BTCEXP_BWT_${envName}`];
+    if (envVal) tList.push(...envVal.split(';'))
+  });
+
+  return options;
+}
+
+module.exports = { setup }

views/address.pug

@@ -90,7 +90,7 @@ block content
 					div.card.shadow-sm.mb-3
 						div.card-body
 							h3.h6.mb-0 Summary
-								if (config.addressApi)
+								if (config.addressApi && config.addressApi != "bwt")
 									small.text-muted.border-dotted.ml-2(title=`Some details for this address were queried from ${config.addressApi}` data-toggle="tooltip") Trust Note
 							hr
 
@@ -252,7 +252,7 @@ block content
 										if (config.addressApi)
 											if (config.addressApi == "electrumx")
 												small.text-muted.border-dotted.ml-2(title=`The list of transaction IDs for this address was queried from ElectrumX (using the configured server(s))` data-toggle="tooltip") Trust Note
-											else
+											else if (config.addressApi != "bwt")
 												small.text-muted.border-dotted.ml-2(title=`The list of transaction IDs for this address was queried from ${config.addressApi}` data-toggle="tooltip") Trust Note
 
 								if (!crawlerBot && txids && txids.length > 1 && addressApiSupport.sortDesc && addressApiSupport.sortAsc)
@@ -318,6 +318,8 @@ block content
 
 								else if (transactions.length == 0)
 									span No transactions found
+									if (config.addressApi == "bwt")
+										p.text-muted btc-rpc-explorer is connected to a personal address index that provides history for wallet addresses only.
 
 								each tx, txIndex in transactions
 									//pre