From 7d81180e852c520be69682feac99e307983d8e37 Mon Sep 17 00:00:00 2001 From: Yifang Ma <33007476+icerove@users.noreply.github.com> Date: Fri, 4 Dec 2020 14:12:43 -0800 Subject: [PATCH] feat(backend): Migrate Explorer to the Indexer for Explorer (#478) # Testing Plan * [x] Manually confirmed that the frontend is compatible with the updated queries * [x] Confirmed that the indexer setup sends the WAMP events just fine --- backend/src/db-utils.js | 237 ++++++++++++++++++++------ backend/src/index.js | 18 ++ frontend/src/context/ListProvider.jsx | 34 +++- 3 files changed, 236 insertions(+), 53 deletions(-) diff --git a/backend/src/db-utils.js b/backend/src/db-utils.js index 2ad11b102..03156058f 100644 --- a/backend/src/db-utils.js +++ b/backend/src/db-utils.js @@ -1,5 +1,6 @@ const { DS_INDEXER_BACKEND } = require("./consts"); const models = require("../models"); +const BN = require("bn.js"); const query = async ([query, replacements], { dataSource }) => { const sequelize = @@ -30,43 +31,6 @@ const getSyncedGenesis = async (options) => { ); }; -const aggregateStats = async (options) => { - async function queryLastDayTxCount({ dataSource }) { - let query; - if (dataSource === DS_INDEXER_BACKEND) { - query = `SELECT COUNT(*) AS total FROM transactions - WHERE DIV(block_timestamp, 1000*1000*1000) > (EXTRACT(EPOCH FROM NOW()) - 60 * 60 * 24)`; - } else { - query = `SELECT COUNT(*) AS total FROM transactions - WHERE block_timestamp > (strftime('%s','now') - 60 * 60 * 24) * 1000`; - } - return await querySingleRow([query], { dataSource }); - } - const [ - totalBlocks, - totalTransactions, - totalAccounts, - lastDayTxCount, - lastBlockHeight, - ] = await Promise.all([ - querySingleRow([`SELECT COUNT(*) as total FROM blocks`], options), - querySingleRow([`SELECT COUNT(*) as total FROM transactions`], options), - querySingleRow([`SELECT COUNT(*) as total FROM accounts`], options), - queryLastDayTxCount(options), - querySingleRow( - [`SELECT height FROM blocks ORDER BY height DESC LIMIT 1`], - options - ), - ]); - return { - totalAccounts: totalAccounts.total, - totalBlocks: totalBlocks.total, - totalTransactions: totalTransactions.total, - lastDayTxCount: lastDayTxCount.total, - lastBlockHeight: lastBlockHeight ? lastBlockHeight.height : 0, - }; -}; - const addNodeInfo = async (nodes) => { const accountArray = nodes.map((node) => node.account_id); let nodesInfo = await queryRows([ @@ -120,17 +84,141 @@ const queryOnlineNodes = async () => { ]); }; +//new query for new dashboard +const queryDashboardBlockInfo = async (options) => { + async function queryLatestBlockHeight({ dataSource }) { + let query; + if (dataSource === DS_INDEXER_BACKEND) { + query = `SELECT block_height FROM blocks ORDER BY block_height DESC LIMIT 1`; + } else { + query = `SELECT height AS block_height FROM blocks ORDER BY height DESC LIMIT 1`; + } + return await querySingleRow([query], { dataSource }); + } + async function queryLatestGasPrice({ dataSource }) { + let query; + if (dataSource === DS_INDEXER_BACKEND) { + query = `SELECT gas_price FROM blocks ORDER BY block_height DESC LIMIT 1`; + } else { + query = `SELECT gas_price FROM blocks ORDER BY height DESC LIMIT 1`; + } + return await querySingleRow([query], { dataSource }); + } + async function queryLastMinuteBlocks({ dataSource }) { + let query; + if (dataSource === DS_INDEXER_BACKEND) { + query = `SELECT block_timestamp AS latest_block_timestamp FROM blocks ORDER BY block_timestamp DESC LIMIT 1`; + } else { + query = `SELECT timestamp AS latest_block_timestamp FROM blocks ORDER BY timestamp DESC LIMIT 1`; + } + const latestBlockTimestampOrNone = await querySingleRow([query], { + dataSource, + }); + if (!latestBlockTimestampOrNone) { + return { total: 0 }; + } + const { + latest_block_timestamp: latestBlockTimestamp, + } = latestBlockTimestampOrNone; + const latestBlockTimestampBN = new BN(latestBlockTimestamp); + const currentUnixTimeBN = new BN(Math.floor(new Date().getTime() / 1000)); + let latestBlockEpochTimeBN; + if (dataSource === DS_INDEXER_BACKEND) { + latestBlockEpochTimeBN = latestBlockTimestampBN.div(new BN("1000000000")); + } else { + latestBlockEpochTimeBN = latestBlockTimestampBN.divn(1000); + } + // If the latest block is older than 1 minute from now, we report 0 + if (currentUnixTimeBN.sub(latestBlockEpochTimeBN).gtn(60)) { + return { total: 0 }; + } + + if (dataSource === DS_INDEXER_BACKEND) { + query = `SELECT COUNT(*) AS total FROM blocks + WHERE block_timestamp > (cast(EXTRACT(EPOCH FROM NOW()) - 60 as bigint) * 1000 * 1000 * 1000)`; + } else { + query = `SELECT COUNT(*) AS total FROM blocks + WHERE timestamp > (:latestBlockTimestamp - 60 * 1000)`; + } + return await querySingleRow( + [ + query, + { + latestBlockTimestamp: + dataSource == DS_INDEXER_BACKEND + ? latestBlockEpochTimeBN.toNumber() + : latestBlockEpochTimeBN.muln(1000).toNumber(), + }, + ], + { + dataSource, + } + ); + } + const [ + lastBlockHeight, + latestGasPrice, + lastMinuteBlocks, + ] = await Promise.all([ + queryLatestBlockHeight(options), + queryLatestGasPrice(options), + queryLastMinuteBlocks(options), + ]); + return { + latestBlockHeight: lastBlockHeight ? lastBlockHeight.block_height : 0, + latestGasPrice: latestGasPrice.gas_price, + numberOfLastMinuteBlocks: lastMinuteBlocks.total, + }; +}; + +const queryDashboardTxInfo = async (options) => { + async function queryTransactionCountArray({ dataSource }) { + let query; + if (dataSource === DS_INDEXER_BACKEND) { + query = `SELECT date_trunc('day', to_timestamp(DIV(block_timestamp, 1000*1000*1000))) as date, count(transaction_hash) as total + FROM transactions + WHERE block_timestamp > (cast(EXTRACT(EPOCH FROM NOW()) - 60 * 60 * 24 * 14 as bigint) * 1000 * 1000 * 1000) + GROUP BY date + ORDER BY date`; + } else { + query = `SELECT strftime('%Y-%m-%d',block_timestamp/1000,'unixepoch') as date, count(hash) as total + FROM transactions + WHERE (block_timestamp/1000) > (strftime('%s','now') - 60 * 60 * 24 * 15) + GROUP BY date + ORDER BY date`; + } + return await queryRows([query], { dataSource }); + } + let transactionCountArray = await queryTransactionCountArray(options); + return transactionCountArray; +}; + +// Old query after refactor finish, we can delete const queryDashboardBlocksAndTxs = async ({ dataSource }) => { const transactionHashColumnName = dataSource === DS_INDEXER_BACKEND ? "transaction_hash" : "hash"; const transactionIndexColumnName = dataSource === DS_INDEXER_BACKEND ? "index_in_chunk" : "transaction_index"; + const transactionSignerAccountIdColumnName = + dataSource === DS_INDEXER_BACKEND ? "signer_account_id" : "signer_id"; + const transactionReceiverAccountIdColumnName = + dataSource === DS_INDEXER_BACKEND ? "receiver_account_id" : "receiver_id"; + const transactionBlockHashColumnName = + dataSource === DS_INDEXER_BACKEND ? "included_in_block_hash" : "block_hash"; + const blockHashColumnName = + dataSource === DS_INDEXER_BACKEND ? "block_hash" : "hash"; + const blockHeightColumnName = + dataSource === DS_INDEXER_BACKEND ? "block_height" : "height"; + const blockTimestampColumnName = + dataSource === DS_INDEXER_BACKEND ? "block_timestamp" : "timestamp"; + const blockPrehashColumnName = + dataSource === DS_INDEXER_BACKEND ? "prev_block_hash" : "prev_hash"; let [transactions, blocks] = await Promise.all([ queryRows( [ - `SELECT ${transactionHashColumnName} as hash, signer_id as signerId, receiver_id as receiverId, - block_hash as blockHash, block_timestamp as blockTimestamp, ${transactionIndexColumnName} as transactionIndex - FROM transactions + `SELECT ${transactionHashColumnName} as hash, ${transactionSignerAccountIdColumnName} as signer_id, ${transactionReceiverAccountIdColumnName} as receiver_id, + ${transactionBlockHashColumnName} as block_hash, block_timestamp, ${transactionIndexColumnName} as transaction_index + FROM transactions ORDER BY block_timestamp DESC, ${transactionIndexColumnName} DESC LIMIT 10`, ], @@ -138,17 +226,19 @@ const queryDashboardBlocksAndTxs = async ({ dataSource }) => { ), queryRows( [ - `SELECT blocks.hash, blocks.height, blocks.timestamp, blocks.prev_hash as prevHash, COUNT(transactions.${transactionHashColumnName}) as transactionsCount + `SELECT blocks.${blockHashColumnName} as hash, blocks.${blockHeightColumnName} as height, blocks.${blockTimestampColumnName}, + blocks.${blockPrehashColumnName} as prev_hash, + COUNT(transactions.${transactionHashColumnName}) as transactions_count FROM ( - SELECT blocks.hash + SELECT blocks.${blockHashColumnName} FROM blocks - ORDER BY blocks.height DESC + ORDER BY blocks.${blockHeightColumnName} DESC LIMIT 8 ) as recent_blocks - LEFT JOIN blocks ON blocks.hash = recent_blocks.hash - LEFT JOIN transactions ON transactions.block_hash = recent_blocks.hash - GROUP BY blocks.hash - ORDER BY blocks.timestamp DESC`, + LEFT JOIN blocks ON blocks.${blockHashColumnName} = recent_blocks.${blockHashColumnName} + LEFT JOIN transactions ON transactions.${transactionBlockHashColumnName} = recent_blocks.${blockHashColumnName} + GROUP BY blocks.${blockHashColumnName} + ORDER BY blocks.${blockTimestampColumnName} DESC`, ], { dataSource } ), @@ -156,15 +246,15 @@ const queryDashboardBlocksAndTxs = async ({ dataSource }) => { let query; let transactionHashes = transactions.map((transaction) => transaction.hash); if (dataSource === DS_INDEXER_BACKEND) { - query = `SELECT transaction_hash, index as action_index, action_kind as kind, args + query = `SELECT transaction_hash, action_kind as kind, args FROM transaction_actions WHERE transaction_hash IN (:transactionHashes) - ORDER BY index`; + ORDER BY index_in_transaction DESC`; } else { - query = `SELECT transaction_hash, action_index, action_type as kind, action_args as args + query = `SELECT transaction_hash, action_type as kind, action_args as args FROM actions WHERE transaction_hash IN (:transactionHashes) - ORDER BY action_index`; + ORDER BY action_index DESC`; } const actionsArray = await queryRows([query, { transactionHashes }], { dataSource, @@ -197,9 +287,54 @@ const queryDashboardBlocksAndTxs = async ({ dataSource }) => { return { transactions, blocks }; }; +const aggregateStats = async (options) => { + async function queryLastDayTxCount({ dataSource }) { + let query; + if (dataSource === DS_INDEXER_BACKEND) { + query = `SELECT COUNT(*) AS total FROM transactions + WHERE block_timestamp > (cast(EXTRACT(EPOCH FROM NOW()) - 60 * 60 * 24 as bigint) * 1000 * 1000 * 1000)`; + } else { + query = `SELECT COUNT(*) AS total FROM transactions + WHERE block_timestamp > (strftime('%s','now') - 60 * 60 * 24) * 1000`; + } + return await querySingleRow([query], { dataSource }); + } + async function queryLatestBlockHeight({ dataSource }) { + let query; + if (dataSource === DS_INDEXER_BACKEND) { + query = `SELECT block_height FROM blocks ORDER BY block_height DESC LIMIT 1`; + } else { + query = `SELECT height AS block_height FROM blocks ORDER BY height DESC LIMIT 1`; + } + return await querySingleRow([query], { dataSource }); + } + const [ + totalBlocks, + totalTransactions, + totalAccounts, + lastDayTxCount, + lastBlockHeight, + ] = await Promise.all([ + querySingleRow([`SELECT COUNT(*) as total FROM blocks`], options), + querySingleRow([`SELECT COUNT(*) as total FROM transactions`], options), + querySingleRow([`SELECT COUNT(*) as total FROM accounts`], options), + queryLastDayTxCount(options), + queryLatestBlockHeight(options), + ]); + return { + totalAccounts: totalAccounts.total, + totalBlocks: totalBlocks.total, + totalTransactions: totalTransactions.total, + lastDayTxCount: lastDayTxCount.total, + lastBlockHeight: lastBlockHeight ? lastBlockHeight.block_height : 0, + }; +}; + exports.queryOnlineNodes = queryOnlineNodes; exports.addNodeInfo = addNodeInfo; exports.aggregateStats = aggregateStats; exports.pickOnlineValidatingNode = pickOnlineValidatingNode; exports.queryDashboardBlocksAndTxs = queryDashboardBlocksAndTxs; exports.getSyncedGenesis = getSyncedGenesis; +exports.queryDashboardTxInfo = queryDashboardTxInfo; +exports.queryDashboardBlockInfo = queryDashboardBlockInfo; diff --git a/backend/src/index.js b/backend/src/index.js index e2b50c4f8..6564964b9 100644 --- a/backend/src/index.js +++ b/backend/src/index.js @@ -31,6 +31,8 @@ const { pickOnlineValidatingNode, queryDashboardBlocksAndTxs, getSyncedGenesis, + queryDashboardBlockInfo, + queryDashboardTxInfo, } = require("./db-utils"); async function startLegacySync() { @@ -142,6 +144,7 @@ function startDataSourceSpecificJobs(wamp, dataSource) { console.log(`Starting regular data stats check from ${dataSource}...`); try { if (wamp.session) { + // old pub-sub data, after refactor, delete properly const dataStats = await aggregateStats({ dataSource }); const { transactions, blocks } = await queryDashboardBlocksAndTxs({ dataSource, @@ -159,6 +162,21 @@ function startDataSourceSpecificJobs(wamp, dataSource) { [{ transactions, blocks }], wamp ); + //new pub-sub data, coexists with old one now + const blockStats = await queryDashboardBlockInfo({ dataSource }); + const transactionCountArray = await queryDashboardTxInfo({ + dataSource, + }); + wampPublish( + getDataSourceSpecificTopicName("chain-block-stats", dataSource), + [{ blockStats }], + wamp + ); + wampPublish( + getDataSourceSpecificTopicName("chain-txs-stats", dataSource), + [{ transactionCountArray }], + wamp + ); } console.log(`Regular data stats check from ${dataSource} is completed.`); } catch (error) { diff --git a/frontend/src/context/ListProvider.jsx b/frontend/src/context/ListProvider.jsx index cf7312282..11f3ec88c 100644 --- a/frontend/src/context/ListProvider.jsx +++ b/frontend/src/context/ListProvider.jsx @@ -13,8 +13,38 @@ export default (props) => { const fetchList = function (stats) { const { blocks, transactions } = stats[0]; - dispatchBlocks(blocks); - dispatchTransactions(transactions); + const preprocessedBlocks = blocks.map( + ({ hash, height, timestamp, prev_hash, transactions_count }) => ({ + hash, + height, + timestamp, + prevHash: prev_hash, + transactionsCount: transactions_count, + }) + ); + + const preprocessedTransactions = transactions.map( + ({ + hash, + signer_id, + receiver_id, + block_hash, + block_timestamp, + transaction_index, + actions, + }) => ({ + hash, + signerId: signer_id, + receiverId: receiver_id, + blockHash: block_hash, + blockTimestamp: block_timestamp, + transactionIndex: transaction_index, + actions, + }) + ); + + dispatchBlocks(preprocessedBlocks); + dispatchTransactions(preprocessedTransactions); }; const Subscription = useCallback(() => {