feat(backend): Migrate Explorer to the Indexer for Explorer

backend/src/db-utils.js

@@ -1,5 +1,6 @@
 const { DS_INDEXER_BACKEND } = require("./consts");
 const models = require("../models");
+const BN = require("bn.js");
 
 const query = async ([query, replacements], { dataSource }) => {
   const sequelize =
@@ -30,43 +31,6 @@ const getSyncedGenesis = async (options) => {
   );
 };
 
-const aggregateStats = async (options) => {
-  async function queryLastDayTxCount({ dataSource }) {
-    let query;
-    if (dataSource === DS_INDEXER_BACKEND) {
-      query = `SELECT COUNT(*) AS total FROM transactions
-        WHERE DIV(block_timestamp, 1000*1000*1000) > (EXTRACT(EPOCH FROM NOW()) - 60 * 60 * 24)`;
-    } else {
-      query = `SELECT COUNT(*) AS total FROM transactions
-        WHERE block_timestamp > (strftime('%s','now') - 60 * 60 * 24) * 1000`;
-    }
-    return await querySingleRow([query], { dataSource });
-  }
-  const [
-    totalBlocks,
-    totalTransactions,
-    totalAccounts,
-    lastDayTxCount,
-    lastBlockHeight,
-  ] = await Promise.all([
-    querySingleRow([`SELECT COUNT(*) as total FROM blocks`], options),
-    querySingleRow([`SELECT COUNT(*) as total FROM transactions`], options),
-    querySingleRow([`SELECT COUNT(*) as total FROM accounts`], options),
-    queryLastDayTxCount(options),
-    querySingleRow(
-      [`SELECT height FROM blocks ORDER BY height DESC LIMIT 1`],
-      options
-    ),
-  ]);
-  return {
-    totalAccounts: totalAccounts.total,
-    totalBlocks: totalBlocks.total,
-    totalTransactions: totalTransactions.total,
-    lastDayTxCount: lastDayTxCount.total,
-    lastBlockHeight: lastBlockHeight ? lastBlockHeight.height : 0,
-  };
-};
-
 const addNodeInfo = async (nodes) => {
   const accountArray = nodes.map((node) => node.account_id);
   let nodesInfo = await queryRows([
@@ -120,51 +84,177 @@ const queryOnlineNodes = async () => {
   ]);
 };
 
+//new query for new dashboard
+const queryDashboardBlockInfo = async (options) => {
+  async function queryLatestBlockHeight({ dataSource }) {
+    let query;
+    if (dataSource === DS_INDEXER_BACKEND) {
+      query = `SELECT block_height FROM blocks ORDER BY block_height DESC LIMIT 1`;
+    } else {
+      query = `SELECT height AS block_height FROM blocks ORDER BY height DESC LIMIT 1`;
+    }
+    return await querySingleRow([query], { dataSource });
+  }
+  async function queryLatestGasPrice({ dataSource }) {
+    let query;
+    if (dataSource === DS_INDEXER_BACKEND) {
+      query = `SELECT gas_price FROM blocks ORDER BY block_height DESC LIMIT 1`;
+    } else {
+      query = `SELECT gas_price FROM blocks ORDER BY height DESC LIMIT 1`;
+    }
+    return await querySingleRow([query], { dataSource });
+  }
+  async function queryLastMinuteBlocks({ dataSource }) {
+    let query;
+    if (dataSource === DS_INDEXER_BACKEND) {
+      query = `SELECT block_timestamp AS latest_block_timestamp FROM blocks ORDER BY block_timestamp DESC LIMIT 1`;
+    } else {
+      query = `SELECT timestamp AS latest_block_timestamp FROM blocks ORDER BY timestamp DESC LIMIT 1`;
+    }
+    const latestBlockTimestampOrNone = await querySingleRow([query], {
+      dataSource,
+    });
+    if (!latestBlockTimestampOrNone) {
+      return { total: 0 };
+    }
+    const {
+      latest_block_timestamp: latestBlockTimestamp,
+    } = latestBlockTimestampOrNone;
+    const latestBlockTimestampBN = new BN(latestBlockTimestamp);
+    const currentUnixTimeBN = new BN(Math.floor(new Date().getTime() / 1000));
+    let latestBlockEpochTimeBN;
+    if (dataSource === DS_INDEXER_BACKEND) {
+      latestBlockEpochTimeBN = latestBlockTimestampBN.div(new BN("1000000000"));
+    } else {
+      latestBlockEpochTimeBN = latestBlockTimestampBN.divn(1000);
+    }
+    // If the latest block is older than 1 minute from now, we report 0
+    if (currentUnixTimeBN.sub(latestBlockEpochTimeBN).gtn(60)) {
+      return { total: 0 };
+    }
+
+    if (dataSource === DS_INDEXER_BACKEND) {
+      query = `SELECT COUNT(*) AS total FROM blocks
+      WHERE block_timestamp > (cast(EXTRACT(EPOCH FROM NOW()) - 60 as bigint) * 1000 * 1000 * 1000)`;
+    } else {
+      query = `SELECT COUNT(*) AS total FROM blocks
+        WHERE timestamp > (:latestBlockTimestamp - 60 * 1000)`;
+    }
+    return await querySingleRow(
+      [
+        query,
+        {
+          latestBlockTimestamp:
+            dataSource == DS_INDEXER_BACKEND
+              ? latestBlockEpochTimeBN.toNumber()
+              : latestBlockEpochTimeBN.muln(1000).toNumber(),
+        },
+      ],
+      {
+        dataSource,
+      }
+    );
+  }
+  const [
+    lastBlockHeight,
+    latestGasPrice,
+    lastMinuteBlocks,
+  ] = await Promise.all([
+    queryLatestBlockHeight(options),
+    queryLatestGasPrice(options),
+    queryLastMinuteBlocks(options),
+  ]);
+  return {
+    latestBlockHeight: lastBlockHeight ? lastBlockHeight.block_height : 0,
+    latestGasPrice: latestGasPrice.gas_price,
+    numberOfLastMinuteBlocks: lastMinuteBlocks.total,
+  };
+};
+
+const queryDashboardTxInfo = async (options) => {
+  async function queryTransactionCountArray({ dataSource }) {
+    let query;
+    if (dataSource === DS_INDEXER_BACKEND) {
+      query = `SELECT date_trunc('day', to_timestamp(DIV(block_timestamp, 1000*1000*1000))) as date, count(transaction_hash) as total
+                FROM transactions
+                WHERE block_timestamp > (cast(EXTRACT(EPOCH FROM NOW()) - 60 * 60 * 24 * 14 as bigint) * 1000 * 1000 * 1000)
+                GROUP BY date 
+                ORDER BY date`;
+    } else {
+      query = `SELECT strftime('%Y-%m-%d',block_timestamp/1000,'unixepoch') as date, count(hash) as total
+                FROM transactions
+                WHERE (block_timestamp/1000) > (strftime('%s','now') - 60 * 60 * 24 * 15)
+                GROUP BY date
+                ORDER BY date`;
+    }
+    return await queryRows([query], { dataSource });
+  }
+  let transactionCountArray = await queryTransactionCountArray(options);
+  return transactionCountArray;
+};
+
+// Old query after refactor finish, we can delete
 const queryDashboardBlocksAndTxs = async ({ dataSource }) => {
   const transactionHashColumnName =
     dataSource === DS_INDEXER_BACKEND ? "transaction_hash" : "hash";
   const transactionIndexColumnName =
     dataSource === DS_INDEXER_BACKEND ? "index_in_chunk" : "transaction_index";
+  const transactionSignerAccountIdColumnName =
+    dataSource === DS_INDEXER_BACKEND ? "signer_account_id" : "signer_id";
+  const transactionReceiverAccountIdColumnName =
+    dataSource === DS_INDEXER_BACKEND ? "receiver_account_id" : "receiver_id";
+  const transactionBlockHashColumnName =
+    dataSource === DS_INDEXER_BACKEND ? "included_in_block_hash" : "block_hash";
+  const blockHashColumnName =
+    dataSource === DS_INDEXER_BACKEND ? "block_hash" : "hash";
+  const blockHeightColumnName =
+    dataSource === DS_INDEXER_BACKEND ? "block_height" : "height";
+  const blockTimestampColumnName =
+    dataSource === DS_INDEXER_BACKEND ? "block_timestamp" : "timestamp";
+  const blockPrehashColumnName =
+    dataSource === DS_INDEXER_BACKEND ? "prev_block_hash" : "prev_hash";
   let [transactions, blocks] = await Promise.all([
     queryRows(
       [
-        `SELECT ${transactionHashColumnName} as hash, signer_id as signerId, receiver_id as receiverId, 
-              block_hash as blockHash, block_timestamp as blockTimestamp, ${transactionIndexColumnName} as transactionIndex
-          FROM transactions
+        `SELECT ${transactionHashColumnName} as hash, ${transactionSignerAccountIdColumnName} as signer_id, ${transactionReceiverAccountIdColumnName} as receiver_id, 
+              ${transactionBlockHashColumnName} as block_hash, block_timestamp, ${transactionIndexColumnName} as transaction_index
+              FROM transactions
           ORDER BY block_timestamp DESC, ${transactionIndexColumnName} DESC
           LIMIT 10`,
       ],
       { dataSource }
     ),
     queryRows(
       [
-        `SELECT blocks.hash, blocks.height, blocks.timestamp, blocks.prev_hash as prevHash, COUNT(transactions.${transactionHashColumnName}) as transactionsCount
+        `SELECT blocks.${blockHashColumnName} as hash, blocks.${blockHeightColumnName} as height, blocks.${blockTimestampColumnName}, 
+              blocks.${blockPrehashColumnName} as prev_hash, 
+            COUNT(transactions.${transactionHashColumnName}) as transactions_count
           FROM (
-            SELECT blocks.hash
+            SELECT blocks.${blockHashColumnName}
             FROM blocks
-            ORDER BY blocks.height DESC
+            ORDER BY blocks.${blockHeightColumnName} DESC
             LIMIT 8
           ) as recent_blocks
-          LEFT JOIN blocks ON blocks.hash = recent_blocks.hash
-          LEFT JOIN transactions ON transactions.block_hash = recent_blocks.hash
-          GROUP BY blocks.hash
-          ORDER BY blocks.timestamp DESC`,
+          LEFT JOIN blocks ON blocks.${blockHashColumnName} = recent_blocks.${blockHashColumnName}
+          LEFT JOIN transactions ON transactions.${transactionBlockHashColumnName} = recent_blocks.${blockHashColumnName}
+          GROUP BY blocks.${blockHashColumnName}
+          ORDER BY blocks.${blockTimestampColumnName} DESC`,
       ],
       { dataSource }
     ),
   ]);
   let query;
   let transactionHashes = transactions.map((transaction) => transaction.hash);
   if (dataSource === DS_INDEXER_BACKEND) {
-    query = `SELECT transaction_hash, index as action_index, action_kind as kind, args
+    query = `SELECT transaction_hash, action_kind as kind, args
               FROM transaction_actions
               WHERE transaction_hash IN (:transactionHashes)
-              ORDER BY index`;
+              ORDER BY index_in_transaction DESC`;
   } else {
-    query = `SELECT transaction_hash, action_index, action_type as kind, action_args as args
+    query = `SELECT transaction_hash, action_type as kind, action_args as args
               FROM actions
               WHERE transaction_hash IN (:transactionHashes)
-              ORDER BY action_index`;
+              ORDER BY action_index DESC`;
   }
   const actionsArray = await queryRows([query, { transactionHashes }], {
     dataSource,
@@ -197,9 +287,54 @@ const queryDashboardBlocksAndTxs = async ({ dataSource }) => {
   return { transactions, blocks };
 };
 
+const aggregateStats = async (options) => {
+  async function queryLastDayTxCount({ dataSource }) {
+    let query;
+    if (dataSource === DS_INDEXER_BACKEND) {
+      query = `SELECT COUNT(*) AS total FROM transactions
+        WHERE block_timestamp > (cast(EXTRACT(EPOCH FROM NOW()) - 60 * 60 * 24 as bigint) * 1000 * 1000 * 1000)`;
+    } else {
+      query = `SELECT COUNT(*) AS total FROM transactions
+        WHERE block_timestamp > (strftime('%s','now') - 60 * 60 * 24) * 1000`;
+    }
+    return await querySingleRow([query], { dataSource });
+  }
+  async function queryLatestBlockHeight({ dataSource }) {
+    let query;
+    if (dataSource === DS_INDEXER_BACKEND) {
+      query = `SELECT block_height FROM blocks ORDER BY block_height DESC LIMIT 1`;
+    } else {
+      query = `SELECT height AS block_height FROM blocks ORDER BY height DESC LIMIT 1`;
+    }
+    return await querySingleRow([query], { dataSource });
+  }
+  const [
+    totalBlocks,
+    totalTransactions,
+    totalAccounts,
+    lastDayTxCount,
+    lastBlockHeight,
+  ] = await Promise.all([
+    querySingleRow([`SELECT COUNT(*) as total FROM blocks`], options),
+    querySingleRow([`SELECT COUNT(*) as total FROM transactions`], options),
+    querySingleRow([`SELECT COUNT(*) as total FROM accounts`], options),
+    queryLastDayTxCount(options),
+    queryLatestBlockHeight(options),
+  ]);
+  return {
+    totalAccounts: totalAccounts.total,
+    totalBlocks: totalBlocks.total,
+    totalTransactions: totalTransactions.total,
+    lastDayTxCount: lastDayTxCount.total,
+    lastBlockHeight: lastBlockHeight ? lastBlockHeight.block_height : 0,
+  };
+};
+
 exports.queryOnlineNodes = queryOnlineNodes;
 exports.addNodeInfo = addNodeInfo;
 exports.aggregateStats = aggregateStats;
 exports.pickOnlineValidatingNode = pickOnlineValidatingNode;
 exports.queryDashboardBlocksAndTxs = queryDashboardBlocksAndTxs;
 exports.getSyncedGenesis = getSyncedGenesis;
+exports.queryDashboardTxInfo = queryDashboardTxInfo;
+exports.queryDashboardBlockInfo = queryDashboardBlockInfo;

backend/src/index.js

@@ -31,6 +31,8 @@ const {
   pickOnlineValidatingNode,
   queryDashboardBlocksAndTxs,
   getSyncedGenesis,
+  queryDashboardBlockInfo,
+  queryDashboardTxInfo,
 } = require("./db-utils");
 
 async function startLegacySync() {
@@ -142,6 +144,7 @@ function startDataSourceSpecificJobs(wamp, dataSource) {
     console.log(`Starting regular data stats check from ${dataSource}...`);
     try {
       if (wamp.session) {
+        // old pub-sub data, after refactor, delete properly
         const dataStats = await aggregateStats({ dataSource });
         const { transactions, blocks } = await queryDashboardBlocksAndTxs({
           dataSource,
@@ -159,6 +162,21 @@ function startDataSourceSpecificJobs(wamp, dataSource) {
           [{ transactions, blocks }],
           wamp
         );
+        //new pub-sub data, coexists with old one now
+        const blockStats = await queryDashboardBlockInfo({ dataSource });
+        const transactionCountArray = await queryDashboardTxInfo({
+          dataSource,
+        });
+        wampPublish(
+          getDataSourceSpecificTopicName("chain-block-stats", dataSource),
+          [{ blockStats }],
+          wamp
+        );
+        wampPublish(
+          getDataSourceSpecificTopicName("chain-txs-stats", dataSource),
+          [{ transactionCountArray }],
+          wamp
+        );
       }
       console.log(`Regular data stats check from ${dataSource} is completed.`);
     } catch (error) {

frontend/src/context/ListProvider.jsx

@@ -13,8 +13,38 @@ export default (props) => {
   const fetchList = function (stats) {
     const { blocks, transactions } = stats[0];
 
-    dispatchBlocks(blocks);
-    dispatchTransactions(transactions);
+    const preprocessedBlocks = blocks.map(
+      ({ hash, height, timestamp, prev_hash, transactions_count }) => ({
+        hash,
+        height,
+        timestamp,
+        prevHash: prev_hash,
+        transactionsCount: transactions_count,
+      })
+    );
+
+    const preprocessedTransactions = transactions.map(
+      ({
+        hash,
+        signer_id,
+        receiver_id,
+        block_hash,
+        block_timestamp,
+        transaction_index,
+        actions,
+      }) => ({
+        hash,
+        signerId: signer_id,
+        receiverId: receiver_id,
+        blockHash: block_hash,
+        blockTimestamp: block_timestamp,
+        transactionIndex: transaction_index,
+        actions,
+      })
+    );
+
+    dispatchBlocks(preprocessedBlocks);
+    dispatchTransactions(preprocessedTransactions);
   };
 
   const Subscription = useCallback(() => {