Skip to content
This repository has been archived by the owner on Sep 26, 2024. It is now read-only.

Commit

Permalink
feat(backend): Migrate Explorer to the Indexer for Explorer (#478)
Browse files Browse the repository at this point in the history 
# Testing Plan

* [x] Manually confirmed that the frontend is compatible with the updated queries
* [x] Confirmed that the indexer setup sends the WAMP events just fine
  • Loading branch information
@icerove
icerove authored Dec 4, 2020
1 parent 912f29a commit 7d81180
Show file tree
Hide file tree
Showing 3 changed files with 236 additions and 53 deletions.
237 changes: 186 additions & 51 deletions backend/src/db-utils.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
const { DS_INDEXER_BACKEND } = require("./consts");
const models = require("../models");
const BN = require("bn.js");

const query = async ([query, replacements], { dataSource }) => {
const sequelize =
Expand Down Expand Up @@ -30,43 +31,6 @@ const getSyncedGenesis = async (options) => {
);
};

const aggregateStats = async (options) => {
async function queryLastDayTxCount({ dataSource }) {
let query;
if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT COUNT(*) AS total FROM transactions
WHERE DIV(block_timestamp, 1000*1000*1000) > (EXTRACT(EPOCH FROM NOW()) - 60 * 60 * 24)`;
} else {
query = `SELECT COUNT(*) AS total FROM transactions
WHERE block_timestamp > (strftime('%s','now') - 60 * 60 * 24) * 1000`;
}
return await querySingleRow([query], { dataSource });
}
const [
totalBlocks,
totalTransactions,
totalAccounts,
lastDayTxCount,
lastBlockHeight,
] = await Promise.all([
querySingleRow([`SELECT COUNT(*) as total FROM blocks`], options),
querySingleRow([`SELECT COUNT(*) as total FROM transactions`], options),
querySingleRow([`SELECT COUNT(*) as total FROM accounts`], options),
queryLastDayTxCount(options),
querySingleRow(
[`SELECT height FROM blocks ORDER BY height DESC LIMIT 1`],
options
),
]);
return {
totalAccounts: totalAccounts.total,
totalBlocks: totalBlocks.total,
totalTransactions: totalTransactions.total,
lastDayTxCount: lastDayTxCount.total,
lastBlockHeight: lastBlockHeight ? lastBlockHeight.height : 0,
};
};

const addNodeInfo = async (nodes) => {
const accountArray = nodes.map((node) => node.account_id);
let nodesInfo = await queryRows([
Expand Down Expand Up @@ -120,51 +84,177 @@ const queryOnlineNodes = async () => {
]);
};

//new query for new dashboard
const queryDashboardBlockInfo = async (options) => {
async function queryLatestBlockHeight({ dataSource }) {
let query;
if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT block_height FROM blocks ORDER BY block_height DESC LIMIT 1`;
} else {
query = `SELECT height AS block_height FROM blocks ORDER BY height DESC LIMIT 1`;
}
return await querySingleRow([query], { dataSource });
}
async function queryLatestGasPrice({ dataSource }) {
let query;
if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT gas_price FROM blocks ORDER BY block_height DESC LIMIT 1`;
} else {
query = `SELECT gas_price FROM blocks ORDER BY height DESC LIMIT 1`;
}
return await querySingleRow([query], { dataSource });
}
async function queryLastMinuteBlocks({ dataSource }) {
let query;
if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT block_timestamp AS latest_block_timestamp FROM blocks ORDER BY block_timestamp DESC LIMIT 1`;
} else {
query = `SELECT timestamp AS latest_block_timestamp FROM blocks ORDER BY timestamp DESC LIMIT 1`;
}
const latestBlockTimestampOrNone = await querySingleRow([query], {
dataSource,
});
if (!latestBlockTimestampOrNone) {
return { total: 0 };
}
const {
latest_block_timestamp: latestBlockTimestamp,
} = latestBlockTimestampOrNone;
const latestBlockTimestampBN = new BN(latestBlockTimestamp);
const currentUnixTimeBN = new BN(Math.floor(new Date().getTime() / 1000));
let latestBlockEpochTimeBN;
if (dataSource === DS_INDEXER_BACKEND) {
latestBlockEpochTimeBN = latestBlockTimestampBN.div(new BN("1000000000"));
} else {
latestBlockEpochTimeBN = latestBlockTimestampBN.divn(1000);
}
// If the latest block is older than 1 minute from now, we report 0
if (currentUnixTimeBN.sub(latestBlockEpochTimeBN).gtn(60)) {
return { total: 0 };
}

if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT COUNT(*) AS total FROM blocks
WHERE block_timestamp > (cast(EXTRACT(EPOCH FROM NOW()) - 60 as bigint) * 1000 * 1000 * 1000)`;
} else {
query = `SELECT COUNT(*) AS total FROM blocks
WHERE timestamp > (:latestBlockTimestamp - 60 * 1000)`;
}
return await querySingleRow(
[
query,
{
latestBlockTimestamp:
dataSource == DS_INDEXER_BACKEND
? latestBlockEpochTimeBN.toNumber()
: latestBlockEpochTimeBN.muln(1000).toNumber(),
},
],
{
dataSource,
}
);
}
const [
lastBlockHeight,
latestGasPrice,
lastMinuteBlocks,
] = await Promise.all([
queryLatestBlockHeight(options),
queryLatestGasPrice(options),
queryLastMinuteBlocks(options),
]);
return {
latestBlockHeight: lastBlockHeight ? lastBlockHeight.block_height : 0,
latestGasPrice: latestGasPrice.gas_price,
numberOfLastMinuteBlocks: lastMinuteBlocks.total,
};
};

const queryDashboardTxInfo = async (options) => {
async function queryTransactionCountArray({ dataSource }) {
let query;
if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT date_trunc('day', to_timestamp(DIV(block_timestamp, 1000*1000*1000))) as date, count(transaction_hash) as total
FROM transactions
WHERE block_timestamp > (cast(EXTRACT(EPOCH FROM NOW()) - 60 * 60 * 24 * 14 as bigint) * 1000 * 1000 * 1000)
GROUP BY date
ORDER BY date`;
} else {
query = `SELECT strftime('%Y-%m-%d',block_timestamp/1000,'unixepoch') as date, count(hash) as total
FROM transactions
WHERE (block_timestamp/1000) > (strftime('%s','now') - 60 * 60 * 24 * 15)
GROUP BY date
ORDER BY date`;
}
return await queryRows([query], { dataSource });
}
let transactionCountArray = await queryTransactionCountArray(options);
return transactionCountArray;
};

// Old query after refactor finish, we can delete
const queryDashboardBlocksAndTxs = async ({ dataSource }) => {
const transactionHashColumnName =
dataSource === DS_INDEXER_BACKEND ? "transaction_hash" : "hash";
const transactionIndexColumnName =
dataSource === DS_INDEXER_BACKEND ? "index_in_chunk" : "transaction_index";
const transactionSignerAccountIdColumnName =
dataSource === DS_INDEXER_BACKEND ? "signer_account_id" : "signer_id";
const transactionReceiverAccountIdColumnName =
dataSource === DS_INDEXER_BACKEND ? "receiver_account_id" : "receiver_id";
const transactionBlockHashColumnName =
dataSource === DS_INDEXER_BACKEND ? "included_in_block_hash" : "block_hash";
const blockHashColumnName =
dataSource === DS_INDEXER_BACKEND ? "block_hash" : "hash";
const blockHeightColumnName =
dataSource === DS_INDEXER_BACKEND ? "block_height" : "height";
const blockTimestampColumnName =
dataSource === DS_INDEXER_BACKEND ? "block_timestamp" : "timestamp";
const blockPrehashColumnName =
dataSource === DS_INDEXER_BACKEND ? "prev_block_hash" : "prev_hash";
let [transactions, blocks] = await Promise.all([
queryRows(
[
`SELECT ${transactionHashColumnName} as hash, signer_id as signerId, receiver_id as receiverId,
block_hash as blockHash, block_timestamp as blockTimestamp, ${transactionIndexColumnName} as transactionIndex
FROM transactions
`SELECT ${transactionHashColumnName} as hash, ${transactionSignerAccountIdColumnName} as signer_id, ${transactionReceiverAccountIdColumnName} as receiver_id,
${transactionBlockHashColumnName} as block_hash, block_timestamp, ${transactionIndexColumnName} as transaction_index
FROM transactions
ORDER BY block_timestamp DESC, ${transactionIndexColumnName} DESC
LIMIT 10`,
],
{ dataSource }
),
queryRows(
[
`SELECT blocks.hash, blocks.height, blocks.timestamp, blocks.prev_hash as prevHash, COUNT(transactions.${transactionHashColumnName}) as transactionsCount
`SELECT blocks.${blockHashColumnName} as hash, blocks.${blockHeightColumnName} as height, blocks.${blockTimestampColumnName},
blocks.${blockPrehashColumnName} as prev_hash,
COUNT(transactions.${transactionHashColumnName}) as transactions_count
FROM (
SELECT blocks.hash
SELECT blocks.${blockHashColumnName}
FROM blocks
ORDER BY blocks.height DESC
ORDER BY blocks.${blockHeightColumnName} DESC
LIMIT 8
) as recent_blocks
LEFT JOIN blocks ON blocks.hash = recent_blocks.hash
LEFT JOIN transactions ON transactions.block_hash = recent_blocks.hash
GROUP BY blocks.hash
ORDER BY blocks.timestamp DESC`,
LEFT JOIN blocks ON blocks.${blockHashColumnName} = recent_blocks.${blockHashColumnName}
LEFT JOIN transactions ON transactions.${transactionBlockHashColumnName} = recent_blocks.${blockHashColumnName}
GROUP BY blocks.${blockHashColumnName}
ORDER BY blocks.${blockTimestampColumnName} DESC`,
],
{ dataSource }
),
]);
let query;
let transactionHashes = transactions.map((transaction) => transaction.hash);
if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT transaction_hash, index as action_index, action_kind as kind, args
query = `SELECT transaction_hash, action_kind as kind, args
FROM transaction_actions
WHERE transaction_hash IN (:transactionHashes)
ORDER BY index`;
ORDER BY index_in_transaction DESC`;
} else {
query = `SELECT transaction_hash, action_index, action_type as kind, action_args as args
query = `SELECT transaction_hash, action_type as kind, action_args as args
FROM actions
WHERE transaction_hash IN (:transactionHashes)
ORDER BY action_index`;
ORDER BY action_index DESC`;
}
const actionsArray = await queryRows([query, { transactionHashes }], {
dataSource,
Expand Down Expand Up @@ -197,9 +287,54 @@ const queryDashboardBlocksAndTxs = async ({ dataSource }) => {
return { transactions, blocks };
};

const aggregateStats = async (options) => {
async function queryLastDayTxCount({ dataSource }) {
let query;
if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT COUNT(*) AS total FROM transactions
WHERE block_timestamp > (cast(EXTRACT(EPOCH FROM NOW()) - 60 * 60 * 24 as bigint) * 1000 * 1000 * 1000)`;
} else {
query = `SELECT COUNT(*) AS total FROM transactions
WHERE block_timestamp > (strftime('%s','now') - 60 * 60 * 24) * 1000`;
}
return await querySingleRow([query], { dataSource });
}
async function queryLatestBlockHeight({ dataSource }) {
let query;
if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT block_height FROM blocks ORDER BY block_height DESC LIMIT 1`;
} else {
query = `SELECT height AS block_height FROM blocks ORDER BY height DESC LIMIT 1`;
}
return await querySingleRow([query], { dataSource });
}
const [
totalBlocks,
totalTransactions,
totalAccounts,
lastDayTxCount,
lastBlockHeight,
] = await Promise.all([
querySingleRow([`SELECT COUNT(*) as total FROM blocks`], options),
querySingleRow([`SELECT COUNT(*) as total FROM transactions`], options),
querySingleRow([`SELECT COUNT(*) as total FROM accounts`], options),
queryLastDayTxCount(options),
queryLatestBlockHeight(options),
]);
return {
totalAccounts: totalAccounts.total,
totalBlocks: totalBlocks.total,
totalTransactions: totalTransactions.total,
lastDayTxCount: lastDayTxCount.total,
lastBlockHeight: lastBlockHeight ? lastBlockHeight.block_height : 0,
};
};

exports.queryOnlineNodes = queryOnlineNodes;
exports.addNodeInfo = addNodeInfo;
exports.aggregateStats = aggregateStats;
exports.pickOnlineValidatingNode = pickOnlineValidatingNode;
exports.queryDashboardBlocksAndTxs = queryDashboardBlocksAndTxs;
exports.getSyncedGenesis = getSyncedGenesis;
exports.queryDashboardTxInfo = queryDashboardTxInfo;
exports.queryDashboardBlockInfo = queryDashboardBlockInfo;
18 changes: 18 additions & 0 deletions backend/src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ const {
pickOnlineValidatingNode,
queryDashboardBlocksAndTxs,
getSyncedGenesis,
queryDashboardBlockInfo,
queryDashboardTxInfo,
} = require("./db-utils");

async function startLegacySync() {
Expand Down Expand Up @@ -142,6 +144,7 @@ function startDataSourceSpecificJobs(wamp, dataSource) {
console.log(`Starting regular data stats check from ${dataSource}...`);
try {
if (wamp.session) {
// old pub-sub data, after refactor, delete properly
const dataStats = await aggregateStats({ dataSource });
const { transactions, blocks } = await queryDashboardBlocksAndTxs({
dataSource,
Expand All @@ -159,6 +162,21 @@ function startDataSourceSpecificJobs(wamp, dataSource) {
[{ transactions, blocks }],
wamp
);
//new pub-sub data, coexists with old one now
const blockStats = await queryDashboardBlockInfo({ dataSource });
const transactionCountArray = await queryDashboardTxInfo({
dataSource,
});
wampPublish(
getDataSourceSpecificTopicName("chain-block-stats", dataSource),
[{ blockStats }],
wamp
);
wampPublish(
getDataSourceSpecificTopicName("chain-txs-stats", dataSource),
[{ transactionCountArray }],
wamp
);
}
console.log(`Regular data stats check from ${dataSource} is completed.`);
} catch (error) {
Expand Down
34 changes: 32 additions & 2 deletions frontend/src/context/ListProvider.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,38 @@ export default (props) => {
const fetchList = function (stats) {
const { blocks, transactions } = stats[0];

dispatchBlocks(blocks);
dispatchTransactions(transactions);
const preprocessedBlocks = blocks.map(
({ hash, height, timestamp, prev_hash, transactions_count }) => ({
hash,
height,
timestamp,
prevHash: prev_hash,
transactionsCount: transactions_count,
})
);

const preprocessedTransactions = transactions.map(
({
hash,
signer_id,
receiver_id,
block_hash,
block_timestamp,
transaction_index,
actions,
}) => ({
hash,
signerId: signer_id,
receiverId: receiver_id,
blockHash: block_hash,
blockTimestamp: block_timestamp,
transactionIndex: transaction_index,
actions,
})
);

dispatchBlocks(preprocessedBlocks);
dispatchTransactions(preprocessedTransactions);
};

const Subscription = useCallback(() => {
Expand Down

0 comments on commit 7d81180

Please sign in to comment.