Skip to content
This repository has been archived by the owner on Sep 26, 2024. It is now read-only.

feat(backend): Migrate Explorer to the Indexer for Explorer #478

Merged
merged 13 commits into from
Dec 4, 2020
Merged

feat(backend): Migrate Explorer to the Indexer for Explorer #478

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
6895de7
backend changes
icerove Nov 13, 2020
2517b30
fix require BN
icerove Nov 13, 2020
91177e4
Merge branch 'master' into refactor/frontend-redesign-query
icerove Nov 19, 2020
451b27e
fix the query
icerove Nov 23, 2020
13b798a
Update backend/src/db-utils.js
icerove Nov 24, 2020
4095ec2
Update backend/src/db-utils.js
icerove Nov 24, 2020
667eb3d
Update backend/src/index.js
icerove Nov 24, 2020
b2a5fbc
Update backend/src/index.js
icerove Nov 24, 2020
e61e044
fix as comments
icerove Nov 24, 2020
3cf8744
fix the query
icerove Nov 25, 2020
caa2e24
fix the query
icerove Nov 25, 2020
f6034b5
fix the query
icerove Nov 25, 2020
b92fe84
fixed frontend and backend compatibility
frol Dec 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
237 changes: 184 additions & 53 deletions backend/src/db-utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
const { DS_INDEXER_BACKEND } = require("./consts");
const models = require("../models");
const BN = require("bn.js");

const query = async ([query, replacements], { dataSource }) => {
const sequelize =
Expand Down Expand Up @@ -30,43 +31,6 @@ const getSyncedGenesis = async (options) => {
);
};

const aggregateStats = async (options) => {
async function queryLastDayTxCount({ dataSource }) {
let query;
if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT COUNT(*) AS total FROM transactions
WHERE DIV(block_timestamp, 1000*1000*1000) > (EXTRACT(EPOCH FROM NOW()) - 60 * 60 * 24)`;
} else {
query = `SELECT COUNT(*) AS total FROM transactions
WHERE block_timestamp > (strftime('%s','now') - 60 * 60 * 24) * 1000`;
}
return await querySingleRow([query], { dataSource });
}
const [
totalBlocks,
totalTransactions,
totalAccounts,
lastDayTxCount,
lastBlockHeight,
] = await Promise.all([
querySingleRow([`SELECT COUNT(*) as total FROM blocks`], options),
querySingleRow([`SELECT COUNT(*) as total FROM transactions`], options),
querySingleRow([`SELECT COUNT(*) as total FROM accounts`], options),
queryLastDayTxCount(options),
querySingleRow(
[`SELECT height FROM blocks ORDER BY height DESC LIMIT 1`],
options
),
]);
return {
totalAccounts: totalAccounts.total,
totalBlocks: totalBlocks.total,
totalTransactions: totalTransactions.total,
lastDayTxCount: lastDayTxCount.total,
lastBlockHeight: lastBlockHeight ? lastBlockHeight.height : 0,
};
};

const addNodeInfo = async (nodes) => {
const accountArray = nodes.map((node) => node.account_id);
let nodesInfo = await queryRows([
Expand Down Expand Up @@ -120,51 +84,173 @@ const queryOnlineNodes = async () => {
]);
};

//new query for new dashboard
const queryDashboardBlockInfo = async (options) => {
async function queryLatestBlockHeight({ dataSource }) {
let query;
if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT block_height FROM blocks ORDER BY block_height DESC LIMIT 1`;
} else {
query = `SELECT height AS block_height FROM blocks ORDER BY height DESC LIMIT 1`;
}
return await querySingleRow([query], { dataSource });
}
async function queryLatestGasPrice({ dataSource }) {
let query;
if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT gas_price FROM blocks ORDER BY block_height DESC LIMIT 1`;
} else {
query = `SELECT gas_price FROM blocks ORDER BY height DESC LIMIT 1`;
}
return await querySingleRow([query], { dataSource });
}
async function queryLastMinuteBlocks({ dataSource }) {
let query;
if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT block_timestamp AS latest_block_timestamp FROM blocks ORDER BY block_timestamp DESC LIMIT 1`;
} else {
query = `SELECT timestamp AS latest_block_timestamp FROM blocks ORDER BY timestamp DESC LIMIT 1`;
}
const latestBlockTimestampOrNone = await querySingleRow([query], {
dataSource,
});
if (!latestBlockTimestampOrNone) {
return { total: 0 };
}
const {
latest_block_timestamp: latestBlockTimestamp,
} = latestBlockTimestampOrNone;
const latestBlockTimestampBN = new BN(latestBlockTimestamp);
const nowEpochTimeBN = new BN(Math.floor(new Date().getTime() / 1000));
icerove marked this conversation as resolved.
Show resolved Hide resolved
let latestBlockEpochTimeBN;
if (dataSource == DS_INDEXER_BACKEND) {
icerove marked this conversation as resolved.
Show resolved Hide resolved
latestBlockEpochTimeBN = latestBlockTimestampBN.div(new BN("1000000000"));
} else {
latestBlockEpochTimeBN = latestBlockTimestampBN.divn(1000);
}
// If the latest block is older than 1 minute from now, we report 0
if (nowEpochTimeBN.sub(latestBlockEpochTimeBN).gtn(60)) {
return { total: 0 };
}

if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT COUNT(*) AS total FROM blocks
WHERE DIV(block_timestamp, 1000*1000*1000) > (:latestBlockTimestamp - 60)`;
icerove marked this conversation as resolved.
Show resolved Hide resolved
} else {
query = `SELECT COUNT(*) AS total FROM blocks
WHERE timestamp > (:latestBlockTimestamp - 60 * 1000)`;
}
return await querySingleRow(
[
query,
{
latestBlockTimestamp:
dataSource == DS_INDEXER_BACKEND
? latestBlockEpochTimeBN.toNumber()
: latestBlockEpochTimeBN.muln(1000).toNumber(),
},
],
{
dataSource,
}
);
}
const [
lastBlockHeight,
latestGasPrice,
lastMinuteBlocks,
] = await Promise.all([
queryLatestBlockHeight(options),
queryLatestGasPrice(options),
queryLastMinuteBlocks(options),
]);
return {
latestBlockHeight: lastBlockHeight ? lastBlockHeight.block_height : 0,
latestGasPrice: latestGasPrice.gas_price,
numberOfLastMinuteBlocks: lastMinuteBlocks.total,
};
};

const queryDashboardTxInfo = async (options) => {
async function queryTransactionCountArray({ dataSource }) {
let query;
if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT date_trunc('day', to_timestamp(DIV(block_timestamp, 1000*1000*1000))) as date, count(1) as total
FROM transactions
WHERE DIV(block_timestamp, 1000*1000*1000) > (EXTRACT(EPOCH FROM NOW()) - 60 * 60 * 24 * 14)
GROUP BY 1`;
icerove marked this conversation as resolved.
Show resolved Hide resolved
} else {
query = `SELECT strftime('%Y-%m-%d',block_timestamp/1000,'unixepoch') as date, count(1) as total
FROM transactions
WHERE (block_timestamp/1000) > (strftime('%s','now') - 60 * 60 * 24 * 15)
GROUP BY 1`;
icerove marked this conversation as resolved.
Show resolved Hide resolved
}
return await queryRows([query], { dataSource });
}
let transactionCountArray = await queryTransactionCountArray(options);
return transactionCountArray;
};

// Old query after refactor finish, we can delete
const queryDashboardBlocksAndTxs = async ({ dataSource }) => {
const transactionHashColumnName =
dataSource === DS_INDEXER_BACKEND ? "transaction_hash" : "hash";
const transactionIndexColumnName =
dataSource === DS_INDEXER_BACKEND ? "index_in_chunk" : "transaction_index";
const transactionSignIdColumnName =
icerove marked this conversation as resolved.
Show resolved Hide resolved
dataSource === DS_INDEXER_BACKEND ? "signer_account_id" : "signer_id";
const transactionReceiveIdColumnName =
dataSource === DS_INDEXER_BACKEND ? "receiver_account_id" : "receiver_id";
const transactionBlockHashColumnName =
dataSource === DS_INDEXER_BACKEND ? "included_in_block_hash" : "block_hash";
const blockHashColumnName =
dataSource === DS_INDEXER_BACKEND ? "block_hash" : "hash";
const blockHeightColumnName =
dataSource === DS_INDEXER_BACKEND ? "block_height" : "height";
const blockTimestampColumnName =
dataSource === DS_INDEXER_BACKEND ? "block_timestamp" : "timestamp";
const blockPrehashColumnName =
dataSource === DS_INDEXER_BACKEND ? "prev_block_hash" : "prev_hash";
let [transactions, blocks] = await Promise.all([
queryRows(
[
`SELECT ${transactionHashColumnName} as hash, signer_id as signerId, receiver_id as receiverId,
block_hash as blockHash, block_timestamp as blockTimestamp, ${transactionIndexColumnName} as transactionIndex
FROM transactions
`SELECT ${transactionHashColumnName} as hash, ${transactionSignIdColumnName} as signer_id, ${transactionReceiveIdColumnName} as receiver_id,
${transactionBlockHashColumnName} as block_hash, block_timestamp as blockTimestamp, ${transactionIndexColumnName} as transaction_index
FROM transactions
ORDER BY block_timestamp DESC, ${transactionIndexColumnName} DESC
LIMIT 10`,
],
{ dataSource }
),
queryRows(
[
`SELECT blocks.hash, blocks.height, blocks.timestamp, blocks.prev_hash as prevHash, COUNT(transactions.${transactionHashColumnName}) as transactionsCount
`SELECT blocks.${blockHashColumnName} as hash, blocks.${blockHeightColumnName} as height, blocks.${blockTimestampColumnName},
blocks.${blockPrehashColumnName} as prev_hash,
COUNT(transactions.${transactionHashColumnName}) as transactionsCount
FROM (
SELECT blocks.hash
SELECT blocks.${blockHashColumnName}
FROM blocks
ORDER BY blocks.height DESC
ORDER BY blocks.${blockHeightColumnName} DESC
LIMIT 8
) as recent_blocks
LEFT JOIN blocks ON blocks.hash = recent_blocks.hash
LEFT JOIN transactions ON transactions.block_hash = recent_blocks.hash
GROUP BY blocks.hash
ORDER BY blocks.timestamp DESC`,
LEFT JOIN blocks ON blocks.${blockHashColumnName} = recent_blocks.${blockHashColumnName}
LEFT JOIN transactions ON transactions.${transactionBlockHashColumnName} = recent_blocks.${blockHashColumnName}
GROUP BY blocks.${blockHashColumnName}
ORDER BY blocks.${blockTimestampColumnName} DESC`,
],
{ dataSource }
),
]);
let query;
let transactionHashes = transactions.map((transaction) => transaction.hash);
if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT transaction_hash, index as action_index, action_kind as kind, args
query = `SELECT transaction_hash, action_kind as kind, args
FROM transaction_actions
WHERE transaction_hash IN (:transactionHashes)
ORDER BY index`;
icerove marked this conversation as resolved.
Show resolved Hide resolved
WHERE transaction_hash IN (:transactionHashes)`;
} else {
query = `SELECT transaction_hash, action_index, action_type as kind, action_args as args
query = `SELECT transaction_hash, action_type as kind, action_args as args
FROM actions
WHERE transaction_hash IN (:transactionHashes)
ORDER BY action_index`;
WHERE transaction_hash IN (:transactionHashes)`;
}
const actionsArray = await queryRows([query, { transactionHashes }], {
dataSource,
Expand Down Expand Up @@ -197,9 +283,54 @@ const queryDashboardBlocksAndTxs = async ({ dataSource }) => {
return { transactions, blocks };
};

const aggregateStats = async (options) => {
async function queryLastDayTxCount({ dataSource }) {
let query;
if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT COUNT(*) AS total FROM transactions
WHERE DIV(block_timestamp, 1000*1000*1000) > (EXTRACT(EPOCH FROM NOW()) - 60 * 60 * 24)`;
icerove marked this conversation as resolved.
Show resolved Hide resolved
} else {
query = `SELECT COUNT(*) AS total FROM transactions
WHERE block_timestamp > (strftime('%s','now') - 60 * 60 * 24) * 1000`;
}
return await querySingleRow([query], { dataSource });
}
async function queryLatestBlockHeight({ dataSource }) {
let query;
if (dataSource === DS_INDEXER_BACKEND) {
query = `SELECT block_height FROM blocks ORDER BY block_height DESC LIMIT 1`;
} else {
query = `SELECT height AS block_height FROM blocks ORDER BY height DESC LIMIT 1`;
}
return await querySingleRow([query], { dataSource });
}
const [
totalBlocks,
totalTransactions,
totalAccounts,
lastDayTxCount,
lastBlockHeight,
] = await Promise.all([
querySingleRow([`SELECT COUNT(*) as total FROM blocks`], options),
querySingleRow([`SELECT COUNT(*) as total FROM transactions`], options),
querySingleRow([`SELECT COUNT(*) as total FROM accounts`], options),
queryLastDayTxCount(options),
queryLatestBlockHeight(options),
]);
return {
totalAccounts: totalAccounts.total,
totalBlocks: totalBlocks.total,
totalTransactions: totalTransactions.total,
lastDayTxCount: lastDayTxCount.total,
lastBlockHeight: lastBlockHeight ? lastBlockHeight.height : 0,
};
};

exports.queryOnlineNodes = queryOnlineNodes;
exports.addNodeInfo = addNodeInfo;
exports.aggregateStats = aggregateStats;
exports.pickOnlineValidatingNode = pickOnlineValidatingNode;
exports.queryDashboardBlocksAndTxs = queryDashboardBlocksAndTxs;
exports.getSyncedGenesis = getSyncedGenesis;
exports.queryDashboardTxInfo = queryDashboardTxInfo;
exports.queryDashboardBlockInfo = queryDashboardBlockInfo;
18 changes: 18 additions & 0 deletions backend/src/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ const {
pickOnlineValidatingNode,
queryDashboardBlocksAndTxs,
getSyncedGenesis,
queryDashboardBlockInfo,
queryDashboardTxInfo,
} = require("./db-utils");

async function startLegacySync() {
Expand Down Expand Up @@ -142,6 +144,7 @@ function startDataSourceSpecificJobs(wamp, dataSource) {
console.log(`Starting regular data stats check from ${dataSource}...`);
try {
if (wamp.session) {
// ole pub-sub data, after refactor, delete properly
icerove marked this conversation as resolved.
Show resolved Hide resolved
const dataStats = await aggregateStats({ dataSource });
const { transactions, blocks } = await queryDashboardBlocksAndTxs({
dataSource,
Expand All @@ -159,6 +162,21 @@ function startDataSourceSpecificJobs(wamp, dataSource) {
[{ transactions, blocks }],
wamp
);
//new pub-sub data, consist with old one now
icerove marked this conversation as resolved.
Show resolved Hide resolved
const blockStats = await queryDashboardBlockInfo({ dataSource });
const transactionCountArray = await queryDashboardTxInfo({
dataSource,
});
wampPublish(
getDataSourceSpecificTopicName("chain-block-stats", dataSource),
[{ blockStats }],
wamp
);
wampPublish(
getDataSourceSpecificTopicName("chain-txs-stats", dataSource),
[{ transactionCountArray }],
wamp
);
}
console.log(`Regular data stats check from ${dataSource} is completed.`);
} catch (error) {
Expand Down