diff --git a/__tests__/integration/core-api/__support__/setup.ts b/__tests__/integration/core-api/__support__/setup.ts index c258b64bfe..f69b4bce08 100644 --- a/__tests__/integration/core-api/__support__/setup.ts +++ b/__tests__/integration/core-api/__support__/setup.ts @@ -1,13 +1,14 @@ import { app } from "@arkecosystem/core-container"; import { Database } from "@arkecosystem/core-interfaces"; import delay from "delay"; -import { registerWithContainer, setUpContainer } from "../../../utils/helpers/container"; import { plugin } from "../../../../packages/core-api/src/plugin"; +import { registerWithContainer, setUpContainer } from "../../../utils/helpers/container"; import { delegates } from "../../../utils/fixtures"; import { generateRound } from "./utils/generate-round"; -import { queries } from "../../../../packages/core-database-postgres/src/queries"; +import { models } from "@arkecosystem/crypto"; +import { sortBy } from "@arkecosystem/utils"; const round = generateRound(delegates.map(delegate => delegate.publicKey), 1); @@ -32,8 +33,7 @@ async function setUp() { const databaseService = app.resolvePlugin("database"); await databaseService.connection.roundsRepository.truncate(); - await databaseService.buildWallets(1); - await databaseService.saveWallets(true); + await databaseService.buildWallets(); await databaseService.saveRound(round); await registerWithContainer(plugin, options); @@ -49,9 +49,11 @@ async function tearDown() { async function calculateRanks() { const databaseService = app.resolvePlugin("database"); - const rows = await (databaseService.connection as any).query.manyOrNone(queries.spv.delegatesRanks); + const delegateWallets = Object.values(databaseService.walletManager.allByUsername()).sort( + (a: models.Wallet, b: models.Wallet) => b.voteBalance.comparedTo(a.voteBalance), + ); - rows.forEach((delegate, i) => { + sortBy(delegateWallets, "publicKey").forEach((delegate, i) => { const wallet = databaseService.walletManager.findByPublicKey(delegate.publicKey); wallet.missedBlocks = +delegate.missedBlocks; (wallet as any).rate = i + 1; diff --git a/__tests__/integration/core-blockchain/blockchain.test.ts b/__tests__/integration/core-blockchain/blockchain.test.ts index afb4ebcd40..a2389d3cdf 100644 --- a/__tests__/integration/core-blockchain/blockchain.test.ts +++ b/__tests__/integration/core-blockchain/blockchain.test.ts @@ -342,7 +342,7 @@ async function __resetToHeight1() { // Make sure the wallet manager has been fed or else revertRound // cannot determine the previous delegates. This is only necessary, because // the database is not dropped after the unit tests are done. - await blockchain.database.buildWallets(lastBlock.data.height); + await blockchain.database.buildWallets(); // Index the genesis wallet or else revert block at height 1 fails const generator = crypto.getAddress(genesisBlock.data.generatorPublicKey); diff --git a/__tests__/unit/core-api/__support__/setup.ts b/__tests__/unit/core-api/__support__/setup.ts index ea67cf8c05..b9da207f5d 100644 --- a/__tests__/unit/core-api/__support__/setup.ts +++ b/__tests__/unit/core-api/__support__/setup.ts @@ -7,7 +7,8 @@ import { registerWithContainer, setUpContainer } from "../../../utils/helpers/co import { delegates } from "../../../utils/fixtures"; import { generateRound } from "./utils/generate-round"; -import { queries } from "../../../../packages/core-database-postgres/src/queries"; +import { models } from "@arkecosystem/crypto"; +import { sortBy } from "@arkecosystem/utils"; const round = generateRound(delegates.map(delegate => delegate.publicKey), 1); @@ -33,8 +34,7 @@ async function setUp() { const databaseService = app.resolvePlugin("database"); await databaseService.connection.roundsRepository.truncate(); - await databaseService.buildWallets(1); - await databaseService.saveWallets(true); + await databaseService.buildWallets(); await databaseService.saveRound(round); await registerWithContainer(plugin, options); @@ -50,9 +50,11 @@ async function tearDown() { async function calculateRanks() { const databaseService = app.resolvePlugin("database"); - const rows = await (databaseService.connection as any).query.manyOrNone(queries.spv.delegatesRanks); + const delegateWallets = Object.values(databaseService.walletManager.allByUsername()).sort( + (a: models.Wallet, b: models.Wallet) => b.voteBalance.comparedTo(a.voteBalance), + ); - rows.forEach((delegate, i) => { + sortBy(delegateWallets, "publicKey").forEach((delegate, i) => { const wallet = databaseService.walletManager.findByPublicKey(delegate.publicKey); wallet.missedBlocks = +delegate.missedBlocks; (wallet as any).rate = i + 1; diff --git a/__tests__/unit/core-blockchain/blockchain.test.ts b/__tests__/unit/core-blockchain/blockchain.test.ts index d8fe6e449a..3826bc17e6 100644 --- a/__tests__/unit/core-blockchain/blockchain.test.ts +++ b/__tests__/unit/core-blockchain/blockchain.test.ts @@ -368,7 +368,7 @@ async function __resetToHeight1() { // Make sure the wallet manager has been fed or else revertRound // cannot determine the previous delegates. This is only necessary, because // the database is not dropped after the unit tests are done. - await blockchain.database.buildWallets(lastBlock.data.height); + await blockchain.database.buildWallets(); // Index the genesis wallet or else revert block at height 1 fails const generator = crypto.getAddress(genesisBlock.data.generatorPublicKey); diff --git a/__tests__/unit/core-blockchain/state-machine.test.ts b/__tests__/unit/core-blockchain/state-machine.test.ts index 5cc73145ef..5a9c76021d 100644 --- a/__tests__/unit/core-blockchain/state-machine.test.ts +++ b/__tests__/unit/core-blockchain/state-machine.test.ts @@ -225,8 +225,6 @@ describe("State Machine", () => { // @ts-ignore buildWallets: jest.spyOn(blockchain.database, "buildWallets").mockReturnValue(true), // @ts-ignore - saveWallets: jest.spyOn(blockchain.database, "saveWallets").mockReturnValue(true), - // @ts-ignore applyRound: jest.spyOn(blockchain.database, "applyRound").mockReturnValue(true), // @ts-ignore getActiveDelegates: jest.spyOn(blockchain.database, "getActiveDelegates").mockReturnValue(true), @@ -290,8 +288,7 @@ describe("State Machine", () => { stateStorage.networkStart = true; await expect(() => actionMap.init()).toDispatch(blockchain, "STARTED"); - expect(databaseMocks.buildWallets).toHaveBeenCalledWith(1); - expect(databaseMocks.saveWallets).toHaveBeenCalledWith(true); + expect(databaseMocks.buildWallets).toHaveBeenCalled(); expect(databaseMocks.applyRound).toHaveBeenCalledWith(1); stateStorage.networkStart = false; // reset to default value @@ -303,7 +300,7 @@ describe("State Machine", () => { const loggerVerbose = jest.spyOn(logger, "verbose"); await expect(() => actionMap.init()).toDispatch(blockchain, "STARTED"); - expect(databaseMocks.buildWallets).toHaveBeenCalledWith(1); + expect(databaseMocks.buildWallets).toHaveBeenCalled(); expect(loggerVerbose).toHaveBeenCalledWith( "TEST SUITE DETECTED! SYNCING WALLETS AND STARTING IMMEDIATELY.", ); @@ -334,7 +331,6 @@ describe("State Machine", () => { expect(loggerWarn).toHaveBeenCalledWith( "Rebuilding wallets table because of some inconsistencies. Most likely due to an unfortunate shutdown.", ); - expect(databaseMocks.saveWallets).toHaveBeenCalledWith(true); }); it("should clean round data if new round starts at block.height + 1 (and dispatch STARTED)", async () => { diff --git a/__tests__/unit/core-database/__fixtures__/database-connection-stub.ts b/__tests__/unit/core-database/__fixtures__/database-connection-stub.ts index 43903162c1..ef2ba50eea 100644 --- a/__tests__/unit/core-database/__fixtures__/database-connection-stub.ts +++ b/__tests__/unit/core-database/__fixtures__/database-connection-stub.ts @@ -10,7 +10,7 @@ export class DatabaseConnectionStub implements Database.IDatabaseConnection { public walletsRepository: Database.IWalletsRepository; public options: any; - public buildWallets(height: number): Promise { + public buildWallets(): Promise { return undefined; } @@ -43,8 +43,4 @@ export class DatabaseConnectionStub implements Database.IDatabaseConnection { public saveBlock(block: models.Block): Promise { return undefined; } - - public saveWallets(wallets: any[], force?: boolean): Promise { - return undefined; - } } diff --git a/__tests__/unit/core-database/database-service.test.ts b/__tests__/unit/core-database/database-service.test.ts index 48c7bfe863..4edc380af5 100644 --- a/__tests__/unit/core-database/database-service.test.ts +++ b/__tests__/unit/core-database/database-service.test.ts @@ -49,7 +49,6 @@ describe("Database Service", () => { expect(emitter.on).toHaveBeenCalledWith("state:started", expect.toBeFunction()); expect(emitter.on).toHaveBeenCalledWith("wallet.created.cold", expect.toBeFunction()); - expect(emitter.once).toHaveBeenCalledWith("shutdown", expect.toBeFunction()); }); describe("applyBlock", () => { diff --git a/packages/core-blockchain/src/state-machine.ts b/packages/core-blockchain/src/state-machine.ts index 86fd0cf198..95413c5bb1 100644 --- a/packages/core-blockchain/src/state-machine.ts +++ b/packages/core-blockchain/src/state-machine.ts @@ -169,8 +169,7 @@ blockchainMachine.actionMap = (blockchain: Blockchain) => ({ stateStorage.lastDownloadedBlock = block; if (stateStorage.networkStart) { - await blockchain.database.buildWallets(block.data.height); - await blockchain.database.saveWallets(true); + await blockchain.database.buildWallets(); await blockchain.database.applyRound(block.data.height); await blockchain.transactionPool.buildWallets(); @@ -181,7 +180,7 @@ blockchainMachine.actionMap = (blockchain: Blockchain) => ({ logger.verbose("TEST SUITE DETECTED! SYNCING WALLETS AND STARTING IMMEDIATELY."); stateStorage.setLastBlock(new Block(config.get("genesisBlock"))); - await blockchain.database.buildWallets(block.data.height); + await blockchain.database.buildWallets(); return blockchain.dispatch("STARTED"); } @@ -198,13 +197,12 @@ blockchainMachine.actionMap = (blockchain: Blockchain) => ({ /** ******************************* * database init * ******************************* */ - // SPV rebuild - const verifiedWalletsIntegrity = await blockchain.database.buildWallets(block.data.height); + // Integrity Verification + const verifiedWalletsIntegrity = await blockchain.database.buildWallets(); if (!verifiedWalletsIntegrity && block.data.height > 1) { logger.warn( "Rebuilding wallets table because of some inconsistencies. Most likely due to an unfortunate shutdown.", ); - await blockchain.database.saveWallets(true); } // NOTE: if the node is shutdown between round, the round has already been applied diff --git a/packages/core-container/src/container.ts b/packages/core-container/src/container.ts index 608ec339d4..f8b650a3e7 100644 --- a/packages/core-container/src/container.ts +++ b/packages/core-container/src/container.ts @@ -265,9 +265,6 @@ export class Container implements container.IContainer { // Wait for event to be emitted and give time to finish await delay(1000); - - // Save dirty wallets - await database.saveWallets(false); } } catch (error) { // tslint:disable-next-line:no-console diff --git a/packages/core-database-postgres/src/index.ts b/packages/core-database-postgres/src/index.ts index b4d0f3e18c..51ab7feb34 100644 --- a/packages/core-database-postgres/src/index.ts +++ b/packages/core-database-postgres/src/index.ts @@ -1,6 +1,6 @@ export * from "./postgres-connection"; export * from "./migrations"; -export * from "./spv"; +export * from "./integrity-verifier"; export * from "./models"; export * from "./repositories"; export * from "./plugin"; diff --git a/packages/core-database-postgres/src/spv.ts b/packages/core-database-postgres/src/integrity-verifier.ts similarity index 61% rename from packages/core-database-postgres/src/spv.ts rename to packages/core-database-postgres/src/integrity-verifier.ts index f9728ee25b..8cb2349b61 100644 --- a/packages/core-database-postgres/src/spv.ts +++ b/packages/core-database-postgres/src/integrity-verifier.ts @@ -1,4 +1,5 @@ -import { Bignum, Transaction } from "@arkecosystem/crypto"; +import { Bignum, models, Transaction } from "@arkecosystem/crypto"; +import { sortBy } from "@arkecosystem/utils"; import { app } from "@arkecosystem/core-container"; import { Database, Logger } from "@arkecosystem/core-interfaces"; @@ -10,53 +11,50 @@ const config = app.getConfig(); const genesisWallets = config.get("genesisBlock.transactions").map(tx => tx.senderId); -export class SPV { +export class IntegrityVerifier { constructor(private query: QueryExecutor, private walletManager: Database.IWalletManager) {} /** - * Perform the SPV (Simple Payment Verification). - * @param {Number} height - * @return {void} + * Perform the State & Integrity Verification. + * @return {Boolean} */ - public async build(height) { - logger.info("SPV Step 1 of 8: Received Transactions"); - await this.__buildReceivedTransactions(); + public async run() { + logger.info("Integrity Verification - Step 1 of 8: Received Transactions"); + await this.buildReceivedTransactions(); - logger.info("SPV Step 2 of 8: Block Rewards"); - await this.__buildBlockRewards(); + logger.info("Integrity Verification - Step 2 of 8: Block Rewards"); + await this.buildBlockRewards(); - logger.info("SPV Step 3 of 8: Last Forged Blocks"); - await this.__buildLastForgedBlocks(); + logger.info("Integrity Verification - Step 3 of 8: Last Forged Blocks"); + await this.buildLastForgedBlocks(); - logger.info("SPV Step 4 of 8: Sent Transactions"); - await this.__buildSentTransactions(); + logger.info("Integrity Verification - Step 4 of 8: Sent Transactions"); + await this.buildSentTransactions(); - logger.info("SPV Step 5 of 8: Second Signatures"); - await this.__buildSecondSignatures(); + logger.info("Integrity Verification - Step 5 of 8: Second Signatures"); + await this.buildSecondSignatures(); - logger.info("SPV Step 6 of 8: Votes"); - await this.__buildVotes(); + logger.info("Integrity Verification - Step 6 of 8: Votes"); + await this.buildVotes(); - logger.info("SPV Step 7 of 8: Delegates"); - await this.__buildDelegates(); + logger.info("Integrity Verification - Step 7 of 8: Delegates"); + await this.buildDelegates(); - logger.info("SPV Step 8 of 8: MultiSignatures"); - await this.__buildMultisignatures(); + logger.info("Integrity Verification - Step 8 of 8: MultiSignatures"); + await this.buildMultisignatures(); - logger.info( - `SPV rebuild finished, wallets in memory: ${Object.keys(this.walletManager.allByAddress()).length}`, - ); + logger.info(`Integrity verified! Wallets in memory: ${Object.keys(this.walletManager.allByAddress()).length}`); logger.info(`Number of registered delegates: ${Object.keys(this.walletManager.allByUsername()).length}`); - return this.__verifyWalletsConsistency(); + return this.verifyWalletsConsistency(); } /** * Load and apply received transactions to wallets. * @return {void} */ - public async __buildReceivedTransactions() { - const transactions = await this.query.many(queries.spv.receivedTransactions); + private async buildReceivedTransactions() { + const transactions = await this.query.many(queries.integrityVerifier.receivedTransactions); for (const transaction of transactions) { const wallet = this.walletManager.findByAddress(transaction.recipientId); @@ -71,8 +69,8 @@ export class SPV { * Load and apply block rewards to wallets. * @return {void} */ - public async __buildBlockRewards() { - const blocks = await this.query.many(queries.spv.blockRewards); + private async buildBlockRewards() { + const blocks = await this.query.many(queries.integrityVerifier.blockRewards); for (const block of blocks) { const wallet = this.walletManager.findByPublicKey(block.generatorPublicKey); @@ -84,8 +82,8 @@ export class SPV { * Load and apply last forged blocks to wallets. * @return {void} */ - public async __buildLastForgedBlocks() { - const blocks = await this.query.many(queries.spv.lastForgedBlocks); + private async buildLastForgedBlocks() { + const blocks = await this.query.many(queries.integrityVerifier.lastForgedBlocks); for (const block of blocks) { const wallet = this.walletManager.findByPublicKey(block.generatorPublicKey); @@ -97,8 +95,8 @@ export class SPV { * Load and apply sent transactions to wallets. * @return {void} */ - public async __buildSentTransactions() { - const transactions = await this.query.many(queries.spv.sentTransactions); + private async buildSentTransactions() { + const transactions = await this.query.many(queries.integrityVerifier.sentTransactions); for (const transaction of transactions) { const wallet = this.walletManager.findByPublicKey(transaction.senderPublicKey); @@ -114,7 +112,7 @@ export class SPV { * Used to determine if a wallet is a Genesis wallet. * @return {Boolean} */ - public isGenesis(wallet) { + private isGenesis(wallet) { return genesisWallets.includes(wallet.address); } @@ -122,8 +120,8 @@ export class SPV { * Load and apply second signature transactions to wallets. * @return {void} */ - public async __buildSecondSignatures() { - const transactions = await this.query.manyOrNone(queries.spv.secondSignatures); + private async buildSecondSignatures() { + const transactions = await this.query.manyOrNone(queries.integrityVerifier.secondSignatures); for (const transaction of transactions) { const wallet = this.walletManager.findByPublicKey(transaction.senderPublicKey); @@ -136,8 +134,8 @@ export class SPV { * Load and apply votes to wallets. * @return {void} */ - public async __buildVotes() { - const transactions = await this.query.manyOrNone(queries.spv.votes); + private async buildVotes() { + const transactions = await this.query.manyOrNone(queries.integrityVerifier.votes); for (const transaction of transactions) { const wallet = this.walletManager.findByPublicKey(transaction.senderPublicKey); @@ -164,9 +162,9 @@ export class SPV { * Load and apply delegate usernames to wallets. * @return {void} */ - public async __buildDelegates() { + private async buildDelegates() { // Register... - const transactions = await this.query.manyOrNone(queries.spv.delegates); + const transactions = await this.query.manyOrNone(queries.integrityVerifier.delegates); transactions.forEach(transaction => { const wallet = this.walletManager.findByPublicKey(transaction.senderPublicKey); @@ -176,7 +174,7 @@ export class SPV { }); // Forged Blocks... - const forgedBlocks = await this.query.manyOrNone(queries.spv.delegatesForgedBlocks); + const forgedBlocks = await this.query.manyOrNone(queries.integrityVerifier.delegatesForgedBlocks); forgedBlocks.forEach(block => { const wallet = this.walletManager.findByPublicKey(block.generatorPublicKey); wallet.forgedFees = wallet.forgedFees.plus(block.totalFees); @@ -184,13 +182,14 @@ export class SPV { wallet.producedBlocks = +block.totalProduced; }); - // NOTE: This is highly NOT reliable, however the number of missed blocks - // is NOT used for the consensus - const delegates = await this.query.manyOrNone(queries.spv.delegatesRanks); - delegates.forEach((delegate, i) => { + // NOTE: This is unreliable but the number of missed blocks is NOT used for the consensus, only for the public API. + const delegateWallets = this.walletManager + .allByUsername() + .sort((a: models.Wallet, b: models.Wallet) => b.voteBalance.comparedTo(a.voteBalance)); + + sortBy(delegateWallets, "publicKey").forEach((delegate, i) => { const wallet = this.walletManager.findByPublicKey(delegate.publicKey); - wallet.missedBlocks = +delegate.missedBlocks; - // TODO: unknown property 'rate' being access on Wallet class + // @TODO: unknown property 'rate' being access on Wallet class (wallet as any).rate = i + 1; this.walletManager.reindex(wallet); }); @@ -200,8 +199,8 @@ export class SPV { * Load and apply multisignatures to wallets. * @return {void} */ - public async __buildMultisignatures() { - const transactions = await this.query.manyOrNone(queries.spv.multiSignatures); + private async buildMultisignatures() { + const transactions = await this.query.manyOrNone(queries.integrityVerifier.multiSignatures); for (const transaction of transactions) { const wallet = this.walletManager.findByPublicKey(transaction.senderPublicKey); @@ -219,37 +218,20 @@ export class SPV { * NOTE: This is faster than rebuilding the entire table from scratch each time. * @returns {Boolean} */ - public async __verifyWalletsConsistency() { - const dbWallets = await this.query.manyOrNone(queries.wallets.all); - const inMemoryWallets = this.walletManager.allByPublicKey(); - + private async verifyWalletsConsistency() { let detectedInconsistency = false; - if (dbWallets.length !== inMemoryWallets.length) { - detectedInconsistency = true; - } else { - for (const dbWallet of dbWallets) { - if (dbWallet.balance < 0 && !this.isGenesis(dbWallet)) { - detectedInconsistency = true; - logger.warn(`Wallet '${dbWallet.address}' has a negative balance of '${dbWallet.balance}'`); - break; - } - if (dbWallet.voteBalance < 0) { - detectedInconsistency = true; - logger.warn(`Wallet ${dbWallet.address} has a negative vote balance of '${dbWallet.voteBalance}'`); - break; - } - - const inMemoryWallet = this.walletManager.findByPublicKey(dbWallet.publicKey); + for (const wallet of this.walletManager.allByAddress()) { + if (wallet.balance.isLessThan(0) && !this.isGenesis(wallet)) { + detectedInconsistency = true; + logger.warn(`Wallet '${wallet.address}' has a negative balance of '${wallet.balance}'`); + break; + } - if ( - !inMemoryWallet.balance.isEqualTo(dbWallet.balance) || - !inMemoryWallet.voteBalance.isEqualTo(dbWallet.voteBalance) || - dbWallet.username !== inMemoryWallet.username - ) { - detectedInconsistency = true; - break; - } + if (wallet.voteBalance.isLessThan(0)) { + detectedInconsistency = true; + logger.warn(`Wallet ${wallet.address} has a negative vote balance of '${wallet.voteBalance}'`); + break; } } diff --git a/packages/core-database-postgres/src/migrations/20190307000000-drop-wallets-table.sql b/packages/core-database-postgres/src/migrations/20190307000000-drop-wallets-table.sql new file mode 100644 index 0000000000..bc21aedbef --- /dev/null +++ b/packages/core-database-postgres/src/migrations/20190307000000-drop-wallets-table.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS "wallets"; diff --git a/packages/core-database-postgres/src/migrations/index.ts b/packages/core-database-postgres/src/migrations/index.ts index d0e51a118e..a469c9cb78 100644 --- a/packages/core-database-postgres/src/migrations/index.ts +++ b/packages/core-database-postgres/src/migrations/index.ts @@ -11,4 +11,5 @@ export const migrations = [ loadQueryFile(__dirname, "./20181204200000-add-timestamp-index-to-blocks-table.sql"), loadQueryFile(__dirname, "./20181204300000-add-sender_public_key-index-to-transactions-table.sql"), loadQueryFile(__dirname, "./20181204400000-add-recipient_id-index-to-transactions-table.sql"), + loadQueryFile(__dirname, "./20190307000000-drop-wallets-table.sql"), ]; diff --git a/packages/core-database-postgres/src/postgres-connection.ts b/packages/core-database-postgres/src/postgres-connection.ts index a8e1b44ca7..5ac229e926 100644 --- a/packages/core-database-postgres/src/postgres-connection.ts +++ b/packages/core-database-postgres/src/postgres-connection.ts @@ -2,15 +2,13 @@ import { app } from "@arkecosystem/core-container"; import { Database, EventEmitter, Logger } from "@arkecosystem/core-interfaces"; import { roundCalculator } from "@arkecosystem/core-utils"; import { models } from "@arkecosystem/crypto"; -import fs from "fs"; -import chunk from "lodash/chunk"; import path from "path"; import pgPromise from "pg-promise"; +import { IntegrityVerifier } from "./integrity-verifier"; import { migrations } from "./migrations"; import { Model } from "./models"; import { repositories } from "./repositories"; import { MigrationsRepository } from "./repositories/migrations"; -import { SPV } from "./spv"; import { QueryExecutor } from "./sql/query-executor"; import { camelizeColumns } from "./utils"; @@ -31,20 +29,10 @@ export class PostgresConnection implements Database.IDatabaseConnection { public constructor(readonly options: any, private walletManager: Database.IWalletManager) {} - public async buildWallets(height: number) { - const spvPath = `${process.env.CORE_PATH_CACHE}/spv.json`; - - if (fs.existsSync(spvPath)) { - (fs as any).removeSync(spvPath); - - this.logger.info("Ark Core ended unexpectedly - resuming from where we left off :runner:"); - - return true; - } - + public async buildWallets() { try { - const spv = new SPV(this.query, this.walletManager); - return await spv.build(height); + const result = await new IntegrityVerifier(this.query, this.walletManager).run(); + return result; } catch (error) { this.logger.error(error.stack); } @@ -193,30 +181,6 @@ export class PostgresConnection implements Database.IDatabaseConnection { } } - public async saveWallets(wallets: any[], force?: boolean) { - if (force) { - // all wallets to be updated, performance is better without upsert - await this.walletsRepository.truncate(); - - try { - const chunks = chunk(wallets, 5000).map(c => this.walletsRepository.insert(c)); // this 5000 figure should be configurable... - await this.db.tx(t => t.batch(chunks)); - } catch (error) { - this.logger.error(error.stack); - } - } else { - // NOTE: The list of delegates is calculated in-memory against the WalletManager, - // so it is safe to perform the costly UPSERT non-blocking during round change only: - // 'await saveWallets(false)' -> 'saveWallets(false)' - try { - const queries = wallets.map(wallet => this.walletsRepository.updateOrCreate(wallet)); - await this.db.tx(t => t.batch(queries)); - } catch (error) { - this.logger.error(error.stack); - } - } - } - /** * Run all migrations. * @return {void} diff --git a/packages/core-database-postgres/src/queries/index.ts b/packages/core-database-postgres/src/queries/index.ts index 39a1933aa8..be522469a2 100644 --- a/packages/core-database-postgres/src/queries/index.ts +++ b/packages/core-database-postgres/src/queries/index.ts @@ -22,17 +22,16 @@ export const queries = { delete: loadQueryFile(__dirname, "./rounds/delete.sql"), find: loadQueryFile(__dirname, "./rounds/find.sql"), }, - spv: { - blockRewards: loadQueryFile(__dirname, "./spv/block-rewards.sql"), - delegates: loadQueryFile(__dirname, "./spv/delegates.sql"), - delegatesForgedBlocks: loadQueryFile(__dirname, "./spv/delegates-forged-blocks.sql"), - delegatesRanks: loadQueryFile(__dirname, "./spv/delegates-ranks.sql"), - lastForgedBlocks: loadQueryFile(__dirname, "./spv/last-forged-blocks.sql"), - multiSignatures: loadQueryFile(__dirname, "./spv/multi-signatures.sql"), - receivedTransactions: loadQueryFile(__dirname, "./spv/received-transactions.sql"), - secondSignatures: loadQueryFile(__dirname, "./spv/second-signatures.sql"), - sentTransactions: loadQueryFile(__dirname, "./spv/sent-transactions.sql"), - votes: loadQueryFile(__dirname, "./spv/votes.sql"), + integrityVerifier: { + blockRewards: loadQueryFile(__dirname, "./integrity-verifier/block-rewards.sql"), + delegates: loadQueryFile(__dirname, "./integrity-verifier/delegates.sql"), + delegatesForgedBlocks: loadQueryFile(__dirname, "./integrity-verifier/delegates-forged-blocks.sql"), + lastForgedBlocks: loadQueryFile(__dirname, "./integrity-verifier/last-forged-blocks.sql"), + multiSignatures: loadQueryFile(__dirname, "./integrity-verifier/multi-signatures.sql"), + receivedTransactions: loadQueryFile(__dirname, "./integrity-verifier/received-transactions.sql"), + secondSignatures: loadQueryFile(__dirname, "./integrity-verifier/second-signatures.sql"), + sentTransactions: loadQueryFile(__dirname, "./integrity-verifier/sent-transactions.sql"), + votes: loadQueryFile(__dirname, "./integrity-verifier/votes.sql"), }, transactions: { findByBlock: loadQueryFile(__dirname, "./transactions/find-by-block.sql"), @@ -43,10 +42,4 @@ export const queries = { findById: loadQueryFile(__dirname, "./transactions/find-by-id.sql"), deleteByBlock: loadQueryFile(__dirname, "./transactions/delete-by-block.sql"), }, - wallets: { - all: loadQueryFile(__dirname, "./wallets/all.sql"), - findByAddress: loadQueryFile(__dirname, "./wallets/find-by-address.sql"), - findNegativeBalances: loadQueryFile(__dirname, "./wallets/find-negative-balances.sql"), - findNegativeVoteBalances: loadQueryFile(__dirname, "./wallets/find-negative-vote-balances.sql"), - }, }; diff --git a/packages/core-database-postgres/src/queries/spv/block-rewards.sql b/packages/core-database-postgres/src/queries/integrity-verifier/block-rewards.sql similarity index 100% rename from packages/core-database-postgres/src/queries/spv/block-rewards.sql rename to packages/core-database-postgres/src/queries/integrity-verifier/block-rewards.sql diff --git a/packages/core-database-postgres/src/queries/spv/delegates-forged-blocks.sql b/packages/core-database-postgres/src/queries/integrity-verifier/delegates-forged-blocks.sql similarity index 100% rename from packages/core-database-postgres/src/queries/spv/delegates-forged-blocks.sql rename to packages/core-database-postgres/src/queries/integrity-verifier/delegates-forged-blocks.sql diff --git a/packages/core-database-postgres/src/queries/spv/delegates.sql b/packages/core-database-postgres/src/queries/integrity-verifier/delegates.sql similarity index 100% rename from packages/core-database-postgres/src/queries/spv/delegates.sql rename to packages/core-database-postgres/src/queries/integrity-verifier/delegates.sql diff --git a/packages/core-database-postgres/src/queries/spv/last-forged-blocks.sql b/packages/core-database-postgres/src/queries/integrity-verifier/last-forged-blocks.sql similarity index 100% rename from packages/core-database-postgres/src/queries/spv/last-forged-blocks.sql rename to packages/core-database-postgres/src/queries/integrity-verifier/last-forged-blocks.sql diff --git a/packages/core-database-postgres/src/queries/spv/multi-signatures.sql b/packages/core-database-postgres/src/queries/integrity-verifier/multi-signatures.sql similarity index 100% rename from packages/core-database-postgres/src/queries/spv/multi-signatures.sql rename to packages/core-database-postgres/src/queries/integrity-verifier/multi-signatures.sql diff --git a/packages/core-database-postgres/src/queries/spv/received-transactions.sql b/packages/core-database-postgres/src/queries/integrity-verifier/received-transactions.sql similarity index 100% rename from packages/core-database-postgres/src/queries/spv/received-transactions.sql rename to packages/core-database-postgres/src/queries/integrity-verifier/received-transactions.sql diff --git a/packages/core-database-postgres/src/queries/spv/second-signatures.sql b/packages/core-database-postgres/src/queries/integrity-verifier/second-signatures.sql similarity index 100% rename from packages/core-database-postgres/src/queries/spv/second-signatures.sql rename to packages/core-database-postgres/src/queries/integrity-verifier/second-signatures.sql diff --git a/packages/core-database-postgres/src/queries/spv/sent-transactions.sql b/packages/core-database-postgres/src/queries/integrity-verifier/sent-transactions.sql similarity index 100% rename from packages/core-database-postgres/src/queries/spv/sent-transactions.sql rename to packages/core-database-postgres/src/queries/integrity-verifier/sent-transactions.sql diff --git a/packages/core-database-postgres/src/queries/spv/votes.sql b/packages/core-database-postgres/src/queries/integrity-verifier/votes.sql similarity index 100% rename from packages/core-database-postgres/src/queries/spv/votes.sql rename to packages/core-database-postgres/src/queries/integrity-verifier/votes.sql diff --git a/packages/core-database-postgres/src/queries/spv/delegates-ranks.sql b/packages/core-database-postgres/src/queries/spv/delegates-ranks.sql deleted file mode 100644 index ea16718904..0000000000 --- a/packages/core-database-postgres/src/queries/spv/delegates-ranks.sql +++ /dev/null @@ -1,7 +0,0 @@ -SELECT public_key, - vote_balance, - missed_blocks -FROM wallets -WHERE username IS NOT NULL -ORDER BY vote_balance DESC, - public_key ASC diff --git a/packages/core-database-postgres/src/queries/wallets/all.sql b/packages/core-database-postgres/src/queries/wallets/all.sql deleted file mode 100644 index 5435ffc400..0000000000 --- a/packages/core-database-postgres/src/queries/wallets/all.sql +++ /dev/null @@ -1,2 +0,0 @@ -SELECT * -FROM wallets diff --git a/packages/core-database-postgres/src/queries/wallets/find-by-address.sql b/packages/core-database-postgres/src/queries/wallets/find-by-address.sql deleted file mode 100644 index 98fe3cdfef..0000000000 --- a/packages/core-database-postgres/src/queries/wallets/find-by-address.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT * -FROM wallets -WHERE address = ${address} diff --git a/packages/core-database-postgres/src/queries/wallets/find-negative-balances.sql b/packages/core-database-postgres/src/queries/wallets/find-negative-balances.sql deleted file mode 100644 index 83ebbad129..0000000000 --- a/packages/core-database-postgres/src/queries/wallets/find-negative-balances.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT COUNT (DISTINCT "address") AS "count" -FROM wallets -WHERE balance < 0; diff --git a/packages/core-database-postgres/src/queries/wallets/find-negative-vote-balances.sql b/packages/core-database-postgres/src/queries/wallets/find-negative-vote-balances.sql deleted file mode 100644 index af79cbad33..0000000000 --- a/packages/core-database-postgres/src/queries/wallets/find-negative-vote-balances.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT COUNT (DISTINCT "address") AS "count" -FROM wallets -WHERE vote_balance < 0; diff --git a/packages/core-database-postgres/src/repositories/index.ts b/packages/core-database-postgres/src/repositories/index.ts index 5dae791f43..43b7c2c080 100644 --- a/packages/core-database-postgres/src/repositories/index.ts +++ b/packages/core-database-postgres/src/repositories/index.ts @@ -2,12 +2,10 @@ import { BlocksRepository } from "./blocks"; import { MigrationsRepository } from "./migrations"; import { RoundsRepository } from "./rounds"; import { TransactionsRepository } from "./transactions"; -import { WalletsRepository } from "./wallets"; export const repositories = { blocks: BlocksRepository, migrations: MigrationsRepository, rounds: RoundsRepository, transactions: TransactionsRepository, - wallets: WalletsRepository, }; diff --git a/packages/core-database-postgres/src/repositories/wallets.ts b/packages/core-database-postgres/src/repositories/wallets.ts deleted file mode 100644 index ca8783ecc8..0000000000 --- a/packages/core-database-postgres/src/repositories/wallets.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { Database } from "@arkecosystem/core-interfaces"; -import { Wallet } from "../models"; -import { queries } from "../queries"; -import { Repository } from "./repository"; - -const { wallets: sql } = queries; - -export class WalletsRepository extends Repository implements Database.IWalletsRepository { - /** - * Get all of the wallets from the database. - * @return {Promise} - */ - public async all() { - return this.db.manyOrNone(sql.all); - } - - /** - * Find a wallet by its address. - * @param {String} address - * @return {Promise} - */ - public async findByAddress(address) { - return this.db.oneOrNone(sql.findByAddress, { address }); - } - - /** - * Get the count of wallets that have a negative balance. - * @return {Promise} - */ - public async tallyWithNegativeBalance() { - return this.db.oneOrNone(sql.findNegativeBalances); - } - - /** - * Get the count of wallets that have a negative vote balance. - * @return {Promise} - */ - public async tallyWithNegativeVoteBalance() { - return this.db.oneOrNone(sql.findNegativeVoteBalances); - } - - /** - * Create or update a record matching the attributes, and fill it with values. - * @param {Object} wallet - * @return {Promise} - */ - public async updateOrCreate(wallet) { - const query = `${this.insertQuery(wallet)} ON CONFLICT(address) DO UPDATE SET ${this.pgp.helpers.sets( - wallet, - this.model.getColumnSet(), - )}`; - - return this.db.none(query); - } - - /** - * Get the model related to this repository. - * @return {Object} - */ - public getModel() { - return new Wallet(this.pgp); - } -} diff --git a/packages/core-database/src/database-service.ts b/packages/core-database/src/database-service.ts index bc8a58f8ac..e47ff3e4e5 100644 --- a/packages/core-database/src/database-service.ts +++ b/packages/core-database/src/database-service.ts @@ -5,7 +5,6 @@ import { roundCalculator } from "@arkecosystem/core-utils"; import { Bignum, constants, crypto, HashAlgorithms, models, Transaction } from "@arkecosystem/crypto"; import assert from "assert"; import cloneDeep from "lodash/cloneDeep"; -import pluralize from "pluralize"; import { WalletManager } from "./wallet-manager"; const { Block } = models; @@ -26,7 +25,6 @@ export class DatabaseService implements Database.IDatabaseService { public restoredDatabaseIntegrity: boolean = false; public forgingDelegates: any[] = null; public cache: Map = new Map(); - private spvFinished: boolean; constructor( options: any, @@ -81,7 +79,6 @@ export class DatabaseService implements Database.IDatabaseService { try { this.updateDelegateStats(this.forgingDelegates); - await this.saveWallets(false); // save only modified wallets during the last round const delegates = this.walletManager.loadActiveDelegateList(maxDelegates, nextHeight); // get active delegate list from in-memory wallet manager await this.saveRound(delegates); // save next round delegate list non-blocking this.forgingDelegates = await this.getActiveDelegates(nextHeight, delegates); // generate the new active delegates list @@ -102,16 +99,16 @@ export class DatabaseService implements Database.IDatabaseService { } } - public async buildWallets(height: number): Promise { + public async buildWallets(): Promise { this.walletManager.reset(); try { - const success = await this.connection.buildWallets(height); - this.spvFinished = true; - return success; + const result = await this.connection.buildWallets(); + return result; } catch (e) { this.logger.error(e.stack); } + return false; } @@ -406,25 +403,6 @@ export class DatabaseService implements Database.IDatabaseService { this.emitter.emit("round.created", activeDelegates); } - public async saveWallets(force: boolean) { - const wallets = this.walletManager - .allByPublicKey() - .filter(wallet => wallet.publicKey && (force || wallet.dirty)); - - // Remove dirty flags first to not save all dirty wallets in the exit handler - // when called during a force insert right after SPV. - this.walletManager.clear(); - - await this.connection.saveWallets(wallets, force); - - this.logger.info(`${wallets.length} modified ${pluralize("wallet", wallets.length)} committed to database`); - - this.emitter.emit("wallet.saved", wallets.length); - - // NOTE: commented out as more use cases to be taken care of - // this.walletManager.purgeEmptyNonDelegates() - } - public updateDelegateStats(delegates: any[]): void { if (!delegates || !this.blocksInCurrentRound) { return; @@ -589,12 +567,5 @@ export class DatabaseService implements Database.IDatabaseService { this.logger.error(err); } }); - - this.emitter.once("shutdown", async () => { - if (!this.spvFinished) { - // Prevent dirty wallets to be saved when SPV didn't finish - this.walletManager.clear(); - } - }); } } diff --git a/packages/core-database/src/wallet-manager.ts b/packages/core-database/src/wallet-manager.ts index 39225074f8..f6211b19b8 100644 --- a/packages/core-database/src/wallet-manager.ts +++ b/packages/core-database/src/wallet-manager.ts @@ -256,7 +256,7 @@ export class WalletManager implements Database.IWalletManager { /** * Build vote balances of all delegates. - * NOTE: Only called during SPV. + * NOTE: Only called during integrity verification on boot. * @return {void} */ public buildVoteBalances() { diff --git a/packages/core-interfaces/src/core-database/database-connection.ts b/packages/core-interfaces/src/core-database/database-connection.ts index 56fbe3b9c2..12ed23649f 100644 --- a/packages/core-interfaces/src/core-database/database-connection.ts +++ b/packages/core-interfaces/src/core-database/database-connection.ts @@ -19,12 +19,7 @@ export interface IDatabaseConnection { disconnect(): Promise; - buildWallets(height: number): Promise; - - /* We have these methods on the connection since they rely on transactions, which is a DB specific detail - Keep DB specifics away from the service layer - */ - saveWallets(wallets: any[], force?: boolean): Promise; + buildWallets(): Promise; saveBlock(block: models.Block): Promise; diff --git a/packages/core-interfaces/src/core-database/database-service.ts b/packages/core-interfaces/src/core-database/database-service.ts index ae1ecd2710..3450990f4c 100644 --- a/packages/core-interfaces/src/core-database/database-service.ts +++ b/packages/core-interfaces/src/core-database/database-service.ts @@ -38,9 +38,7 @@ export interface IDatabaseService { getActiveDelegates(height: number, delegates?: any[]): Promise; - buildWallets(height: number): Promise; - - saveWallets(force: boolean): Promise; + buildWallets(): Promise; saveBlock(block: models.Block): Promise;