diff --git a/elements/lisk-api-client/test/unit/block.spec.ts b/elements/lisk-api-client/test/unit/block.spec.ts index 8a4d07dbfc2..7c4ffb82f1d 100644 --- a/elements/lisk-api-client/test/unit/block.spec.ts +++ b/elements/lisk-api-client/test/unit/block.spec.ts @@ -24,7 +24,7 @@ describe('block', () => { let block: Block; const sampleHeight = 1; const encodedBlock = - '0ac001080110c4d23d18c4d23d22144a462ea57a8c9f72d866c09770e5ec70cef187272a14be63fb1c0426573352556f18b21efd5b6183c39c3214b27ca21f40d44113c2090ca8f05fb706c54e87dd3a14b27ca21f40d44113c2090ca8f05fb706c54e87dd42147f9d96a09a3fd17f3478eb7bef3a8bda00e1238b489c8c3d509c8c3d5a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8556206080012001a006a146da88e2fd4435e26e02682435f108002ccc3ddd5'; + '0ae201080110c4d23d18c4d23d22144a462ea57a8c9f72d866c09770e5ec70cef187272a14be63fb1c0426573352556f18b21efd5b6183c39c3214b27ca21f40d44113c2090ca8f05fb706c54e87dd3a14b27ca21f40d44113c2090ca8f05fb706c54e87dd422030dda4fbc395828e5a9f2f8824771e434fce4945a1e7820012440d09dd1e2b6d4a147f9d96a09a3fd17f3478eb7bef3a8bda00e1238b509c8c3d589c8c3d6220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8556a06080012001a0072146da88e2fd4435e26e02682435f108002ccc3ddd5'; const encodedBlockBuffer = Buffer.from(encodedBlock, 'hex'); const sampleBlock = { header: { @@ -35,6 +35,10 @@ describe('block', () => { stateRoot: Buffer.from('7f9d96a09a3fd17f3478eb7bef3a8bda00e1238b', 'hex'), transactionRoot: Buffer.from('b27ca21f40d44113c2090ca8f05fb706c54e87dd', 'hex'), assetsRoot: Buffer.from('b27ca21f40d44113c2090ca8f05fb706c54e87dd', 'hex'), + eventRoot: Buffer.from( + '30dda4fbc395828e5a9f2f8824771e434fce4945a1e7820012440d09dd1e2b6d', + 'hex', + ), generatorAddress: Buffer.from('be63fb1c0426573352556f18b21efd5b6183c39c', 'hex'), maxHeightPrevoted: 1000988, maxHeightGenerated: 1000988, @@ -45,7 +49,7 @@ describe('block', () => { certificateSignature: Buffer.alloc(0), }, signature: Buffer.from('6da88e2fd4435e26e02682435f108002ccc3ddd5', 'hex'), - id: Buffer.from('097ce5adc1a34680d6c939287011dec9b70a3bc1f5f896a3f9024fc9bed59992', 'hex'), + id: Buffer.from('f14104e384546adaba487af56e658188eea07bb534a61b4cbb9ccaee54139b8c', 'hex'), }, assets: [], transactions: [], diff --git a/elements/lisk-chain/src/block_header.ts b/elements/lisk-chain/src/block_header.ts index 98ad2c6f52a..388d7fedaad 100644 --- a/elements/lisk-chain/src/block_header.ts +++ b/elements/lisk-chain/src/block_header.ts @@ -36,6 +36,7 @@ export interface BlockHeaderAttrs { readonly stateRoot?: Buffer; readonly transactionRoot?: Buffer; readonly assetsRoot?: Buffer; + readonly eventRoot?: Buffer; signature?: Buffer; id?: Buffer; } @@ -59,6 +60,7 @@ export class BlockHeader { private _stateRoot?: Buffer; private _transactionRoot?: Buffer; private _assetsRoot?: Buffer; + private _eventRoot?: Buffer; private _signature?: Buffer; private _id?: Buffer; @@ -73,6 +75,7 @@ export class BlockHeader { aggregateCommit, validatorsHash, stateRoot, + eventRoot, assetsRoot, transactionRoot, signature, @@ -87,6 +90,7 @@ export class BlockHeader { this.maxHeightGenerated = maxHeightGenerated; this._aggregateCommit = aggregateCommit; this._validatorsHash = validatorsHash; + this._eventRoot = eventRoot; this._stateRoot = stateRoot; this._transactionRoot = transactionRoot; this._assetsRoot = assetsRoot; @@ -112,6 +116,15 @@ export class BlockHeader { this._resetComputedValues(); } + public get eventRoot() { + return this._eventRoot; + } + + public set eventRoot(val) { + this._eventRoot = val; + this._resetComputedValues(); + } + public get assetsRoot() { return this._assetsRoot; } @@ -335,6 +348,9 @@ export class BlockHeader { if (!this.assetsRoot) { throw new Error('Asset root is empty.'); } + if (!this.eventRoot) { + throw new Error('Event root is empty.'); + } if (!this.stateRoot) { throw new Error('State root is empty.'); } @@ -351,6 +367,7 @@ export class BlockHeader { previousBlockID: this.previousBlockID, stateRoot: this.stateRoot, assetsRoot: this.assetsRoot, + eventRoot: this.eventRoot, transactionRoot: this.transactionRoot, validatorsHash: this.validatorsHash, aggregateCommit: this.aggregateCommit, diff --git a/elements/lisk-chain/src/chain.ts b/elements/lisk-chain/src/chain.ts index c190f51e183..05be8a4bc7f 100644 --- a/elements/lisk-chain/src/chain.ts +++ b/elements/lisk-chain/src/chain.ts @@ -17,6 +17,7 @@ import { KVStore, NotFoundError } from '@liskhq/lisk-db'; import * as createDebug from 'debug'; import { regularMerkleTree } from '@liskhq/lisk-tree'; import { + DEFAULT_KEEP_EVENTS_FOR_HEIGHTS, DEFAULT_MAX_BLOCK_HEADER_CACHE, DEFAULT_MIN_BLOCK_HEADER_CACHE, GENESIS_BLOCK_VERSION, @@ -29,11 +30,13 @@ import { stateDiffSchema, } from './schema'; import { Block } from './block'; +import { Event } from './event'; import { BlockHeader } from './block_header'; import { CurrentState } from './state_store/smt_store'; interface ChainConstructor { // Constants + readonly keepEventsForHeights: number; readonly maxTransactionsSize: number; readonly minBlockHeaderCache?: number; readonly maxBlockHeaderCache?: number; @@ -58,6 +61,7 @@ export class Chain { readonly maxTransactionsSize: number; readonly minBlockHeaderCache: number; readonly maxBlockHeaderCache: number; + readonly keepEventsForHeights: number; }; private _lastBlock?: Block; @@ -68,6 +72,7 @@ export class Chain { public constructor({ // Constants maxTransactionsSize, + keepEventsForHeights = DEFAULT_KEEP_EVENTS_FOR_HEIGHTS, minBlockHeaderCache = DEFAULT_MIN_BLOCK_HEADER_CACHE, maxBlockHeaderCache = DEFAULT_MAX_BLOCK_HEADER_CACHE, }: ChainConstructor) { @@ -81,6 +86,7 @@ export class Chain { maxTransactionsSize, maxBlockHeaderCache, minBlockHeaderCache, + keepEventsForHeights, }; } @@ -112,6 +118,7 @@ export class Chain { db: args.db, minBlockHeaderCache: this.constants.minBlockHeaderCache, maxBlockHeaderCache: this.constants.maxBlockHeaderCache, + keepEventsForHeights: this.constants.keepEventsForHeights, }); } @@ -191,13 +198,14 @@ export class Chain { public async saveBlock( block: Block, + events: Event[], state: CurrentState, finalizedHeight: number, { removeFromTempTable } = { removeFromTempTable: false, }, ): Promise { - await this.dataAccess.saveBlock(block, state, finalizedHeight, removeFromTempTable); + await this.dataAccess.saveBlock(block, events, state, finalizedHeight, removeFromTempTable); this.dataAccess.addBlockHeader(block.header); this._finalizedHeight = finalizedHeight; this._lastBlock = block; diff --git a/elements/lisk-chain/src/constants.ts b/elements/lisk-chain/src/constants.ts index 5c4ae37f01c..612755b57e6 100644 --- a/elements/lisk-chain/src/constants.ts +++ b/elements/lisk-chain/src/constants.ts @@ -14,6 +14,7 @@ import { createMessageTag, hash } from '@liskhq/lisk-cryptography'; +export const DEFAULT_KEEP_EVENTS_FOR_HEIGHTS = 300; export const DEFAULT_MIN_BLOCK_HEADER_CACHE = 309; export const DEFAULT_MAX_BLOCK_HEADER_CACHE = 515; @@ -34,3 +35,16 @@ export const MAX_ASSET_DATA_SIZE_BYTES = 64; export const SIGNATURE_LENGTH_BYTES = 64; export const SMT_PREFIX_SIZE = 6; + +export const EVENT_TOPIC_HASH_LENGTH_BYTES = 8; +export const EVENT_INDEX_LENGTH_BITS = 30; +export const EVENT_TOPIC_INDEX_LENGTH_BITS = 2; +export const EVENT_MAX_EVENT_SIZE_BYTES = 1024; + +export const EVENT_TOTAL_INDEX_LENGTH_BYTES = + (EVENT_INDEX_LENGTH_BITS + EVENT_TOPIC_INDEX_LENGTH_BITS) / 8; + +export const EVENT_MAX_TOPICS_PER_EVENT = 2 ** EVENT_TOPIC_INDEX_LENGTH_BITS; + +export const MAX_EVENTS_PER_BLOCK = 2 ** EVENT_INDEX_LENGTH_BITS; +export const EVENT_ID_LENGTH_BYTES = 4 + EVENT_TOTAL_INDEX_LENGTH_BYTES; diff --git a/elements/lisk-chain/src/data_access/data_access.ts b/elements/lisk-chain/src/data_access/data_access.ts index c9f38d983ee..cac595f8b37 100644 --- a/elements/lisk-chain/src/data_access/data_access.ts +++ b/elements/lisk-chain/src/data_access/data_access.ts @@ -17,6 +17,7 @@ import { Transaction } from '../transaction'; import { RawBlock } from '../types'; import { BlockHeader } from '../block_header'; import { Block } from '../block'; +import { Event } from '../event'; import { BlockCache } from './cache'; import { Storage as StorageAccess } from './storage'; @@ -27,14 +28,20 @@ interface DAConstructor { readonly db: KVStore; readonly minBlockHeaderCache: number; readonly maxBlockHeaderCache: number; + readonly keepEventsForHeights: number; } export class DataAccess { private readonly _storage: StorageAccess; private readonly _blocksCache: BlockCache; - public constructor({ db, minBlockHeaderCache, maxBlockHeaderCache }: DAConstructor) { - this._storage = new StorageAccess(db); + public constructor({ + db, + minBlockHeaderCache, + maxBlockHeaderCache, + keepEventsForHeights, + }: DAConstructor) { + this._storage = new StorageAccess(db, { keepEventsForHeights }); this._blocksCache = new BlockCache(minBlockHeaderCache, maxBlockHeaderCache); } @@ -230,6 +237,12 @@ export class DataAccess { return this._decodeRawBlock(block); } + public async getEvents(height: number): Promise { + const events = await this._storage.getEvents(height); + + return events; + } + public async isBlockPersisted(blockId: Buffer): Promise { const isPersisted = await this._storage.isBlockPersisted(blockId); @@ -283,6 +296,7 @@ export class DataAccess { */ public async saveBlock( block: Block, + events: Event[], state: CurrentState, finalizedHeight: number, removeFromTemp = false, @@ -296,12 +310,14 @@ export class DataAccess { const encodedTx = tx.getBytes(); encodedTransactions.push({ id: txID, value: encodedTx }); } + const encodedEvents = events.map(e => e.getBytes()); await this._storage.saveBlock( blockID, height, finalizedHeight, encodedHeader, encodedTransactions, + encodedEvents, block.assets.getBytes(), state, removeFromTemp, diff --git a/elements/lisk-chain/src/data_access/storage.ts b/elements/lisk-chain/src/data_access/storage.ts index 8a55c57fd43..36bc2b6ad6a 100644 --- a/elements/lisk-chain/src/data_access/storage.ts +++ b/elements/lisk-chain/src/data_access/storage.ts @@ -15,7 +15,7 @@ import { KVStore, formatInt, getFirstPrefix, getLastPrefix, NotFoundError } from import { codec } from '@liskhq/lisk-codec'; import { hash } from '@liskhq/lisk-cryptography'; import { RawBlock, StateDiff } from '../types'; - +import { Event } from '../event'; import { DB_KEY_BLOCKS_ID, DB_KEY_BLOCKS_HEIGHT, @@ -25,11 +25,13 @@ import { DB_KEY_DIFF_STATE, DB_KEY_FINALIZED_HEIGHT, DB_KEY_BLOCK_ASSETS_BLOCK_ID, + DB_KEY_BLOCK_EVENTS, } from '../db_keys'; import { concatDBKeys } from '../utils'; import { stateDiffSchema } from '../schema'; import { CurrentState } from '../state_store'; import { toSMTKey } from '../state_store/utils'; +import { DEFAULT_KEEP_EVENTS_FOR_HEIGHTS } from '../constants'; const bytesArraySchema = { $id: 'lisk-chain/bytesarray', @@ -55,13 +57,20 @@ const decodeByteArray = (val: Buffer): Buffer[] => { return decoded.list; }; -const encodeByteArray = (val: Buffer[]): Buffer => codec.encode(bytesArraySchema, { list: val }); +export const encodeByteArray = (val: Buffer[]): Buffer => + codec.encode(bytesArraySchema, { list: val }); + +interface StorageOption { + keepEventsForHeights?: number; +} export class Storage { private readonly _db: KVStore; + private readonly _keepEventsForHeights: number; - public constructor(db: KVStore) { + public constructor(db: KVStore, options?: StorageOption) { this._db = db; + this._keepEventsForHeights = options?.keepEventsForHeights ?? DEFAULT_KEEP_EVENTS_FOR_HEIGHTS; } /* @@ -235,6 +244,13 @@ export class Storage { }; } + public async getEvents(height: number): Promise { + const eventsByte = await this._db.get(concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(height))); + const events = decodeByteArray(eventsByte); + + return events.map(e => Event.fromBytes(e)); + } + public async getTempBlocks(): Promise { const stream = this._db.createReadStream({ gte: getFirstPrefix(DB_KEY_TEMPBLOCKS_HEIGHT), @@ -338,6 +354,7 @@ export class Storage { finalizedHeight: number, header: Buffer, transactions: { id: Buffer; value: Buffer }[], + events: Buffer[], assets: Buffer[], state: CurrentState, removeFromTemp = false, @@ -354,6 +371,10 @@ export class Storage { } batch.put(concatDBKeys(DB_KEY_TRANSACTIONS_BLOCK_ID, id), Buffer.concat(ids)); } + if (events.length > 0) { + const encodedEvents = encodeByteArray(events); + batch.put(concatDBKeys(DB_KEY_BLOCK_EVENTS, heightBuf), encodedEvents); + } if (assets.length > 0) { const encodedAsset = encodeByteArray(assets); batch.put(concatDBKeys(DB_KEY_BLOCK_ASSETS_BLOCK_ID, id), encodedAsset); @@ -369,7 +390,7 @@ export class Storage { batch.put(DB_KEY_FINALIZED_HEIGHT, finalizedHeightBytes); await batch.write(); - await this._cleanUntil(finalizedHeight); + await this._cleanUntil(height, finalizedHeight); } public async deleteBlock( @@ -391,6 +412,7 @@ export class Storage { } batch.del(concatDBKeys(DB_KEY_TRANSACTIONS_BLOCK_ID, id)); } + batch.del(concatDBKeys(DB_KEY_BLOCK_EVENTS, heightBuf)); if (assets.length > 0) { batch.del(concatDBKeys(DB_KEY_BLOCK_ASSETS_BLOCK_ID, id)); } @@ -438,11 +460,28 @@ export class Storage { } // This function is out of batch, but even if it fails, it will run again next time - private async _cleanUntil(height: number): Promise { + private async _cleanUntil(currentHeight: number, finalizedHeight: number): Promise { await this._db.clear({ gte: concatDBKeys(DB_KEY_DIFF_STATE, formatInt(0)), - lt: concatDBKeys(DB_KEY_DIFF_STATE, formatInt(height)), + lt: concatDBKeys(DB_KEY_DIFF_STATE, formatInt(finalizedHeight)), }); + + // Delete outdated events if keepEventsForHeights is positive. + if (this._keepEventsForHeights > -1) { + // events are removed only if finalized and below height - keepEventsForHeights + const minEventDeleteHeight = Math.min( + finalizedHeight, + Math.max(0, currentHeight - this._keepEventsForHeights), + ); + if (minEventDeleteHeight > 0) { + const endHeight = Buffer.alloc(4); + endHeight.writeUInt32BE(minEventDeleteHeight - 1, 0); + await this._db.clear({ + gte: Buffer.concat([DB_KEY_BLOCK_EVENTS, Buffer.alloc(4, 0)]), + lte: Buffer.concat([DB_KEY_BLOCK_EVENTS, endHeight]), + }); + } + } } private async _getBlockAssets(blockID: Buffer): Promise { diff --git a/elements/lisk-chain/src/db_keys.ts b/elements/lisk-chain/src/db_keys.ts index 0b5450e1ee2..6b2ff2d17d1 100644 --- a/elements/lisk-chain/src/db_keys.ts +++ b/elements/lisk-chain/src/db_keys.ts @@ -19,6 +19,7 @@ export const DB_KEY_TRANSACTIONS_BLOCK_ID = Buffer.from([5]); export const DB_KEY_TRANSACTIONS_ID = Buffer.from([6]); export const DB_KEY_TEMPBLOCKS_HEIGHT = Buffer.from([7]); export const DB_KEY_BLOCK_ASSETS_BLOCK_ID = Buffer.from([8]); +export const DB_KEY_BLOCK_EVENTS = Buffer.from([8]); export const DB_KEY_STATE_STORE = Buffer.from([10]); diff --git a/elements/lisk-chain/src/event.ts b/elements/lisk-chain/src/event.ts new file mode 100644 index 00000000000..cfbd6b6e61d --- /dev/null +++ b/elements/lisk-chain/src/event.ts @@ -0,0 +1,109 @@ +/* + * Copyright © 2022 Lisk Foundation + * + * See the LICENSE file at the top-level directory of this distribution + * for licensing information. + * + * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, + * no part of this software, including this file, may be copied, modified, + * propagated, or distributed except according to the terms contained in the + * LICENSE file. + * + * Removal or modification of this copyright notice is prohibited. + */ + +import { codec } from '@liskhq/lisk-codec'; +import { hash } from '@liskhq/lisk-cryptography'; +import { + EVENT_ID_LENGTH_BYTES, + EVENT_TOPIC_HASH_LENGTH_BYTES, + EVENT_TOPIC_INDEX_LENGTH_BITS, + EVENT_TOTAL_INDEX_LENGTH_BYTES, +} from './constants'; +import { eventSchema } from './schema'; +import { JSONObject } from './types'; + +export interface EventAttr { + moduleID: Buffer; + typeID: Buffer; + topics: Buffer[]; + index: number; + data: Buffer; +} + +type EventJSON = JSONObject; + +export class Event { + private readonly _index: number; + private readonly _moduleID: Buffer; + private readonly _topics: Buffer[]; + private readonly _typeID: Buffer; + private readonly _data: Buffer; + + public constructor({ index, moduleID, topics, typeID, data }: EventAttr) { + this._index = index; + this._moduleID = moduleID; + this._topics = topics; + this._typeID = typeID; + this._data = data; + } + + public static fromBytes(value: Buffer): Event { + const decoded = codec.decode(eventSchema, value); + return new Event(decoded); + } + + public id(height: number): Buffer { + const id = Buffer.alloc(EVENT_ID_LENGTH_BYTES); + id.writeUInt32BE(height, 0); + id.writeUIntBE( + // eslint-disable-next-line no-bitwise + this._index << EVENT_TOPIC_INDEX_LENGTH_BITS, + 4, + EVENT_TOTAL_INDEX_LENGTH_BYTES, + ); + return id; + } + + public getBytes(): Buffer { + return codec.encode(eventSchema, this._getAllProps()); + } + + public keyPair(): { key: Buffer; value: Buffer }[] { + const result = []; + const value = this.getBytes(); + for (let i = 0; i < this._topics.length; i += 1) { + // eslint-disable-next-line no-bitwise + const indexBit = (this._index << EVENT_TOPIC_INDEX_LENGTH_BITS) + i; + const indexBytes = Buffer.alloc(EVENT_TOTAL_INDEX_LENGTH_BYTES); + indexBytes.writeUIntBE(indexBit, 0, EVENT_TOTAL_INDEX_LENGTH_BYTES); + const key = Buffer.concat([ + hash(this._topics[i]).slice(0, EVENT_TOPIC_HASH_LENGTH_BYTES), + indexBytes, + ]); + result.push({ + key, + value, + }); + } + return result; + } + + public toJSON(): EventJSON { + return codec.toJSON(eventSchema, this._getAllProps()); + } + + public toObject(): EventAttr { + return this._getAllProps(); + } + + private _getAllProps(): EventAttr { + return { + data: this._data, + index: this._index, + moduleID: this._moduleID, + topics: this._topics, + typeID: this._typeID, + }; + } +} diff --git a/elements/lisk-chain/src/index.ts b/elements/lisk-chain/src/index.ts index d3f44250de0..5a4b584e794 100644 --- a/elements/lisk-chain/src/index.ts +++ b/elements/lisk-chain/src/index.ts @@ -25,7 +25,13 @@ export { signingBlockHeaderSchema, stateDiffSchema, } from './schema'; -export { TAG_BLOCK_HEADER, TAG_TRANSACTION } from './constants'; +export { + TAG_BLOCK_HEADER, + TAG_TRANSACTION, + EVENT_MAX_TOPICS_PER_EVENT, + EVENT_MAX_EVENT_SIZE_BYTES, + MAX_EVENTS_PER_BLOCK, +} from './constants'; export * from './db_keys'; export type { RawBlock } from './types'; export { Slots } from './slots'; @@ -36,3 +42,4 @@ export { Block, BlockJSON } from './block'; export { BlockAsset, BlockAssets, BlockAssetJSON } from './block_assets'; export { BlockHeader, BlockHeaderAttrs, BlockHeaderJSON } from './block_header'; export { DataAccess } from './data_access'; +export { Event } from './event'; diff --git a/elements/lisk-chain/src/schema.ts b/elements/lisk-chain/src/schema.ts index f42765aeaf1..9923f94f14d 100644 --- a/elements/lisk-chain/src/schema.ts +++ b/elements/lisk-chain/src/schema.ts @@ -48,13 +48,14 @@ export const signingBlockHeaderSchema = { generatorAddress: { dataType: 'bytes', fieldNumber: 5 }, transactionRoot: { dataType: 'bytes', fieldNumber: 6 }, assetsRoot: { dataType: 'bytes', fieldNumber: 7 }, - stateRoot: { dataType: 'bytes', fieldNumber: 8 }, - maxHeightPrevoted: { dataType: 'uint32', fieldNumber: 9 }, - maxHeightGenerated: { dataType: 'uint32', fieldNumber: 10 }, - validatorsHash: { dataType: 'bytes', fieldNumber: 11 }, + eventRoot: { dataType: 'bytes', fieldNumber: 8 }, + stateRoot: { dataType: 'bytes', fieldNumber: 9 }, + maxHeightPrevoted: { dataType: 'uint32', fieldNumber: 10 }, + maxHeightGenerated: { dataType: 'uint32', fieldNumber: 11 }, + validatorsHash: { dataType: 'bytes', fieldNumber: 12 }, aggregateCommit: { type: 'object', - fieldNumber: 12, + fieldNumber: 13, required: ['height', 'aggregationBits', 'certificateSignature'], properties: { height: { @@ -80,6 +81,7 @@ export const signingBlockHeaderSchema = { 'generatorAddress', 'transactionRoot', 'assetsRoot', + 'eventRoot', 'stateRoot', 'maxHeightPrevoted', 'maxHeightGenerated', @@ -94,7 +96,7 @@ export const blockHeaderSchema = { required: [...signingBlockHeaderSchema.required, 'signature'], properties: { ...signingBlockHeaderSchema.properties, - signature: { dataType: 'bytes', fieldNumber: 13 }, + signature: { dataType: 'bytes', fieldNumber: 14 }, }, }; @@ -104,7 +106,7 @@ export const blockHeaderSchemaWithId = { required: [...blockHeaderSchema.required, 'id'], properties: { ...blockHeaderSchema.properties, - id: { dataType: 'bytes', fieldNumber: 14 }, + id: { dataType: 'bytes', fieldNumber: 15 }, }, }; @@ -172,3 +174,47 @@ export const stateDiffSchema = { }, }, }; + +export const eventSchema = { + $id: '/block/event', + type: 'object', + required: ['moduleID', 'typeID', 'data', 'topics', 'index'], + properties: { + moduleID: { + dataType: 'bytes', + fieldNumber: 1, + }, + typeID: { + dataType: 'bytes', + fieldNumber: 2, + }, + data: { + dataType: 'bytes', + fieldNumber: 3, + }, + topics: { + type: 'array', + fieldNumber: 4, + items: { + maxItems: 4, + dataType: 'bytes', + }, + }, + index: { + dataType: 'uint32', + fieldNumber: 5, + }, + }, +}; + +export const standardEventDataSchema = { + $id: '/block/event/standard', + type: 'object', + required: ['success'], + properties: { + success: { + dataType: 'bool', + fieldNumber: 1, + }, + }, +}; diff --git a/elements/lisk-chain/test/integration/data_access/blocks.spec.ts b/elements/lisk-chain/test/integration/data_access/blocks.spec.ts index 13fb5b02d37..cef9764e219 100644 --- a/elements/lisk-chain/test/integration/data_access/blocks.spec.ts +++ b/elements/lisk-chain/test/integration/data_access/blocks.spec.ts @@ -18,11 +18,12 @@ import { KVStore, formatInt, NotFoundError, InMemoryKVStore } from '@liskhq/lisk import { codec } from '@liskhq/lisk-codec'; import { getRandomBytes, hash, intToBuffer } from '@liskhq/lisk-cryptography'; import { SparseMerkleTree } from '@liskhq/lisk-tree'; -import { Storage } from '../../../src/data_access/storage'; +import { encodeByteArray, Storage } from '../../../src/data_access/storage'; import { createValidDefaultBlock } from '../../utils/block'; import { getTransaction } from '../../utils/transaction'; import { Block, BlockAssets, CurrentState, SMTStore, StateStore, Transaction } from '../../../src'; import { DataAccess } from '../../../src/data_access'; +import { Event } from '../../../src/event'; import { stateDiffSchema } from '../../../src/schema'; import { DB_KEY_BLOCKS_ID, @@ -32,6 +33,7 @@ import { DB_KEY_DIFF_STATE, DB_KEY_TRANSACTIONS_BLOCK_ID, DB_KEY_STATE_STORE, + DB_KEY_BLOCK_EVENTS, } from '../../../src/db_keys'; import { concatDBKeys } from '../../../src/utils'; import { toSMTKey } from '../../../src/state_store/utils'; @@ -59,6 +61,7 @@ describe('dataAccess.blocks', () => { db, minBlockHeaderCache: 3, maxBlockHeaderCache: 5, + keepEventsForHeights: -1, }); // Prepare sample data const block300 = await createValidDefaultBlock({ @@ -79,6 +82,23 @@ describe('dataAccess.blocks', () => { assets: new BlockAssets([{ moduleID: 3, data: getRandomBytes(64) }]), }); + const events = [ + new Event({ + data: getRandomBytes(20), + index: 0, + moduleID: Buffer.from([0, 0, 0, 2]), + topics: [getRandomBytes(32)], + typeID: Buffer.from([0, 0, 0, 0]), + }), + new Event({ + data: getRandomBytes(20), + index: 1, + moduleID: Buffer.from([0, 0, 0, 3]), + topics: [getRandomBytes(32)], + typeID: Buffer.from([0, 0, 0, 0]), + }), + ]; + blocks = [block300, block301, block302, block303]; const batch = db.batch(); for (const block of blocks) { @@ -95,6 +115,10 @@ describe('dataAccess.blocks', () => { batch.put(concatDBKeys(DB_KEY_TRANSACTIONS_ID, tx.id), tx.getBytes()); } } + batch.put( + concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(block.header.height)), + encodeByteArray(events.map(e => e.getBytes())), + ); batch.put( concatDBKeys(DB_KEY_TEMPBLOCKS_HEIGHT, formatInt(blocks[2].header.height)), blocks[2].getBytes(), @@ -343,6 +367,23 @@ describe('dataAccess.blocks', () => { let block: Block; let currentState: CurrentState; + const events = [ + new Event({ + data: getRandomBytes(20), + index: 0, + moduleID: Buffer.from([0, 0, 0, 2]), + topics: [getRandomBytes(32)], + typeID: Buffer.from([0, 0, 0, 0]), + }), + new Event({ + data: getRandomBytes(20), + index: 1, + moduleID: Buffer.from([0, 0, 0, 3]), + topics: [getRandomBytes(32)], + typeID: Buffer.from([0, 0, 0, 0]), + }), + ]; + beforeEach(async () => { const stateStore = new StateStore(db); block = await createValidDefaultBlock({ @@ -368,7 +409,7 @@ describe('dataAccess.blocks', () => { }); it('should create block with all index required', async () => { - await dataAccess.saveBlock(block, currentState, 0); + await dataAccess.saveBlock(block, events, currentState, 0); await expect(db.exists(concatDBKeys(DB_KEY_BLOCKS_ID, block.header.id))).resolves.toBeTrue(); await expect( @@ -386,6 +427,9 @@ describe('dataAccess.blocks', () => { await expect( db.exists(concatDBKeys(DB_KEY_TEMPBLOCKS_HEIGHT, formatInt(block.header.height))), ).resolves.toBeTrue(); + await expect( + db.exists(concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(block.header.height))), + ).resolves.toBeTrue(); const createdBlock = await dataAccess.getBlockByID(block.header.id); expect(createdBlock.header.toObject()).toStrictEqual(block.header.toObject()); expect(createdBlock.transactions[0]).toBeInstanceOf(Transaction); @@ -395,7 +439,7 @@ describe('dataAccess.blocks', () => { }); it('should create block with all index required and remove the same height block from temp', async () => { - await dataAccess.saveBlock(block, currentState, 0, true); + await dataAccess.saveBlock(block, events, currentState, 0, true); await expect(db.exists(concatDBKeys(DB_KEY_BLOCKS_ID, block.header.id))).resolves.toBeTrue(); await expect( @@ -424,11 +468,48 @@ describe('dataAccess.blocks', () => { it('should delete diff before the finalized height', async () => { await db.put(concatDBKeys(DB_KEY_DIFF_STATE, formatInt(99)), Buffer.from('random diff')); await db.put(concatDBKeys(DB_KEY_DIFF_STATE, formatInt(100)), Buffer.from('random diff 2')); - await dataAccess.saveBlock(block, currentState, 100, true); + await dataAccess.saveBlock(block, events, currentState, 100, true); await expect(db.exists(concatDBKeys(DB_KEY_DIFF_STATE, formatInt(100)))).resolves.toBeTrue(); await expect(db.exists(concatDBKeys(DB_KEY_DIFF_STATE, formatInt(99)))).resolves.toBeFalse(); }); + + it('should not delete events before finalized height when keepEventsForHeights == -1', async () => { + await db.put(concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(99)), Buffer.from('random diff')); + await db.put(concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(100)), Buffer.from('random diff 2')); + await dataAccess.saveBlock(block, events, currentState, 100, true); + + await expect( + db.exists(concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(100))), + ).resolves.toBeTrue(); + await expect(db.exists(concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(99)))).resolves.toBeTrue(); + }); + + it('should delete events before finalized height when keepEventsForHeights == 1', async () => { + (dataAccess['_storage']['_keepEventsForHeights'] as any) = 1; + await db.put(concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(99)), Buffer.from('random diff')); + await db.put(concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(100)), Buffer.from('random diff 2')); + await dataAccess.saveBlock(block, events, currentState, 100, true); + + await expect( + db.exists(concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(100))), + ).resolves.toBeTrue(); + await expect( + db.exists(concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(99))), + ).resolves.toBeFalse(); + }); + + it('should maintain events for not finalized blocks', async () => { + (dataAccess['_storage']['_keepEventsForHeights'] as any) = 0; + await db.put(concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(99)), Buffer.from('random diff')); + await db.put(concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(100)), Buffer.from('random diff 2')); + await dataAccess.saveBlock(block, events, currentState, 50, true); + + await expect( + db.exists(concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(100))), + ).resolves.toBeTrue(); + await expect(db.exists(concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(99)))).resolves.toBeTrue(); + }); }); describe('deleteBlock', () => { @@ -474,6 +555,9 @@ describe('dataAccess.blocks', () => { await expect( db.exists(concatDBKeys(DB_KEY_TEMPBLOCKS_HEIGHT, formatInt(blocks[2].header.height))), ).resolves.toBeFalse(); + await expect( + db.exists(concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(blocks[2].header.height))), + ).resolves.toBeFalse(); }); it('should throw an error when there is no diff', async () => { @@ -607,7 +691,7 @@ describe('dataAccess.blocks', () => { diff: diff1, stateStore, }; - await dataAccess.saveBlock(firstBlock, currentState, 100); + await dataAccess.saveBlock(firstBlock, [], currentState, 100); expect(smt.rootHash).toEqual(firstBlock.header.stateRoot); @@ -634,7 +718,7 @@ describe('dataAccess.blocks', () => { diff: diff2, stateStore, }; - await dataAccess.saveBlock(secondBlock, newCurrentState, 100); + await dataAccess.saveBlock(secondBlock, [], newCurrentState, 100); expect(secondSMT.rootHash).toEqual(secondBlock.header.stateRoot); }); @@ -687,7 +771,7 @@ describe('dataAccess.blocks', () => { diff: diff1, stateStore, }; - await dataAccess.saveBlock(firstBlock, currentState, 100); + await dataAccess.saveBlock(firstBlock, [], currentState, 100); expect(smt.rootHash).toEqual(firstBlock.header.stateRoot); diff --git a/elements/lisk-chain/test/integration/data_access/transactions.spec.ts b/elements/lisk-chain/test/integration/data_access/transactions.spec.ts index 994a0211973..8702721dfe9 100644 --- a/elements/lisk-chain/test/integration/data_access/transactions.spec.ts +++ b/elements/lisk-chain/test/integration/data_access/transactions.spec.ts @@ -33,6 +33,7 @@ describe('dataAccess.transactions', () => { db, minBlockHeaderCache: 3, maxBlockHeaderCache: 5, + keepEventsForHeights: -1, }); }); diff --git a/elements/lisk-chain/test/unit/__snapshots__/block_header.spec.ts.snap b/elements/lisk-chain/test/unit/__snapshots__/block_header.spec.ts.snap index cc2085383de..fe3a54daed5 100644 --- a/elements/lisk-chain/test/unit/__snapshots__/block_header.spec.ts.snap +++ b/elements/lisk-chain/test/unit/__snapshots__/block_header.spec.ts.snap @@ -5,7 +5,7 @@ Object { "$id": "/block/header/3", "properties": Object { "aggregateCommit": Object { - "fieldNumber": 12, + "fieldNumber": 13, "properties": Object { "aggregationBits": Object { "dataType": "bytes", @@ -31,6 +31,10 @@ Object { "dataType": "bytes", "fieldNumber": 7, }, + "eventRoot": Object { + "dataType": "bytes", + "fieldNumber": 8, + }, "generatorAddress": Object { "dataType": "bytes", "fieldNumber": 5, @@ -41,11 +45,11 @@ Object { }, "maxHeightGenerated": Object { "dataType": "uint32", - "fieldNumber": 10, + "fieldNumber": 11, }, "maxHeightPrevoted": Object { "dataType": "uint32", - "fieldNumber": 9, + "fieldNumber": 10, }, "previousBlockID": Object { "dataType": "bytes", @@ -53,11 +57,11 @@ Object { }, "signature": Object { "dataType": "bytes", - "fieldNumber": 13, + "fieldNumber": 14, }, "stateRoot": Object { "dataType": "bytes", - "fieldNumber": 8, + "fieldNumber": 9, }, "timestamp": Object { "dataType": "uint32", @@ -69,7 +73,7 @@ Object { }, "validatorsHash": Object { "dataType": "bytes", - "fieldNumber": 11, + "fieldNumber": 12, }, "version": Object { "dataType": "uint32", @@ -84,6 +88,7 @@ Object { "generatorAddress", "transactionRoot", "assetsRoot", + "eventRoot", "stateRoot", "maxHeightPrevoted", "maxHeightGenerated", @@ -100,7 +105,7 @@ Object { "$id": "/block/header/3", "properties": Object { "aggregateCommit": Object { - "fieldNumber": 12, + "fieldNumber": 13, "properties": Object { "aggregationBits": Object { "dataType": "bytes", @@ -126,6 +131,10 @@ Object { "dataType": "bytes", "fieldNumber": 7, }, + "eventRoot": Object { + "dataType": "bytes", + "fieldNumber": 8, + }, "generatorAddress": Object { "dataType": "bytes", "fieldNumber": 5, @@ -136,15 +145,15 @@ Object { }, "id": Object { "dataType": "bytes", - "fieldNumber": 14, + "fieldNumber": 15, }, "maxHeightGenerated": Object { "dataType": "uint32", - "fieldNumber": 10, + "fieldNumber": 11, }, "maxHeightPrevoted": Object { "dataType": "uint32", - "fieldNumber": 9, + "fieldNumber": 10, }, "previousBlockID": Object { "dataType": "bytes", @@ -152,11 +161,11 @@ Object { }, "signature": Object { "dataType": "bytes", - "fieldNumber": 13, + "fieldNumber": 14, }, "stateRoot": Object { "dataType": "bytes", - "fieldNumber": 8, + "fieldNumber": 9, }, "timestamp": Object { "dataType": "uint32", @@ -168,7 +177,7 @@ Object { }, "validatorsHash": Object { "dataType": "bytes", - "fieldNumber": 11, + "fieldNumber": 12, }, "version": Object { "dataType": "uint32", @@ -183,6 +192,7 @@ Object { "generatorAddress", "transactionRoot", "assetsRoot", + "eventRoot", "stateRoot", "maxHeightPrevoted", "maxHeightGenerated", @@ -200,7 +210,7 @@ Object { "$id": "/block/header/signing/3", "properties": Object { "aggregateCommit": Object { - "fieldNumber": 12, + "fieldNumber": 13, "properties": Object { "aggregationBits": Object { "dataType": "bytes", @@ -226,6 +236,10 @@ Object { "dataType": "bytes", "fieldNumber": 7, }, + "eventRoot": Object { + "dataType": "bytes", + "fieldNumber": 8, + }, "generatorAddress": Object { "dataType": "bytes", "fieldNumber": 5, @@ -236,11 +250,11 @@ Object { }, "maxHeightGenerated": Object { "dataType": "uint32", - "fieldNumber": 10, + "fieldNumber": 11, }, "maxHeightPrevoted": Object { "dataType": "uint32", - "fieldNumber": 9, + "fieldNumber": 10, }, "previousBlockID": Object { "dataType": "bytes", @@ -248,7 +262,7 @@ Object { }, "stateRoot": Object { "dataType": "bytes", - "fieldNumber": 8, + "fieldNumber": 9, }, "timestamp": Object { "dataType": "uint32", @@ -260,7 +274,7 @@ Object { }, "validatorsHash": Object { "dataType": "bytes", - "fieldNumber": 11, + "fieldNumber": 12, }, "version": Object { "dataType": "uint32", @@ -275,6 +289,7 @@ Object { "generatorAddress", "transactionRoot", "assetsRoot", + "eventRoot", "stateRoot", "maxHeightPrevoted", "maxHeightGenerated", diff --git a/elements/lisk-chain/test/unit/block_header.spec.ts b/elements/lisk-chain/test/unit/block_header.spec.ts index 0280834f893..27f5e101a80 100644 --- a/elements/lisk-chain/test/unit/block_header.spec.ts +++ b/elements/lisk-chain/test/unit/block_header.spec.ts @@ -28,6 +28,7 @@ const getBlockAttrs = () => ({ stateRoot: Buffer.from('7f9d96a09a3fd17f3478eb7bef3a8bda00e1238b', 'hex'), transactionRoot: Buffer.from('b27ca21f40d44113c2090ca8f05fb706c54e87dd', 'hex'), assetsRoot: Buffer.from('b27ca21f40d44113c2090ca8f05fb706c54e87dd', 'hex'), + eventRoot: Buffer.from('30dda4fbc395828e5a9f2f8824771e434fce4945a1e7820012440d09dd1e2b6d', 'hex'), generatorAddress: Buffer.from('be63fb1c0426573352556f18b21efd5b6183c39c', 'hex'), maxHeightPrevoted: 1000988, maxHeightGenerated: 1000988, @@ -48,6 +49,7 @@ const getGenesisBlockAttrs = () => ({ stateRoot: Buffer.from('7f9d96a09a3fd17f3478eb7bef3a8bda00e1238b', 'hex'), transactionRoot: EMPTY_HASH, assetsRoot: EMPTY_HASH, + eventRoot: EMPTY_HASH, generatorAddress: EMPTY_BUFFER, maxHeightPrevoted: 1009988, maxHeightGenerated: 0, @@ -61,12 +63,12 @@ const getGenesisBlockAttrs = () => ({ }); const blockId = Buffer.from( - '097ce5adc1a34680d6c939287011dec9b70a3bc1f5f896a3f9024fc9bed59992', + 'f14104e384546adaba487af56e658188eea07bb534a61b4cbb9ccaee54139b8c', 'hex', ); const blockHeaderBytes = Buffer.from( - '080110c4d23d18c4d23d22144a462ea57a8c9f72d866c09770e5ec70cef187272a14be63fb1c0426573352556f18b21efd5b6183c39c3214b27ca21f40d44113c2090ca8f05fb706c54e87dd3a14b27ca21f40d44113c2090ca8f05fb706c54e87dd42147f9d96a09a3fd17f3478eb7bef3a8bda00e1238b489c8c3d509c8c3d5a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8556206080012001a006a146da88e2fd4435e26e02682435f108002ccc3ddd5', + '080110c4d23d18c4d23d22144a462ea57a8c9f72d866c09770e5ec70cef187272a14be63fb1c0426573352556f18b21efd5b6183c39c3214b27ca21f40d44113c2090ca8f05fb706c54e87dd3a14b27ca21f40d44113c2090ca8f05fb706c54e87dd422030dda4fbc395828e5a9f2f8824771e434fce4945a1e7820012440d09dd1e2b6d4a147f9d96a09a3fd17f3478eb7bef3a8bda00e1238b509c8c3d589c8c3d6220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8556a06080012001a0072146da88e2fd4435e26e02682435f108002ccc3ddd5', 'hex', ); @@ -135,6 +137,7 @@ describe('block_header', () => { expect(blockHeader.generatorAddress).toEqual(data.generatorAddress); expect(blockHeader.previousBlockID).toEqual(data.previousBlockID); expect(blockHeader.stateRoot).toEqual(data.stateRoot); + expect(blockHeader.eventRoot).toEqual(data.eventRoot); expect(blockHeader.validatorsHash).toEqual(data.validatorsHash); expect(blockHeader.aggregateCommit).toEqual(data.aggregateCommit); expect(blockHeader.maxHeightPrevoted).toEqual(data.maxHeightPrevoted); diff --git a/elements/lisk-chain/test/unit/chain.spec.ts b/elements/lisk-chain/test/unit/chain.spec.ts index cf17cb76f54..fca2856d6bd 100644 --- a/elements/lisk-chain/test/unit/chain.spec.ts +++ b/elements/lisk-chain/test/unit/chain.spec.ts @@ -39,6 +39,7 @@ import { BlockAssets } from '../../src'; describe('chain', () => { const constants = { maxTransactionsSize: 15 * 1024, + keepEventsForHeights: 300, }; const emptyEncodedDiff = codec.encode(stateDiffSchema, { created: [], @@ -176,7 +177,7 @@ describe('chain', () => { }); it('should remove diff until finalized height', async () => { - await chainInstance.saveBlock(savingBlock, currentState, 1, { + await chainInstance.saveBlock(savingBlock, [], currentState, 1, { removeFromTempTable: true, }); expect(db.clear).toHaveBeenCalledWith({ @@ -186,7 +187,7 @@ describe('chain', () => { }); it('should remove tempBlock by height when removeFromTempTable is true', async () => { - await chainInstance.saveBlock(savingBlock, currentState, 0, { + await chainInstance.saveBlock(savingBlock, [], currentState, 0, { removeFromTempTable: true, }); expect(batch.del).toHaveBeenCalledWith( @@ -195,7 +196,7 @@ describe('chain', () => { }); it('should save block', async () => { - await chainInstance.saveBlock(savingBlock, currentState, 0); + await chainInstance.saveBlock(savingBlock, [], currentState, 0); expect(batch.put).toHaveBeenCalledWith( concatDBKeys(DB_KEY_BLOCKS_ID, savingBlock.header.id), expect.anything(), diff --git a/elements/lisk-chain/test/unit/data_access/data_access.spec.ts b/elements/lisk-chain/test/unit/data_access/data_access.spec.ts index b9e6b0f5ff8..49c6e247e09 100644 --- a/elements/lisk-chain/test/unit/data_access/data_access.spec.ts +++ b/elements/lisk-chain/test/unit/data_access/data_access.spec.ts @@ -14,6 +14,7 @@ import { Readable } from 'stream'; import { when } from 'jest-when'; import { formatInt, NotFoundError, getFirstPrefix, getLastPrefix, KVStore } from '@liskhq/lisk-db'; +import { getRandomBytes } from '@liskhq/lisk-cryptography'; import { DataAccess } from '../../../src/data_access'; import { createFakeBlockHeader, createValidDefaultBlock } from '../../utils/block'; import { Transaction } from '../../../src/transaction'; @@ -23,9 +24,12 @@ import { DB_KEY_BLOCKS_ID, DB_KEY_TRANSACTIONS_ID, DB_KEY_TEMPBLOCKS_HEIGHT, + DB_KEY_BLOCK_EVENTS, } from '../../../src/db_keys'; import { Block } from '../../../src/block'; +import { Event } from '../../../src/event'; import { BlockAssets, BlockHeader } from '../../../src'; +import { encodeByteArray } from '../../../src/data_access/storage'; jest.mock('@liskhq/lisk-db'); @@ -48,6 +52,7 @@ describe('data_access', () => { db, minBlockHeaderCache: 3, maxBlockHeaderCache: 5, + keepEventsForHeights: 1, }); block = await createValidDefaultBlock({ header: { height: 1 } }); }); @@ -367,6 +372,32 @@ describe('data_access', () => { }); }); + describe('#getEvents', () => { + it('should get the events related to heights', async () => { + const original = [ + new Event({ + data: getRandomBytes(20), + index: 0, + moduleID: Buffer.from([0, 0, 0, 2]), + topics: [getRandomBytes(32)], + typeID: Buffer.from([0, 0, 0, 0]), + }), + new Event({ + data: getRandomBytes(20), + index: 1, + moduleID: Buffer.from([0, 0, 0, 3]), + topics: [getRandomBytes(32)], + typeID: Buffer.from([0, 0, 0, 0]), + }), + ]; + db.get.mockResolvedValue(encodeByteArray(original.map(e => e.getBytes())) as never); + + const resp = await dataAccess.getEvents(30); + expect(db.get).toHaveBeenCalledWith(concatDBKeys(DB_KEY_BLOCK_EVENTS, formatInt(30))); + expect(resp).toEqual(original); + }); + }); + describe('#isBlockPersisted', () => { it('should call check if the id exists in the database', async () => { // Act diff --git a/elements/lisk-chain/test/unit/event.spec.ts b/elements/lisk-chain/test/unit/event.spec.ts new file mode 100644 index 00000000000..3f765b2af37 --- /dev/null +++ b/elements/lisk-chain/test/unit/event.spec.ts @@ -0,0 +1,109 @@ +/* + * Copyright © 2022 Lisk Foundation + * + * See the LICENSE file at the top-level directory of this distribution + * for licensing information. + * + * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, + * no part of this software, including this file, may be copied, modified, + * propagated, or distributed except according to the terms contained in the + * LICENSE file. + * + * Removal or modification of this copyright notice is prohibited. + */ +/* eslint-disable no-bitwise */ + +import { codec } from '@liskhq/lisk-codec'; +import { getRandomBytes } from '@liskhq/lisk-cryptography'; +import { eventSchema } from '../../src/schema'; +import { Event } from '../../src/event'; +import { EVENT_TOPIC_HASH_LENGTH_BYTES, EVENT_TOTAL_INDEX_LENGTH_BYTES } from '../../src/constants'; + +describe('event', () => { + const eventObj = { + moduleID: Buffer.from([0, 0, 0, 2]), + typeID: Buffer.from([0, 0, 0, 1]), + topics: [getRandomBytes(32), getRandomBytes(20), getRandomBytes(2)], + index: 3, + data: getRandomBytes(200), + }; + const encodedEvent = codec.encode(eventSchema, eventObj); + + describe('fromBytes', () => { + it('should create eventObject from encoded bytes', () => { + expect(Event.fromBytes(encodedEvent).toObject()).toEqual(eventObj); + }); + }); + + describe('id', () => { + it('should return event id', () => { + const event = Event.fromBytes(encodedEvent); + const id = event.id(30); + + expect(id.slice(0, 4)).toEqual(Buffer.from([0, 0, 0, 30])); + const indexBytes = Buffer.alloc(4); + // eslint-disable-next-line no-bitwise + indexBytes.writeUInt32BE(eventObj.index << 2, 0); + expect(id.slice(4)).toEqual(indexBytes); + }); + }); + + describe('keyPair', () => { + it('should return number of pairs for topics', () => { + const event = Event.fromBytes(encodedEvent); + const pairs = event.keyPair(); + + expect(pairs).toHaveLength(eventObj.topics.length); + }); + + it('should return the values for all key pairs', () => { + const event = Event.fromBytes(encodedEvent); + const pairs = event.keyPair(); + + expect.assertions(pairs.length - 1); + for (let i = 1; i < pairs.length; i += 1) { + expect(pairs[i].value).toEqual(pairs[0].value); + } + }); + + it('should return key with correct size and index', () => { + const event = Event.fromBytes(encodedEvent); + const pairs = event.keyPair(); + // eslint-disable-next-line @typescript-eslint/prefer-for-of + for (let i = 0; i < pairs.length; i += 1) { + const { key } = pairs[i]; + expect(key).toHaveLength(EVENT_TOPIC_HASH_LENGTH_BYTES + EVENT_TOTAL_INDEX_LENGTH_BYTES); + + const index = key.slice(EVENT_TOPIC_HASH_LENGTH_BYTES); + + // Check index + const indexNum = index.readUInt32BE(0); + expect((indexNum - i) >> 2).toEqual(eventObj.index); + + // Check topic index + const topicIndex = indexNum - (eventObj.index << 2); + expect(topicIndex).toEqual(i); + } + }); + }); + + describe('toJSON', () => { + it('should return all values in JSON compatible format', () => { + const event = Event.fromBytes(encodedEvent).toJSON(); + expect(event).toEqual({ + moduleID: eventObj.moduleID.toString('hex'), + typeID: eventObj.typeID.toString('hex'), + index: 3, + topics: eventObj.topics.map(t => t.toString('hex')), + data: expect.any(String), + }); + }); + }); + + describe('toObject', () => { + it('should return all values contained', () => { + const event = Event.fromBytes(encodedEvent); + expect(event.toObject()).toEqual(eventObj); + }); + }); +}); diff --git a/elements/lisk-chain/test/utils/block.ts b/elements/lisk-chain/test/utils/block.ts index d59b1f918de..e0e94d37739 100644 --- a/elements/lisk-chain/test/utils/block.ts +++ b/elements/lisk-chain/test/utils/block.ts @@ -45,6 +45,7 @@ export const createFakeBlockHeader = (header?: Partial): Block generatorAddress: header?.generatorAddress ?? getRandomBytes(32), maxHeightGenerated: header?.maxHeightGenerated ?? 0, maxHeightPrevoted: header?.maxHeightPrevoted ?? 0, + eventRoot: header?.eventRoot ?? hash(getRandomBytes(32)), stateRoot: header?.stateRoot ?? hash(getRandomBytes(32)), assetsRoot: header?.assetsRoot ?? hash(getRandomBytes(32)), validatorsHash: header?.validatorsHash ?? hash(getRandomBytes(32)), @@ -82,6 +83,7 @@ export const createValidDefaultBlock = async ( timestamp: genesis.header.timestamp + 10, transactionRoot: txTree.root, stateRoot: getRandomBytes(32), + eventRoot: getRandomBytes(32), assetsRoot, validatorsHash: hash(getRandomBytes(32)), maxHeightGenerated: 0, diff --git a/framework-plugins/lisk-framework-faucet-plugin/test/unit/plugin/auth.spec.ts b/framework-plugins/lisk-framework-faucet-plugin/test/unit/plugin/auth.spec.ts index eb32bb2749a..567f6065787 100644 --- a/framework-plugins/lisk-framework-faucet-plugin/test/unit/plugin/auth.spec.ts +++ b/framework-plugins/lisk-framework-faucet-plugin/test/unit/plugin/auth.spec.ts @@ -19,6 +19,9 @@ import { configSchema } from '../../../src/plugin/schemas'; const appConfigForPlugin: ApplicationConfigForPlugin = { rootPath: '~/.lisk', label: 'my-app', + system: { + keepEventsForHeights: -1, + }, logger: { consoleLogLevel: 'debug', fileLogLevel: 'none', diff --git a/framework-plugins/lisk-framework-faucet-plugin/test/unit/plugin/faucet_plugin.spec.ts b/framework-plugins/lisk-framework-faucet-plugin/test/unit/plugin/faucet_plugin.spec.ts index cb76299b049..4c690b5e897 100644 --- a/framework-plugins/lisk-framework-faucet-plugin/test/unit/plugin/faucet_plugin.spec.ts +++ b/framework-plugins/lisk-framework-faucet-plugin/test/unit/plugin/faucet_plugin.spec.ts @@ -18,6 +18,9 @@ import { FaucetPlugin } from '../../../src/plugin'; const appConfigForPlugin: ApplicationConfigForPlugin = { rootPath: '~/.lisk', label: 'my-app', + system: { + keepEventsForHeights: -1, + }, logger: { consoleLogLevel: 'info', fileLogLevel: 'none', logFileName: 'plugin-FaucetPlugin.log' }, rpc: { modes: ['ipc'], diff --git a/framework-plugins/lisk-framework-monitor-plugin/test/unit/blocks.spec.ts b/framework-plugins/lisk-framework-monitor-plugin/test/unit/blocks.spec.ts index ec90edeaba5..6e251698a8e 100644 --- a/framework-plugins/lisk-framework-monitor-plugin/test/unit/blocks.spec.ts +++ b/framework-plugins/lisk-framework-monitor-plugin/test/unit/blocks.spec.ts @@ -26,6 +26,9 @@ import { configSchema } from '../../src/schemas'; const appConfigForPlugin: ApplicationConfigForPlugin = { rootPath: '~/.lisk', label: 'my-app', + system: { + keepEventsForHeights: -1, + }, logger: { consoleLogLevel: 'info', fileLogLevel: 'none', @@ -81,6 +84,7 @@ describe('_handlePostBlock', () => { previousBlockID: Buffer.alloc(0), timestamp: Math.floor(Date.now() / 1000 - 24 * 60 * 60), stateRoot: cryptography.hash(Buffer.alloc(0)), + eventRoot: cryptography.hash(Buffer.alloc(0)), maxHeightGenerated: 0, maxHeightPrevoted: 0, assetsRoot: cryptography.hash(Buffer.alloc(0)), diff --git a/framework-plugins/lisk-framework-monitor-plugin/test/unit/forks.spec.ts b/framework-plugins/lisk-framework-monitor-plugin/test/unit/forks.spec.ts index 5aa993c0737..753f79c10e2 100644 --- a/framework-plugins/lisk-framework-monitor-plugin/test/unit/forks.spec.ts +++ b/framework-plugins/lisk-framework-monitor-plugin/test/unit/forks.spec.ts @@ -31,6 +31,9 @@ const appConfigForPlugin: ApplicationConfigForPlugin = { fileLogLevel: 'none', logFileName: 'plugin-MisbehaviourPlugin.log', }, + system: { + keepEventsForHeights: -1, + }, rpc: { modes: ['ipc'], ws: { @@ -80,6 +83,7 @@ describe('_handleFork', () => { previousBlockID: Buffer.alloc(0), timestamp: Math.floor(Date.now() / 1000 - 24 * 60 * 60), stateRoot: cryptography.hash(Buffer.alloc(0)), + eventRoot: cryptography.hash(Buffer.alloc(0)), maxHeightGenerated: 0, maxHeightPrevoted: 0, assetsRoot: cryptography.hash(Buffer.alloc(0)), diff --git a/framework-plugins/lisk-framework-monitor-plugin/test/unit/network.spec.ts b/framework-plugins/lisk-framework-monitor-plugin/test/unit/network.spec.ts index 9a61614a6fc..d0cb1e6e0ad 100644 --- a/framework-plugins/lisk-framework-monitor-plugin/test/unit/network.spec.ts +++ b/framework-plugins/lisk-framework-monitor-plugin/test/unit/network.spec.ts @@ -26,6 +26,9 @@ const appConfigForPlugin: ApplicationConfigForPlugin = { fileLogLevel: 'none', logFileName: 'plugin-MisbehaviourPlugin.log', }, + system: { + keepEventsForHeights: -1, + }, rpc: { modes: ['ipc'], ws: { diff --git a/framework-plugins/lisk-framework-monitor-plugin/test/unit/subscribe_event.spec.ts b/framework-plugins/lisk-framework-monitor-plugin/test/unit/subscribe_event.spec.ts index ea7c9d67819..25d02972db9 100644 --- a/framework-plugins/lisk-framework-monitor-plugin/test/unit/subscribe_event.spec.ts +++ b/framework-plugins/lisk-framework-monitor-plugin/test/unit/subscribe_event.spec.ts @@ -23,6 +23,9 @@ const appConfigForPlugin: ApplicationConfigForPlugin = { fileLogLevel: 'none', logFileName: 'plugin-MisbehaviourPlugin.log', }, + system: { + keepEventsForHeights: -1, + }, rpc: { modes: ['ipc'], ws: { diff --git a/framework-plugins/lisk-framework-monitor-plugin/test/unit/transaction.spec.ts b/framework-plugins/lisk-framework-monitor-plugin/test/unit/transaction.spec.ts index a91f6758ea4..b156388398a 100644 --- a/framework-plugins/lisk-framework-monitor-plugin/test/unit/transaction.spec.ts +++ b/framework-plugins/lisk-framework-monitor-plugin/test/unit/transaction.spec.ts @@ -25,6 +25,9 @@ const appConfigForPlugin: ApplicationConfigForPlugin = { fileLogLevel: 'none', logFileName: 'plugin-MisbehaviourPlugin.log', }, + system: { + keepEventsForHeights: -1, + }, rpc: { modes: ['ipc'], ws: { diff --git a/framework-plugins/lisk-framework-report-misbehavior-plugin/test/integration/cleanup_job.spec.ts b/framework-plugins/lisk-framework-report-misbehavior-plugin/test/integration/cleanup_job.spec.ts index f89edc5ff72..58202445601 100644 --- a/framework-plugins/lisk-framework-report-misbehavior-plugin/test/integration/cleanup_job.spec.ts +++ b/framework-plugins/lisk-framework-report-misbehavior-plugin/test/integration/cleanup_job.spec.ts @@ -35,6 +35,9 @@ const appConfigForPlugin: ApplicationConfigForPlugin = { fileLogLevel: 'none', logFileName: 'plugin-reportMisbehavior.log', }, + system: { + keepEventsForHeights: -1, + }, rpc: { modes: ['ipc'], ws: { diff --git a/framework-plugins/lisk-framework-report-misbehavior-plugin/test/unit/__snapshots__/send_pom_transaction.spec.ts.snap b/framework-plugins/lisk-framework-report-misbehavior-plugin/test/unit/__snapshots__/send_pom_transaction.spec.ts.snap index 42425bddcbb..e9716d1d0fa 100644 --- a/framework-plugins/lisk-framework-report-misbehavior-plugin/test/unit/__snapshots__/send_pom_transaction.spec.ts.snap +++ b/framework-plugins/lisk-framework-report-misbehavior-plugin/test/unit/__snapshots__/send_pom_transaction.spec.ts.snap @@ -1,3 +1,3 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`Send PoM transaction should create pom transaction for given block headers 1`] = `"0805100318002080c2d72f2a200fe9a3f1a21b5530f27f87a414b549e79a940bf24fdf2b2f05e7f22aeeecc86a32ce020aa401080010bb87b28b06180022002a003220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8553a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8554220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855480050005a20d800954794e0882c2419fe4736c2a191e6515859a7a894043ba5c911da6b72e76206080012001a006a0012a401080010fe86b28b06180022002a003220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8553a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8554220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855480050005a20f8da7f49e92286b0129fd75a9208eed942ef1d79df93c42c9b87e8b6bb9fc84f6206080012001a006a003a40614ae816c7ced020d00a4260c1980e2353f3ccdaea4e4054c903e0fa2cc6ba1ce462821cfe38958a9f96c92631f0629bf244151c2bb7b1bd7b96c2395174890b"`; +exports[`Send PoM transaction should create pom transaction for given block headers 1`] = `"0805100318002080c2d72f2a200fe9a3f1a21b5530f27f87a414b549e79a940bf24fdf2b2f05e7f22aeeecc86a32b2040a960208021064186422203d1b5dd1ef4ff7b22359598ebdf58966a51adcc03e02ad356632743e658989902a1440ff452fae2affe6eeef3c30e53e9eac35a1bc4332203d1b5dd1ef4ff7b22359598ebdf58966a51adcc03e02ad356632743e658989903a203d1b5dd1ef4ff7b22359598ebdf58966a51adcc03e02ad356632743e6589899042203d1b5dd1ef4ff7b22359598ebdf58966a51adcc03e02ad356632743e658989904a203d1b5dd1ef4ff7b22359598ebdf58966a51adcc03e02ad356632743e658989905020580062203d1b5dd1ef4ff7b22359598ebdf58966a51adcc03e02ad356632743e658989906a06080012001a0072203d1b5dd1ef4ff7b22359598ebdf58966a51adcc03e02ad356632743e6589899012960208021064186422203d1b5dd1ef4ff7b22359598ebdf58966a51adcc03e02ad356632743e658989902a1440ff452fae2affe6eeef3c30e53e9eac35a1bc4332203d1b5dd1ef4ff7b22359598ebdf58966a51adcc03e02ad356632743e658989903a203d1b5dd1ef4ff7b22359598ebdf58966a51adcc03e02ad356632743e6589899042203d1b5dd1ef4ff7b22359598ebdf58966a51adcc03e02ad356632743e658989904a203d1b5dd1ef4ff7b22359598ebdf58966a51adcc03e02ad356632743e658989905032580062203d1b5dd1ef4ff7b22359598ebdf58966a51adcc03e02ad356632743e658989906a06080012001a0072203d1b5dd1ef4ff7b22359598ebdf58966a51adcc03e02ad356632743e658989903a40ca689d875c50c49ab0c2dbcb182bdc38a4f8b48358f1ec29e8dde673a3a9baa08e6c19f8b7dfe1f4df9b66f032bfbc2bc0c6adbd031f5ec146ff55a5aa64160f"`; diff --git a/framework-plugins/lisk-framework-report-misbehavior-plugin/test/unit/auth.spec.ts b/framework-plugins/lisk-framework-report-misbehavior-plugin/test/unit/auth.spec.ts index f9b93cba1b2..ac76bb5a1e6 100644 --- a/framework-plugins/lisk-framework-report-misbehavior-plugin/test/unit/auth.spec.ts +++ b/framework-plugins/lisk-framework-report-misbehavior-plugin/test/unit/auth.spec.ts @@ -24,6 +24,9 @@ const appConfigForPlugin: ApplicationConfigForPlugin = { fileLogLevel: 'none', logFileName: 'plugin-MisbehaviourPlugin.log', }, + system: { + keepEventsForHeights: -1, + }, rpc: { modes: ['ipc'], ws: { diff --git a/framework-plugins/lisk-framework-report-misbehavior-plugin/test/unit/send_pom_transaction.spec.ts b/framework-plugins/lisk-framework-report-misbehavior-plugin/test/unit/send_pom_transaction.spec.ts index 56ec623eabf..b57b3f3c6e5 100644 --- a/framework-plugins/lisk-framework-report-misbehavior-plugin/test/unit/send_pom_transaction.spec.ts +++ b/framework-plugins/lisk-framework-report-misbehavior-plugin/test/unit/send_pom_transaction.spec.ts @@ -26,6 +26,9 @@ const appConfigForPlugin: ApplicationConfigForPlugin = { fileLogLevel: 'none', logFileName: 'plugin-MisbehaviourPlugin.log', }, + system: { + keepEventsForHeights: -1, + }, rpc: { modes: ['ipc'], ws: { @@ -71,6 +74,11 @@ describe('Send PoM transaction', () => { let reportMisbehaviorPlugin: ReportMisbehaviorPlugin; const defaultNetworkIdentifier = '93d00fe5be70d90e7ae247936a2e7d83b50809c79b73fa14285f02c842348b3e'; + const random32Bytes = Buffer.from( + '3d1b5dd1ef4ff7b22359598ebdf58966a51adcc03e02ad356632743e65898990', + 'hex', + ); + const random20Bytes = Buffer.from('40ff452fae2affe6eeef3c30e53e9eac35a1bc43', 'hex'); const channelMock = { registerToBus: jest.fn(), once: jest.fn(), @@ -85,18 +93,31 @@ describe('Send PoM transaction', () => { moduleName: '', options: {}, } as any; - const header1 = chain.BlockHeader.fromBytes( - Buffer.from( - '080010fe86b28b06180022002a003220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8553a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8554220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855480050005a20f8da7f49e92286b0129fd75a9208eed942ef1d79df93c42c9b87e8b6bb9fc84f6206080012001a006a00', - 'hex', - ), - ); - const header2 = chain.BlockHeader.fromBytes( - Buffer.from( - '080010bb87b28b06180022002a003220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8553a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8554220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855480050005a20d800954794e0882c2419fe4736c2a191e6515859a7a894043ba5c911da6b72e76206080012001a006a00', - 'hex', - ), - ); + const header1 = new chain.BlockHeader({ + height: 100, + aggregateCommit: { + aggregationBits: Buffer.alloc(0), + certificateSignature: Buffer.alloc(0), + height: 0, + }, + generatorAddress: random20Bytes, + maxHeightGenerated: 0, + maxHeightPrevoted: 50, + previousBlockID: random32Bytes, + timestamp: 100, + version: 2, + assetsRoot: random32Bytes, + eventRoot: random32Bytes, + stateRoot: random32Bytes, + transactionRoot: random32Bytes, + validatorsHash: random32Bytes, + signature: random32Bytes, + }); + const header2 = new chain.BlockHeader({ + ...header1.toObject(), + height: 100, + maxHeightPrevoted: 32, + }); beforeEach(async () => { reportMisbehaviorPlugin = new ReportMisbehaviorPlugin(); diff --git a/framework-plugins/lisk-framework-report-misbehavior-plugin/test/unit/subscribe_event.spec.ts b/framework-plugins/lisk-framework-report-misbehavior-plugin/test/unit/subscribe_event.spec.ts index d2d7893746b..88c730b5ce0 100644 --- a/framework-plugins/lisk-framework-report-misbehavior-plugin/test/unit/subscribe_event.spec.ts +++ b/framework-plugins/lisk-framework-report-misbehavior-plugin/test/unit/subscribe_event.spec.ts @@ -24,6 +24,9 @@ const appConfigForPlugin: ApplicationConfigForPlugin = { fileLogLevel: 'none', logFileName: 'plugin-MisbehaviourPlugin.log', }, + system: { + keepEventsForHeights: -1, + }, rpc: { modes: ['ipc'], ws: { diff --git a/framework/src/node/consensus/consensus.ts b/framework/src/node/consensus/consensus.ts index d0d193b6716..820e77baa43 100644 --- a/framework/src/node/consensus/consensus.ts +++ b/framework/src/node/consensus/consensus.ts @@ -221,7 +221,7 @@ export class Consensus { ) { throw new Error('Genesis block validators hash is invalid'); } - await this._chain.saveBlock(args.genesisBlock, state, args.genesisBlock.header.height); + await this._chain.saveBlock(args.genesisBlock, [], state, args.genesisBlock.header.height); } await this._chain.loadLastBlocks(args.genesisBlock); this._genesisBlockTimestamp = args.genesisBlock.header.timestamp; @@ -541,7 +541,7 @@ export class Consensus { ); this._verifyStateRoot(block, currentState.smt.rootHash); - await this._chain.saveBlock(block, currentState, finalizedHeight, { + await this._chain.saveBlock(block, [], currentState, finalizedHeight, { removeFromTempTable: options.removeFromTempTable ?? false, }); diff --git a/framework/src/node/generator/generator.ts b/framework/src/node/generator/generator.ts index 1e2fcf36b10..9afc7736e63 100644 --- a/framework/src/node/generator/generator.ts +++ b/framework/src/node/generator/generator.ts @@ -340,6 +340,8 @@ export class Generator { const apiContext = createAPIContext({ stateStore, eventQueue }); const bftParams = await this._bftAPI.getBFTParameters(apiContext, height + 1); header.stateRoot = smt.rootHash; + // TODO: Add event root calculation + header.eventRoot = EMPTY_HASH; header.validatorsHash = bftParams.validatorsHash; return new Block(header, [], assets); @@ -628,6 +630,8 @@ export class Generator { blockHeader.assetsRoot = await blockAssets.getRoot(); // Assign root hash calculated in SMT to state root of block header blockHeader.stateRoot = smt.rootHash; + // TODO: Add event root calculation + blockHeader.eventRoot = EMPTY_HASH; // Set validatorsHash const { validatorsHash } = await this._bftAPI.getBFTParameters(apiContext, height + 1); blockHeader.validatorsHash = validatorsHash; diff --git a/framework/src/node/node.ts b/framework/src/node/node.ts index 67aa6ae3497..f29372d3389 100644 --- a/framework/src/node/node.ts +++ b/framework/src/node/node.ts @@ -80,6 +80,7 @@ export class Node { this._chain = new Chain({ maxTransactionsSize: this._options.genesis.maxTransactionsSize, + keepEventsForHeights: this._options.system.keepEventsForHeights, }); this._stateMachine = new StateMachine(); diff --git a/framework/src/node/state_machine/event_queue.ts b/framework/src/node/state_machine/event_queue.ts index 619f7cf1e25..83527dd692c 100644 --- a/framework/src/node/state_machine/event_queue.ts +++ b/framework/src/node/state_machine/event_queue.ts @@ -12,34 +12,68 @@ * Removal or modification of this copyright notice is prohibited. */ -interface Event { - key: string; - value: Buffer; +import { Event, EVENT_MAX_EVENT_SIZE_BYTES, EVENT_MAX_TOPICS_PER_EVENT } from '@liskhq/lisk-chain'; + +interface RevertibleEvent { + event: Event; + noRevert: boolean; } export class EventQueue { - private _events: Event[]; - private _originalEvents: Event[]; + private readonly _events: RevertibleEvent[]; + private _snapshotIndex = -1; public constructor() { this._events = []; - this._originalEvents = []; } - public add(key: string, value: Buffer): void { - this._events.push({ key, value }); + public add( + moduleID: number, + typeID: Buffer, + data: Buffer, + topics: Buffer[], + noRevert?: boolean, + ): void { + if (data.length > EVENT_MAX_EVENT_SIZE_BYTES) { + throw new Error( + `Max size of event data is ${EVENT_MAX_EVENT_SIZE_BYTES} but received ${data.length}`, + ); + } + if (!topics.length) { + throw new Error('Topics must have at least one element.'); + } + if (topics.length > EVENT_MAX_TOPICS_PER_EVENT) { + throw new Error( + `Max topics per event is ${EVENT_MAX_TOPICS_PER_EVENT} but received ${topics.length}`, + ); + } + // TODO: Remove once moduleID becomes bytes + const moduleIDBytes = Buffer.alloc(4); + moduleIDBytes.writeUInt32BE(moduleID, 0); + this._events.push({ + event: new Event({ + moduleID: moduleIDBytes, + typeID, + index: this._events.length, + data, + topics, + }), + noRevert: noRevert ?? false, + }); } public createSnapshot(): void { - this._originalEvents = this._events.map(eventData => ({ ...eventData })); + this._snapshotIndex = this._events.length; } public restoreSnapshot(): void { - this._events = this._originalEvents; - this._originalEvents = []; + const newEvents = this._events.splice(this._snapshotIndex); + const nonRevertableEvents = newEvents.filter(eventData => eventData.noRevert); + this._events.push(...nonRevertableEvents); + this._snapshotIndex = -1; } public getEvents(): Event[] { - return this._events; + return this._events.map(e => e.event); } } diff --git a/framework/src/schema/application_config_schema.ts b/framework/src/schema/application_config_schema.ts index 45b8dbae9bb..e52a419e1f6 100644 --- a/framework/src/schema/application_config_schema.ts +++ b/framework/src/schema/application_config_schema.ts @@ -89,6 +89,15 @@ export const applicationConfigSchema = { description: 'The root path for storing temporary pid and socket file and data. Restricted length due to unix domain socket path length limitations.', }, + system: { + type: 'object', + required: ['keepEventsForHeights'], + properties: { + keepEventsForHeights: { + type: 'integer', + }, + }, + }, logger: { type: 'object', required: ['fileLogLevel', 'logFileName', 'consoleLogLevel'], @@ -372,6 +381,9 @@ export const applicationConfigSchema = { version: '0.0.0', networkVersion: '1.1', rootPath: '~/.lisk', + system: { + keepEventsForHeights: 300, + }, logger: { fileLogLevel: 'info', consoleLogLevel: 'none', diff --git a/framework/src/testing/create_block.ts b/framework/src/testing/create_block.ts index e26c1ced5f6..891f3ec6e63 100644 --- a/framework/src/testing/create_block.ts +++ b/framework/src/testing/create_block.ts @@ -40,6 +40,7 @@ export const createBlockHeaderWithDefaults = (header?: Partial previousBlockID: header?.previousBlockID ?? hash(getRandomBytes(4)), transactionRoot: header?.transactionRoot ?? hash(getRandomBytes(4)), stateRoot: header?.stateRoot ?? hash(getRandomBytes(4)), + eventRoot: header?.eventRoot ?? hash(getRandomBytes(4)), generatorAddress: header?.generatorAddress ?? getRandomBytes(32), aggregateCommit: header?.aggregateCommit ?? { height: 0, @@ -76,6 +77,7 @@ export const createBlock = async ({ previousBlockID, timestamp, transactionRoot: header?.transactionRoot ?? txTree.root, + eventRoot: header?.eventRoot, stateRoot: header?.stateRoot, generatorAddress: getAddressFromPublicKey(publicKey), ...header, diff --git a/framework/src/testing/fixtures/config.ts b/framework/src/testing/fixtures/config.ts index 53e0c6ad28e..5c3663f7aa5 100644 --- a/framework/src/testing/fixtures/config.ts +++ b/framework/src/testing/fixtures/config.ts @@ -27,6 +27,9 @@ export const defaultConfig = { consoleLogLevel: 'none', logFileName: 'lisk.log', }, + system: { + keepEventsForHeights: -1, + }, genesis: { blockTime: 10, communityIdentifier: 'sdk', diff --git a/framework/src/types.ts b/framework/src/types.ts index 53e4a548159..b9ba4b0748b 100644 --- a/framework/src/types.ts +++ b/framework/src/types.ts @@ -83,6 +83,10 @@ export interface TransactionPoolConfig { readonly minReplacementFeeDifference?: string; } +export interface SystemConfig { + keepEventsForHeights: number; +} + type RecursivePartial = { [P in keyof T]?: RecursivePartial; }; @@ -128,6 +132,7 @@ export interface ApplicationConfig { genesis: GenesisConfig; generation: GenerationConfig; network: NetworkConfig; + system: SystemConfig; logger: { logFileName: string; fileLogLevel: string; diff --git a/framework/test/fixtures/blocks.ts b/framework/test/fixtures/blocks.ts index e3f3d590a33..ed3d4a269f5 100644 --- a/framework/test/fixtures/blocks.ts +++ b/framework/test/fixtures/blocks.ts @@ -35,6 +35,7 @@ export const genesisBlock = (): Block => { previousBlockID: getRandomBytes(32), timestamp: Math.floor(Date.now() / 1000 - 24 * 60 * 60), stateRoot: hash(Buffer.alloc(0)), + eventRoot: hash(Buffer.alloc(0)), maxHeightGenerated: 0, maxHeightPrevoted: 0, assetsRoot: hash(Buffer.alloc(0)), @@ -73,6 +74,7 @@ export const createFakeBlockHeader = (header?: Partial): Block }, validatorsHash: header?.validatorsHash ?? getRandomBytes(32), stateRoot: header?.stateRoot ?? hash(getRandomBytes(4)), + eventRoot: header?.eventRoot ?? hash(getRandomBytes(4)), generatorAddress: header?.generatorAddress ?? getRandomBytes(32), signature: header?.signature ?? getRandomBytes(64), }); @@ -101,6 +103,7 @@ export const createValidDefaultBlock = async ( timestamp: 1000, transactionRoot: txTree.root, stateRoot: getRandomBytes(32), + eventRoot: getRandomBytes(32), generatorAddress: getAddressFromPublicKey(keypair.publicKey), aggregateCommit: { height: 0, diff --git a/framework/test/fixtures/node.ts b/framework/test/fixtures/node.ts index f8bbb8e9a09..efac3d292c3 100644 --- a/framework/test/fixtures/node.ts +++ b/framework/test/fixtures/node.ts @@ -21,6 +21,9 @@ export const nodeOptions = ({ networkVersion: '1.0', rootPath: '~/.lisk', label: 'default', + system: { + keepEventsForHeights: 300, + }, network: { maxInboundConnections: 0, seedPeers: [{ ip: '127.0.0.1', port: 5000 }], diff --git a/framework/test/integration/node/genesis_block.spec.ts b/framework/test/integration/node/genesis_block.spec.ts index c77ab9e3468..09168721f4c 100644 --- a/framework/test/integration/node/genesis_block.spec.ts +++ b/framework/test/integration/node/genesis_block.spec.ts @@ -123,6 +123,7 @@ describe('genesis block', () => { const chain = new Chain({ maxTransactionsSize: 15 * 1024, + keepEventsForHeights: -1, }); const newConsensus = new Consensus({ bftAPI: consensus['_bftAPI'], diff --git a/framework/test/unit/__snapshots__/application.spec.ts.snap b/framework/test/unit/__snapshots__/application.spec.ts.snap index 95a05bafbc5..4ff470f3afe 100644 --- a/framework/test/unit/__snapshots__/application.spec.ts.snap +++ b/framework/test/unit/__snapshots__/application.spec.ts.snap @@ -2223,6 +2223,9 @@ Object { "port": 8080, }, }, + "system": Object { + "keepEventsForHeights": 300, + }, "transactionPool": Object { "maxTransactions": 4096, "maxTransactionsPerAccount": 64, diff --git a/framework/test/unit/node/consensus/synchronizer/block_synchronization_mechanism/block_synchronization_mechanism.spec.ts b/framework/test/unit/node/consensus/synchronizer/block_synchronization_mechanism/block_synchronization_mechanism.spec.ts index fb3002cd6c6..81b7973ed3d 100644 --- a/framework/test/unit/node/consensus/synchronizer/block_synchronization_mechanism/block_synchronization_mechanism.spec.ts +++ b/framework/test/unit/node/consensus/synchronizer/block_synchronization_mechanism/block_synchronization_mechanism.spec.ts @@ -80,6 +80,7 @@ describe('block_synchronization_mechanism', () => { chainModule = new Chain({ maxTransactionsSize: 15000, + keepEventsForHeights: -1, }); chainModule.init({ db: new InMemoryKVStore(), diff --git a/framework/test/unit/node/consensus/synchronizer/fast_chain_switching_mechanism/fast_chain_switching_mechanism.spec.ts b/framework/test/unit/node/consensus/synchronizer/fast_chain_switching_mechanism/fast_chain_switching_mechanism.spec.ts index 02b711e5bcd..743c7036945 100644 --- a/framework/test/unit/node/consensus/synchronizer/fast_chain_switching_mechanism/fast_chain_switching_mechanism.spec.ts +++ b/framework/test/unit/node/consensus/synchronizer/fast_chain_switching_mechanism/fast_chain_switching_mechanism.spec.ts @@ -61,6 +61,7 @@ describe('fast_chain_switching_mechanism', () => { chainModule = new Chain({ maxTransactionsSize: 15000, + keepEventsForHeights: -1, }); chainModule.init({ db: new InMemoryKVStore(), diff --git a/framework/test/unit/node/consensus/synchronizer/synchronizer.spec.ts b/framework/test/unit/node/consensus/synchronizer/synchronizer.spec.ts index 7b53d12da25..e0b59c10420 100644 --- a/framework/test/unit/node/consensus/synchronizer/synchronizer.spec.ts +++ b/framework/test/unit/node/consensus/synchronizer/synchronizer.spec.ts @@ -50,6 +50,7 @@ describe('Synchronizer', () => { chainModule = new Chain({ maxTransactionsSize: applicationConfigSchema.default.genesis.maxTransactionsSize, + keepEventsForHeights: applicationConfigSchema.default.system.keepEventsForHeights, }); chainModule.init({ db: new InMemoryKVStore(), diff --git a/framework/test/unit/node/state_machine/event_queue.spec.ts b/framework/test/unit/node/state_machine/event_queue.spec.ts index 5f1d725f333..2d2c802022f 100644 --- a/framework/test/unit/node/state_machine/event_queue.spec.ts +++ b/framework/test/unit/node/state_machine/event_queue.spec.ts @@ -12,15 +12,37 @@ * Removal or modification of this copyright notice is prohibited. */ +import { EVENT_MAX_EVENT_SIZE_BYTES } from '@liskhq/lisk-chain'; +import { getRandomBytes, intToBuffer } from '@liskhq/lisk-cryptography'; import { EventQueue } from '../../../../src/node/state_machine/event_queue'; describe('EventQueue', () => { // Arrange const events = [ - { key: 'moduleA:beforeX', value: Buffer.from('Module A Before X started', 'utf-8') }, - { key: 'moduleB:beforeX', value: Buffer.from('Module B Before X started', 'utf-8') }, - { key: 'moduleB:afterX', value: Buffer.from('Module B Before X end', 'utf-8') }, - { key: 'moduleA:afterX', value: Buffer.from('Module A Before X end', 'utf-8') }, + { + moduleID: 3, + typeID: Buffer.from([0, 0, 0, 0]), + data: getRandomBytes(20), + topics: [getRandomBytes(32), getRandomBytes(20)], + }, + { + moduleID: 4, + typeID: Buffer.from([0, 0, 0, 0]), + data: getRandomBytes(20), + topics: [getRandomBytes(32), getRandomBytes(20)], + }, + { + moduleID: 2, + typeID: Buffer.from([0, 0, 0, 0]), + data: getRandomBytes(20), + topics: [getRandomBytes(32)], + }, + { + moduleID: 1, + typeID: Buffer.from([0, 0, 0, 0]), + data: getRandomBytes(20), + topics: [getRandomBytes(32), getRandomBytes(20), getRandomBytes(20), getRandomBytes(20)], + }, ]; let eventQueue: EventQueue; @@ -28,66 +50,75 @@ describe('EventQueue', () => { eventQueue = new EventQueue(); }); - it('should expose interface for add, createSnapshot, restoreSnapshot, and getEvents', () => { - // Asset - expect(eventQueue.add).toBeFunction(); - expect(eventQueue.createSnapshot).toBeFunction(); - expect(eventQueue.restoreSnapshot).toBeFunction(); - return expect(eventQueue.getEvents).toBeFunction(); + it('should throw error if data size exceeds maximum allowed', () => { + expect(() => + eventQueue.add(2, Buffer.from([0, 0, 0, 1]), getRandomBytes(EVENT_MAX_EVENT_SIZE_BYTES + 1), [ + getRandomBytes(32), + ]), + ).toThrow('Max size of event data is'); + }); + + it('should throw error if topics is empty', () => { + expect(() => + eventQueue.add(2, Buffer.from([0, 0, 0, 1]), getRandomBytes(EVENT_MAX_EVENT_SIZE_BYTES), []), + ).toThrow('Topics must have at least one element'); + }); + + it('should throw error if topics length exceeds maxumum allowed', () => { + expect(() => + eventQueue.add( + 2, + Buffer.from([0, 0, 0, 1]), + getRandomBytes(EVENT_MAX_EVENT_SIZE_BYTES), + new Array(5).fill(0).map(() => getRandomBytes(32)), + ), + ).toThrow('Max topics per event is'); }); it('should be able to add events to queue', () => { // Act - events.map(e => eventQueue.add(e.key, e.value)); + events.map(e => eventQueue.add(e.moduleID, e.typeID, e.data, e.topics)); const addedEvents = eventQueue.getEvents(); // Asset + expect(addedEvents).toHaveLength(events.length); addedEvents.forEach((e, i) => { - expect(e.key).toEqual(events[i].key); - expect(e.value.equals(events[i].value)).toBeTrue(); + expect(e.toObject()).toEqual({ + ...events[i], + moduleID: intToBuffer(events[i].moduleID, 4), + index: i, + }); }); - return expect(addedEvents).toHaveLength(events.length); }); - it('should be able to createSnapshot for events', () => { - // Act - events.map(e => eventQueue.add(e.key, e.value)); - eventQueue.createSnapshot(); - const originalEvents = eventQueue['_originalEvents']; - - // Asset - originalEvents.forEach((e, i) => { - expect(e.key).toEqual(events[i].key); - expect(e.value.equals(events[i].value)).toBeTrue(); - }); - return expect(originalEvents).toHaveLength(events.length); - }); + it('should return original set of events when create and restore snapshot', () => { + events.map(e => eventQueue.add(e.moduleID, e.typeID, e.data, e.topics)); + expect(eventQueue.getEvents()).toHaveLength(events.length); - it('should be able to restoreSnapshot for events', () => { - // Act - events.map(e => eventQueue.add(e.key, e.value)); - const addedEvents = eventQueue.getEvents(); + eventQueue.createSnapshot(); + eventQueue.add(3, Buffer.from([0, 0, 0, 1]), getRandomBytes(100), [getRandomBytes(32)]); eventQueue.restoreSnapshot(); - // Asset - expect(eventQueue['_originalEvents']).toBeEmpty(); - addedEvents.forEach((e, i) => { - expect(e.key).toEqual(events[i].key); - expect(e.value.equals(events[i].value)).toBeTrue(); + expect(eventQueue.getEvents()).toHaveLength(events.length); + eventQueue.getEvents().forEach((e, i) => { + expect(e.toObject()).toEqual({ + ...events[i], + moduleID: intToBuffer(events[i].moduleID, 4), + index: i, + }); }); - return expect(addedEvents).toHaveLength(events.length); }); - it('should be able to getEvents added', () => { - // Act - events.map(e => eventQueue.add(e.key, e.value)); - const addedEvents = eventQueue.getEvents(); + it('should maintain new nonRevertible events when restoring the snapshot', () => { + events.map(e => eventQueue.add(e.moduleID, e.typeID, e.data, e.topics)); + expect(eventQueue.getEvents()).toHaveLength(events.length); - // Asset - addedEvents.forEach((e, i) => { - expect(e.key).toEqual(events[i].key); - expect(e.value.equals(events[i].value)).toBeTrue(); - }); - return expect(addedEvents).toHaveLength(events.length); + eventQueue.createSnapshot(); + eventQueue.add(3, Buffer.from([0, 0, 0, 1]), getRandomBytes(100), [getRandomBytes(32)], false); + eventQueue.add(3, Buffer.from([0, 0, 0, 1]), getRandomBytes(100), [getRandomBytes(32)], true); + eventQueue.add(3, Buffer.from([0, 0, 0, 1]), getRandomBytes(100), [getRandomBytes(32)], false); + eventQueue.restoreSnapshot(); + + expect(eventQueue.getEvents()).toHaveLength(events.length + 1); }); }); diff --git a/framework/test/unit/plugins/base_plugin.spec.ts b/framework/test/unit/plugins/base_plugin.spec.ts index e59664a077f..3b75b87256d 100644 --- a/framework/test/unit/plugins/base_plugin.spec.ts +++ b/framework/test/unit/plugins/base_plugin.spec.ts @@ -24,6 +24,9 @@ const appConfigForPlugin = { rootPath: '/my/path', label: 'my-app', logger: { consoleLogLevel: 'debug', fileLogLevel: '123', logFileName: 'plugin1.log' }, + system: { + keepEventsForHeights: -1, + }, rpc: { modes: ['ipc'], ws: { diff --git a/framework/test/unit/schema/__snapshots__/application_config_schema.spec.ts.snap b/framework/test/unit/schema/__snapshots__/application_config_schema.spec.ts.snap index d7bcfe7805e..5089703f802 100644 --- a/framework/test/unit/schema/__snapshots__/application_config_schema.spec.ts.snap +++ b/framework/test/unit/schema/__snapshots__/application_config_schema.spec.ts.snap @@ -46,6 +46,9 @@ Object { "port": 8080, }, }, + "system": Object { + "keepEventsForHeights": 300, + }, "transactionPool": Object { "maxTransactions": 4096, "maxTransactionsPerAccount": 64, @@ -436,6 +439,17 @@ Object { }, "type": "object", }, + "system": Object { + "properties": Object { + "keepEventsForHeights": Object { + "type": "integer", + }, + }, + "required": Array [ + "keepEventsForHeights", + ], + "type": "object", + }, "transactionPool": Object { "properties": Object { "maxTransactions": Object {