From adadfa5be51a33d6b089c59d3bad5d46073f8bc3 Mon Sep 17 00:00:00 2001 From: PhilWindle <60546371+PhilWindle@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:05:54 +0000 Subject: [PATCH] feat: DB Metrics now use labels for easier querying (#10572) This PR changes the way we generate DB metrics such that tree and db types are now labels to allow for easier querying --- .../archiver/src/archiver/instrumentation.ts | 6 +- yarn-project/kv-store/src/lmdb/store.ts | 18 +-- .../p2p/src/mem_pools/instrumentation.ts | 6 +- .../proving_broker_database/persisted.ts | 9 +- .../telemetry-client/src/attributes.ts | 6 + .../telemetry-client/src/lmdb_metrics.ts | 23 ++- yarn-project/telemetry-client/src/metrics.ts | 120 ++------------- yarn-project/telemetry-client/src/otel.ts | 16 +- .../src/synchronizer/instrumentation.ts | 138 +++++++----------- 9 files changed, 117 insertions(+), 225 deletions(-) diff --git a/yarn-project/archiver/src/archiver/instrumentation.ts b/yarn-project/archiver/src/archiver/instrumentation.ts index a3096ee0088..11cc378fdcf 100644 --- a/yarn-project/archiver/src/archiver/instrumentation.ts +++ b/yarn-project/archiver/src/archiver/instrumentation.ts @@ -62,17 +62,17 @@ export class ArchiverInstrumentation { this.dbMetrics = new LmdbMetrics( meter, { - name: Metrics.ARCHIVER_DB_MAP_SIZE, description: 'Database map size for the archiver', }, { - name: Metrics.ARCHIVER_DB_USED_SIZE, description: 'Database used size for the archiver', }, { - name: Metrics.ARCHIVER_DB_NUM_ITEMS, description: 'Num items in the archiver database', }, + { + [Attributes.DB_DATA_TYPE]: 'archiver', + }, lmdbStats, ); } diff --git a/yarn-project/kv-store/src/lmdb/store.ts b/yarn-project/kv-store/src/lmdb/store.ts index 59cd59f5ad7..ad66ec76691 100644 --- a/yarn-project/kv-store/src/lmdb/store.ts +++ b/yarn-project/kv-store/src/lmdb/store.ts @@ -212,11 +212,7 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore { private estimateSubDBSize(db: Database): { actualSize: number; numItems: number } { const stats = db.getStats(); - let branchPages = 0; - let leafPages = 0; - let overflowPages = 0; - let pageSize = 0; - let totalSize = 0; + let actualSize = 0; let numItems = 0; // This is the total number of key/value pairs present in the DB if ('entryCount' in stats && typeof stats.entryCount === 'number') { @@ -233,12 +229,12 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore { 'pageSize' in stats && typeof stats.pageSize === 'number' ) { - branchPages = stats.treeBranchPageCount; - leafPages = stats.treeLeafPageCount; - overflowPages = stats.overflowPages; - pageSize = stats.pageSize; - totalSize = (branchPages + leafPages + overflowPages) * pageSize; + const branchPages = stats.treeBranchPageCount; + const leafPages = stats.treeLeafPageCount; + const overflowPages = stats.overflowPages; + const pageSize = stats.pageSize; + actualSize = (branchPages + leafPages + overflowPages) * pageSize; } - return { actualSize: totalSize, numItems }; + return { actualSize, numItems }; } } diff --git a/yarn-project/p2p/src/mem_pools/instrumentation.ts b/yarn-project/p2p/src/mem_pools/instrumentation.ts index d80b2f69d55..5597bf6393e 100644 --- a/yarn-project/p2p/src/mem_pools/instrumentation.ts +++ b/yarn-project/p2p/src/mem_pools/instrumentation.ts @@ -88,17 +88,17 @@ export class PoolInstrumentation { this.dbMetrics = new LmdbMetrics( meter, { - name: Metrics.MEMPOOL_DB_MAP_SIZE, description: 'Database map size for the Tx mempool', }, { - name: Metrics.MEMPOOL_DB_USED_SIZE, description: 'Database used size for the Tx mempool', }, { - name: Metrics.MEMPOOL_DB_NUM_ITEMS, description: 'Num items in database for the Tx mempool', }, + { + [Attributes.DB_DATA_TYPE]: 'tx-pool', + }, dbStats, ); } diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker_database/persisted.ts b/yarn-project/prover-client/src/proving_broker/proving_broker_database/persisted.ts index 61ca5232015..b6c314cc106 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker_database/persisted.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker_database/persisted.ts @@ -1,7 +1,7 @@ import { type ProofUri, ProvingJob, type ProvingJobId, ProvingJobSettledResult } from '@aztec/circuit-types'; import { jsonParseWithSchema, jsonStringify } from '@aztec/foundation/json-rpc'; import { type AztecKVStore, type AztecMap } from '@aztec/kv-store'; -import { LmdbMetrics, Metrics, type TelemetryClient } from '@aztec/telemetry-client'; +import { Attributes, LmdbMetrics, type TelemetryClient } from '@aztec/telemetry-client'; import { type ProvingBrokerDatabase } from '../proving_broker_database.js'; @@ -14,14 +14,15 @@ export class KVBrokerDatabase implements ProvingBrokerDatabase { this.metrics = new LmdbMetrics( client.getMeter('KVBrokerDatabase'), { - name: Metrics.PROVING_QUEUE_DB_MAP_SIZE, description: 'Database map size for the proving broker', }, { - name: Metrics.PROVING_QUEUE_DB_USED_SIZE, description: 'Database used size for the proving broker', }, - { name: Metrics.PROVING_QUEUE_DB_NUM_ITEMS, description: 'Number of items in the broker database' }, + { description: 'Number of items in the broker database' }, + { + [Attributes.DB_DATA_TYPE]: 'prover-broker', + }, () => store.estimateSize(), ); this.jobs = store.openMap('proving_jobs'); diff --git a/yarn-project/telemetry-client/src/attributes.ts b/yarn-project/telemetry-client/src/attributes.ts index 87c4be24ce0..1fcfdbcf926 100644 --- a/yarn-project/telemetry-client/src/attributes.ts +++ b/yarn-project/telemetry-client/src/attributes.ts @@ -88,3 +88,9 @@ export const SIMULATOR_PHASE = 'aztec.simulator.phase'; export const TARGET_ADDRESS = 'aztec.address.target'; export const SENDER_ADDRESS = 'aztec.address.sender'; export const MANA_USED = 'aztec.mana.used'; + +/** Identifier for the tables in a world state DB */ +export const WS_DB_DATA_TYPE = 'aztec.world_state.db_type'; + +/** Identifier for component database (e.g. archiver, tx pool) */ +export const DB_DATA_TYPE = 'aztec.db_type'; diff --git a/yarn-project/telemetry-client/src/lmdb_metrics.ts b/yarn-project/telemetry-client/src/lmdb_metrics.ts index a8e70662d65..1b1729295aa 100644 --- a/yarn-project/telemetry-client/src/lmdb_metrics.ts +++ b/yarn-project/telemetry-client/src/lmdb_metrics.ts @@ -1,7 +1,13 @@ -import { type BatchObservableResult, type Meter, type Metrics, type ObservableGauge, ValueType } from './telemetry.js'; +import { DB_MAP_SIZE, DB_NUM_ITEMS, DB_USED_SIZE } from './metrics.js'; +import { + type Attributes, + type BatchObservableResult, + type Meter, + type ObservableGauge, + ValueType, +} from './telemetry.js'; export type LmdbMetricDescriptor = { - name: Metrics; description: string; }; @@ -17,17 +23,18 @@ export class LmdbMetrics { dbMapSizeDescriptor: LmdbMetricDescriptor, dbUsedSizeDescriptor: LmdbMetricDescriptor, dbNumItemsDescriptor: LmdbMetricDescriptor, + private attributes?: Attributes, private getStats?: LmdbStatsCallback, ) { - this.dbMapSize = meter.createObservableGauge(dbMapSizeDescriptor.name, { + this.dbMapSize = meter.createObservableGauge(DB_MAP_SIZE, { description: dbMapSizeDescriptor.description, valueType: ValueType.INT, }); - this.dbUsedSize = meter.createObservableGauge(dbUsedSizeDescriptor.name, { + this.dbUsedSize = meter.createObservableGauge(DB_USED_SIZE, { description: dbUsedSizeDescriptor.description, valueType: ValueType.INT, }); - this.dbNumItems = meter.createObservableGauge(dbNumItemsDescriptor.name, { + this.dbNumItems = meter.createObservableGauge(DB_NUM_ITEMS, { description: dbNumItemsDescriptor.description, valueType: ValueType.INT, }); @@ -40,8 +47,8 @@ export class LmdbMetrics { return; } const metrics = this.getStats(); - observable.observe(this.dbMapSize, metrics.mappingSize); - observable.observe(this.dbNumItems, metrics.numItems); - observable.observe(this.dbUsedSize, metrics.actualSize); + observable.observe(this.dbMapSize, metrics.mappingSize, this.attributes); + observable.observe(this.dbNumItems, metrics.numItems, this.attributes); + observable.observe(this.dbUsedSize, metrics.actualSize, this.attributes); }; } diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index d737e6dd863..22b83427ec5 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -25,9 +25,9 @@ export const CIRCUIT_SIZE = 'aztec.circuit.size'; export const MEMPOOL_TX_COUNT = 'aztec.mempool.tx_count'; export const MEMPOOL_TX_SIZE = 'aztec.mempool.tx_size'; -export const MEMPOOL_DB_NUM_ITEMS = 'aztec.mempool.db.num_items'; -export const MEMPOOL_DB_MAP_SIZE = 'aztec.mempool.db.map_size'; -export const MEMPOOL_DB_USED_SIZE = 'aztec.mempool.db.used_size'; +export const DB_NUM_ITEMS = 'aztec.db.num_items'; +export const DB_MAP_SIZE = 'aztec.db.map_size'; +export const DB_USED_SIZE = 'aztec.db.used_size'; export const MEMPOOL_ATTESTATIONS_COUNT = 'aztec.mempool.attestations_count'; export const MEMPOOL_ATTESTATIONS_SIZE = 'aztec.mempool.attestations_size'; @@ -40,9 +40,6 @@ export const ARCHIVER_BLOCK_HEIGHT = 'aztec.archiver.block_height'; export const ARCHIVER_BLOCK_SIZE = 'aztec.archiver.block_size'; export const ARCHIVER_ROLLUP_PROOF_DELAY = 'aztec.archiver.rollup_proof_delay'; export const ARCHIVER_ROLLUP_PROOF_COUNT = 'aztec.archiver.rollup_proof_count'; -export const ARCHIVER_DB_NUM_ITEMS = 'aztec.archiver.db.num_items'; -export const ARCHIVER_DB_MAP_SIZE = 'aztec.archiver.db.map_size'; -export const ARCHIVER_DB_USED_SIZE = 'aztec.archiver.db.used_size'; export const NODE_RECEIVE_TX_DURATION = 'aztec.node.receive_tx.duration'; export const NODE_RECEIVE_TX_COUNT = 'aztec.node.receive_tx.count'; @@ -98,110 +95,13 @@ export const WORLD_STATE_FORK_DURATION = 'aztec.world_state.fork.duration'; export const WORLD_STATE_SYNC_DURATION = 'aztec.world_state.sync.duration'; export const WORLD_STATE_MERKLE_TREE_SIZE = 'aztec.world_state.merkle_tree_size'; export const WORLD_STATE_DB_SIZE = 'aztec.world_state.db_size'; - -export const WORLD_STATE_DB_MAP_SIZE_NULLIFIER = 'aztec.world_state.db_map_size.nullifier'; -export const WORLD_STATE_DB_MAP_SIZE_PUBLIC_DATA = 'aztec.world_state.db_map_size.public_data'; -export const WORLD_STATE_DB_MAP_SIZE_ARCHIVE = 'aztec.world_state.db_map_size.archive'; -export const WORLD_STATE_DB_MAP_SIZE_MESSAGE = 'aztec.world_state.db_map_size.message'; -export const WORLD_STATE_DB_MAP_SIZE_NOTE_HASH = 'aztec.world_state.db_map_size.note_hash'; - -export const WORLD_STATE_TREE_SIZE_NULLIFIER = 'aztec.world_state.tree_size.nullifier'; -export const WORLD_STATE_TREE_SIZE_PUBLIC_DATA = 'aztec.world_state.tree_size.public_data'; -export const WORLD_STATE_TREE_SIZE_ARCHIVE = 'aztec.world_state.tree_size.archive'; -export const WORLD_STATE_TREE_SIZE_MESSAGE = 'aztec.world_state.tree_size.message'; -export const WORLD_STATE_TREE_SIZE_NOTE_HASH = 'aztec.world_state.tree_size.note_hash'; - -export const WORLD_STATE_UNFINALISED_HEIGHT_NULLIFIER = 'aztec.world_state.unfinalised_height.nullifier'; -export const WORLD_STATE_UNFINALISED_HEIGHT_PUBLIC_DATA = 'aztec.world_state.unfinalised_height.public_data'; -export const WORLD_STATE_UNFINALISED_HEIGHT_ARCHIVE = 'aztec.world_state.unfinalised_height.archive'; -export const WORLD_STATE_UNFINALISED_HEIGHT_MESSAGE = 'aztec.world_state.unfinalised_height.message'; -export const WORLD_STATE_UNFINALISED_HEIGHT_NOTE_HASH = 'aztec.world_state.unfinalised_height.note_hash'; - -export const WORLD_STATE_FINALISED_HEIGHT_NULLIFIER = 'aztec.world_state.finalised_height.nullifier'; -export const WORLD_STATE_FINALISED_HEIGHT_PUBLIC_DATA = 'aztec.world_state.finalised_height.public_data'; -export const WORLD_STATE_FINALISED_HEIGHT_ARCHIVE = 'aztec.world_state.finalised_height.archive'; -export const WORLD_STATE_FINALISED_HEIGHT_MESSAGE = 'aztec.world_state.finalised_height.message'; -export const WORLD_STATE_FINALISED_HEIGHT_NOTE_HASH = 'aztec.world_state.finalised_height.note_hash'; - -export const WORLD_STATE_OLDEST_BLOCK_NULLIFIER = 'aztec.world_state.oldest_block.nullifier'; -export const WORLD_STATE_OLDEST_BLOCK_PUBLIC_DATA = 'aztec.world_state.oldest_block.public_data'; -export const WORLD_STATE_OLDEST_BLOCK_ARCHIVE = 'aztec.world_state.oldest_block.archive'; -export const WORLD_STATE_OLDEST_BLOCK_MESSAGE = 'aztec.world_state.oldest_block.message'; -export const WORLD_STATE_OLDEST_BLOCK_NOTE_HASH = 'aztec.world_state.oldest_block.note_hash'; - -export const WORLD_STATE_BLOCKS_DB_USED_SIZE_NULLIFIER = 'aztec.world_state.db_used_size.blocks.nullifier'; -export const WORLD_STATE_BLOCKS_DB_USED_SIZE_PUBLIC_DATA = 'aztec.world_state.db_used_size.blocks.public_data'; -export const WORLD_STATE_BLOCKS_DB_USED_SIZE_ARCHIVE = 'aztec.world_state.db_used_size.blocks.archive'; -export const WORLD_STATE_BLOCKS_DB_USED_SIZE_MESSAGE = 'aztec.world_state.db_used_size.blocks.message'; -export const WORLD_STATE_BLOCKS_DB_USED_SIZE_NOTE_HASH = 'aztec.world_state.db_used_size.blocks.note_hash'; - -export const WORLD_STATE_BLOCKS_DB_NUM_ITEMS_NULLIFIER = 'aztec.world_state.db_num_items.blocks.nullifier'; -export const WORLD_STATE_BLOCKS_DB_NUM_ITEMS_PUBLIC_DATA = 'aztec.world_state.db_num_items.blocks.public_data'; -export const WORLD_STATE_BLOCKS_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state.db_num_items.blocks.archive'; -export const WORLD_STATE_BLOCKS_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.blocks.message'; -export const WORLD_STATE_BLOCKS_DB_NUM_ITEMS_NOTE_HASH = 'aztec.world_state.db_num_items.blocks.note_hash'; - -export const WORLD_STATE_NODES_DB_USED_SIZE_NULLIFIER = 'aztec.world_state.db_used_size.nodes.nullifier'; -export const WORLD_STATE_NODES_DB_USED_SIZE_PUBLIC_DATA = 'aztec.world_state.db_used_size.nodes.public_data'; -export const WORLD_STATE_NODES_DB_USED_SIZE_ARCHIVE = 'aztec.world_state.db_used_size.nodes.archive'; -export const WORLD_STATE_NODES_DB_USED_SIZE_MESSAGE = 'aztec.world_state.db_used_size.nodes.message'; -export const WORLD_STATE_NODES_DB_USED_SIZE_NOTE_HASH = 'aztec.world_state.db_used_size.nodes.note_hash'; - -export const WORLD_STATE_NODES_DB_NUM_ITEMS_NULLIFIER = 'aztec.world_state.db_num_items.nodes.nullifier'; -export const WORLD_STATE_NODES_DB_NUM_ITEMS_PUBLIC_DATA = 'aztec.world_state.db_num_items.nodes.public_data'; -export const WORLD_STATE_NODES_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state.db_num_items.nodes.archive'; -export const WORLD_STATE_NODES_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.nodes.message'; -export const WORLD_STATE_NODES_DB_NUM_ITEMS_NOTE_HASH = 'aztec.world_state.db_num_items.nodes.note_hash'; - -export const WORLD_STATE_LEAF_PREIMAGE_DB_USED_SIZE_NULLIFIER = - 'aztec.world_state.db_used_size.leaf_preimage.nullifier'; -export const WORLD_STATE_LEAF_PREIMAGE_DB_USED_SIZE_PUBLIC_DATA = - 'aztec.world_state.db_used_size.leaf_preimage.public_data'; -export const WORLD_STATE_LEAF_PREIMAGE_DB_USED_SIZE_ARCHIVE = 'aztec.world_state.db_used_size.leaf_preimage.archive'; -export const WORLD_STATE_LEAF_PREIMAGE_DB_USED_SIZE_MESSAGE = 'aztec.world_state.db_used_size.leaf_preimage.message'; -export const WORLD_STATE_LEAF_PREIMAGE_DB_USED_SIZE_NOTE_HASH = - 'aztec.world_state.db_used_size.leaf_preimage.note_hash'; - -export const WORLD_STATE_LEAF_PREIMAGE_DB_NUM_ITEMS_NULLIFIER = - 'aztec.world_state.db_num_items.leaf_preimage.nullifier'; -export const WORLD_STATE_LEAF_PREIMAGE_DB_NUM_ITEMS_PUBLIC_DATA = - 'aztec.world_state.db_num_items.leaf_preimage.public_data'; -export const WORLD_STATE_LEAF_PREIMAGE_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state.db_num_items.leaf_preimage.archive'; -export const WORLD_STATE_LEAF_PREIMAGE_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.leaf_preimage.message'; -export const WORLD_STATE_LEAF_PREIMAGE_DB_NUM_ITEMS_NOTE_HASH = - 'aztec.world_state.db_num_items.leaf_preimage.note_hash'; - -export const WORLD_STATE_LEAF_INDICES_DB_USED_SIZE_NULLIFIER = 'aztec.world_state.db_used_size.leaf_indices.nullifier'; -export const WORLD_STATE_LEAF_INDICES_DB_USED_SIZE_PUBLIC_DATA = - 'aztec.world_state.db_used_size.leaf_indices.public_data'; -export const WORLD_STATE_LEAF_INDICES_DB_USED_SIZE_ARCHIVE = 'aztec.world_state.db_used_size.leaf_indices.archive'; -export const WORLD_STATE_LEAF_INDICES_DB_USED_SIZE_MESSAGE = 'aztec.world_state.db_used_size.leaf_indices.message'; -export const WORLD_STATE_LEAF_INDICES_DB_USED_SIZE_NOTE_HASH = 'aztec.world_state.db_used_size.leaf_indices.note_hash'; - -export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_NULLIFIER = 'aztec.world_state.db_num_items.leaf_indices.nullifier'; -export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_PUBLIC_DATA = - 'aztec.world_state.db_num_items.leaf_indices.public_data'; -export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state.db_num_items.leaf_indices.archive'; -export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.leaf_indices.message'; -export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_NOTE_HASH = 'aztec.world_state.db_num_items.leaf_indices.note_hash'; - -export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_NULLIFIER = - 'aztec.world_state.db_used_size.block_indices.nullifier'; -export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_PUBLIC_DATA = - 'aztec.world_state.db_used_size.block_indices.public_data'; -export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_ARCHIVE = 'aztec.world_state.db_used_size.block_indices.archive'; -export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_MESSAGE = 'aztec.world_state.db_used_size.block_indices.message'; -export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_NOTE_HASH = - 'aztec.world_state.db_used_size.block_indices.note_hash'; - -export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_NULLIFIER = - 'aztec.world_state.db_num_items.block_indices.nullifier'; -export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_PUBLIC_DATA = - 'aztec.world_state.db_num_items.block_indices.public_data'; -export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state.db_num_items.block_indices.archive'; -export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.block_indices.message'; -export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_NOTE_HASH = - 'aztec.world_state.db_num_items.block_indices.note_hash'; +export const WORLD_STATE_DB_MAP_SIZE = 'aztec.world_state.db_map_size'; +export const WORLD_STATE_TREE_SIZE = 'aztec.world_state.tree_size'; +export const WORLD_STATE_UNFINALISED_HEIGHT = 'aztec.world_state.unfinalised_height'; +export const WORLD_STATE_FINALISED_HEIGHT = 'aztec.world_state.finalised_height'; +export const WORLD_STATE_OLDEST_BLOCK = 'aztec.world_state.oldest_block'; +export const WORLD_STATE_DB_USED_SIZE = 'aztec.world_state.db_used_size'; +export const WORLD_STATE_DB_NUM_ITEMS = 'aztec.world_state.db_num_items'; export const PROOF_VERIFIER_COUNT = 'aztec.proof_verifier.count'; diff --git a/yarn-project/telemetry-client/src/otel.ts b/yarn-project/telemetry-client/src/otel.ts index c09895697b7..e5b8be2f224 100644 --- a/yarn-project/telemetry-client/src/otel.ts +++ b/yarn-project/telemetry-client/src/otel.ts @@ -28,6 +28,8 @@ import { type Gauge, type TelemetryClient } from './telemetry.js'; export class OpenTelemetryClient implements TelemetryClient { hostMetrics: HostMetrics | undefined; targetInfo: Gauge | undefined; + private meters: Map = new Map(); + private tracers: Map = new Map(); protected constructor( private resource: IResource, @@ -38,11 +40,21 @@ export class OpenTelemetryClient implements TelemetryClient { ) {} getMeter(name: string): Meter { - return this.meterProvider.getMeter(name, this.resource.attributes[ATTR_SERVICE_VERSION] as string); + let meter = this.meters.get(name); + if (!meter) { + meter = this.meterProvider.getMeter(name, this.resource.attributes[ATTR_SERVICE_VERSION] as string); + this.meters.set(name, meter); + } + return meter; } getTracer(name: string): Tracer { - return this.traceProvider.getTracer(name, this.resource.attributes[ATTR_SERVICE_VERSION] as string); + let tracer = this.tracers.get(name); + if (!tracer) { + tracer = this.traceProvider.getTracer(name, this.resource.attributes[ATTR_SERVICE_VERSION] as string); + this.tracers.set(name, tracer); + } + return tracer; } public start() { diff --git a/yarn-project/world-state/src/synchronizer/instrumentation.ts b/yarn-project/world-state/src/synchronizer/instrumentation.ts index b4c6ad750f0..93fc76823ce 100644 --- a/yarn-project/world-state/src/synchronizer/instrumentation.ts +++ b/yarn-project/world-state/src/synchronizer/instrumentation.ts @@ -1,121 +1,91 @@ import { MerkleTreeId } from '@aztec/circuit-types'; -import { type Logger, createLogger } from '@aztec/foundation/log'; -import { type Gauge, type Meter, type TelemetryClient, ValueType } from '@aztec/telemetry-client'; +import { createLogger } from '@aztec/foundation/log'; +import { Attributes, type Gauge, type TelemetryClient, ValueType } from '@aztec/telemetry-client'; import { type DBStats, type TreeDBStats, type TreeMeta, type WorldStateStatusFull } from '../native/message.js'; -type TreeTypeString = 'nullifier' | 'note_hash' | 'archive' | 'message' | 'public_data'; type DBTypeString = 'leaf_preimage' | 'leaf_indices' | 'nodes' | 'blocks' | 'block_indices'; -class TreeDBInstrumentation { +export class WorldStateInstrumentation { + private dbMapSize: Gauge; + private treeSize: Gauge; + private unfinalisedHeight: Gauge; + private finalisedHeight: Gauge; + private oldestBlock: Gauge; private dbNumItems: Gauge; private dbUsedSize: Gauge; - constructor(meter: Meter, treeName: TreeTypeString, dbName: DBTypeString) { - this.dbUsedSize = meter.createGauge(`aztec.world_state.db_used_size.${dbName}.${treeName}`, { - description: `The current used database size for the ${treeName} tree ${dbName} database`, + constructor(telemetry: TelemetryClient, private log = createLogger('world-state:instrumentation')) { + const meter = telemetry.getMeter('World State'); + this.dbMapSize = meter.createGauge(`aztec.world_state.db_map_size`, { + description: `The current configured map size for each merkle tree`, valueType: ValueType.INT, }); - this.dbNumItems = meter.createGauge(`aztec.world_state.db_num_items.${dbName}.${treeName}`, { - description: `The current number of items in the ${treeName} tree ${dbName} database`, + this.treeSize = meter.createGauge(`aztec.world_state.tree_size`, { + description: `The current number of leaves in each merkle tree`, valueType: ValueType.INT, }); - } - - public updateMetrics(treeDbStats: DBStats) { - this.dbNumItems.record(Number(treeDbStats.numDataItems)); - this.dbUsedSize.record(Number(treeDbStats.totalUsedSize)); - } -} - -class TreeInstrumentation { - private treeDbInstrumentation: Map = new Map< - DBTypeString, - TreeDBInstrumentation - >(); - private dbMapSize: Gauge; - private treeSize: Gauge; - private unfinalisedHeight: Gauge; - private finalisedHeight: Gauge; - private oldestBlock: Gauge; - constructor(meter: Meter, treeName: TreeTypeString, private log: Logger) { - this.dbMapSize = meter.createGauge(`aztec.world_state.db_map_size.${treeName}`, { - description: `The current configured map size for the ${treeName} tree`, + this.unfinalisedHeight = meter.createGauge(`aztec.world_state.unfinalised_height`, { + description: `The unfinalised block height of each merkle tree`, valueType: ValueType.INT, }); - this.treeSize = meter.createGauge(`aztec.world_state.tree_size.${treeName}`, { - description: `The current number of leaves in the ${treeName} tree`, + this.finalisedHeight = meter.createGauge(`aztec.world_state.finalised_height`, { + description: `The finalised block height of each merkle tree`, valueType: ValueType.INT, }); - this.unfinalisedHeight = meter.createGauge(`aztec.world_state.unfinalised_height.${treeName}`, { - description: `The unfinalised block height of the ${treeName} tree`, + this.oldestBlock = meter.createGauge(`aztec.world_state.oldest_block`, { + description: `The oldest historical block of each merkle tree`, valueType: ValueType.INT, }); - this.finalisedHeight = meter.createGauge(`aztec.world_state.finalised_height.${treeName}`, { - description: `The finalised block height of the ${treeName} tree`, + this.dbUsedSize = meter.createGauge(`aztec.world_state.db_used_size`, { + description: `The current used database size for each db of each merkle tree`, valueType: ValueType.INT, }); - this.oldestBlock = meter.createGauge(`aztec.world_state.oldest_block.${treeName}`, { - description: `The oldest historical block of the ${treeName} tree`, + this.dbNumItems = meter.createGauge(`aztec.world_state.db_num_items`, { + description: `The current number of items in each database of each merkle tree`, valueType: ValueType.INT, }); - - this.treeDbInstrumentation.set('blocks', new TreeDBInstrumentation(meter, treeName, 'blocks')); - this.treeDbInstrumentation.set('nodes', new TreeDBInstrumentation(meter, treeName, 'nodes')); - this.treeDbInstrumentation.set('leaf_preimage', new TreeDBInstrumentation(meter, treeName, 'leaf_preimage')); - this.treeDbInstrumentation.set('leaf_indices', new TreeDBInstrumentation(meter, treeName, 'leaf_indices')); - this.treeDbInstrumentation.set('block_indices', new TreeDBInstrumentation(meter, treeName, 'block_indices')); - } - - private updateDBMetrics(dbName: DBTypeString, dbStats: DBStats) { - const inst = this.treeDbInstrumentation.get(dbName); - if (!inst) { - this.log.error(`Failed to find instrumentation for ${dbName}`); - return; - } - inst.updateMetrics(dbStats); } - public updateMetrics(treeDbStats: TreeDBStats, treeMeta: TreeMeta) { - this.dbMapSize.record(Number(treeDbStats.mapSize)); - this.treeSize.record(Number(treeMeta.committedSize)); - this.finalisedHeight.record(Number(treeMeta.finalisedBlockHeight)); - this.unfinalisedHeight.record(Number(treeMeta.unfinalisedBlockHeight)); - this.oldestBlock.record(Number(treeMeta.oldestHistoricBlock)); - - this.updateDBMetrics('leaf_indices', treeDbStats.leafIndicesDBStats); - this.updateDBMetrics('leaf_preimage', treeDbStats.leafPreimagesDBStats); - this.updateDBMetrics('blocks', treeDbStats.blocksDBStats); - this.updateDBMetrics('nodes', treeDbStats.nodesDBStats); - this.updateDBMetrics('block_indices', treeDbStats.blockIndicesDBStats); - } -} - -export class WorldStateInstrumentation { - private treeInstrumentation: Map = new Map(); + private updateTreeStats(treeDbStats: TreeDBStats, treeMeta: TreeMeta, tree: MerkleTreeId) { + this.dbMapSize.record(Number(treeDbStats.mapSize), { + [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree], + }); + this.treeSize.record(Number(treeMeta.size), { + [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree], + }); + this.unfinalisedHeight.record(Number(treeMeta.unfinalisedBlockHeight), { + [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree], + }); + this.finalisedHeight.record(Number(treeMeta.finalisedBlockHeight), { + [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree], + }); + this.oldestBlock.record(Number(treeMeta.oldestHistoricBlock), { + [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree], + }); - constructor(telemetry: TelemetryClient, private log = createLogger('world-state:instrumentation')) { - const meter = telemetry.getMeter('World State'); - this.treeInstrumentation.set(MerkleTreeId.ARCHIVE, new TreeInstrumentation(meter, 'archive', log)); - this.treeInstrumentation.set(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, new TreeInstrumentation(meter, 'message', log)); - this.treeInstrumentation.set(MerkleTreeId.NOTE_HASH_TREE, new TreeInstrumentation(meter, 'note_hash', log)); - this.treeInstrumentation.set(MerkleTreeId.NULLIFIER_TREE, new TreeInstrumentation(meter, 'nullifier', log)); - this.treeInstrumentation.set(MerkleTreeId.PUBLIC_DATA_TREE, new TreeInstrumentation(meter, 'public_data', log)); + this.updateTreeDBStats(treeDbStats.blockIndicesDBStats, 'block_indices', tree); + this.updateTreeDBStats(treeDbStats.blocksDBStats, 'blocks', tree); + this.updateTreeDBStats(treeDbStats.leafIndicesDBStats, 'leaf_indices', tree); + this.updateTreeDBStats(treeDbStats.leafPreimagesDBStats, 'leaf_preimage', tree); + this.updateTreeDBStats(treeDbStats.nodesDBStats, 'nodes', tree); } - private updateTreeStats(treeDbStats: TreeDBStats, treeMeta: TreeMeta, tree: MerkleTreeId) { - const instrumentation = this.treeInstrumentation.get(tree); - if (!instrumentation) { - this.log.error(`Failed to retrieve instrumentation for tree ${MerkleTreeId[tree]}`); - return; - } - instrumentation.updateMetrics(treeDbStats, treeMeta); + private updateTreeDBStats(dbStats: DBStats, dbType: DBTypeString, tree: MerkleTreeId) { + this.dbNumItems.record(Number(dbStats.numDataItems), { + [Attributes.WS_DB_DATA_TYPE]: dbType, + [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree], + }); + this.dbUsedSize.record(Number(dbStats.totalUsedSize), { + [Attributes.WS_DB_DATA_TYPE]: dbType, + [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree], + }); } public updateWorldStateMetrics(worldStateStatus: WorldStateStatusFull) {