From ae8d79129175cb84804d9a8b84fc4bfa6eadd176 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Mon, 4 Nov 2024 14:10:31 +0800 Subject: [PATCH 01/69] fix: gossipsub metrics adapter --- yarn-project/p2p/package.json | 2 + .../p2p/src/service/libp2p_service.ts | 23 ++- yarn-project/telemetry-client/src/index.ts | 1 + .../telemetry-client/src/prom_otel_adapter.ts | 167 ++++++++++++++++++ yarn-project/yarn.lock | 96 +++++++++- 5 files changed, 279 insertions(+), 10 deletions(-) create mode 100644 yarn-project/telemetry-client/src/prom_otel_adapter.ts diff --git a/yarn-project/p2p/package.json b/yarn-project/p2p/package.json index 284973b6c85..f2e14f5a3fc 100644 --- a/yarn-project/p2p/package.json +++ b/yarn-project/p2p/package.json @@ -82,12 +82,14 @@ "@libp2p/peer-id": "4.0.7", "@libp2p/peer-id-factory": "4.1.1", "@libp2p/peer-store": "10.0.16", + "@libp2p/prometheus-metrics": "^4.2.4", "@libp2p/tcp": "9.0.24", "@multiformats/multiaddr": "12.1.14", "interface-datastore": "^8.2.11", "interface-store": "^5.1.8", "it-pipe": "^3.0.1", "libp2p": "1.5.0", + "prom-client": "^15.1.3", "semver": "^7.6.0", "sha3": "^2.1.4", "tslib": "^2.4.0" diff --git a/yarn-project/p2p/src/service/libp2p_service.ts b/yarn-project/p2p/src/service/libp2p_service.ts index 463471041bb..317a94ce9f0 100644 --- a/yarn-project/p2p/src/service/libp2p_service.ts +++ b/yarn-project/p2p/src/service/libp2p_service.ts @@ -18,7 +18,7 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { SerialQueue } from '@aztec/foundation/queue'; import { RunningPromise } from '@aztec/foundation/running-promise'; import type { AztecKVStore } from '@aztec/kv-store'; -import { Attributes, type TelemetryClient, WithTracer, trackSpan } from '@aztec/telemetry-client'; +import { Attributes, OtelMetricsAdapter, type TelemetryClient, WithTracer, trackSpan } from '@aztec/telemetry-client'; import { type ENR } from '@chainsafe/enr'; import { type GossipSub, type GossipSubComponents, gossipsub } from '@chainsafe/libp2p-gossipsub'; @@ -32,6 +32,7 @@ import { mplex } from '@libp2p/mplex'; import { createFromJSON, createSecp256k1PeerId } from '@libp2p/peer-id-factory'; import { tcp } from '@libp2p/tcp'; import { createLibp2p } from 'libp2p'; +import { prometheusMetrics } from "@libp2p/prometheus-metrics"; import { type P2PConfig } from '../config.js'; import { type MemPools } from '../mem_pools/interface.js'; @@ -218,6 +219,8 @@ export class LibP2PService extends WithTracer implements P2PService { const datastore = new AztecDatastore(store); + const otelMetricsAdapter = new OtelMetricsAdapter(telemetry); + const node = await createLibp2p({ start: false, peerId, @@ -245,6 +248,11 @@ export class LibP2PService extends WithTracer implements P2PService { minConnections: minPeerCount, maxConnections: maxPeerCount, }, + metrics: prometheusMetrics({ + registry: otelMetricsAdapter, + collectDefaultMetrics: false, + preserveExistingMetrics: true + }), services: { identify: identify({ protocolPrefix: 'aztec', @@ -257,6 +265,7 @@ export class LibP2PService extends WithTracer implements P2PService { heartbeatInterval: config.gossipsubInterval, mcacheLength: config.gossipsubMcacheLength, mcacheGossip: config.gossipsubMcacheGossip, + metricsRegister: otelMetricsAdapter, scoreParams: createPeerScoreParams({ topics: { [Tx.p2pTopic]: createTopicScoreParams({ @@ -285,6 +294,12 @@ export class LibP2PService extends WithTracer implements P2PService { }, }); + // Register libp2p gossipsub metrics to our telemetry client + node.services.pubsub.g + + + + // Create request response protocol handlers /** * Handler for tx requests @@ -317,6 +332,12 @@ export class LibP2PService extends WithTracer implements P2PService { ); } + registerGossipSubMetrics() { + // + + + } + /** * Send Request via the ReqResp service * The subprotocol defined will determine the request and response types diff --git a/yarn-project/telemetry-client/src/index.ts b/yarn-project/telemetry-client/src/index.ts index 2fd66afb9eb..3eabd8eb1c9 100644 --- a/yarn-project/telemetry-client/src/index.ts +++ b/yarn-project/telemetry-client/src/index.ts @@ -1,3 +1,4 @@ export * from './telemetry.js'; export * from './histogram_utils.js'; export * from './with_tracer.js'; +export * from './prom_otel_adapter.js'; \ No newline at end of file diff --git a/yarn-project/telemetry-client/src/prom_otel_adapter.ts b/yarn-project/telemetry-client/src/prom_otel_adapter.ts new file mode 100644 index 00000000000..b251d701523 --- /dev/null +++ b/yarn-project/telemetry-client/src/prom_otel_adapter.ts @@ -0,0 +1,167 @@ +import { type MeterProvider } from "@opentelemetry/sdk-metrics"; +import { TelemetryClient } from "./telemetry.js"; + +import { Registry } from "prom-client"; + + +type TopicStr = string; +export type TopicLabel = string +export type TopicStrToLabel = Map + +export enum MessageSource { + forward = 'forward', + publish = 'publish' +} + +type NoLabels = Record +type LabelsGeneric = Record +type LabelKeys = Extract +interface CollectFn { (metric: Gauge): void } + +interface Gauge { + inc: NoLabels extends Labels ? (value?: number) => void : (labels: Labels, value?: number) => void + set: NoLabels extends Labels ? (value: number) => void : (labels: Labels, value: number) => void + + addCollect(collectFn: CollectFn): void +} + +interface Histogram { + startTimer(): () => void + + observe: NoLabels extends Labels ? (value: number) => void : (labels: Labels, value: number) => void + + reset(): void +} + +interface AvgMinMax { + set: NoLabels extends Labels ? (values: number[]) => void : (labels: Labels, values: number[]) => void +} + +export type GaugeConfig = { + name: string + help: string +} & (NoLabels extends Labels ? { labelNames?: never } : { labelNames: [LabelKeys, ...Array>] }) + +export type HistogramConfig = GaugeConfig & { + buckets?: number[] +} + +export type AvgMinMaxConfig = GaugeConfig + +export interface MetricsRegister { + gauge(config: GaugeConfig): Gauge + histogram(config: HistogramConfig): Histogram + avgMinMax(config: AvgMinMaxConfig): AvgMinMax +} + +/**Otel Metrics Adapter + * + * Some dependencies we use export metrics directly in a Prometheus format + * This adapter is used to convert those metrics to a format that we can use with OpenTelemetry + * + * Affected services include: + * - chainsafe/gossipsub + * - libp2p + */ +export class OtelMetricsAdapter extends Registry implements MetricsRegister { + private readonly meter: any; + + constructor(telemetryClient: TelemetryClient) { + super(); + this.meter = telemetryClient.getMeter('gossipsub'); + } + + gauge(config: GaugeConfig): Gauge { + const otelGauge = this.meter.createGauge(config.name, config.help); + + return { + inc: ((labels?: Labels, value = 1) => { + if (labels) { + otelGauge.add(value, labels); + } else { + otelGauge.add(value); + } + }) as any, + + set: ((labels: Labels, value: number) => { + if (labels) { + otelGauge.set(value, labels); + } else { + otelGauge.set(value); + } + }) as any, + + + // TOOD: deal with this part + addCollect: (_collectFn: (metric: Gauge) => void) => { + // OpenTelemetry handles collection internally + }, + } + } + + histogram(config: HistogramConfig): Histogram { + const otelHistogram = this.meter.createHistogram(config.name, { + description: config.help, + unit: '1', + boundaries: config.buckets, + }); + + return { + observe: ((labels: Labels, value: number) => { + if (labels) { + otelHistogram.record(value, labels); + } else { + otelHistogram.record(value); + } + }) as any, + + startTimer: () => { + const startTime = performance.now(); + return () => { + const duration = performance.now() - startTime; + otelHistogram.record(duration); + }; + }, + + reset: () => { + // OpenTelemetry histograms are immutable, reset not needed + }, + }; + } + + avgMinMax(config: AvgMinMaxConfig): AvgMinMax { + // Create a single gauge with additional attributes for min/max/avg + const otelGauge = this.meter.createGauge(config.name, { + description: config.help, + unit: '1', + }); + + return { + set: ((labels: Labels, values: number[]) => { + if (values.length > 0) { + const avg = values.reduce((a, b) => a + b) / values.length; + const min = Math.min(...values); + const max = Math.max(...values); + + const attributes = { + ...(labels as object), + type: 'avg', + }; + + if (labels) { + otelGauge.set(avg, { ...attributes, type: 'avg' }); + otelGauge.set(min, { ...attributes, type: 'min' }); + otelGauge.set(max, { ...attributes, type: 'max' }); + } else { + otelGauge.set(avg, { type: 'avg' }); + otelGauge.set(min, { type: 'min' }); + otelGauge.set(max, { type: 'max' }); + } + } + }) as any, + }; + } + + // static(meter: Meter, config: GaugeConfig): void { + +} \ No newline at end of file diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 16c668952a9..34316ad3e91 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -868,6 +868,7 @@ __metadata: "@libp2p/peer-id": 4.0.7 "@libp2p/peer-id-factory": 4.1.1 "@libp2p/peer-store": 10.0.16 + "@libp2p/prometheus-metrics": ^4.2.4 "@libp2p/tcp": 9.0.24 "@multiformats/multiaddr": 12.1.14 "@types/jest": ^29.5.0 @@ -881,6 +882,7 @@ __metadata: jest: ^29.5.0 jest-mock-extended: ^3.0.4 libp2p: 1.5.0 + prom-client: ^15.1.3 semver: ^7.6.0 sha3: ^2.1.4 ts-node: ^10.9.1 @@ -2792,6 +2794,20 @@ __metadata: languageName: node linkType: hard +"@libp2p/interface@npm:^2.2.0": + version: 2.2.0 + resolution: "@libp2p/interface@npm:2.2.0" + dependencies: + "@multiformats/multiaddr": ^12.2.3 + it-pushable: ^3.2.3 + it-stream-types: ^2.0.1 + multiformats: ^13.1.0 + progress-events: ^1.0.0 + uint8arraylist: ^2.4.8 + checksum: 277634721147384af134232fe68c6d904eb6794f48b29eb070a85f414a4a3d362151f0d7c1d76ad013f09f9efaf9010ca6d7fe4ab99b5f5c98dc1c22e184d40f + languageName: node + linkType: hard + "@libp2p/kad-dht@npm:10.0.4": version: 10.0.4 resolution: "@libp2p/kad-dht@npm:10.0.4" @@ -3034,6 +3050,19 @@ __metadata: languageName: node linkType: hard +"@libp2p/prometheus-metrics@npm:^4.2.4": + version: 4.2.4 + resolution: "@libp2p/prometheus-metrics@npm:4.2.4" + dependencies: + "@libp2p/interface": ^2.2.0 + it-foreach: ^2.1.0 + it-stream-types: ^2.0.1 + prom-client: ^15.1.2 + uint8arraylist: ^2.4.8 + checksum: f5aef58ede8371e7f6d451b3d3cc1e3136fd26acc12609030dc03218fdaeada0a03c5a2ee404c16e15f9b0191f0017a31d8b909012f9094c4c95b12cf6b978ee + languageName: node + linkType: hard + "@libp2p/pubsub@npm:^9.0.8": version: 9.0.17 resolution: "@libp2p/pubsub@npm:9.0.17" @@ -3299,6 +3328,20 @@ __metadata: languageName: node linkType: hard +"@multiformats/multiaddr@npm:^12.2.3": + version: 12.3.1 + resolution: "@multiformats/multiaddr@npm:12.3.1" + dependencies: + "@chainsafe/is-ip": ^2.0.1 + "@chainsafe/netmask": ^2.0.0 + "@multiformats/dns": ^1.0.3 + multiformats: ^13.0.0 + uint8-varint: ^2.0.1 + uint8arrays: ^5.0.0 + checksum: 086a71f86caeb441ea8895e497d7ce1e46d8bda881bc275812054b02a94d08323f100ca78d1ff9979347488145563b492d1d4ffc8d800baf5ce5c726d02af451 + languageName: node + linkType: hard + "@noble/ciphers@npm:^0.4.0": version: 0.4.1 resolution: "@noble/ciphers@npm:0.4.1" @@ -3391,7 +3434,7 @@ __metadata: version: 0.0.0-use.local resolution: "@noir-lang/noir_codegen@portal:../noir/packages/noir_codegen::locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: - "@noir-lang/types": 0.36.0 + "@noir-lang/types": 0.24.0 glob: ^10.3.10 ts-command-line-args: ^2.5.1 bin: @@ -3400,13 +3443,13 @@ __metadata: linkType: soft "@noir-lang/noir_js@file:../noir/packages/noir_js::locator=%40aztec%2Faztec3-packages%40workspace%3A.": - version: 0.36.0 - resolution: "@noir-lang/noir_js@file:../noir/packages/noir_js#../noir/packages/noir_js::hash=b897b0&locator=%40aztec%2Faztec3-packages%40workspace%3A." + version: 0.24.0 + resolution: "@noir-lang/noir_js@file:../noir/packages/noir_js#../noir/packages/noir_js::hash=ea1618&locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: - "@noir-lang/acvm_js": 0.52.0 - "@noir-lang/noirc_abi": 0.36.0 - "@noir-lang/types": 0.36.0 - checksum: 80eab489930f98827554644d8031a4b884042e24a0786bea001cd8ad4aca7b15b95bc97efa00dcd2301f7efd10dcc35ce8cb9dce0c4e492127701dbafaca34db + "@noir-lang/acvm_js": 0.40.0 + "@noir-lang/noirc_abi": 0.24.0 + "@noir-lang/types": 0.24.0 + checksum: e3fa36670395c98bd7778e0f03aec0f4d90f01dfd71ecac0ea8be6ec0c1dbd2d105e3603f1b3fe39e36089a92a5ff13c5ace77c8c6c4149fe275a856e666a73a languageName: node linkType: hard @@ -3414,7 +3457,7 @@ __metadata: version: 0.0.0-use.local resolution: "@noir-lang/noirc_abi@portal:../noir/packages/noirc_abi::locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: - "@noir-lang/types": 0.36.0 + "@noir-lang/types": 0.24.0 languageName: node linkType: soft @@ -3464,7 +3507,7 @@ __metadata: languageName: node linkType: hard -"@opentelemetry/api@npm:^1.0.0, @opentelemetry/api@npm:^1.3.0, @opentelemetry/api@npm:^1.9.0": +"@opentelemetry/api@npm:^1.0.0, @opentelemetry/api@npm:^1.3.0, @opentelemetry/api@npm:^1.4.0, @opentelemetry/api@npm:^1.9.0": version: 1.9.0 resolution: "@opentelemetry/api@npm:1.9.0" checksum: 9e88e59d53ced668f3daaecfd721071c5b85a67dd386f1c6f051d1be54375d850016c881f656ffbe9a03bedae85f7e89c2f2b635313f9c9b195ad033cdc31020 @@ -6130,6 +6173,13 @@ __metadata: languageName: node linkType: hard +"bintrees@npm:1.0.2": + version: 1.0.2 + resolution: "bintrees@npm:1.0.2" + checksum: 56a52b7d3634e30002b1eda740d2517a22fa8e9e2eb088e919f37c030a0ed86e364ab59e472fc770fc8751308054bb1c892979d150e11d9e11ac33bcc1b5d16e + languageName: node + linkType: hard + "bl@npm:^4.1.0": version: 4.1.0 resolution: "bl@npm:4.1.0" @@ -10603,6 +10653,15 @@ __metadata: languageName: node linkType: hard +"it-foreach@npm:^2.1.0": + version: 2.1.1 + resolution: "it-foreach@npm:2.1.1" + dependencies: + it-peekable: ^3.0.0 + checksum: 17cb7917fdecd7a8e2106a26be13d7edea5f02ce1163f866afd9bb3dab2acf3b919064bd383c89fcb2843073474257ecf21dc3d086f695ed3e2d45bae6e03f0c + languageName: node + linkType: hard + "it-length-prefixed-stream@npm:^1.0.0, it-length-prefixed-stream@npm:^1.1.6": version: 1.1.7 resolution: "it-length-prefixed-stream@npm:1.1.7" @@ -13648,6 +13707,16 @@ __metadata: languageName: node linkType: hard +"prom-client@npm:^15.1.2, prom-client@npm:^15.1.3": + version: 15.1.3 + resolution: "prom-client@npm:15.1.3" + dependencies: + "@opentelemetry/api": ^1.4.0 + tdigest: ^0.1.1 + checksum: 9a57f3c16f39aa9a03da021883a4231c0bb56fc9d02f6ef9c28f913379f275640a5a33b98d9946ebf53c71011a29b580e9d2d6e3806cb1c229a3f59c65993968 + languageName: node + linkType: hard + "promise-retry@npm:^2.0.1": version: 2.0.1 resolution: "promise-retry@npm:2.0.1" @@ -15339,6 +15408,15 @@ __metadata: languageName: node linkType: hard +"tdigest@npm:^0.1.1": + version: 0.1.2 + resolution: "tdigest@npm:0.1.2" + dependencies: + bintrees: 1.0.2 + checksum: 44de8246752b6f8c2924685f969fd3d94c36949f22b0907e99bef2b2220726dd8467f4730ea96b06040b9aa2587c0866049640039d1b956952dfa962bc2075a3 + languageName: node + linkType: hard + "terser-webpack-plugin@npm:^5.3.10": version: 5.3.10 resolution: "terser-webpack-plugin@npm:5.3.10" From 923db38212c19135eb72d6dfb47d737096ab6922 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Mon, 4 Nov 2024 10:16:45 +0000 Subject: [PATCH 02/69] check point --- .../aztec/src/cli/cmds/start_p2p_bootstrap.ts | 2 + .../circuit-types/src/p2p/topic_type.ts | 18 ++++ .../scripts/docker-compose.metrics.yml | 100 ++++++++++++++++++ .../end-to-end/src/e2e_p2p/gerousia.test.ts | 6 +- .../src/e2e_p2p/gossip_network.test.ts | 10 +- .../end-to-end/src/e2e_p2p/p2p_network.ts | 19 +++- .../src/e2e_p2p/rediscovery.test.ts | 6 +- .../end-to-end/src/e2e_p2p/reqresp_tx.test.ts | 6 +- .../end-to-end/src/fixtures/fixtures.ts | 2 + yarn-project/end-to-end/src/fixtures/index.ts | 1 + .../end-to-end/src/fixtures/setup_p2p_test.ts | 10 +- .../src/fixtures/snapshot_manager.ts | 7 +- yarn-project/end-to-end/src/fixtures/utils.ts | 2 + .../src/fixtures/with_telemetry_utils.ts | 24 +++++ yarn-project/p2p-bootstrap/src/index.ts | 7 +- yarn-project/p2p/src/bootstrap/bootstrap.ts | 8 +- yarn-project/p2p/src/mocks/index.ts | 13 +-- .../p2p/src/service/discv5_service.test.ts | 4 +- .../p2p/src/service/libp2p_service.ts | 7 +- yarn-project/telemetry-client/src/config.ts | 2 +- .../telemetry-client/src/prom_otel_adapter.ts | 2 +- yarn-project/yarn.lock | 16 +-- 22 files changed, 237 insertions(+), 35 deletions(-) create mode 100644 yarn-project/end-to-end/scripts/docker-compose.metrics.yml create mode 100644 yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts diff --git a/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts b/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts index 60ffada5411..efe93868503 100644 --- a/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts +++ b/yarn-project/aztec/src/cli/cmds/start_p2p_bootstrap.ts @@ -8,6 +8,8 @@ import { extractRelevantOptions } from '../util.js'; export const startP2PBootstrap = async (options: any, userLog: LogFn, debugLogger: DebugLogger) => { // Start a P2P bootstrap node. const config = extractRelevantOptions(options, bootnodeConfigMappings, 'p2p'); + + // TODO: fix up telemetry client config here await runBootstrapNode(config, debugLogger); userLog(`P2P bootstrap node started on ${config.udpListenAddress}`); }; diff --git a/yarn-project/circuit-types/src/p2p/topic_type.ts b/yarn-project/circuit-types/src/p2p/topic_type.ts index 88aed5b8bc1..8094905276c 100644 --- a/yarn-project/circuit-types/src/p2p/topic_type.ts +++ b/yarn-project/circuit-types/src/p2p/topic_type.ts @@ -17,3 +17,21 @@ export enum TopicType { block_attestation = 'block_attestation', epoch_proof_quote = 'epoch_proof_quote', } + +/** + * Convert the topic string into a set of labels + * + * In the form: + * { + * "/aztec/tx/0.1.0": "tx", + * ... + * } + */ +export function metricsTopicStrToLabels() { + const topicStrToLabel = new Map(); + for (const topic in TopicType) { + topicStrToLabel.set(createTopicString(TopicType[topic as keyof typeof TopicType]), topic); + } + + return topicStrToLabel; +} diff --git a/yarn-project/end-to-end/scripts/docker-compose.metrics.yml b/yarn-project/end-to-end/scripts/docker-compose.metrics.yml new file mode 100644 index 00000000000..fb99a31796e --- /dev/null +++ b/yarn-project/end-to-end/scripts/docker-compose.metrics.yml @@ -0,0 +1,100 @@ +# OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: ${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-http://otel-collector:4318/v1/metrics} +# OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-http://otel-collector:4318/v1/traces} + +# A docker-compose to run with an end-to-end to collect metrics +# If testing metrics collection, run this compose file along side your end to end test +# name: end-to-end-metrics +services: + otel-collector: + image: otel/opentelemetry-collector-contrib + pull_policy: always + configs: + - source: otel-collector-config + target: /etc/otelcol-contrib/config.yaml + ports: + - 4318:4318 + + prometheus: + image: prom/prometheus + configs: + - source: prometheus-config + target: /etc/prometheus/prometheus.yml + + grafana: + pull_policy: always + image: grafana/grafana + ports: + - 3000:3000 + volumes: + - ./spartan/aztec-network/files/grafana_dashboards:/etc/grafana/provisioning/dashboards + - grafana:/var/lib/grafana + configs: + - source: grafana-sources + target: /etc/grafana/provisioning/datasources/default.yml + + jaeger: + pull_policy: always + image: jaegertracing/all-in-one + ports: + - 16686:16686 + +volumes: + grafana: + +configs: + grafana-sources: + content: | + apiVersion: 1 + datasources: + - name: Prometheus + uid: aztec-node-metrics + type: prometheus + url: http://prometheus:9090 + editable: false + isDefault: true + jsonData: + timeInterval: 10s + + prometheus-config: + content: | + global: + evaluation_interval: 15s + scrape_interval: 15s + scrape_configs: + - job_name: otel-collector + static_configs: + - targets: ['otel-collector:8888'] + - job_name: aztec + static_configs: + - targets: ['otel-collector:8889'] + otel-collector-config: + content: | + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + + processors: + batch: + + exporters: + prometheus: + endpoint: 0.0.0.0:8889 + metric_expiration: 5m + otlp/jaeger: + endpoint: "jaeger:4317" + tls: + insecure: true + + service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp/jaeger] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheus] + diff --git a/yarn-project/end-to-end/src/e2e_p2p/gerousia.test.ts b/yarn-project/end-to-end/src/e2e_p2p/gerousia.test.ts index 7a6e90d6258..7ee5d7c94f2 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/gerousia.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/gerousia.test.ts @@ -30,7 +30,11 @@ describe('e2e_p2p_gerousia', () => { let nodes: AztecNodeService[]; beforeEach(async () => { - t = await P2PNetworkTest.create('e2e_p2p_network', NUM_NODES, BOOT_NODE_UDP_PORT); + t = await P2PNetworkTest.create({ + testName: 'e2e_p2p_gerousia', + numberOfNodes: NUM_NODES, + basePort: BOOT_NODE_UDP_PORT, + }); await t.applyBaseSnapshots(); await t.setup(); }); diff --git a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts index b59a5a7ed8b..b4585f9d141 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts @@ -6,6 +6,7 @@ import fs from 'fs'; import { type NodeContext, createNodes } from '../fixtures/setup_p2p_test.js'; import { P2PNetworkTest, WAIT_FOR_TX_TIMEOUT } from './p2p_network.js'; import { createPXEServiceAndSubmitTransactions } from './shared.js'; +import { METRICS_PORT } from '../fixtures/fixtures.js'; // Don't set this to a higher value than 9 because each node will use a different L1 publisher account and anvil seeds const NUM_NODES = 4; @@ -19,7 +20,12 @@ describe('e2e_p2p_network', () => { let nodes: AztecNodeService[]; beforeEach(async () => { - t = await P2PNetworkTest.create('e2e_p2p_network', NUM_NODES, BOOT_NODE_UDP_PORT); + t = await P2PNetworkTest.create({ + testName: 'e2e_p2p_network', + numberOfNodes: NUM_NODES, + basePort: BOOT_NODE_UDP_PORT, + metricsPort: METRICS_PORT, + }); await t.applyBaseSnapshots(); await t.setup(); }); @@ -33,7 +39,7 @@ describe('e2e_p2p_network', () => { }); // TODO(https://github.com/AztecProtocol/aztec-packages/issues/9164): Currently flakey - it.skip('should rollup txs from all peers', async () => { + it('should rollup txs from all peers', async () => { // create the bootstrap node for the network if (!t.bootstrapNodeEnr) { throw new Error('Bootstrap node ENR is not available'); diff --git a/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts b/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts index 84a2aa53b8f..cd6a525dd97 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts @@ -17,6 +17,7 @@ import { } from '../fixtures/setup_p2p_test.js'; import { type ISnapshotManager, type SubsystemsContext, createSnapshotManager } from '../fixtures/snapshot_manager.js'; import { getPrivateKeyFromIndex } from '../fixtures/utils.js'; +import { getEndToEndTestTelemetryClient } from '../fixtures/with_telemetry_utils.js'; // Use a fixed bootstrap node private key so that we can re-use the same snapshot and the nodes can find each other const BOOTSTRAP_NODE_PRIVATE_KEY = '080212208f988fc0899e4a73a5aee4d271a5f20670603a756ad8d84f2c94263a6427c591'; @@ -41,6 +42,8 @@ export class P2PNetworkTest { private numberOfNodes: number, initialValidatorAddress: string, initialValidatorConfig: AztecNodeConfig, + // If set enable metrics collection + metricsPort?: number, ) { this.logger = createDebugLogger(`aztec:e2e_p2p:${testName}`); @@ -58,13 +61,25 @@ export class P2PNetworkTest { l1BlockTime: ETHEREUM_SLOT_DURATION, salt: 420, initialValidators, + metricsPort: metricsPort, }); } - static async create(testName: string, numberOfNodes: number, basePort?: number) { + static async create({ + testName, + numberOfNodes, + basePort, + metricsPort, + }: { + testName: string; + numberOfNodes: number; + basePort?: number; + metricsPort?: number; + }) { const port = basePort || (await getPort()); - const bootstrapNode = await createBootstrapNodeFromPrivateKey(BOOTSTRAP_NODE_PRIVATE_KEY, port); + const telemetry = await getEndToEndTestTelemetryClient(metricsPort); + const bootstrapNode = await createBootstrapNodeFromPrivateKey(BOOTSTRAP_NODE_PRIVATE_KEY, port, telemetry); const bootstrapNodeEnr = bootstrapNode.getENR().encodeTxt(); const initialValidatorConfig = await createValidatorConfig({} as AztecNodeConfig, bootstrapNodeEnr); diff --git a/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts b/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts index c9d78f5d1ca..7e270ee5975 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts @@ -19,7 +19,11 @@ describe('e2e_p2p_rediscovery', () => { let nodes: AztecNodeService[]; beforeEach(async () => { - t = await P2PNetworkTest.create('e2e_p2p_rediscovery', NUM_NODES, BOOT_NODE_UDP_PORT); + t = await P2PNetworkTest.create({ + testName: 'e2e_p2p_rediscovery', + numberOfNodes: NUM_NODES, + basePort: BOOT_NODE_UDP_PORT, + }); await t.applyBaseSnapshots(); await t.setup(); }); diff --git a/yarn-project/end-to-end/src/e2e_p2p/reqresp_tx.test.ts b/yarn-project/end-to-end/src/e2e_p2p/reqresp_tx.test.ts index 8000c72bc09..9de73adca98 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/reqresp_tx.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/reqresp_tx.test.ts @@ -20,7 +20,11 @@ describe('e2e_p2p_reqresp_tx', () => { let nodes: AztecNodeService[]; beforeEach(async () => { - t = await P2PNetworkTest.create('e2e_p2p_reqresp_tx', NUM_NODES, BOOT_NODE_UDP_PORT); + t = await P2PNetworkTest.create({ + testName: 'e2e_p2p_reqresp_tx', + numberOfNodes: NUM_NODES, + basePort: BOOT_NODE_UDP_PORT, + }); await t.applyBaseSnapshots(); await t.setup(); }); diff --git a/yarn-project/end-to-end/src/fixtures/fixtures.ts b/yarn-project/end-to-end/src/fixtures/fixtures.ts index 6470f015229..c9ee5dd9601 100644 --- a/yarn-project/end-to-end/src/fixtures/fixtures.ts +++ b/yarn-project/end-to-end/src/fixtures/fixtures.ts @@ -1,3 +1,5 @@ +export const METRICS_PORT = 4318; + export const MNEMONIC = 'test test test test test test test test test test test junk'; export const privateKey = Buffer.from('ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80', 'hex'); export const privateKey2 = Buffer.from('59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d', 'hex'); diff --git a/yarn-project/end-to-end/src/fixtures/index.ts b/yarn-project/end-to-end/src/fixtures/index.ts index c2a32f7e035..05ab9ddbfff 100644 --- a/yarn-project/end-to-end/src/fixtures/index.ts +++ b/yarn-project/end-to-end/src/fixtures/index.ts @@ -2,3 +2,4 @@ export * from './fixtures.js'; export * from './logging.js'; export * from './utils.js'; export * from './token_utils.js'; +export * from './with_telemetry_utils.js'; \ No newline at end of file diff --git a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts index 48e89c6512d..ea1881ec1d8 100644 --- a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts +++ b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts @@ -11,6 +11,7 @@ import getPort from 'get-port'; import { generatePrivateKey } from 'viem/accounts'; import { getPrivateKeyFromIndex } from './utils.js'; +import { getEndToEndTestTelemetryClient } from './with_telemetry_utils.js'; export interface NodeContext { node: AztecNodeService; @@ -48,6 +49,7 @@ export function createNodes( numNodes: number, bootNodePort: number, dataDirectory?: string, + metricsPort?: number, ): Promise { const nodePromises = []; for (let i = 0; i < numNodes; i++) { @@ -55,7 +57,7 @@ export function createNodes( const port = bootNodePort + i + 1; const dataDir = dataDirectory ? `${dataDirectory}-${i}` : undefined; - const nodePromise = createNode(config, peerIdPrivateKeys[i], port, bootstrapNodeEnr, i, dataDir); + const nodePromise = createNode(config, peerIdPrivateKeys[i], port, bootstrapNodeEnr, i, dataDir, metricsPort); nodePromises.push(nodePromise); } return Promise.all(nodePromises); @@ -69,6 +71,7 @@ export async function createNode( bootstrapNode: string | undefined, publisherAddressIndex: number, dataDirectory?: string, + metricsPort?: number, ) { const validatorConfig = await createValidatorConfig( config, @@ -78,9 +81,12 @@ export async function createNode( publisherAddressIndex, dataDirectory, ); + + const telemetryClient = await getEndToEndTestTelemetryClient(metricsPort); + return await AztecNodeService.createAndSync( validatorConfig, - new NoopTelemetryClient(), + telemetryClient, createDebugLogger(`aztec:node-${tcpPort}`), ); } diff --git a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts index d8a9502d340..ac709e7699d 100644 --- a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts +++ b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts @@ -24,7 +24,7 @@ import { resolver, reviver } from '@aztec/foundation/serialize'; import { type ProverNode, type ProverNodeConfig, createProverNode } from '@aztec/prover-node'; import { type PXEService, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; -import { createAndStartTelemetryClient, getConfigEnvVars as getTelemetryConfig } from '@aztec/telemetry-client/start'; +import { createAndStartTelemetryClient, getConfigEnvVars as getTelemetryConfig, TelemetryClientConfig } from '@aztec/telemetry-client/start'; import { type Anvil, createAnvil } from '@viem/anvil'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; @@ -39,6 +39,7 @@ import { getACVMConfig } from './get_acvm_config.js'; import { getBBConfig } from './get_bb_config.js'; import { setupL1Contracts } from './setup_l1_contracts.js'; import { type SetupOptions, getPrivateKeyFromIndex, startAnvil } from './utils.js'; +import { getEndToEndTestTelemetryClient } from './with_telemetry_utils.js'; export type SubsystemsContext = { anvil: Anvil; @@ -312,6 +313,7 @@ async function setupFromFresh( const aztecNodeConfig: AztecNodeConfig & SetupOptions = { ...getConfigEnvVars(), ...opts }; aztecNodeConfig.dataDirectory = statePath; + // Start anvil. We go via a wrapper script to ensure if the parent dies, anvil dies. logger.verbose('Starting anvil...'); const res = await startAnvil(opts.l1BlockTime); @@ -386,7 +388,8 @@ async function setupFromFresh( aztecNodeConfig.bbWorkingDirectory = bbConfig.bbWorkingDirectory; } - const telemetry = await createAndStartTelemetryClient(getTelemetryConfig()); + const telemetry = await getEndToEndTestTelemetryClient(opts.metricsPort); + logger.verbose('Creating and synching an aztec node...'); const aztecNode = await AztecNodeService.createAndSync(aztecNodeConfig, telemetry); diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index b5321c702b1..6929900dddd 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -230,6 +230,8 @@ async function setupWithRemoteEnvironment( export type SetupOptions = { /** State load */ stateLoad?: string; + /** Whether to enable metrics collection, if undefined, metrics collection is disabled */ + metricsPort?: number | undefined; /** Previously deployed contracts on L1 */ deployL1ContractsValues?: DeployL1Contracts; /** Whether to skip deployment of protocol contracts (auth registry, etc) */ diff --git a/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts b/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts new file mode 100644 index 00000000000..880f83b6307 --- /dev/null +++ b/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts @@ -0,0 +1,24 @@ +import { TelemetryClient } from '@aztec/telemetry-client'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; +import { TelemetryClientConfig, getConfigEnvVars as getTelemetryConfig, createAndStartTelemetryClient } from '@aztec/telemetry-client/start'; + +export function getEndToEndTestTelemetryClient(metricsPort?: number): Promise { + return !metricsPort + ? Promise.resolve(new NoopTelemetryClient()) + : createAndStartTelemetryClient(getEndToEndTestTelemetryConfig(metricsPort)); +} + +/** + * Utility functions for setting up end-to-end tests with telemetry. + * + * Read from env vars, override if metricsPort is set + */ +export function getEndToEndTestTelemetryConfig(metricsPort?: number) { + const telemetryConfig: TelemetryClientConfig = getTelemetryConfig(); + if (metricsPort) { + telemetryConfig.metricsCollectorUrl = new URL(`http://localhost:${metricsPort}/v1/metrics`); + telemetryConfig.tracesCollectorUrl = new URL(`http://localhost:${metricsPort}/v1/traces`); + telemetryConfig.logsCollectorUrl = new URL(`http://localhost:${metricsPort}/v1/logs`); + } + return telemetryConfig; +} \ No newline at end of file diff --git a/yarn-project/p2p-bootstrap/src/index.ts b/yarn-project/p2p-bootstrap/src/index.ts index b22492d3cac..4fba93ee68f 100644 --- a/yarn-project/p2p-bootstrap/src/index.ts +++ b/yarn-project/p2p-bootstrap/src/index.ts @@ -3,6 +3,8 @@ import { type BootnodeConfig, BootstrapNode } from '@aztec/p2p'; import Koa from 'koa'; import Router from 'koa-router'; +import { TelemetryClient } from '../../telemetry-client/src/telemetry.js'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; const debugLogger = createDebugLogger('aztec:bootstrap_node'); @@ -12,7 +14,10 @@ const { HTTP_PORT } = process.env; * The application entry point. */ async function main(config: BootnodeConfig, logger = debugLogger) { - const bootstrapNode = new BootstrapNode(logger); + // TODO: insert real telemetry client here + const telemetryClient = new NoopTelemetryClient(); + + const bootstrapNode = new BootstrapNode(telemetryClient, logger); await bootstrapNode.start(config); logger.info('DiscV5 Bootnode started'); diff --git a/yarn-project/p2p/src/bootstrap/bootstrap.ts b/yarn-project/p2p/src/bootstrap/bootstrap.ts index b80ed067a4f..36167fdac64 100644 --- a/yarn-project/p2p/src/bootstrap/bootstrap.ts +++ b/yarn-project/p2p/src/bootstrap/bootstrap.ts @@ -9,6 +9,7 @@ import type { BootnodeConfig } from '../config.js'; import { AZTEC_ENR_KEY, AZTEC_NET } from '../service/discV5_service.js'; import { createLibP2PPeerId } from '../service/index.js'; import { convertToMultiaddr } from '../util.js'; +import { OtelMetricsAdapter, TelemetryClient } from '@aztec/telemetry-client'; /** * Encapsulates a 'Bootstrap' node, used for the purpose of assisting new joiners in acquiring peers. @@ -17,7 +18,10 @@ export class BootstrapNode { private node?: Discv5 = undefined; private peerId?: PeerId; - constructor(private logger = createDebugLogger('aztec:p2p_bootstrap')) {} + constructor( + private telemetry: TelemetryClient, + private logger = createDebugLogger('aztec:p2p_bootstrap'), + ) {} /** * Starts the bootstrap node. @@ -42,6 +46,7 @@ export class BootstrapNode { this.logger.info(`Starting bootstrap node ${peerId}, listening on ${listenAddrUdp.toString()}`); + const metricsRegistry = new OtelMetricsAdapter(this.telemetry); this.node = Discv5.create({ enr, peerId, @@ -50,6 +55,7 @@ export class BootstrapNode { lookupTimeout: 2000, allowUnverifiedSessions: true, }, + metricsRegistry, }); (this.node as Discv5EventEmitter).on('multiaddrUpdated', (addr: Multiaddr) => { diff --git a/yarn-project/p2p/src/mocks/index.ts b/yarn-project/p2p/src/mocks/index.ts index 5127a8c6238..69a0a6e8868 100644 --- a/yarn-project/p2p/src/mocks/index.ts +++ b/yarn-project/p2p/src/mocks/index.ts @@ -35,6 +35,7 @@ import { } from '../service/reqresp/interface.js'; import { ReqResp } from '../service/reqresp/reqresp.js'; import { type PubSubLibp2p } from '../util.js'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; /** * Creates a libp2p node, pre configured. @@ -233,20 +234,20 @@ export function createBootstrapNodeConfig(privateKey: string, port: number): Boo }; } -export function createBootstrapNodeFromPrivateKey(privateKey: string, port: number): Promise { +export function createBootstrapNodeFromPrivateKey(privateKey: string, port: number, telemetry: TelemetryClient = new NoopTelemetryClient()): Promise { const config = createBootstrapNodeConfig(privateKey, port); - return startBootstrapNode(config); + return startBootstrapNode(config, telemetry); } -export async function createBootstrapNode(port: number): Promise { +export async function createBootstrapNode(port: number, telemetry: TelemetryClient = new NoopTelemetryClient()): Promise { const peerId = await createLibP2PPeerId(); const config = createBootstrapNodeConfig(Buffer.from(peerId.privateKey!).toString('hex'), port); - return startBootstrapNode(config); + return startBootstrapNode(config, telemetry); } -async function startBootstrapNode(config: BootnodeConfig) { - const bootstrapNode = new BootstrapNode(); +async function startBootstrapNode(config: BootnodeConfig, telemetry: TelemetryClient) { + const bootstrapNode = new BootstrapNode(telemetry); await bootstrapNode.start(config); return bootstrapNode; } diff --git a/yarn-project/p2p/src/service/discv5_service.test.ts b/yarn-project/p2p/src/service/discv5_service.test.ts index 8a4106d4f4f..69e359268fa 100644 --- a/yarn-project/p2p/src/service/discv5_service.test.ts +++ b/yarn-project/p2p/src/service/discv5_service.test.ts @@ -8,6 +8,7 @@ import { type P2PConfig, getP2PDefaultConfig } from '../config.js'; import { DiscV5Service } from './discV5_service.js'; import { createLibP2PPeerId } from './libp2p_service.js'; import { PeerDiscoveryState } from './service.js'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; const waitForPeers = (node: DiscV5Service, expectedCount: number): Promise => { const timeout = 7_000; @@ -42,7 +43,8 @@ describe('Discv5Service', () => { }; beforeEach(async () => { - bootNode = new BootstrapNode(); + const telemetryClient = new NoopTelemetryClient(); + bootNode = new BootstrapNode(telemetryClient); await bootNode.start(baseConfig); bootNodePeerId = bootNode.getPeerId(); }); diff --git a/yarn-project/p2p/src/service/libp2p_service.ts b/yarn-project/p2p/src/service/libp2p_service.ts index 317a94ce9f0..eae936c1791 100644 --- a/yarn-project/p2p/src/service/libp2p_service.ts +++ b/yarn-project/p2p/src/service/libp2p_service.ts @@ -6,6 +6,7 @@ import { type Gossipable, type L2BlockSource, MerkleTreeId, + metricsTopicStrToLabels, type RawGossipMessage, TopicType, TopicTypeMap, @@ -266,6 +267,7 @@ export class LibP2PService extends WithTracer implements P2PService { mcacheLength: config.gossipsubMcacheLength, mcacheGossip: config.gossipsubMcacheGossip, metricsRegister: otelMetricsAdapter, + metricsTopicStrToLabel: metricsTopicStrToLabels(), scoreParams: createPeerScoreParams({ topics: { [Tx.p2pTopic]: createTopicScoreParams({ @@ -294,11 +296,6 @@ export class LibP2PService extends WithTracer implements P2PService { }, }); - // Register libp2p gossipsub metrics to our telemetry client - node.services.pubsub.g - - - // Create request response protocol handlers /** diff --git a/yarn-project/telemetry-client/src/config.ts b/yarn-project/telemetry-client/src/config.ts index c48a9be6bc4..099fff27a0e 100644 --- a/yarn-project/telemetry-client/src/config.ts +++ b/yarn-project/telemetry-client/src/config.ts @@ -26,7 +26,7 @@ export const telemetryClientConfigMappings: ConfigMappingsType(config: GaugeConfig): Gauge { - const otelGauge = this.meter.createGauge(config.name, config.help); + const otelGauge = this.meter.createObservableUpDownCounter(config.name, config.help); return { inc: ((labels?: Labels, value = 1) => { diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 34316ad3e91..a527dc0bc8e 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -3434,7 +3434,7 @@ __metadata: version: 0.0.0-use.local resolution: "@noir-lang/noir_codegen@portal:../noir/packages/noir_codegen::locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: - "@noir-lang/types": 0.24.0 + "@noir-lang/types": 0.36.0 glob: ^10.3.10 ts-command-line-args: ^2.5.1 bin: @@ -3443,13 +3443,13 @@ __metadata: linkType: soft "@noir-lang/noir_js@file:../noir/packages/noir_js::locator=%40aztec%2Faztec3-packages%40workspace%3A.": - version: 0.24.0 - resolution: "@noir-lang/noir_js@file:../noir/packages/noir_js#../noir/packages/noir_js::hash=ea1618&locator=%40aztec%2Faztec3-packages%40workspace%3A." + version: 0.36.0 + resolution: "@noir-lang/noir_js@file:../noir/packages/noir_js#../noir/packages/noir_js::hash=b897b0&locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: - "@noir-lang/acvm_js": 0.40.0 - "@noir-lang/noirc_abi": 0.24.0 - "@noir-lang/types": 0.24.0 - checksum: e3fa36670395c98bd7778e0f03aec0f4d90f01dfd71ecac0ea8be6ec0c1dbd2d105e3603f1b3fe39e36089a92a5ff13c5ace77c8c6c4149fe275a856e666a73a + "@noir-lang/acvm_js": 0.52.0 + "@noir-lang/noirc_abi": 0.36.0 + "@noir-lang/types": 0.36.0 + checksum: 80eab489930f98827554644d8031a4b884042e24a0786bea001cd8ad4aca7b15b95bc97efa00dcd2301f7efd10dcc35ce8cb9dce0c4e492127701dbafaca34db languageName: node linkType: hard @@ -3457,7 +3457,7 @@ __metadata: version: 0.0.0-use.local resolution: "@noir-lang/noirc_abi@portal:../noir/packages/noirc_abi::locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: - "@noir-lang/types": 0.24.0 + "@noir-lang/types": 0.36.0 languageName: node linkType: soft From 02873267eb11fd40632f4440d864ba9ea0e3d209 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:45:46 +0000 Subject: [PATCH 03/69] fix: view discv5 metric gauges --- .../scripts/docker-compose.metrics.yml | 100 ----- .../src/e2e_p2p/gossip_network.test.ts | 1 + .../src/fixtures/with_telemetry_utils.ts | 6 +- yarn-project/p2p/src/bootstrap/bootstrap.ts | 1 - .../p2p/src/service/libp2p_service.ts | 16 +- .../telemetry-client/src/prom_otel_adapter.ts | 415 +++++++++++++----- 6 files changed, 329 insertions(+), 210 deletions(-) delete mode 100644 yarn-project/end-to-end/scripts/docker-compose.metrics.yml diff --git a/yarn-project/end-to-end/scripts/docker-compose.metrics.yml b/yarn-project/end-to-end/scripts/docker-compose.metrics.yml deleted file mode 100644 index fb99a31796e..00000000000 --- a/yarn-project/end-to-end/scripts/docker-compose.metrics.yml +++ /dev/null @@ -1,100 +0,0 @@ -# OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: ${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-http://otel-collector:4318/v1/metrics} -# OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-http://otel-collector:4318/v1/traces} - -# A docker-compose to run with an end-to-end to collect metrics -# If testing metrics collection, run this compose file along side your end to end test -# name: end-to-end-metrics -services: - otel-collector: - image: otel/opentelemetry-collector-contrib - pull_policy: always - configs: - - source: otel-collector-config - target: /etc/otelcol-contrib/config.yaml - ports: - - 4318:4318 - - prometheus: - image: prom/prometheus - configs: - - source: prometheus-config - target: /etc/prometheus/prometheus.yml - - grafana: - pull_policy: always - image: grafana/grafana - ports: - - 3000:3000 - volumes: - - ./spartan/aztec-network/files/grafana_dashboards:/etc/grafana/provisioning/dashboards - - grafana:/var/lib/grafana - configs: - - source: grafana-sources - target: /etc/grafana/provisioning/datasources/default.yml - - jaeger: - pull_policy: always - image: jaegertracing/all-in-one - ports: - - 16686:16686 - -volumes: - grafana: - -configs: - grafana-sources: - content: | - apiVersion: 1 - datasources: - - name: Prometheus - uid: aztec-node-metrics - type: prometheus - url: http://prometheus:9090 - editable: false - isDefault: true - jsonData: - timeInterval: 10s - - prometheus-config: - content: | - global: - evaluation_interval: 15s - scrape_interval: 15s - scrape_configs: - - job_name: otel-collector - static_configs: - - targets: ['otel-collector:8888'] - - job_name: aztec - static_configs: - - targets: ['otel-collector:8889'] - otel-collector-config: - content: | - receivers: - otlp: - protocols: - http: - endpoint: 0.0.0.0:4318 - - processors: - batch: - - exporters: - prometheus: - endpoint: 0.0.0.0:8889 - metric_expiration: 5m - otlp/jaeger: - endpoint: "jaeger:4317" - tls: - insecure: true - - service: - pipelines: - traces: - receivers: [otlp] - processors: [batch] - exporters: [otlp/jaeger] - metrics: - receivers: [otlp] - processors: [batch] - exporters: [prometheus] - diff --git a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts index b4585f9d141..2f24ffa9531 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts @@ -57,6 +57,7 @@ describe('e2e_p2p_network', () => { NUM_NODES, BOOT_NODE_UDP_PORT, DATA_DIR, + METRICS_PORT, ); // wait a bit for peers to discover each other diff --git a/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts b/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts index 880f83b6307..450458ede76 100644 --- a/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts +++ b/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts @@ -16,9 +16,9 @@ export function getEndToEndTestTelemetryClient(metricsPort?: number): Promise { inc: NoLabels extends Labels ? (value?: number) => void : (labels: Labels, value?: number) => void set: NoLabels extends Labels ? (value: number) => void : (labels: Labels, value: number) => void + collect?(): void addCollect(collectFn: CollectFn): void } @@ -63,105 +62,325 @@ export interface MetricsRegister { * - chainsafe/gossipsub * - libp2p */ -export class OtelMetricsAdapter extends Registry implements MetricsRegister { - private readonly meter: any; - constructor(telemetryClient: TelemetryClient) { - super(); - this.meter = telemetryClient.getMeter('gossipsub'); +class OtelGauge implements Gauge { + private gauge: ObservableGauge; + private currentValue: number = 0; + private collectCallback?: () => void; + + constructor( + meter: Meter, + // TODO: be more strict on metrics name types + name: string, + help: string, + attributes: Array = [] + ) { + console.log("registering gauge", name); + + this.gauge = meter.createObservableGauge(name as Metrics, { + description: help + }); + + // Register callback for the observable gauge + this.gauge.addCallback((result) => { + if (this.collectCallback) { + this.collectCallback(); + } + // TODO: fix labels + result.observe(this.currentValue); + }); + } + + inc(value?: number): void; + inc(labels: Labels, value?: number): void; + inc(labelsOrValue?: Labels | number, value?: number): void { + const { labels, value: actualValue } = getLabelAndValue(labelsOrValue, value); + const typedValue = actualValue as number; + + this.currentValue += typedValue; + } + + dec(labels?: Labels): void { + this.currentValue -= 1; + } + + // In prom, set is for an observable value?? + set(value: number): void; + set(labels: Labels, value: number): void; + // Implementation + set(labelsOrValue: Labels | number, value?: number): void { + if (typeof labelsOrValue === 'number') { + // Case: set(value) + this.currentValue = labelsOrValue; + } else { + // Case: set(labels, value) + if (value === undefined) { + throw new Error('Value must be provided when using labels'); + } + this.currentValue = value; } + } + + setToCurrentTime(labels?: Labels): void { + this.set(labels || {} as Labels, Date.now()); + } + + startTimer(labels?: Labels): (labels?: Labels) => void { + const start = Date.now(); + return (endLabels?: Labels) => { + const duration = Date.now() - start; + this.set(endLabels || labels || {} as Labels, duration); + }; + } + + reset(): void { + this.currentValue = 0; + } + + set collect(callback: () => void) { + this.collectCallback = callback; + } + + // TODO: implement + addCollect(): void {} +} - gauge(config: GaugeConfig): Gauge { - const otelGauge = this.meter.createObservableUpDownCounter(config.name, config.help); - - return { - inc: ((labels?: Labels, value = 1) => { - if (labels) { - otelGauge.add(value, labels); - } else { - otelGauge.add(value); - } - }) as any, - - set: ((labels: Labels, value: number) => { - if (labels) { - otelGauge.set(value, labels); - } else { - otelGauge.set(value); - } - }) as any, - - - // TOOD: deal with this part - addCollect: (_collectFn: (metric: Gauge) => void) => { - // OpenTelemetry handles collection internally - }, - } - } - histogram(config: HistogramConfig): Histogram { - const otelHistogram = this.meter.createHistogram(config.name, { - description: config.help, - unit: '1', - boundaries: config.buckets, - }); - - return { - observe: ((labels: Labels, value: number) => { - if (labels) { - otelHistogram.record(value, labels); - } else { - otelHistogram.record(value); - } - }) as any, - - startTimer: () => { - const startTime = performance.now(); - return () => { - const duration = performance.now() - startTime; - otelHistogram.record(duration); - }; - }, - - reset: () => { - // OpenTelemetry histograms are immutable, reset not needed - }, - }; - } +class OtelHistogram implements Histogram { + private histogram; + + constructor( + meter: Meter, + name: Metrics, // Metrics must be registered in the aztec labels registry + help: string, + private buckets: number[] = [], + labelNames: Array = [] + ) { + // TODO: deal with buckets + this.histogram = meter.createHistogram(name, { + description: help, + }); + } + + // Overload signatures + observe(value: number): void; + observe(labels: Labels, value: number): void; + // Implementation + observe(valueOrLabels: number | Labels, value?: number): void { + const { labels, value: actualValue } = getLabelAndValue(valueOrLabels, value); + const typedValue = actualValue as number; + + this.histogram.record(typedValue, labels); + } + + startTimer(labels?: Labels): (labels?: Labels) => number { + const start = performance.now(); + return (endLabels?: Labels) => { + const duration = performance.now() - start; + this.observe(endLabels || labels || {} as Labels, duration); + return duration; + }; + } + + reset(): void { + // OpenTelemetry histograms cannot be reset, but we implement the interface + console.warn('OpenTelemetry histograms cannot be reset'); + } +} + - avgMinMax(config: AvgMinMaxConfig): AvgMinMax { - // Create a single gauge with additional attributes for min/max/avg - const otelGauge = this.meter.createGauge(config.name, { - description: config.help, - unit: '1', - }); - - return { - set: ((labels: Labels, values: number[]) => { - if (values.length > 0) { - const avg = values.reduce((a, b) => a + b) / values.length; - const min = Math.min(...values); - const max = Math.max(...values); - - const attributes = { - ...(labels as object), - type: 'avg', - }; - - if (labels) { - otelGauge.set(avg, { ...attributes, type: 'avg' }); - otelGauge.set(min, { ...attributes, type: 'min' }); - otelGauge.set(max, { ...attributes, type: 'max' }); - } else { - otelGauge.set(avg, { type: 'avg' }); - otelGauge.set(min, { type: 'min' }); - otelGauge.set(max, { type: 'max' }); - } - } - }) as any, - }; +class OtelAvgMinMax implements AvgMinMax { + private minGauge; + private maxGauge; + private avgGauge; + private count: number = 0; + private sum: number = 0; + private min: number = Infinity; + private max: number = -Infinity; + + constructor( + meter: Meter, + // TODO: be more strict on metrics name types + name: string, // Metrics must be registered in the aztec labels registry + help: string, + labelNames: Array = [] + ) { + this.minGauge = meter.createObservableGauge(`${name}_min` as Metrics, { + description: `${help} (minimum)`, + }); + + this.maxGauge = meter.createObservableGauge(`${name}_max` as Metrics, { + description: `${help} (maximum)`, + }); + + this.avgGauge = meter.createObservableGauge(`${name}_avg` as Metrics, { + description: `${help} (average)`, + }); + + this.setupCallbacks(); + } + + private setupCallbacks(): void { + this.minGauge.addCallback((result) => { + result.observe(this.min, {}); + }); + + this.maxGauge.addCallback((result) => { + result.observe(this.max, {}); + }); + + this.avgGauge.addCallback((result) => { + const avg = this.getAverage(); + if (avg !== undefined) { + result.observe(avg, {}); } + }); + } + + private getLabelKey(labels?: Labels): string { + return JSON.stringify(labels || {}); + } + + private getAverage(): number | undefined { + if (this.count > 0) { + return this.sum / this.count; + } + return undefined; + } - // static(meter: Meter, config: GaugeConfig): void { + // TODO(md): just set up two different classes, one with labels and one without that can inherit from each other + + + set(values: number[]): void; + set(labels: Labels, values: number[]): void; + set(valueOrLabels: number[] | Labels, values?: number[]): void { + const { labels, value: actualValue } = getLabelAndValue(valueOrLabels, values); + const actualValues = actualValue as number[]; + + this.count += actualValues.length; + + const sorted = actualValues.sort((a, b) => a - b); + this.sum += sorted.reduce((acc, curr) => acc + curr, 0); + this.min = sorted[0]; + this.max = sorted[sorted.length - 1]; + } + + reset(): void { + this.count = 0; + this.sum = 0; + this.min = Infinity; + this.max = -Infinity; + } +} + +export class OtelMetricsAdapter extends Registry implements MetricsRegister { + private readonly meter: Meter; + + constructor(telemetryClient: TelemetryClient) { + super(); + this.meter = telemetryClient.getMeter('metrics-adapter'); + } + + gauge( + configuration: GaugeConfig + ): Gauge { + return new OtelGauge( + this.meter, + configuration.name as Metrics, + configuration.help, + configuration.labelNames + ); + } + + histogram( + configuration: HistogramConfig + ): Histogram { + return new OtelHistogram( + this.meter, + configuration.name as Metrics, + configuration.help, + configuration.buckets, + configuration.labelNames + ); + } + + avgMinMax( + configuration: AvgMinMaxConfig + ): AvgMinMax { + return new OtelAvgMinMax( + this.meter, + configuration.name as Metrics, + configuration.help, + configuration.labelNames + ); + } + + // static({ + // name, + // help, + // value + // }: StaticConfig): void { + // const gauge = this.meter.createObservableGauge(name, { + // description: help, + // unit: '1', + // }); + + // gauge.addCallback((result) => { + // result.observe(1, value); + // }); + // } + + // counter( + // configuration: CounterConfig + // ): ICounter { + // return new OtelCounter( + // this.meter, + // configuration.name, + // configuration.help, + // configuration.labelNames + // ); + // } +} + +function getLabelAndValue(valueOrLabels?: number | number[]| Labels, value?: number | number[]): { labels: Labels | undefined, value: number | number[] } { + let labels: Labels | undefined; + let actualValue: number | number[]; + if (typeof valueOrLabels === 'number') { + actualValue = valueOrLabels; + // it is an array + } else if (typeof valueOrLabels === 'object') { + actualValue = valueOrLabels as number[]; + } else if (valueOrLabels !== undefined) { + labels = valueOrLabels; + actualValue = value ?? 1; + } else { + actualValue = 1; + } + return { labels, value: actualValue }; +} -} \ No newline at end of file +// class OtelCounter implements Counter { +// private counter; + +// constructor( +// meter: Meter, +// name: string, // Metrics must be registered in the aztec labels registry +// help: string, +// labelNames: Array = [] +// ) { +// // TODO: be more strict on metrics name types +// this.counter = meter.createUpDownCounter(name as Metrics, { +// description: help, +// unit: '1', +// }); +// } + +// inc(labels?: Labels, value: number = 1): void { +// this.counter.add(value, labels); +// } + +// reset(): void { +// // OpenTelemetry counters cannot be reset, but we implement the interface +// console.warn('OpenTelemetry counters cannot be reset'); +// } +// } \ No newline at end of file From 33f45f07bca670b848f14de7acf7bb8d98a8bcfe Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:48:20 +0000 Subject: [PATCH 04/69] fix: enable for other discv5 service --- yarn-project/p2p/src/client/index.ts | 2 +- yarn-project/p2p/src/mocks/index.ts | 2 +- yarn-project/p2p/src/service/discV5_service.ts | 5 ++++- yarn-project/p2p/src/service/discv5_service.test.ts | 2 +- yarn-project/telemetry-client/src/prom_otel_adapter.ts | 2 -- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/yarn-project/p2p/src/client/index.ts b/yarn-project/p2p/src/client/index.ts index e6a4d273f0e..2b4c498ca59 100644 --- a/yarn-project/p2p/src/client/index.ts +++ b/yarn-project/p2p/src/client/index.ts @@ -49,7 +49,7 @@ export const createP2PClient = async ( // Create peer discovery service const peerId = await createLibP2PPeerId(config.peerIdPrivateKey); - const discoveryService = new DiscV5Service(peerId, config); + const discoveryService = new DiscV5Service(peerId, config, telemetry); p2pService = await LibP2PService.new( config, diff --git a/yarn-project/p2p/src/mocks/index.ts b/yarn-project/p2p/src/mocks/index.ts index 69a0a6e8868..7c0a00fbe83 100644 --- a/yarn-project/p2p/src/mocks/index.ts +++ b/yarn-project/p2p/src/mocks/index.ts @@ -115,7 +115,7 @@ export async function createTestLibP2PService( p2pEnabled: true, peerIdPrivateKey: Buffer.from(peerId.privateKey!).toString('hex'), } as P2PConfig & DataStoreConfig; - const discoveryService = new DiscV5Service(peerId, config); + const discoveryService = new DiscV5Service(peerId, config, telemetry); const proofVerifier = new AlwaysTrueCircuitVerifier(); // No bootstrap nodes provided as the libp2p service will register them in the constructor diff --git a/yarn-project/p2p/src/service/discV5_service.ts b/yarn-project/p2p/src/service/discV5_service.ts index 70983cb6656..091d9e2ee20 100644 --- a/yarn-project/p2p/src/service/discV5_service.ts +++ b/yarn-project/p2p/src/service/discV5_service.ts @@ -10,6 +10,7 @@ import EventEmitter from 'events'; import type { P2PConfig } from '../config.js'; import { convertToMultiaddr } from '../util.js'; import { type PeerDiscoveryService, PeerDiscoveryState } from './service.js'; +import { OtelMetricsAdapter, TelemetryClient } from '@aztec/telemetry-client'; export const AZTEC_ENR_KEY = 'aztec_network'; @@ -41,7 +42,7 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService private startTime = 0; - constructor(private peerId: PeerId, config: P2PConfig, private logger = createDebugLogger('aztec:discv5_service')) { + constructor(private peerId: PeerId, config: P2PConfig, telemetry: TelemetryClient, private logger = createDebugLogger('aztec:discv5_service')) { super(); const { tcpAnnounceAddress, udpAnnounceAddress, udpListenAddress, bootstrapNodes } = config; this.bootstrapNodes = bootstrapNodes; @@ -66,6 +67,7 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService this.enr.setLocationMultiaddr(multiAddrUdp); this.enr.setLocationMultiaddr(multiAddrTcp); + const metricsRegistry = new OtelMetricsAdapter(telemetry); this.discv5 = Discv5.create({ enr: this.enr, peerId, @@ -74,6 +76,7 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService lookupTimeout: 2000, allowUnverifiedSessions: true, }, + metricsRegistry, }); this.logger.info(`ENR NodeId: ${this.enr.nodeId}`); diff --git a/yarn-project/p2p/src/service/discv5_service.test.ts b/yarn-project/p2p/src/service/discv5_service.test.ts index 69e359268fa..1f7e6fb169a 100644 --- a/yarn-project/p2p/src/service/discv5_service.test.ts +++ b/yarn-project/p2p/src/service/discv5_service.test.ts @@ -140,6 +140,6 @@ describe('Discv5Service', () => { keepProvenTxsInPoolFor: 0, l1ChainId: 31337, }; - return new DiscV5Service(peerId, config); + return new DiscV5Service(peerId, config, new NoopTelemetryClient()); }; }); diff --git a/yarn-project/telemetry-client/src/prom_otel_adapter.ts b/yarn-project/telemetry-client/src/prom_otel_adapter.ts index 514254ac1ee..8fc996c496b 100644 --- a/yarn-project/telemetry-client/src/prom_otel_adapter.ts +++ b/yarn-project/telemetry-client/src/prom_otel_adapter.ts @@ -75,8 +75,6 @@ class OtelGauge implements Gauge = [] ) { - console.log("registering gauge", name); - this.gauge = meter.createObservableGauge(name as Metrics, { description: help }); From 09e27f2c11237bd7e638a6c7332080927ac152a7 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Mon, 4 Nov 2024 16:40:48 +0000 Subject: [PATCH 05/69] temp --- .../end-to-end/src/e2e_p2p/p2p_network.ts | 2 +- .../end-to-end/src/fixtures/setup_p2p_test.ts | 2 +- .../src/fixtures/snapshot_manager.ts | 2 +- .../src/fixtures/with_telemetry_utils.ts | 9 +- .../telemetry-client/src/prom_otel_adapter.ts | 215 ++++++++++++------ 5 files changed, 153 insertions(+), 77 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts b/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts index cd6a525dd97..18abae31c6d 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts @@ -78,7 +78,7 @@ export class P2PNetworkTest { }) { const port = basePort || (await getPort()); - const telemetry = await getEndToEndTestTelemetryClient(metricsPort); + const telemetry = await getEndToEndTestTelemetryClient(metricsPort, /*service name*/ `bootstrapnode`); const bootstrapNode = await createBootstrapNodeFromPrivateKey(BOOTSTRAP_NODE_PRIVATE_KEY, port, telemetry); const bootstrapNodeEnr = bootstrapNode.getENR().encodeTxt(); diff --git a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts index ea1881ec1d8..1887c699157 100644 --- a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts +++ b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts @@ -82,7 +82,7 @@ export async function createNode( dataDirectory, ); - const telemetryClient = await getEndToEndTestTelemetryClient(metricsPort); + const telemetryClient = await getEndToEndTestTelemetryClient(metricsPort, /*serviceName*/ `node:${tcpPort}`); return await AztecNodeService.createAndSync( validatorConfig, diff --git a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts index ac709e7699d..89c82a0f153 100644 --- a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts +++ b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts @@ -388,7 +388,7 @@ async function setupFromFresh( aztecNodeConfig.bbWorkingDirectory = bbConfig.bbWorkingDirectory; } - const telemetry = await getEndToEndTestTelemetryClient(opts.metricsPort); + const telemetry = await getEndToEndTestTelemetryClient(opts.metricsPort, /*serviceName*/ 'basenode'); logger.verbose('Creating and synching an aztec node...'); const aztecNode = await AztecNodeService.createAndSync(aztecNodeConfig, telemetry); diff --git a/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts b/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts index 450458ede76..7c5d7255a50 100644 --- a/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts +++ b/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts @@ -2,10 +2,10 @@ import { TelemetryClient } from '@aztec/telemetry-client'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { TelemetryClientConfig, getConfigEnvVars as getTelemetryConfig, createAndStartTelemetryClient } from '@aztec/telemetry-client/start'; -export function getEndToEndTestTelemetryClient(metricsPort?: number): Promise { +export function getEndToEndTestTelemetryClient(metricsPort?: number, serviceName?: string): Promise { return !metricsPort ? Promise.resolve(new NoopTelemetryClient()) - : createAndStartTelemetryClient(getEndToEndTestTelemetryConfig(metricsPort)); + : createAndStartTelemetryClient(getEndToEndTestTelemetryConfig(metricsPort, serviceName)); } /** @@ -13,12 +13,15 @@ export function getEndToEndTestTelemetryClient(metricsPort?: number): Promise type LabelsGeneric = Record type LabelKeys = Extract -interface CollectFn { (metric: Gauge): void } +interface CollectFn { (metric: IGauge): void } -interface Gauge { +interface IGauge { inc: NoLabels extends Labels ? (value?: number) => void : (labels: Labels, value?: number) => void set: NoLabels extends Labels ? (value: number) => void : (labels: Labels, value: number) => void @@ -24,7 +24,7 @@ interface Gauge { addCollect(collectFn: CollectFn): void } -interface Histogram { +interface IHistogram { startTimer(): () => void observe: NoLabels extends Labels ? (value: number) => void : (labels: Labels, value: number) => void @@ -32,7 +32,7 @@ interface Histogram { reset(): void } -interface AvgMinMax { +interface IAvgMinMax { set: NoLabels extends Labels ? (values: number[]) => void : (labels: Labels, values: number[]) => void } @@ -48,9 +48,9 @@ export type HistogramConfig = GaugeConfig export type AvgMinMaxConfig = GaugeConfig export interface MetricsRegister { - gauge(config: GaugeConfig): Gauge - histogram(config: HistogramConfig): Histogram - avgMinMax(config: AvgMinMaxConfig): AvgMinMax + gauge(config: GaugeConfig): IGauge + histogram(config: HistogramConfig): IHistogram + avgMinMax(config: AvgMinMaxConfig): IAvgMinMax } /**Otel Metrics Adapter @@ -63,88 +63,157 @@ export interface MetricsRegister { * - libp2p */ -class OtelGauge implements Gauge { +class OtelGauge implements IGauge { private gauge: ObservableGauge; private currentValue: number = 0; - private collectCallback?: () => void; + private labeledValues: Map = new Map(); + private collectFns: CollectFn[] = []; + + private _collect: () => void = () => {}; + get collect(): () => void { + return this._collect; + } + set collect(fn: () => void) { + this._collect = fn; + } constructor( meter: Meter, - // TODO: be more strict on metrics name types name: string, help: string, - attributes: Array = [] + private labelNames: Array = [] ) { this.gauge = meter.createObservableGauge(name as Metrics, { description: help }); - // Register callback for the observable gauge + // Only observe in the callback when collect() is called this.gauge.addCallback((result) => { - if (this.collectCallback) { - this.collectCallback(); + // Execute the main collect function if assigned + this._collect(); + + // Execute any additional collect callbacks + // this.collectFns.forEach(fn => fn()); + + // Report the current values + if (this.labelNames.length === 0) { + result.observe(this.currentValue); + return; + } + + for (const [labelStr, value] of this.labeledValues.entries()) { + const labels = this.parseLabelsSafely(labelStr); + if (labels) { + result.observe(value, labels); + } } - // TODO: fix labels - result.observe(this.currentValue); }); } + addCollect(collectFn: CollectFn): void { + this.collectFns.push(collectFn); + } + +/** + * Increments the gauge value + * @param labelsOrValue - Labels object or numeric value + * @param value - Value to increment by (defaults to 1) + */ inc(value?: number): void; inc(labels: Labels, value?: number): void; inc(labelsOrValue?: Labels | number, value?: number): void { - const { labels, value: actualValue } = getLabelAndValue(labelsOrValue, value); - const typedValue = actualValue as number; - - this.currentValue += typedValue; + if (typeof labelsOrValue === 'number') { + this.currentValue += labelsOrValue; + return; } - dec(labels?: Labels): void { - this.currentValue -= 1; + if (labelsOrValue) { + this.validateLabels(labelsOrValue); + const labelKey = JSON.stringify(labelsOrValue); + const currentValue = this.labeledValues.get(labelKey) ?? 0; + this.labeledValues.set(labelKey, currentValue + (value ?? 1)); + return; } - // In prom, set is for an observable value?? - set(value: number): void; - set(labels: Labels, value: number): void; - // Implementation - set(labelsOrValue: Labels | number, value?: number): void { - if (typeof labelsOrValue === 'number') { - // Case: set(value) - this.currentValue = labelsOrValue; - } else { - // Case: set(labels, value) - if (value === undefined) { - throw new Error('Value must be provided when using labels'); - } - this.currentValue = value; - } - } + this.currentValue += value ?? 1; +} - setToCurrentTime(labels?: Labels): void { - this.set(labels || {} as Labels, Date.now()); +/** + * Sets the gauge value + * @param labelsOrValue - Labels object or numeric value + * @param value - Value to set + */ +set(value: number): void; +set(labels: Labels, value: number): void; +set(labelsOrValue: Labels | number, value?: number): void { + if (typeof labelsOrValue === 'number') { + this.currentValue = labelsOrValue; + return; } - startTimer(labels?: Labels): (labels?: Labels) => void { - const start = Date.now(); - return (endLabels?: Labels) => { - const duration = Date.now() - start; - this.set(endLabels || labels || {} as Labels, duration); - }; + this.validateLabels(labelsOrValue); + const labelKey = JSON.stringify(labelsOrValue); + this.labeledValues.set(labelKey, value!); +} + +/** + * Decrements the gauge value + * @param labels - Optional labels object + */ +dec(labels?: Labels): void { + if (labels) { + this.validateLabels(labels); + const labelKey = JSON.stringify(labels); + const currentValue = this.labeledValues.get(labelKey) ?? 0; + this.labeledValues.set(labelKey, currentValue - 1); + return; } - reset(): void { - this.currentValue = 0; + this.currentValue -= 1; +} + +/** + * Resets the gauge to initial state + */ +reset(): void { + this.currentValue = 0; + this.labeledValues.clear(); +} + +/** + * Validates that provided labels match the expected schema + * @param labels - Labels object to validate + * @throws Error if invalid labels are provided + */ +private validateLabels(labels: Labels): void { + if (this.labelNames.length === 0) { + throw new Error('Gauge was initialized without labels support'); } - set collect(callback: () => void) { - this.collectCallback = callback; + for (const key of Object.keys(labels)) { + if (!this.labelNames.includes(key as keyof Labels)) { + throw new Error(`Invalid label key: ${key}`); + } } +} - // TODO: implement - addCollect(): void {} +/** + * Safely parses label string back to object + * @param labelStr - Stringified labels object + * @returns Labels object or null if parsing fails + */ +private parseLabelsSafely(labelStr: string): Labels | null { + try { + return JSON.parse(labelStr) as Labels; + } catch { + console.error(`Failed to parse label string: ${labelStr}`); + return null; + } +} } -class OtelHistogram implements Histogram { +class OtelHistogram implements IHistogram { private histogram; constructor( @@ -187,7 +256,7 @@ class OtelHistogram implements Histogra } -class OtelAvgMinMax implements AvgMinMax { +class OtelAvgMinMax implements IAvgMinMax { private minGauge; private maxGauge; private avgGauge; @@ -281,7 +350,7 @@ export class OtelMetricsAdapter extends Registry implements MetricsRegister { gauge( configuration: GaugeConfig - ): Gauge { + ): IGauge { return new OtelGauge( this.meter, configuration.name as Metrics, @@ -292,7 +361,7 @@ export class OtelMetricsAdapter extends Registry implements MetricsRegister { histogram( configuration: HistogramConfig - ): Histogram { + ): IHistogram { return new OtelHistogram( this.meter, configuration.name as Metrics, @@ -304,7 +373,7 @@ export class OtelMetricsAdapter extends Registry implements MetricsRegister { avgMinMax( configuration: AvgMinMaxConfig - ): AvgMinMax { + ): IAvgMinMax { return new OtelAvgMinMax( this.meter, configuration.name as Metrics, @@ -340,21 +409,25 @@ export class OtelMetricsAdapter extends Registry implements MetricsRegister { // } } -function getLabelAndValue(valueOrLabels?: number | number[]| Labels, value?: number | number[]): { labels: Labels | undefined, value: number | number[] } { - let labels: Labels | undefined; - let actualValue: number | number[]; +function getLabelAndValue( + valueOrLabels?: number | number[] | Labels, + value?: number | number[] +): { labels: Labels | undefined, value: number | number[] } { + // If it's a number, it's a direct value if (typeof valueOrLabels === 'number') { - actualValue = valueOrLabels; - // it is an array - } else if (typeof valueOrLabels === 'object') { - actualValue = valueOrLabels as number[]; - } else if (valueOrLabels !== undefined) { - labels = valueOrLabels; - actualValue = value ?? 1; - } else { - actualValue = 1; + return { labels: undefined, value: valueOrLabels }; + } + + // If it's an array, it's a value array + if (Array.isArray(valueOrLabels)) { + return { labels: undefined, value: valueOrLabels }; } - return { labels, value: actualValue }; + + // Otherwise it's a labels object + return { + labels: valueOrLabels as Labels, + value: value ?? 1 + }; } // class OtelCounter implements Counter { From 54d9b6c92d337d61e5d1797b757c59fcaee4b0f2 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Mon, 4 Nov 2024 19:17:37 +0000 Subject: [PATCH 06/69] fix: noop non gauge for now --- .../end-to-end/src/fixtures/setup_p2p_test.ts | 6 +- yarn-project/p2p/src/mocks/index.ts | 8 +- .../p2p/src/service/libp2p_service.ts | 23 +-- yarn-project/telemetry-client/src/otel.ts | 7 +- .../telemetry-client/src/prom_otel_adapter.ts | 176 ++++-------------- 5 files changed, 64 insertions(+), 156 deletions(-) diff --git a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts index 1887c699157..287a0290ae9 100644 --- a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts +++ b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts @@ -2,7 +2,7 @@ * Test fixtures and utilities to set up and run a test using multiple validators */ import { type AztecNodeConfig, AztecNodeService } from '@aztec/aztec-node'; -import { type SentTx, createDebugLogger } from '@aztec/aztec.js'; +import { type SentTx, createDebugLogger, sleep } from '@aztec/aztec.js'; import { type AztecAddress } from '@aztec/circuits.js'; import { type PXEService } from '@aztec/pxe'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; @@ -42,7 +42,7 @@ export function generatePeerIdPrivateKeys(numberOfPeers: number): string[] { return peerIdPrivateKeys; } -export function createNodes( +export async function createNodes( config: AztecNodeConfig, peerIdPrivateKeys: string[], bootstrapNodeEnr: string, @@ -58,6 +58,8 @@ export function createNodes( const dataDir = dataDirectory ? `${dataDirectory}-${i}` : undefined; const nodePromise = createNode(config, peerIdPrivateKeys[i], port, bootstrapNodeEnr, i, dataDir, metricsPort); + // toO:D remove + await sleep(3000); nodePromises.push(nodePromise); } return Promise.all(nodePromises); diff --git a/yarn-project/p2p/src/mocks/index.ts b/yarn-project/p2p/src/mocks/index.ts index 7c0a00fbe83..eddb1714a58 100644 --- a/yarn-project/p2p/src/mocks/index.ts +++ b/yarn-project/p2p/src/mocks/index.ts @@ -5,7 +5,7 @@ import { type WorldStateSynchronizer, } from '@aztec/circuit-types'; import { type DataStoreConfig } from '@aztec/kv-store/utils'; -import { type TelemetryClient } from '@aztec/telemetry-client'; +import { OtelMetricsAdapter, type TelemetryClient } from '@aztec/telemetry-client'; import { gossipsub } from '@chainsafe/libp2p-gossipsub'; import { noise } from '@chainsafe/libp2p-noise'; @@ -26,6 +26,7 @@ import { type PeerManager } from '../service/peer_manager.js'; import { type P2PReqRespConfig } from '../service/reqresp/config.js'; import { pingHandler, statusHandler } from '../service/reqresp/handlers.js'; import { + DEFAULT_SUB_PROTOCOL_HANDLERS, PING_PROTOCOL, type ReqRespSubProtocolHandlers, type ReqRespSubProtocolValidators, @@ -121,6 +122,8 @@ export async function createTestLibP2PService( // No bootstrap nodes provided as the libp2p service will register them in the constructor const p2pNode = await createLibp2pNode([], peerId, port, /*enable gossip */ true, /**start */ false); + const otelMetricsAdapter = new OtelMetricsAdapter(telemetry); + return new LibP2PService( config, p2pNode as PubSubLibp2p, @@ -130,6 +133,9 @@ export async function createTestLibP2PService( proofVerifier, worldStateSynchronizer, telemetry, + // TODO: remove below here + DEFAULT_SUB_PROTOCOL_HANDLERS, + otelMetricsAdapter, ); } diff --git a/yarn-project/p2p/src/service/libp2p_service.ts b/yarn-project/p2p/src/service/libp2p_service.ts index 6c03ba7ac86..58d5e220d1f 100644 --- a/yarn-project/p2p/src/service/libp2p_service.ts +++ b/yarn-project/p2p/src/service/libp2p_service.ts @@ -105,6 +105,9 @@ export class LibP2PService extends WithTracer implements P2PService { private worldStateSynchronizer: WorldStateSynchronizer, telemetry: TelemetryClient, private requestResponseHandlers: ReqRespSubProtocolHandlers = DEFAULT_SUB_PROTOCOL_HANDLERS, + + // Temp + private metricsAdapter: OtelMetricsAdapter, private logger = createDebugLogger('aztec:libp2p_service'), ) { // Instatntiate tracer @@ -220,7 +223,7 @@ export class LibP2PService extends WithTracer implements P2PService { const datastore = new AztecDatastore(store); - // const otelMetricsAdapter = new OtelMetricsAdapter(telemetry); + const otelMetricsAdapter = new OtelMetricsAdapter(telemetry); const node = await createLibp2p({ start: false, @@ -249,11 +252,6 @@ export class LibP2PService extends WithTracer implements P2PService { minConnections: minPeerCount, maxConnections: maxPeerCount, }, - // metrics: prometheusMetrics({ - // registry: otelMetricsAdapter, - // collectDefaultMetrics: false, - // preserveExistingMetrics: true - // }), services: { identify: identify({ protocolPrefix: 'aztec', @@ -266,8 +264,8 @@ export class LibP2PService extends WithTracer implements P2PService { heartbeatInterval: config.gossipsubInterval, mcacheLength: config.gossipsubMcacheLength, mcacheGossip: config.gossipsubMcacheGossip, - // metricsRegister: otelMetricsAdapter, - // metricsTopicStrToLabel: metricsTopicStrToLabels(), + metricsRegister: otelMetricsAdapter, + metricsTopicStrToLabel: metricsTopicStrToLabels(), scoreParams: createPeerScoreParams({ topics: { [Tx.p2pTopic]: createTopicScoreParams({ @@ -326,15 +324,10 @@ export class LibP2PService extends WithTracer implements P2PService { worldStateSynchronizer, telemetry, requestResponseHandlers, + otelMetricsAdapter, ); } - registerGossipSubMetrics() { - // - - - } - /** * Send Request via the ReqResp service * The subprotocol defined will determine the request and response types @@ -617,6 +610,8 @@ export class LibP2PService extends WithTracer implements P2PService { // Libp2p seems to hang sometimes if new peers are initiating connections. private async stopLibP2P() { + console.log(this.metricsAdapter.outputNames()); + const TIMEOUT_MS = 5000; // 5 seconds timeout const timeout = new Promise((resolve, reject) => { setTimeout(() => reject(new Error('Timeout during libp2p.stop()')), TIMEOUT_MS); diff --git a/yarn-project/telemetry-client/src/otel.ts b/yarn-project/telemetry-client/src/otel.ts index 422fee32efe..93dbddb13af 100644 --- a/yarn-project/telemetry-client/src/otel.ts +++ b/yarn-project/telemetry-client/src/otel.ts @@ -3,6 +3,7 @@ import { type DebugLogger } from '@aztec/foundation/log'; import { DiagConsoleLogger, DiagLogLevel, + DiagLogger, type Meter, type Tracer, type TracerProvider, @@ -28,6 +29,7 @@ import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION } from '@opentele import { aztecDetector } from './aztec_resource_detector.js'; import { registerOtelLoggerProvider } from './otelLoggerProvider.js'; import { type Gauge, type TelemetryClient } from './telemetry.js'; +import { toFriendlyJSON } from '@aztec/foundation/serialize'; export class OpenTelemetryClient implements TelemetryClient { hostMetrics: HostMetrics | undefined; @@ -51,7 +53,7 @@ export class OpenTelemetryClient implements TelemetryClient { public start() { this.log.info('Starting OpenTelemetry client'); - diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO); + diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG); this.hostMetrics = new HostMetrics({ name: this.resource.attributes[SEMRESATTRS_SERVICE_NAME] as string, @@ -117,6 +119,9 @@ export class OpenTelemetryClient implements TelemetryClient { exporter: new OTLPMetricExporter({ url: metricsCollector.href, }), + // TODO: make this configurable for debug builds + exportIntervalMillis: 5000, + exportTimeoutMillis: 2500 }), ], }); diff --git a/yarn-project/telemetry-client/src/prom_otel_adapter.ts b/yarn-project/telemetry-client/src/prom_otel_adapter.ts index 8b147d5bede..534a9c7fdf6 100644 --- a/yarn-project/telemetry-client/src/prom_otel_adapter.ts +++ b/yarn-project/telemetry-client/src/prom_otel_adapter.ts @@ -1,6 +1,6 @@ import { Meter, Metrics, TelemetryClient, ObservableGauge, Attributes, Gauge } from "./telemetry.js"; -import { Registry, Counter as PromCounter, Gauge as PromGauge, Histogram as PromHistogram } from 'prom-client'; +import { Registry, Gauge as PromGauge, Histogram as PromHistogram } from 'prom-client'; type TopicStr = string; export type TopicLabel = string @@ -213,40 +213,23 @@ private parseLabelsSafely(labelStr: string): Labels | null { } -class OtelHistogram implements IHistogram { - private histogram; +class MockOtelHistogram implements IHistogram { constructor( - meter: Meter, - name: Metrics, // Metrics must be registered in the aztec labels registry - help: string, - private buckets: number[] = [], - labelNames: Array = [] - ) { - // TODO: deal with buckets - this.histogram = meter.createHistogram(name, { - description: help, - }); - } + _meter: Meter, + _name: string, // Metrics must be registered in the aztec labels registry + _help: string, + _buckets: number[] = [], + _labelNames: Array = [] + ) {} // Overload signatures - observe(value: number): void; - observe(labels: Labels, value: number): void; - // Implementation - observe(valueOrLabels: number | Labels, value?: number): void { - const { labels, value: actualValue } = getLabelAndValue(valueOrLabels, value); - const typedValue = actualValue as number; - - this.histogram.record(typedValue, labels); - } + observe(_value: number): void; + observe(_labels: Labels, _value: number): void; + observe(_valueOrLabels: number | Labels, _value?: number): void {} - startTimer(labels?: Labels): (labels?: Labels) => number { - const start = performance.now(); - return (endLabels?: Labels) => { - const duration = performance.now() - start; - this.observe(endLabels || labels || {} as Labels, duration); - return duration; - }; + startTimer(_labels?: Labels): (_labels?: Labels) => number { + return () => 0; } reset(): void { @@ -255,102 +238,45 @@ class OtelHistogram implements IHistogr } } - -class OtelAvgMinMax implements IAvgMinMax { - private minGauge; - private maxGauge; - private avgGauge; - private count: number = 0; - private sum: number = 0; - private min: number = Infinity; - private max: number = -Infinity; - +class MockOtelAvgMinMax implements IAvgMinMax { constructor( meter: Meter, - // TODO: be more strict on metrics name types - name: string, // Metrics must be registered in the aztec labels registry - help: string, - labelNames: Array = [] - ) { - this.minGauge = meter.createObservableGauge(`${name}_min` as Metrics, { - description: `${help} (minimum)`, - }); - - this.maxGauge = meter.createObservableGauge(`${name}_max` as Metrics, { - description: `${help} (maximum)`, - }); - - this.avgGauge = meter.createObservableGauge(`${name}_avg` as Metrics, { - description: `${help} (average)`, - }); - - this.setupCallbacks(); - } - - private setupCallbacks(): void { - this.minGauge.addCallback((result) => { - result.observe(this.min, {}); - }); - - this.maxGauge.addCallback((result) => { - result.observe(this.max, {}); - }); - - this.avgGauge.addCallback((result) => { - const avg = this.getAverage(); - if (avg !== undefined) { - result.observe(avg, {}); - } - }); - } - - private getLabelKey(labels?: Labels): string { - return JSON.stringify(labels || {}); - } - - private getAverage(): number | undefined { - if (this.count > 0) { - return this.sum / this.count; - } - return undefined; - } - - // TODO(md): just set up two different classes, one with labels and one without that can inherit from each other - - - set(values: number[]): void; - set(labels: Labels, values: number[]): void; - set(valueOrLabels: number[] | Labels, values?: number[]): void { - const { labels, value: actualValue } = getLabelAndValue(valueOrLabels, values); - const actualValues = actualValue as number[]; + _name: string, // Metrics must be registered in the aztec labels registry + _help: string, + _labelNames: Array = [] + ) {} - this.count += actualValues.length; + set(_values: number[]): void; + set(_labels: Labels, _values: number[]): void; + set(_valueOrLabels: number[] | Labels, _values?: number[]): void {} - const sorted = actualValues.sort((a, b) => a - b); - this.sum += sorted.reduce((acc, curr) => acc + curr, 0); - this.min = sorted[0]; - this.max = sorted[sorted.length - 1]; - } - - reset(): void { - this.count = 0; - this.sum = 0; - this.min = Infinity; - this.max = -Infinity; - } + reset(): void {} } export class OtelMetricsAdapter extends Registry implements MetricsRegister { private readonly meter: Meter; + // Check if any labels we receive are the same as any we've already seen + private trackNames: Map = new Map(); + constructor(telemetryClient: TelemetryClient) { super(); this.meter = telemetryClient.getMeter('metrics-adapter'); } + outputNames(): [string, number][] { + return Array.from(this.trackNames.entries()); + } + + private trackName(name: string) { + const labelKey = this.trackNames.get(name) ?? 0; + this.trackNames.set(name, labelKey + 1); + } + gauge( configuration: GaugeConfig ): IGauge { + this.trackName(configuration.name); return new OtelGauge( this.meter, configuration.name as Metrics, @@ -362,7 +288,7 @@ export class OtelMetricsAdapter extends Registry implements MetricsRegister { histogram( configuration: HistogramConfig ): IHistogram { - return new OtelHistogram( + return new MockOtelHistogram( this.meter, configuration.name as Metrics, configuration.help, @@ -374,7 +300,7 @@ export class OtelMetricsAdapter extends Registry implements MetricsRegister { avgMinMax( configuration: AvgMinMaxConfig ): IAvgMinMax { - return new OtelAvgMinMax( + return new MockOtelAvgMinMax( this.meter, configuration.name as Metrics, configuration.help, @@ -428,30 +354,4 @@ function getLabelAndValue( labels: valueOrLabels as Labels, value: value ?? 1 }; -} - -// class OtelCounter implements Counter { -// private counter; - -// constructor( -// meter: Meter, -// name: string, // Metrics must be registered in the aztec labels registry -// help: string, -// labelNames: Array = [] -// ) { -// // TODO: be more strict on metrics name types -// this.counter = meter.createUpDownCounter(name as Metrics, { -// description: help, -// unit: '1', -// }); -// } - -// inc(labels?: Labels, value: number = 1): void { -// this.counter.add(value, labels); -// } - -// reset(): void { -// // OpenTelemetry counters cannot be reset, but we implement the interface -// console.warn('OpenTelemetry counters cannot be reset'); -// } -// } \ No newline at end of file +} \ No newline at end of file From fe6974cbf2373304dbfef3c7308a5c9955d71607 Mon Sep 17 00:00:00 2001 From: Maddiaa0 <47148561+Maddiaa0@users.noreply.github.com> Date: Tue, 5 Nov 2024 05:35:26 +0000 Subject: [PATCH 07/69] fmt --- .../src/e2e_p2p/gossip_network.test.ts | 7 +- yarn-project/end-to-end/src/fixtures/index.ts | 2 +- .../end-to-end/src/fixtures/setup_p2p_test.ts | 7 +- .../src/fixtures/snapshot_manager.ts | 3 +- .../src/fixtures/with_telemetry_utils.ts | 13 +++- yarn-project/p2p/src/bootstrap/bootstrap.ts | 7 +- yarn-project/p2p/src/mocks/index.ts | 13 +++- .../p2p/src/service/discV5_service.ts | 9 ++- .../p2p/src/service/discv5_service.test.ts | 2 +- .../p2p/src/service/libp2p_service.ts | 6 +- yarn-project/telemetry-client/src/otel.ts | 4 +- .../telemetry-client/src/prom_otel_adapter.ts | 67 +++++++------------ 12 files changed, 65 insertions(+), 75 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts index 2f24ffa9531..f34989ab16f 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts @@ -3,10 +3,10 @@ import { sleep } from '@aztec/aztec.js'; import fs from 'fs'; +import { METRICS_PORT } from '../fixtures/fixtures.js'; import { type NodeContext, createNodes } from '../fixtures/setup_p2p_test.js'; import { P2PNetworkTest, WAIT_FOR_TX_TIMEOUT } from './p2p_network.js'; import { createPXEServiceAndSubmitTransactions } from './shared.js'; -import { METRICS_PORT } from '../fixtures/fixtures.js'; // Don't set this to a higher value than 9 because each node will use a different L1 publisher account and anvil seeds const NUM_NODES = 4; @@ -24,7 +24,8 @@ describe('e2e_p2p_network', () => { testName: 'e2e_p2p_network', numberOfNodes: NUM_NODES, basePort: BOOT_NODE_UDP_PORT, - metricsPort: METRICS_PORT, + // Uncomment to collect metrics - run in aztec-packages `docker compose --profile metrics up` + // metricsPort: METRICS_PORT, }); await t.applyBaseSnapshots(); await t.setup(); @@ -39,7 +40,7 @@ describe('e2e_p2p_network', () => { }); // TODO(https://github.com/AztecProtocol/aztec-packages/issues/9164): Currently flakey - it('should rollup txs from all peers', async () => { + it.skip('should rollup txs from all peers', async () => { // create the bootstrap node for the network if (!t.bootstrapNodeEnr) { throw new Error('Bootstrap node ENR is not available'); diff --git a/yarn-project/end-to-end/src/fixtures/index.ts b/yarn-project/end-to-end/src/fixtures/index.ts index 05ab9ddbfff..af43dc38012 100644 --- a/yarn-project/end-to-end/src/fixtures/index.ts +++ b/yarn-project/end-to-end/src/fixtures/index.ts @@ -2,4 +2,4 @@ export * from './fixtures.js'; export * from './logging.js'; export * from './utils.js'; export * from './token_utils.js'; -export * from './with_telemetry_utils.js'; \ No newline at end of file +export * from './with_telemetry_utils.js'; diff --git a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts index 287a0290ae9..46994fbfd7f 100644 --- a/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts +++ b/yarn-project/end-to-end/src/fixtures/setup_p2p_test.ts @@ -2,10 +2,9 @@ * Test fixtures and utilities to set up and run a test using multiple validators */ import { type AztecNodeConfig, AztecNodeService } from '@aztec/aztec-node'; -import { type SentTx, createDebugLogger, sleep } from '@aztec/aztec.js'; +import { type SentTx, createDebugLogger } from '@aztec/aztec.js'; import { type AztecAddress } from '@aztec/circuits.js'; import { type PXEService } from '@aztec/pxe'; -import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import getPort from 'get-port'; import { generatePrivateKey } from 'viem/accounts'; @@ -42,7 +41,7 @@ export function generatePeerIdPrivateKeys(numberOfPeers: number): string[] { return peerIdPrivateKeys; } -export async function createNodes( +export function createNodes( config: AztecNodeConfig, peerIdPrivateKeys: string[], bootstrapNodeEnr: string, @@ -58,8 +57,6 @@ export async function createNodes( const dataDir = dataDirectory ? `${dataDirectory}-${i}` : undefined; const nodePromise = createNode(config, peerIdPrivateKeys[i], port, bootstrapNodeEnr, i, dataDir, metricsPort); - // toO:D remove - await sleep(3000); nodePromises.push(nodePromise); } return Promise.all(nodePromises); diff --git a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts index 89c82a0f153..29a2f11e367 100644 --- a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts +++ b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts @@ -24,7 +24,7 @@ import { resolver, reviver } from '@aztec/foundation/serialize'; import { type ProverNode, type ProverNodeConfig, createProverNode } from '@aztec/prover-node'; import { type PXEService, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; -import { createAndStartTelemetryClient, getConfigEnvVars as getTelemetryConfig, TelemetryClientConfig } from '@aztec/telemetry-client/start'; +import { createAndStartTelemetryClient, getConfigEnvVars as getTelemetryConfig } from '@aztec/telemetry-client/start'; import { type Anvil, createAnvil } from '@viem/anvil'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; @@ -313,7 +313,6 @@ async function setupFromFresh( const aztecNodeConfig: AztecNodeConfig & SetupOptions = { ...getConfigEnvVars(), ...opts }; aztecNodeConfig.dataDirectory = statePath; - // Start anvil. We go via a wrapper script to ensure if the parent dies, anvil dies. logger.verbose('Starting anvil...'); const res = await startAnvil(opts.l1BlockTime); diff --git a/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts b/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts index 7c5d7255a50..1b8792dff9e 100644 --- a/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts +++ b/yarn-project/end-to-end/src/fixtures/with_telemetry_utils.ts @@ -1,6 +1,10 @@ -import { TelemetryClient } from '@aztec/telemetry-client'; +import { type TelemetryClient } from '@aztec/telemetry-client'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; -import { TelemetryClientConfig, getConfigEnvVars as getTelemetryConfig, createAndStartTelemetryClient } from '@aztec/telemetry-client/start'; +import { + type TelemetryClientConfig, + createAndStartTelemetryClient, + getConfigEnvVars as getTelemetryConfig, +} from '@aztec/telemetry-client/start'; export function getEndToEndTestTelemetryClient(metricsPort?: number, serviceName?: string): Promise { return !metricsPort @@ -19,9 +23,12 @@ export function getEndToEndTestTelemetryConfig(metricsPort?: number, serviceName telemetryConfig.metricsCollectorUrl = new URL(`http://127.0.0.1:${metricsPort}/v1/metrics`); telemetryConfig.tracesCollectorUrl = new URL(`http://127.0.0.1:${metricsPort}/v1/traces`); telemetryConfig.logsCollectorUrl = new URL(`http://127.0.0.1:${metricsPort}/v1/logs`); + // If we are in an end to end test, we want to collect metrics more frequently + telemetryConfig.otelCollectIntervalMs = 5000; + telemetryConfig.otelExportTimeoutMs = 2500; } if (serviceName) { telemetryConfig.serviceName = serviceName; } return telemetryConfig; -} \ No newline at end of file +} diff --git a/yarn-project/p2p/src/bootstrap/bootstrap.ts b/yarn-project/p2p/src/bootstrap/bootstrap.ts index 22c8409d3f9..065574030fa 100644 --- a/yarn-project/p2p/src/bootstrap/bootstrap.ts +++ b/yarn-project/p2p/src/bootstrap/bootstrap.ts @@ -1,4 +1,5 @@ import { createDebugLogger } from '@aztec/foundation/log'; +import { OtelMetricsAdapter, type TelemetryClient } from '@aztec/telemetry-client'; import { Discv5, type Discv5EventEmitter } from '@chainsafe/discv5'; import { SignableENR } from '@chainsafe/enr'; @@ -9,7 +10,6 @@ import type { BootnodeConfig } from '../config.js'; import { AZTEC_ENR_KEY, AZTEC_NET } from '../service/discV5_service.js'; import { createLibP2PPeerId } from '../service/index.js'; import { convertToMultiaddr } from '../util.js'; -import { OtelMetricsAdapter, TelemetryClient } from '@aztec/telemetry-client'; /** * Encapsulates a 'Bootstrap' node, used for the purpose of assisting new joiners in acquiring peers. @@ -18,10 +18,7 @@ export class BootstrapNode { private node?: Discv5 = undefined; private peerId?: PeerId; - constructor( - private telemetry: TelemetryClient, - private logger = createDebugLogger('aztec:p2p_bootstrap'), - ) {} + constructor(private telemetry: TelemetryClient, private logger = createDebugLogger('aztec:p2p_bootstrap')) {} /** * Starts the bootstrap node. diff --git a/yarn-project/p2p/src/mocks/index.ts b/yarn-project/p2p/src/mocks/index.ts index eddb1714a58..ae920ed61bf 100644 --- a/yarn-project/p2p/src/mocks/index.ts +++ b/yarn-project/p2p/src/mocks/index.ts @@ -6,6 +6,7 @@ import { } from '@aztec/circuit-types'; import { type DataStoreConfig } from '@aztec/kv-store/utils'; import { OtelMetricsAdapter, type TelemetryClient } from '@aztec/telemetry-client'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { gossipsub } from '@chainsafe/libp2p-gossipsub'; import { noise } from '@chainsafe/libp2p-noise'; @@ -36,7 +37,6 @@ import { } from '../service/reqresp/interface.js'; import { ReqResp } from '../service/reqresp/reqresp.js'; import { type PubSubLibp2p } from '../util.js'; -import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; /** * Creates a libp2p node, pre configured. @@ -240,12 +240,19 @@ export function createBootstrapNodeConfig(privateKey: string, port: number): Boo }; } -export function createBootstrapNodeFromPrivateKey(privateKey: string, port: number, telemetry: TelemetryClient = new NoopTelemetryClient()): Promise { +export function createBootstrapNodeFromPrivateKey( + privateKey: string, + port: number, + telemetry: TelemetryClient = new NoopTelemetryClient(), +): Promise { const config = createBootstrapNodeConfig(privateKey, port); return startBootstrapNode(config, telemetry); } -export async function createBootstrapNode(port: number, telemetry: TelemetryClient = new NoopTelemetryClient()): Promise { +export async function createBootstrapNode( + port: number, + telemetry: TelemetryClient = new NoopTelemetryClient(), +): Promise { const peerId = await createLibP2PPeerId(); const config = createBootstrapNodeConfig(Buffer.from(peerId.privateKey!).toString('hex'), port); diff --git a/yarn-project/p2p/src/service/discV5_service.ts b/yarn-project/p2p/src/service/discV5_service.ts index 091d9e2ee20..324dd60ad13 100644 --- a/yarn-project/p2p/src/service/discV5_service.ts +++ b/yarn-project/p2p/src/service/discV5_service.ts @@ -1,5 +1,6 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { sleep } from '@aztec/foundation/sleep'; +import { OtelMetricsAdapter, type TelemetryClient } from '@aztec/telemetry-client'; import { Discv5, type Discv5EventEmitter } from '@chainsafe/discv5'; import { ENR, SignableENR } from '@chainsafe/enr'; @@ -10,7 +11,6 @@ import EventEmitter from 'events'; import type { P2PConfig } from '../config.js'; import { convertToMultiaddr } from '../util.js'; import { type PeerDiscoveryService, PeerDiscoveryState } from './service.js'; -import { OtelMetricsAdapter, TelemetryClient } from '@aztec/telemetry-client'; export const AZTEC_ENR_KEY = 'aztec_network'; @@ -42,7 +42,12 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService private startTime = 0; - constructor(private peerId: PeerId, config: P2PConfig, telemetry: TelemetryClient, private logger = createDebugLogger('aztec:discv5_service')) { + constructor( + private peerId: PeerId, + config: P2PConfig, + telemetry: TelemetryClient, + private logger = createDebugLogger('aztec:discv5_service'), + ) { super(); const { tcpAnnounceAddress, udpAnnounceAddress, udpListenAddress, bootstrapNodes } = config; this.bootstrapNodes = bootstrapNodes; diff --git a/yarn-project/p2p/src/service/discv5_service.test.ts b/yarn-project/p2p/src/service/discv5_service.test.ts index 1f7e6fb169a..75ae683676f 100644 --- a/yarn-project/p2p/src/service/discv5_service.test.ts +++ b/yarn-project/p2p/src/service/discv5_service.test.ts @@ -1,4 +1,5 @@ import { sleep } from '@aztec/foundation/sleep'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { jest } from '@jest/globals'; import type { PeerId } from '@libp2p/interface'; @@ -8,7 +9,6 @@ import { type P2PConfig, getP2PDefaultConfig } from '../config.js'; import { DiscV5Service } from './discV5_service.js'; import { createLibP2PPeerId } from './libp2p_service.js'; import { PeerDiscoveryState } from './service.js'; -import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; const waitForPeers = (node: DiscV5Service, expectedCount: number): Promise => { const timeout = 7_000; diff --git a/yarn-project/p2p/src/service/libp2p_service.ts b/yarn-project/p2p/src/service/libp2p_service.ts index 58d5e220d1f..7224a53c218 100644 --- a/yarn-project/p2p/src/service/libp2p_service.ts +++ b/yarn-project/p2p/src/service/libp2p_service.ts @@ -6,13 +6,13 @@ import { type Gossipable, type L2BlockSource, MerkleTreeId, - metricsTopicStrToLabels, type RawGossipMessage, TopicType, TopicTypeMap, Tx, TxHash, type WorldStateSynchronizer, + metricsTopicStrToLabels, } from '@aztec/circuit-types'; import { Fr } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; @@ -33,7 +33,6 @@ import { mplex } from '@libp2p/mplex'; import { createFromJSON, createSecp256k1PeerId } from '@libp2p/peer-id-factory'; import { tcp } from '@libp2p/tcp'; import { createLibp2p } from 'libp2p'; -import { prometheusMetrics } from "@libp2p/prometheus-metrics"; import { type P2PConfig } from '../config.js'; import { type MemPools } from '../mem_pools/interface.js'; @@ -294,7 +293,6 @@ export class LibP2PService extends WithTracer implements P2PService { }, }); - // Create request response protocol handlers /** * Handler for tx requests @@ -610,8 +608,6 @@ export class LibP2PService extends WithTracer implements P2PService { // Libp2p seems to hang sometimes if new peers are initiating connections. private async stopLibP2P() { - console.log(this.metricsAdapter.outputNames()); - const TIMEOUT_MS = 5000; // 5 seconds timeout const timeout = new Promise((resolve, reject) => { setTimeout(() => reject(new Error('Timeout during libp2p.stop()')), TIMEOUT_MS); diff --git a/yarn-project/telemetry-client/src/otel.ts b/yarn-project/telemetry-client/src/otel.ts index 93dbddb13af..0314481da12 100644 --- a/yarn-project/telemetry-client/src/otel.ts +++ b/yarn-project/telemetry-client/src/otel.ts @@ -3,7 +3,6 @@ import { type DebugLogger } from '@aztec/foundation/log'; import { DiagConsoleLogger, DiagLogLevel, - DiagLogger, type Meter, type Tracer, type TracerProvider, @@ -29,7 +28,6 @@ import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION } from '@opentele import { aztecDetector } from './aztec_resource_detector.js'; import { registerOtelLoggerProvider } from './otelLoggerProvider.js'; import { type Gauge, type TelemetryClient } from './telemetry.js'; -import { toFriendlyJSON } from '@aztec/foundation/serialize'; export class OpenTelemetryClient implements TelemetryClient { hostMetrics: HostMetrics | undefined; @@ -53,7 +51,7 @@ export class OpenTelemetryClient implements TelemetryClient { public start() { this.log.info('Starting OpenTelemetry client'); - diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG); + diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO); this.hostMetrics = new HostMetrics({ name: this.resource.attributes[SEMRESATTRS_SERVICE_NAME] as string, diff --git a/yarn-project/telemetry-client/src/prom_otel_adapter.ts b/yarn-project/telemetry-client/src/prom_otel_adapter.ts index 534a9c7fdf6..f15c6aa97a4 100644 --- a/yarn-project/telemetry-client/src/prom_otel_adapter.ts +++ b/yarn-project/telemetry-client/src/prom_otel_adapter.ts @@ -1,7 +1,10 @@ -import { Meter, Metrics, TelemetryClient, ObservableGauge, Attributes, Gauge } from "./telemetry.js"; +import { type Meter, type Metrics, type TelemetryClient, type ObservableGauge, Attributes, Gauge } from "./telemetry.js"; -import { Registry, Gauge as PromGauge, Histogram as PromHistogram } from 'prom-client'; +import { Registry } from 'prom-client'; +/** + * Types matching the gossipsub and libp2p services + */ type TopicStr = string; export type TopicLabel = string export type TopicStrToLabel = Map @@ -21,7 +24,6 @@ interface IGauge { set: NoLabels extends Labels ? (value: number) => void : (labels: Labels, value: number) => void collect?(): void - addCollect(collectFn: CollectFn): void } interface IHistogram { @@ -53,7 +55,7 @@ export interface MetricsRegister { avgMinMax(config: AvgMinMaxConfig): IAvgMinMax } -/**Otel Metrics Adapter +/**Otel Metrics Adapters * * Some dependencies we use export metrics directly in a Prometheus format * This adapter is used to convert those metrics to a format that we can use with OpenTelemetry @@ -67,7 +69,6 @@ class OtelGauge implements IGauge