From f54ba9a39c1346e3bd6af5d3ba42f22a7d5125ec Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 22 Aug 2024 14:39:00 +0000 Subject: [PATCH] fix: start trace exporter only if required --- docker-compose.yml | 10 ++++++---- yarn-project/aztec/terraform/node/main.tf | 4 ++-- .../aztec/terraform/prover-node/main.tf | 2 +- yarn-project/aztec/terraform/prover/main.tf | 4 ++-- yarn-project/foundation/src/config/env_var.ts | 3 ++- yarn-project/telemetry-client/src/config.ts | 14 ++++++++++---- yarn-project/telemetry-client/src/otel.ts | 17 ++++++++++++----- yarn-project/telemetry-client/src/start.ts | 4 ++-- 8 files changed, 37 insertions(+), 21 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 44b53fbdb29..c894ed45b8e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,7 +62,8 @@ services: P2P_ENABLED: true PEER_ID_PRIVATE_KEY: AZTEC_PORT: 8999 - OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} + OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: ${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-http://otel-collector:4318/v1/metrics} + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-http://otel-collector:4318/v1/traces} secrets: - ethereum-host - p2p-boot-node @@ -77,13 +78,14 @@ services: # if the stack is started with --profile metrics --profile node, give the collector a chance to start before the node i=0 max=3 - while ! curl --head --silent $$OTEL_EXPORTER_OTLP_ENDPOINT > /dev/null; do + while ! curl --head --silent $$OTEL_EXPORTER_OTLP_METRICS_ENDPOINT > /dev/null; do echo "OpenTelemetry collector not up. Retrying after 1s"; sleep 1; i=$$((i+1)); if [ $$i -eq $$max ]; then - echo "OpenTelemetry collector at $$OTEL_EXPORTER_OTLP_ENDPOINT not up after $${max}s. Running without metrics"; - unset OTEL_EXPORTER_OTLP_ENDPOINT; + echo "OpenTelemetry collector at $$OTEL_EXPORTER_METRICS_ENDPOINT not up after $${max}s. Running without metrics"; + unset OTEL_EXPORTER_METRICS_ENDPOINT; + unset OTEL_EXPORTER_TRACES_ENDPOINT; break fi; done; diff --git a/yarn-project/aztec/terraform/node/main.tf b/yarn-project/aztec/terraform/node/main.tf index 4a963fa174d..ef2638d1bbe 100644 --- a/yarn-project/aztec/terraform/node/main.tf +++ b/yarn-project/aztec/terraform/node/main.tf @@ -337,8 +337,8 @@ resource "aws_ecs_task_definition" "aztec-node" { value = tostring(var.PROVING_ENABLED) }, { - name = "OTEL_EXPORTER_OTLP_ENDPOINT" - value = "http://aztec-otel.local:4318" + name = "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT" + value = "http://aztec-otel.local:4318/v1/metrics" }, { name = "OTEL_SERVICE_NAME" diff --git a/yarn-project/aztec/terraform/prover-node/main.tf b/yarn-project/aztec/terraform/prover-node/main.tf index 5cca8711717..e499eacb050 100644 --- a/yarn-project/aztec/terraform/prover-node/main.tf +++ b/yarn-project/aztec/terraform/prover-node/main.tf @@ -202,7 +202,7 @@ resource "aws_ecs_task_definition" "aztec-prover-node" { { name = "PROVER_NODE_MAX_PENDING_JOBS", value = tostring(var.PROVER_NODE_MAX_PENDING_JOBS) }, // Metrics - { name = "OTEL_EXPORTER_OTLP_ENDPOINT", value = "http://aztec-otel.local:4318" }, + { name = "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", value = "http://aztec-otel.local:4318/v1/metrics" }, { name = "OTEL_SERVICE_NAME", value = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" }, // L1 addresses diff --git a/yarn-project/aztec/terraform/prover/main.tf b/yarn-project/aztec/terraform/prover/main.tf index a288f13d8bf..924aae3e6ea 100644 --- a/yarn-project/aztec/terraform/prover/main.tf +++ b/yarn-project/aztec/terraform/prover/main.tf @@ -274,8 +274,8 @@ resource "aws_ecs_task_definition" "aztec-proving-agent" { "value": "${var.PROVING_ENABLED}" }, { - "name": "OTEL_EXPORTER_OTLP_ENDPOINT", - "value": "http://aztec-otel.local:4318" + "name": "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + "value": "http://aztec-otel.local:4318/v1/metrics" }, { "name": "OTEL_SERVICE_NAME", diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index fdb6394eb30..2d9a5029351 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -37,8 +37,9 @@ export type EnvVar = | 'P2P_QUERY_FOR_IP' | 'P2P_TX_POOL_KEEP_PROVEN_FOR' | 'TELEMETRY' - | 'OTEL_EXPORTER_OTLP_ENDPOINT' | 'OTEL_SERVICE_NAME' + | 'OTEL_EXPORTER_OTLP_METRICS_ENDPOINT' + | 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT' | 'NETWORK_NAME' | 'NETWORK' | 'API_KEY' diff --git a/yarn-project/telemetry-client/src/config.ts b/yarn-project/telemetry-client/src/config.ts index 7a11f4e1a8a..c7789ba05bc 100644 --- a/yarn-project/telemetry-client/src/config.ts +++ b/yarn-project/telemetry-client/src/config.ts @@ -1,15 +1,21 @@ import { type ConfigMappingsType, getConfigFromMappings } from '@aztec/foundation/config'; export interface TelemetryClientConfig { - collectorBaseUrl?: URL; + metricsCollectorUrl?: URL; + tracesCollectorUrl?: URL; serviceName: string; networkName: string; } export const telemetryClientConfigMappings: ConfigMappingsType = { - collectorBaseUrl: { - env: 'OTEL_EXPORTER_OTLP_ENDPOINT', - description: 'The URL of the telemetry collector', + metricsCollectorUrl: { + env: 'OTEL_EXPORTER_OTLP_METRICS_ENDPOINT', + description: 'The URL of the telemetry collector for metrics', + parseEnv: (val: string) => new URL(val), + }, + tracesCollectorUrl: { + env: 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT', + description: 'The URL of the telemetry collector for traces', parseEnv: (val: string) => new URL(val), }, serviceName: { diff --git a/yarn-project/telemetry-client/src/otel.ts b/yarn-project/telemetry-client/src/otel.ts index 0a95433dc2b..fb2218d7ac5 100644 --- a/yarn-project/telemetry-client/src/otel.ts +++ b/yarn-project/telemetry-client/src/otel.ts @@ -77,7 +77,11 @@ export class OpenTelemetryClient implements TelemetryClient { await Promise.all([this.meterProvider.shutdown()]); } - public static createAndStart(collectorBaseUrl: URL, log: DebugLogger): OpenTelemetryClient { + public static createAndStart( + metricsCollector: URL, + tracesCollector: URL | undefined, + log: DebugLogger, + ): OpenTelemetryClient { const resource = detectResourcesSync({ detectors: [ osDetectorSync, @@ -93,9 +97,12 @@ export class OpenTelemetryClient implements TelemetryClient { const tracerProvider = new NodeTracerProvider({ resource, }); - tracerProvider.addSpanProcessor( - new BatchSpanProcessor(new OTLPTraceExporter({ url: new URL('/v1/traces', collectorBaseUrl).href })), - ); + + // optionally push traces to an OTEL collector instance + if (tracesCollector) { + tracerProvider.addSpanProcessor(new BatchSpanProcessor(new OTLPTraceExporter({ url: tracesCollector.href }))); + } + tracerProvider.register(); const meterProvider = new MeterProvider({ @@ -103,7 +110,7 @@ export class OpenTelemetryClient implements TelemetryClient { readers: [ new PeriodicExportingMetricReader({ exporter: new OTLPMetricExporter({ - url: new URL('/v1/metrics', collectorBaseUrl).href, + url: metricsCollector.href, }), }), ], diff --git a/yarn-project/telemetry-client/src/start.ts b/yarn-project/telemetry-client/src/start.ts index 5425107e7ee..8ff2dc06ee1 100644 --- a/yarn-project/telemetry-client/src/start.ts +++ b/yarn-project/telemetry-client/src/start.ts @@ -9,9 +9,9 @@ export * from './config.js'; export function createAndStartTelemetryClient(config: TelemetryClientConfig): TelemetryClient { const log = createDebugLogger('aztec:telemetry-client'); - if (config.collectorBaseUrl) { + if (config.metricsCollectorUrl) { log.info('Using OpenTelemetry client'); - return OpenTelemetryClient.createAndStart(config.collectorBaseUrl, log); + return OpenTelemetryClient.createAndStart(config.metricsCollectorUrl, config.tracesCollectorUrl, log); } else { log.info('Using NoopTelemetryClient'); return new NoopTelemetryClient();