Set up otel pino stream to deliver logs to otel collector

yarn-project/accounts/package.json

@@ -102,4 +102,4 @@
   "engines": {
     "node": ">=18"
   }
-}
+}

yarn-project/circuits.js/package.json

@@ -101,4 +101,4 @@
       ]
     ]
   }
-}
+}

yarn-project/foundation/src/log/pino-logger.ts

@@ -2,6 +2,7 @@ import { bold, reset } from 'colorette';
 import { type LoggerOptions, pino } from 'pino';
 import { inspect } from 'util';
 
+import { compactArray } from '../collection/array.js';
 import { getLogLevelFromFilters, parseEnv } from './log-filters.js';
 import { type LogLevel } from './log-levels.js';
 import { type LogData, type LogFn } from './log_fn.js';
@@ -40,31 +41,54 @@ export function createDebugLogger(module: string): DebugLogger {
 const defaultLogLevel = process.env.NODE_ENV === 'test' ? 'silent' : 'info';
 const [logLevel, logFilters] = parseEnv(process.env.LOG_LEVEL, defaultLogLevel);
 
-const pretty: Pick<LoggerOptions, 'transport'> = {
-  transport: {
-    target: 'pino-pretty',
-    options: {
-      sync: true,
-      ignore: 'module,pid,hostname',
-      messageFormat: `${bold('{module}')} ${reset('{msg}')}`,
-      customLevels: 'fatal:60,error:50,warn:40,info:30,verbose:25,debug:20,trace:10',
-      customColors: 'fatal:bgRed,error:red,warn:yellow,info:green,verbose:magenta,debug:blue,trace:gray',
-    },
+// Transport options for pretty logging to stdout via pino-pretty.
+const prettyTransport: LoggerOptions['transport'] = {
+  target: 'pino-pretty',
+  options: {
+    sync: true,
+    ignore: 'module,pid,hostname',
+    messageFormat: `${bold('{module}')} ${reset('{msg}')}`,
+    customLevels: 'fatal:60,error:50,warn:40,info:30,verbose:25,debug:20,trace:10',
+    customColors: 'fatal:bgRed,error:red,warn:yellow,info:green,verbose:magenta,debug:blue,trace:gray',
   },
 };
 
-const logger = pino({
-  customLevels: {
-    verbose: 25,
-  },
-  useOnlyCustomLevels: false,
-  level: logLevel,
-  ...(['1', 'true', 'TRUE'].includes(process.env.LOG_JSON ?? '') ? {} : pretty),
-});
+// Transport for vanilla stdout logging as JSON.
+const stdoutTransport: LoggerOptions['transport'] = {
+  target: 'pino/file',
+  options: { destination: 1 },
+};
+
+// Transport for OpenTelemetry logging. While defining this here is an abstraction leakage since this
+// should live in the telemetry-client, it is necessary to ensure that the logger is initialized with
+// the correct transport. Tweaking transports of a live pino instance is tricky, and creating a new instance
+// would mean that all child loggers created before the telemetry-client is initialized would not have
+// this transport configured. Note that the target is defined as the export in the telemetry-client,
+// since pino will load this transport separately on a worker thread, to minimize disruption to the main loop.
+const customLevels = { verbose: 25 };
+const { levels } = pino({ customLevels, useOnlyCustomLevels: false });
+const otelTransport: LoggerOptions['transport'] = {
+  target: '@aztec/telemetry-client/otel-pino-stream',
+  options: { levels, messageKey: 'msg' },
+};
+
+// Create a new pino instance with an stdout transport (either vanilla or json), and optionally
+// an OTLP transport if the OTLP endpoint is provided. Note that transports are initialized in a
+// worker thread.
+const otlpEndpoint = process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT;
+const logger = pino(
+  { customLevels, useOnlyCustomLevels: false, level: logLevel },
+  pino.transport({
+    targets: compactArray([
+      ['1', 'true', 'TRUE'].includes(process.env.LOG_JSON ?? '') ? stdoutTransport : prettyTransport,
+      process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT ? otelTransport : undefined,
+    ]),
+  }),
+);
 
 logger.info(
   { module: 'logger', ...logFilters.reduce((accum, [module, level]) => ({ ...accum, [`log.${module}`]: level }), {}) },
-  `Console logger initialized with level ${logLevel}`,
+  `Logger initialized with level ${logLevel}` + (otlpEndpoint ? ` with OTLP exporter to ${otlpEndpoint}` : ''),
 );
 
 /** Log function that accepts an exception object */

yarn-project/telemetry-client/package.json

@@ -7,7 +7,8 @@
   "exports": {
     ".": "./dest/index.js",
     "./start": "./dest/start.js",
-    "./noop": "./dest/noop.js"
+    "./noop": "./dest/noop.js",
+    "./otel-pino-stream": "./dest/vendor/otel-pino-stream.js"
   },
   "scripts": {
     "build": "yarn clean && tsc -b",
@@ -28,19 +29,20 @@
   "dependencies": {
     "@aztec/foundation": "workspace:^",
     "@opentelemetry/api": "^1.9.0",
-    "@opentelemetry/api-logs": "^0.54.0",
-    "@opentelemetry/exporter-logs-otlp-http": "^0.54.0",
-    "@opentelemetry/exporter-metrics-otlp-http": "^0.52.0",
-    "@opentelemetry/exporter-trace-otlp-http": "^0.54.0",
-    "@opentelemetry/host-metrics": "^0.35.2",
+    "@opentelemetry/api-logs": "^0.55.0",
+    "@opentelemetry/core": "^1.28.0",
+    "@opentelemetry/exporter-logs-otlp-http": "^0.55.0",
+    "@opentelemetry/exporter-metrics-otlp-http": "^0.55.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.55.0",
+    "@opentelemetry/host-metrics": "^0.35.4",
     "@opentelemetry/instrumentation-pino": "^0.44.0",
-    "@opentelemetry/otlp-exporter-base": "^0.54.0",
-    "@opentelemetry/resource-detector-aws": "^1.5.2",
-    "@opentelemetry/resources": "^1.25.0",
-    "@opentelemetry/sdk-logs": "^0.54.0",
-    "@opentelemetry/sdk-metrics": "^1.25.0",
-    "@opentelemetry/sdk-trace-node": "^1.25.0",
-    "@opentelemetry/semantic-conventions": "^1.25.0",
+    "@opentelemetry/otlp-exporter-base": "^0.55.0",
+    "@opentelemetry/resource-detector-aws": "^1.8.0",
+    "@opentelemetry/resources": "^1.28.0",
+    "@opentelemetry/sdk-logs": "^0.55.0",
+    "@opentelemetry/sdk-metrics": "^1.28.0",
+    "@opentelemetry/sdk-trace-node": "^1.28.0",
+    "@opentelemetry/semantic-conventions": "^1.28.0",
     "prom-client": "^15.1.3"
   },
   "devDependencies": {

yarn-project/telemetry-client/src/otel.ts

@@ -11,7 +11,6 @@ import {
 import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
 import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
 import { HostMetrics } from '@opentelemetry/host-metrics';
-import { PinoInstrumentation } from '@opentelemetry/instrumentation-pino';
 import { awsEc2Detector, awsEcsDetector } from '@opentelemetry/resource-detector-aws';
 import {
   type IResource,
@@ -29,6 +28,7 @@ import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION } from '@opentele
 import { aztecDetector } from './aztec_resource_detector.js';
 import { type TelemetryClientConfig } from './config.js';
 import { registerOtelLoggerProvider } from './otel_logger_provider.js';
+import { getOtelResource } from './otel_resource.js';
 import { type Gauge, type TelemetryClient } from './telemetry.js';
 
 export class OpenTelemetryClient implements TelemetryClient {
@@ -89,25 +89,9 @@ export class OpenTelemetryClient implements TelemetryClient {
   }
 
   public static async createAndStart(config: TelemetryClientConfig, log: DebugLogger): Promise<OpenTelemetryClient> {
-    const resource = detectResourcesSync({
-      detectors: [
-        osDetectorSync,
-        envDetectorSync,
-        processDetectorSync,
-        serviceInstanceIdDetectorSync,
-        awsEc2Detector,
-        awsEcsDetector,
-        aztecDetector,
-      ],
-    });
-
-    if (resource.asyncAttributesPending) {
-      await resource.waitForAsyncAttributes!();
-    }
+    const resource = await getOtelResource();
 
-    const tracerProvider = new NodeTracerProvider({
-      resource,
-    });
+    const tracerProvider = new NodeTracerProvider({ resource });
 
     // optionally push traces to an OTEL collector instance
     if (config.tracesCollectorUrl) {
@@ -131,25 +115,11 @@ export class OpenTelemetryClient implements TelemetryClient {
       ],
     });
 
-    const loggerProvider = registerOtelLoggerProvider(resource, config.logsCollectorUrl);
-    instrumentLogger(loggerProvider, tracerProvider, meterProvider);
+    const loggerProvider = await registerOtelLoggerProvider(resource, config.logsCollectorUrl);
 
     const service = new OpenTelemetryClient(resource, meterProvider, tracerProvider, loggerProvider, log);
     service.start();
 
     return service;
   }
 }
-
-function instrumentLogger(
-  loggerProvider: LoggerProvider,
-  tracerProvider: NodeTracerProvider,
-  meterProvider: MeterProvider,
-) {
-  // We disable log sending since we have a batch log processor already configured
-  const instrumentation = new PinoInstrumentation({ disableLogSending: true });
-  instrumentation.setLoggerProvider(loggerProvider);
-  instrumentation.setTracerProvider(tracerProvider);
-  instrumentation.setMeterProvider(meterProvider);
-  instrumentation.enable();
-}

yarn-project/telemetry-client/src/otel_logger_provider.ts

@@ -4,7 +4,11 @@ import { CompressionAlgorithm } from '@opentelemetry/otlp-exporter-base';
 import { type IResource } from '@opentelemetry/resources';
 import { BatchLogRecordProcessor, LoggerProvider } from '@opentelemetry/sdk-logs';
 
-export function registerOtelLoggerProvider(resource: IResource, otelLogsUrl?: URL) {
+import { getOtelResource } from './otel_resource.js';
+
+export async function registerOtelLoggerProvider(resource?: IResource, otelLogsUrl?: URL) {
+  resource ??= await getOtelResource();
+
   const loggerProvider = new LoggerProvider({ resource });
   if (!otelLogsUrl) {
     // If no URL provided, return it disconnected.

yarn-project/telemetry-client/src/otel_resource.ts

@@ -0,0 +1,31 @@
+import { awsEc2Detector, awsEcsDetector } from '@opentelemetry/resource-detector-aws';
+import {
+  type IResource,
+  detectResourcesSync,
+  envDetectorSync,
+  osDetectorSync,
+  processDetectorSync,
+  serviceInstanceIdDetectorSync,
+} from '@opentelemetry/resources';
+
+import { aztecDetector } from './aztec_resource_detector.js';
+
+export async function getOtelResource(): Promise<IResource> {
+  const resource = detectResourcesSync({
+    detectors: [
+      osDetectorSync,
+      envDetectorSync,
+      processDetectorSync,
+      serviceInstanceIdDetectorSync,
+      awsEc2Detector,
+      awsEcsDetector,
+      aztecDetector,
+    ],
+  });
+
+  if (resource.asyncAttributesPending) {
+    await resource.waitForAsyncAttributes!();
+  }
+
+  return resource;
+}