From a4caca708245de8dbb2d4e6a6861de33547f9aa5 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Thu, 16 Nov 2023 17:52:27 -0500 Subject: [PATCH 01/20] Move addMetricsIfPresent into the metrics builder as a first class method for others to leverage. Signed-off-by: Greg Schohn --- .../migrations/coreutils/MetricsLogBuilder.java | 10 ++++++++-- .../replay/ParsedHttpMessagesAsDicts.java | 15 +++++---------- .../migrations/replay/ReplayEngine.java | 1 - .../replay/netty/BacksideHttpWatcherHandler.java | 1 - 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogBuilder.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogBuilder.java index 0e18e6218..3126eb4e5 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogBuilder.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogBuilder.java @@ -5,10 +5,11 @@ import org.slf4j.Logger; import org.slf4j.spi.LoggingEventBuilder; +import java.util.Optional; + @Slf4j -public -class MetricsLogBuilder { +public class MetricsLogBuilder { private Logger logger; private LoggingEventBuilder loggingEventBuilder; @@ -16,6 +17,11 @@ public MetricsLogBuilder(Logger logger) { this.logger = logger; } + public static MetricsLogBuilder addMetricIfPresent(MetricsLogBuilder metricBuilder, + MetricsAttributeKey key, Optional value) { + return value.map(v -> metricBuilder.setAttribute(key, v)).orElse(metricBuilder); + } + public MetricsLogBuilder setAttribute(MetricsAttributeKey key, Object value) { loggingEventBuilder.addKeyValue(key.getKeyName(), value); return this; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ParsedHttpMessagesAsDicts.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ParsedHttpMessagesAsDicts.java index df2037eb6..4307ec6ea 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ParsedHttpMessagesAsDicts.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ParsedHttpMessagesAsDicts.java @@ -89,11 +89,6 @@ public ParsedHttpMessagesAsDicts(Optional> sourceRequestOp1, this.targetResponseOp = targetResponseOp4; } - private static MetricsLogBuilder addMetricIfPresent(MetricsLogBuilder metricBuilder, - MetricsAttributeKey key, Optional value) { - return value.map(v -> metricBuilder.setAttribute(key, v)).orElse(metricBuilder); - } - public MetricsLogBuilder buildStatusCodeMetrics(MetricsLogger logger, UniqueSourceRequestKey requestKey) { var builder = logger.atSuccess(MetricsEvent.STATUS_CODE_COMPARISON); return buildStatusCodeMetrics(builder, requestKey); @@ -111,14 +106,14 @@ public static MetricsLogBuilder buildStatusCodeMetrics(MetricsLogBuilder builder var targetStatus = targetResponseOp.map(r -> r.get(STATUS_CODE_KEY)); builder = builder.setAttribute(MetricsAttributeKey.REQUEST_ID, requestKey.getTrafficStreamKey().getConnectionId() + "." + requestKey.getSourceRequestIndex()); - builder = addMetricIfPresent(builder, MetricsAttributeKey.SOURCE_HTTP_STATUS, sourceStatus); - builder = addMetricIfPresent(builder, MetricsAttributeKey.TARGET_HTTP_STATUS, targetStatus); - builder = addMetricIfPresent(builder, MetricsAttributeKey.HTTP_STATUS_MATCH, + builder = MetricsLogBuilder.addMetricIfPresent(builder, MetricsAttributeKey.SOURCE_HTTP_STATUS, sourceStatus); + builder = MetricsLogBuilder.addMetricIfPresent(builder, MetricsAttributeKey.TARGET_HTTP_STATUS, targetStatus); + builder = MetricsLogBuilder.addMetricIfPresent(builder, MetricsAttributeKey.HTTP_STATUS_MATCH, sourceStatus.flatMap(ss -> targetStatus.map(ts -> ss.equals(ts))) .filter(x -> x).map(b -> (Object) 1).or(() -> Optional.of(Integer.valueOf(0)))); - builder = addMetricIfPresent(builder, MetricsAttributeKey.HTTP_METHOD, + builder = MetricsLogBuilder.addMetricIfPresent(builder, MetricsAttributeKey.HTTP_METHOD, sourceResponseOp.map(r -> r.get("Method"))); - builder = addMetricIfPresent(builder, MetricsAttributeKey.HTTP_ENDPOINT, + builder = MetricsLogBuilder.addMetricIfPresent(builder, MetricsAttributeKey.HTTP_ENDPOINT, sourceResponseOp.map(r -> r.get("Request-URI"))); return builder; } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java index dd823156a..542243d56 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java @@ -1,7 +1,6 @@ package org.opensearch.migrations.replay; import io.netty.buffer.ByteBuf; -import io.netty.util.concurrent.Future; import io.netty.util.concurrent.ScheduledFuture; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsAttributeKey; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/netty/BacksideHttpWatcherHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/netty/BacksideHttpWatcherHandler.java index a552fe876..7cbb36864 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/netty/BacksideHttpWatcherHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/netty/BacksideHttpWatcherHandler.java @@ -3,7 +3,6 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.channel.SimpleChannelInboundHandler; import io.netty.handler.codec.http.FullHttpResponse; -import lombok.extern.log4j.Log4j2; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; From c026588ef5a9cb1bca3aaf302481bd41444a7041 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Fri, 17 Nov 2023 11:02:14 -0500 Subject: [PATCH 02/20] WIP to play with OpenTelemetry metric instruments and tracer spans. Most of this is just playing, but making the StreamManager implement AutoCloseable gives a place to end spans to show how long a serializer/connection factory was relevant for. Signed-off-by: Greg Schohn --- .../captureKafkaOffloader/build.gradle | 14 ++- .../kafkaoffloader/KafkaCaptureFactory.java | 42 +++++++- .../FileConnectionCaptureFactory.java | 3 + ...eamChannelConnectionCaptureSerializer.java | 32 +++--- .../StreamLifecycleManager.java | 4 +- ...hannelConnectionCaptureSerializerTest.java | 3 + .../InMemoryConnectionCaptureFactory.java | 3 + TrafficCapture/coreUtilities/build.gradle | 8 +- .../migrations/coreutils/MetricsLogger.java | 97 ++++++++++++++++--- .../src/main/docker/docker-compose.yml | 5 + .../src/main/docker/otelcol/otel-config.yml | 4 +- ...ReliableLoggingHttpRequestHandlerTest.java | 3 + .../proxyserver/CaptureProxy.java | 8 +- .../src/main/resources/log4j2.properties | 7 +- 14 files changed, 184 insertions(+), 49 deletions(-) diff --git a/TrafficCapture/captureKafkaOffloader/build.gradle b/TrafficCapture/captureKafkaOffloader/build.gradle index 11f4e3a06..d7bb474e9 100644 --- a/TrafficCapture/captureKafkaOffloader/build.gradle +++ b/TrafficCapture/captureKafkaOffloader/build.gradle @@ -12,11 +12,15 @@ dependencies { api 'io.netty:netty-buffer:4.1.100.Final' implementation project(':captureOffloader') implementation project(':coreUtilities') - implementation 'org.projectlombok:lombok:1.18.26' - implementation 'com.google.protobuf:protobuf-java:3.22.2' - implementation 'org.apache.kafka:kafka-clients:3.6.0' - implementation 'software.amazon.msk:aws-msk-iam-auth:1.1.9' - implementation 'org.slf4j:slf4j-api:2.0.7' + + implementation group: 'com.google.protobuf', name:'protobuf-java', version:'3.22.2' + implementation group: 'io.opentelemetry', name:'opentelemetry-api', version: '1.30.0' + implementation group: 'org.projectlombok', name:'lombok', version:'1.18.26' + implementation group: 'org.apache.kafka', name:'kafka-clients', version:'3.6.0' + implementation group: 'software.amazon.msk', name:'aws-msk-iam-auth', version:'1.1.9' + + implementation group: 'org.slf4j', name:'slf4j-api', version:'2.0.7' + testImplementation project(':captureProtobufs') testImplementation 'org.mockito:mockito-core:4.6.1' testImplementation 'org.mockito:mockito-junit-jupiter:4.6.1' diff --git a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java index 756ec9739..acbe92c10 100644 --- a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java +++ b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java @@ -1,6 +1,12 @@ package org.opensearch.migrations.trafficcapture.kafkaoffloader; import com.google.protobuf.CodedOutputStream; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; import lombok.AllArgsConstructor; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -17,7 +23,10 @@ import org.opensearch.migrations.trafficcapture.StreamChannelConnectionCaptureSerializer; import org.opensearch.migrations.coreutils.MetricsLogger; +import java.io.IOException; import java.nio.ByteBuffer; +import java.time.Duration; +import java.time.Instant; import java.util.Arrays; import java.util.concurrent.CompletableFuture; @@ -31,6 +40,7 @@ public class KafkaCaptureFactory implements IConnectionCaptureFactory producer, int @Override public IChannelConnectionCaptureSerializer createOffloader(String connectionId) { - return new StreamChannelConnectionCaptureSerializer<>(nodeId, connectionId, new StreamManager(connectionId)); + var tracer = GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME); + Span connectionSpan = tracer.spanBuilder("connection").startSpan(); + + try (var namedOnlyForAutoClose = Context.current().with(connectionSpan).makeCurrent()) { + var meter = GlobalOpenTelemetry.get().getMeter(TELEMETRY_SCOPE_NAME); + meter.counterBuilder("connection_created").build().add(1); + } + + return new StreamChannelConnectionCaptureSerializer<>(nodeId, connectionId, + new StreamManager(connectionSpan, connectionId)); } @AllArgsConstructor @@ -65,9 +84,28 @@ static class CodedOutputStreamWrapper implements CodedOutputStreamHolder { } } - @AllArgsConstructor class StreamManager extends OrderedStreamLifecyleManager { + Span telemetrySpan; String connectionId; + Instant startTime; + + public StreamManager(Span telemetrySpan, String connectionId) { + this.telemetrySpan = telemetrySpan; + this.connectionId = connectionId; + this.startTime = Instant.now(); + } + + @Override + public void close() throws IOException { + try (var namedOnlyForAutoClose = Context.current().with(telemetrySpan).makeCurrent()) { + var histogram = GlobalOpenTelemetry.get().getMeter(TELEMETRY_SCOPE_NAME) + .histogramBuilder("connection_lifetime").build(); + telemetrySpan.setAttribute("connectionId", connectionId); + histogram.record((double) Duration.between(startTime, Instant.now()).toMillis(), Attributes.empty(), + Context.current().with(telemetrySpan)); + telemetrySpan.end(); + } + } @Override public CodedOutputStreamWrapper createStream() { diff --git a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java index 26cee767d..751d39f63 100644 --- a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java +++ b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java @@ -51,6 +51,9 @@ public FileConnectionCaptureFactory(String nodeId, String path, int bufferSize) @AllArgsConstructor class StreamManager extends OrderedStreamLifecyleManager { String connectionId; + @Override + public void close() {} + @Override public CodedOutputStreamAndByteBufferWrapper createStream() { return new CodedOutputStreamAndByteBufferWrapper(bufferSize); diff --git a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializer.java b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializer.java index 14c501d37..d59bcc512 100644 --- a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializer.java +++ b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializer.java @@ -188,20 +188,23 @@ public CompletableFuture flushCommitAndResetStream(boolean isFinal) throws IO if (streamHasBeenClosed || (currentCodedOutputStreamHolderOrNull == null && !isFinal)) { return CompletableFuture.completedFuture(null); } - CodedOutputStream currentStream = getOrCreateCodedOutputStream(); - var fieldNum = isFinal ? TrafficStream.NUMBEROFTHISLASTCHUNK_FIELD_NUMBER : TrafficStream.NUMBER_FIELD_NUMBER; - // e.g. 3: 1 - currentStream.writeInt32(fieldNum, ++numFlushesSoFar); - log.trace("Flushing the current CodedOutputStream for {}.{}", connectionIdString, numFlushesSoFar); - currentStream.flush(); - assert currentStream == currentCodedOutputStreamHolderOrNull.getOutputStream() : "Expected the stream that " + - "is being finalized to be the same stream contained by currentCodedOutputStreamHolderOrNull"; - var future = streamManager.closeStream(currentCodedOutputStreamHolderOrNull, numFlushesSoFar); - currentCodedOutputStreamHolderOrNull = null; - if (isFinal) { - streamHasBeenClosed = true; + try { + CodedOutputStream currentStream = getOrCreateCodedOutputStream(); + var fieldNum = isFinal ? TrafficStream.NUMBEROFTHISLASTCHUNK_FIELD_NUMBER : TrafficStream.NUMBER_FIELD_NUMBER; + // e.g. 3: 1 + currentStream.writeInt32(fieldNum, ++numFlushesSoFar); + log.trace("Flushing the current CodedOutputStream for {}.{}", connectionIdString, numFlushesSoFar); + currentStream.flush(); + assert currentStream == currentCodedOutputStreamHolderOrNull.getOutputStream() : "Expected the stream that " + + "is being finalized to be the same stream contained by currentCodedOutputStreamHolderOrNull"; + return streamManager.closeStream(currentCodedOutputStreamHolderOrNull, numFlushesSoFar); + } finally { + currentCodedOutputStreamHolderOrNull = null; + if (isFinal) { + streamHasBeenClosed = true; + streamManager.close(); + } } - return future; } @Override @@ -222,7 +225,8 @@ public void addDisconnectEvent(Instant timestamp) throws IOException { @Override public void addCloseEvent(Instant timestamp) throws IOException { beginSubstreamObservation(timestamp, TrafficObservation.CLOSE_FIELD_NUMBER, 1); - getOrCreateCodedOutputStream().writeMessage(TrafficObservation.CLOSE_FIELD_NUMBER, CloseObservation.getDefaultInstance()); + getOrCreateCodedOutputStream().writeMessage(TrafficObservation.CLOSE_FIELD_NUMBER, + CloseObservation.getDefaultInstance()); } @Override diff --git a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/StreamLifecycleManager.java b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/StreamLifecycleManager.java index 18db43cc4..b41af74a5 100644 --- a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/StreamLifecycleManager.java +++ b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/StreamLifecycleManager.java @@ -1,9 +1,11 @@ package org.opensearch.migrations.trafficcapture; +import java.io.IOException; import java.util.concurrent.CompletableFuture; -public interface StreamLifecycleManager { +public interface StreamLifecycleManager extends AutoCloseable { CodedOutputStreamHolder createStream(); CompletableFuture closeStream(CodedOutputStreamHolder outputStreamHolder, int index); + void close() throws IOException; } diff --git a/TrafficCapture/captureOffloader/src/test/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializerTest.java b/TrafficCapture/captureOffloader/src/test/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializerTest.java index 4a25aa4d7..97def992c 100644 --- a/TrafficCapture/captureOffloader/src/test/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializerTest.java +++ b/TrafficCapture/captureOffloader/src/test/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializerTest.java @@ -320,6 +320,9 @@ class StreamManager extends OrderedStreamLifecyleManager { int bufferSize; ConcurrentLinkedQueue outputBuffers; + @Override + public void close() {} + @Override public CodedOutputStreamHolder createStream() { return new CodedOutputStreamAndByteBufferWrapper(bufferSize); diff --git a/TrafficCapture/captureOffloader/src/testFixtures/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java b/TrafficCapture/captureOffloader/src/testFixtures/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java index 8af6b3a89..b63ef52af 100644 --- a/TrafficCapture/captureOffloader/src/testFixtures/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java +++ b/TrafficCapture/captureOffloader/src/testFixtures/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java @@ -36,6 +36,9 @@ public InMemoryConnectionCaptureFactory(String nodeId, int bufferSize, Runnable @AllArgsConstructor class StreamManager extends OrderedStreamLifecyleManager { + @Override + public void close() {} + @Override public CodedOutputStreamHolder createStream() { return new CodedOutputStreamAndByteBufferWrapper(bufferSize); diff --git a/TrafficCapture/coreUtilities/build.gradle b/TrafficCapture/coreUtilities/build.gradle index 1c4404477..da76d0f15 100644 --- a/TrafficCapture/coreUtilities/build.gradle +++ b/TrafficCapture/coreUtilities/build.gradle @@ -56,9 +56,11 @@ dependencies { implementation("org.apache.logging.log4j:log4j-slf4j2-impl:2.20.0") // OpenTelemetry core - implementation("io.opentelemetry:opentelemetry-sdk:1.30.0") - implementation("io.opentelemetry:opentelemetry-exporter-otlp:1.30.0") - implementation("io.opentelemetry:opentelemetry-semconv:1.30.1-alpha") + implementation group: 'io.opentelemetry', name:'opentelemetry-api', version: '1.30.0' + implementation group: 'io.opentelemetry', name:'opentelemetry-exporter-otlp', version: '1.30.0' + implementation group: 'io.opentelemetry', name:'opentelemetry-sdk', version: '1.30.0' + implementation group: 'io.opentelemetry.instrumentation', name:'opentelemetry-log4j-appender-2.17', version: '1.30.0-alpha' + implementation group: 'io.opentelemetry', name:'opentelemetry-semconv', version: '1.30.0-alpha' // OpenTelemetry log4j appender implementation("io.opentelemetry.instrumentation:opentelemetry-log4j-appender-2.17:1.30.0-alpha") diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogger.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogger.java index ec62c88d1..23f73aa29 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogger.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogger.java @@ -1,17 +1,30 @@ package org.opensearch.migrations.coreutils; + import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter; -import io.opentelemetry.instrumentation.log4j.appender.v2_17.OpenTelemetryAppender; +import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.logs.SdkLoggerProvider; import io.opentelemetry.sdk.logs.export.BatchLogRecordProcessor; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; import org.slf4j.Logger; import lombok.extern.slf4j.Slf4j; import org.slf4j.LoggerFactory; +import java.time.Duration; +import java.util.concurrent.TimeUnit; + @Slf4j public class MetricsLogger { @@ -29,14 +42,15 @@ public MetricsLogger(String source) { } public static void initializeOpenTelemetry(String serviceName, String collectorEndpoint) { - OpenTelemetrySdk sdk = + var serviceResource = Resource.getDefault().toBuilder() + .put(ResourceAttributes.SERVICE_NAME, serviceName) + .build(); + + OpenTelemetrySdk openTelemetrySdk = OpenTelemetrySdk.builder() .setLoggerProvider( SdkLoggerProvider.builder() - .setResource( - Resource.getDefault().toBuilder() - .put(ResourceAttributes.SERVICE_NAME, serviceName) - .build()) + .setResource(serviceResource) .addLogRecordProcessor( BatchLogRecordProcessor.builder( OtlpGrpcLogRecordExporter.builder() @@ -44,14 +58,68 @@ public static void initializeOpenTelemetry(String serviceName, String collectorE .build()) .build()) .build()) - .build(); - GlobalOpenTelemetry.set(sdk); + .setTracerProvider( + SdkTracerProvider.builder() + .setResource(serviceResource) + .addSpanProcessor( + BatchSpanProcessor.builder( + OtlpGrpcSpanExporter.builder() + .setEndpoint(collectorEndpoint) + .setTimeout(2, TimeUnit.SECONDS) + .build()) + .setScheduleDelay(100, TimeUnit.MILLISECONDS) + .build()) + .build()) + .setMeterProvider( + SdkMeterProvider.builder() + .setResource(serviceResource) + .registerMetricReader( + PeriodicMetricReader.builder( + OtlpGrpcMetricExporter.builder() + .setEndpoint(collectorEndpoint) + .build()) + .setInterval(Duration.ofMillis(1000)) + .build()) + .build()) + .buildAndRegisterGlobal(); // Add hook to close SDK, which flushes logs - Runtime.getRuntime().addShutdownHook(new Thread(sdk::close)); - OpenTelemetryAppender.install(GlobalOpenTelemetry.get()); + Runtime.getRuntime().addShutdownHook(new Thread(openTelemetrySdk::close)); + //OpenTelemetryAppender.install(GlobalOpenTelemetry.get()); } + public static class SimpleMeteringClosure { + public final Meter meter; + public final Tracer tracer; + public SimpleMeteringClosure(String scopeName) { + meter = GlobalOpenTelemetry.getMeter(scopeName); + tracer = GlobalOpenTelemetry.getTracer(scopeName); + } + public void meterIncrementEvent(Context ctx, String eventName) { + meterIncrementEvent(ctx, eventName, 1); + } + public void meterIncrementEvent(Context ctx, String eventName, long increment) { + if (ctx == null) { return; } + try (var namedOnlyForAutoClose = ctx.makeCurrent()) { + meter.counterBuilder(eventName).build().add(increment); + } + } + public void meterDeltaEvent(Context ctx, String eventName, long delta) { + if (ctx == null) { return; } + try (var namedOnlyForAutoClose = ctx.makeCurrent()) { + meter.upDownCounterBuilder(eventName).build().add(delta); + } + } + public void meterHistogramMillis(Context ctx, String eventName, Duration between) { + meterHistogram(ctx, eventName, (double) between.toMillis()); + } + public void meterHistogram(Context ctx, String eventName, double value) { + if (ctx == null) { return; } + try (var namedOnlyForAutoClose = ctx.makeCurrent()) { + meter.histogramBuilder(eventName).build().record(value); + } + } + } /** * To indicate a successful event (e.g. data received or data sent) that may be a helpful @@ -61,7 +129,7 @@ public static void initializeOpenTelemetry(String serviceName, String collectorE * metricsLogger.atSuccess().addKeyValue("key", "value").setMessage("Task succeeded").log(); */ public MetricsLogBuilder atSuccess(MetricsEvent event) { - return new MetricsLogBuilder(logger).atSuccess(event); + return new MetricsLogBuilder().atSuccess(event); } /** @@ -74,7 +142,7 @@ public MetricsLogBuilder atError(MetricsEvent event, Throwable cause) { if (cause == null) { return atError(event); } - return new MetricsLogBuilder(logger).atError(event) + return new MetricsLogBuilder().atError(event) .setAttribute(MetricsAttributeKey.EXCEPTION_MESSAGE, cause.getMessage()) .setAttribute(MetricsAttributeKey.EXCEPTION_TYPE, cause.getClass().getName()); } @@ -84,10 +152,11 @@ public MetricsLogBuilder atError(MetricsEvent event, Throwable cause) { * there is a failure that isn't indicated by an Exception being thrown. */ public MetricsLogBuilder atError(MetricsEvent event) { - return new MetricsLogBuilder(logger).atError(event); + + return new MetricsLogBuilder().atError(event); } public MetricsLogBuilder atTrace(MetricsEvent event) { - return new MetricsLogBuilder(logger).atTrace(event); + return new MetricsLogBuilder().atTrace(event); } } diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 14c1b00ff..283f42ded 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -9,6 +9,8 @@ services: ports: - "9200:9200" - "19200:19200" + volumes: + - /Users/schohn/dev/opensearch-migrations/TrafficCapture/containerLogs:/logs environment: - http.port=19200 # Run processes for elasticsearch and capture proxy, and exit if either one ends @@ -70,6 +72,7 @@ services: - migrations volumes: - sharedReplayerOutput:/shared-replayer-output + - /Users/schohn/dev/opensearch-migrations/TrafficCapture/containerLogs:/logs environment: - TUPLE_DIR_PATH=/shared-replayer-output/traffic-replayer-default depends_on: @@ -119,10 +122,12 @@ services: - "13133:13133" volumes: - ./otelcol/otel-config.yml:/etc/otel-config.yml + - /Users/schohn/dev/opensearch-migrations/TrafficCapture/containerLogs:/logs networks: - migrations depends_on: - opensearchanalytics + command: tail -f /dev/null migration-console: image: 'migrations/migration_console:latest' diff --git a/TrafficCapture/dockerSolution/src/main/docker/otelcol/otel-config.yml b/TrafficCapture/dockerSolution/src/main/docker/otelcol/otel-config.yml index f6ccc70e6..e84720698 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/otelcol/otel-config.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/otelcol/otel-config.yml @@ -52,6 +52,8 @@ exporters: insecure_skip_verify: true logging: verbosity: detailed + file: + path: /logs/filename.json debug: service: @@ -63,4 +65,4 @@ service: logs: receivers: [otlp] processors: [attributes] - exporters: [logging, debug, opensearch] \ No newline at end of file + exporters: [logging, debug, opensearch, file] \ No newline at end of file diff --git a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java index 531d206ad..3110d0188 100644 --- a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java +++ b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java @@ -39,6 +39,9 @@ static class StreamManager extends OrderedStreamLifecyleManager { AtomicReference byteBufferAtomicReference; AtomicInteger flushCount = new AtomicInteger(); + @Override + public void close() {} + @Override public CodedOutputStreamAndByteBufferWrapper createStream() { return new CodedOutputStreamAndByteBufferWrapper(1024*1024); diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java index 504cdf275..2d400d036 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java @@ -17,6 +17,7 @@ import org.apache.kafka.common.config.SaslConfigs; import org.apache.logging.log4j.core.util.NullOutputStream; import org.opensearch.common.settings.Settings; +import org.opensearch.migrations.coreutils.MetricsLogger; import org.opensearch.migrations.trafficcapture.CodedOutputStreamHolder; import org.opensearch.migrations.trafficcapture.FileConnectionCaptureFactory; import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; @@ -43,8 +44,6 @@ import java.util.function.Supplier; import java.util.stream.Stream; -import static org.opensearch.migrations.coreutils.MetricsLogger.initializeOpenTelemetry; - @Slf4j public class CaptureProxy { @@ -177,6 +176,9 @@ private static IConnectionCaptureFactory getNullConnectionCaptureFactory System.err.println("No trace log directory specified. Logging to /dev/null"); return connectionId -> new StreamChannelConnectionCaptureSerializer<>(null, connectionId, new StreamLifecycleManager<>() { + @Override + public void close() {} + @Override public CodedOutputStreamHolder createStream() { return () -> CodedOutputStream.newInstance(NullOutputStream.getInstance()); @@ -282,7 +284,7 @@ public static void main(String[] args) throws InterruptedException, IOException var backsideUri = convertStringToUri(params.backsideUriString); if (params.otelCollectorEndpoint != null) { - initializeOpenTelemetry("capture-proxy", params.otelCollectorEndpoint); + MetricsLogger.initializeOpenTelemetry("capture-proxy", params.otelCollectorEndpoint); } var sksOp = Optional.ofNullable(params.sslConfigFilePath) diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/resources/log4j2.properties b/TrafficCapture/trafficCaptureProxyServer/src/main/resources/log4j2.properties index 6bd32ae07..c35476b38 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/resources/log4j2.properties +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/resources/log4j2.properties @@ -1,4 +1,4 @@ -status = info +status = debug packages = io.opentelemetry.instrumentation.log4j.appender.v2_17 appenders = console, METRICS @@ -18,8 +18,3 @@ appender.METRICS.captureContextDataAttributes = * rootLogger.level = info rootLogger.appenderRefs = stderr rootLogger.appenderRef.stderr.ref = STDERR - -logger.MetricsLogger.name = MetricsLogger -logger.MetricsLogger.level = info -logger.MetricsLogger.additivity = false -logger.MetricsLogger.appenderRef.METRICS.ref = METRICS From f3c007707109f70dc8c92aa774a6d84d03cedd54 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Mon, 27 Nov 2023 15:56:52 -0500 Subject: [PATCH 03/20] Get gradle files and docker-compose in order to support otlp exports to the collector to prometheus, zipkin, etc Signed-off-by: Greg Schohn --- .../captureKafkaOffloader/build.gradle | 10 +-- .../src/main/docker/docker-compose.yml | 80 +++++++++++++++---- .../docker/otel-collector-config-demo.yaml | 50 ++++++++++++ .../src/main/docker/prometheus.yaml | 6 ++ TrafficCapture/nettyWireLogging/build.gradle | 14 +++- .../trafficCaptureProxyServer/build.gradle | 5 ++ 6 files changed, 143 insertions(+), 22 deletions(-) create mode 100644 TrafficCapture/dockerSolution/src/main/docker/otel-collector-config-demo.yaml create mode 100644 TrafficCapture/dockerSolution/src/main/docker/prometheus.yaml diff --git a/TrafficCapture/captureKafkaOffloader/build.gradle b/TrafficCapture/captureKafkaOffloader/build.gradle index d7bb474e9..0b516f677 100644 --- a/TrafficCapture/captureKafkaOffloader/build.gradle +++ b/TrafficCapture/captureKafkaOffloader/build.gradle @@ -10,23 +10,23 @@ repositories { dependencies { api 'io.netty:netty-buffer:4.1.100.Final' + implementation platform("io.opentelemetry:opentelemetry-bom:1.32.0") + implementation project(':captureOffloader') implementation project(':coreUtilities') - implementation group: 'com.google.protobuf', name:'protobuf-java', version:'3.22.2' implementation group: 'io.opentelemetry', name:'opentelemetry-api', version: '1.30.0' implementation group: 'org.projectlombok', name:'lombok', version:'1.18.26' implementation group: 'org.apache.kafka', name:'kafka-clients', version:'3.6.0' - implementation group: 'software.amazon.msk', name:'aws-msk-iam-auth', version:'1.1.9' - implementation group: 'org.slf4j', name:'slf4j-api', version:'2.0.7' + implementation group: 'software.amazon.msk', name:'aws-msk-iam-auth', version:'1.1.9' testImplementation project(':captureProtobufs') - testImplementation 'org.mockito:mockito-core:4.6.1' - testImplementation 'org.mockito:mockito-junit-jupiter:4.6.1' testImplementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.20.0' testImplementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.20.0' testImplementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl', version: '2.20.0' + testImplementation group: 'org.mockito', name: 'mockito-core', version: '4.6.1' + testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '4.6.1' testImplementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7' } diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 283f42ded..e220c357d 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -1,6 +1,58 @@ version: '3.7' services: + + prometheus: + container_name: prometheus + image: prom/prometheus:latest + networks: + - migrations + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + + # Jaeger + jaeger-all-in-one: + image: jaegertracing/all-in-one:latest + networks: + - migrations + ports: + - "16686:16686" + - "14268" + - "14250:14250" + environment: + - COLLECTOR_OTLP_ENABLED=true + + # Zipkin + zipkin-all-in-one: + image: openzipkin/zipkin:latest + networks: + - migrations + ports: + - "9411:9411" + + # Collector + otel-collector: + image: otel/opentelemetry-collector:latest +# command: ["--config=/etc/otel-collector-config-demo.yaml", "${OTELCOL_ARGS}"] + networks: + - migrations + volumes: +# - ./otel-collector-config-demo.yaml:/etc/otel-collector-config-demo.yaml + - ./otel-collector-config-demo.yaml:/etc/otelcol/config.yaml + - /Users/schohn/dev/opensearch-migrations/TrafficCapture/containerLogs:/logs + ports: + - "1888:1888" # pprof extension + - "8888:8888" # Prometheus metrics exposed by the collector + - "8889:8889" # Prometheus exporter metrics + - "13133:13133" # health_check extension + - "55679:55679" # zpages extension + - "4317:4317" # otlp receiver + depends_on: + - jaeger-all-in-one + - zipkin-all-in-one + # Run combined instance of Capture Proxy and Elasticsearch capture-proxy-es: image: 'migrations/capture_proxy:latest' @@ -114,20 +166,20 @@ services: depends_on: - opensearchanalytics - otel-collector: - image: public.ecr.aws/a0w2c5q7/otelcol-with-opensearch:latest - container_name: otel-collector - ports: - - "4317:4317" - - "13133:13133" - volumes: - - ./otelcol/otel-config.yml:/etc/otel-config.yml - - /Users/schohn/dev/opensearch-migrations/TrafficCapture/containerLogs:/logs - networks: - - migrations - depends_on: - - opensearchanalytics - command: tail -f /dev/null +# otel-collector: +# image: public.ecr.aws/a0w2c5q7/otelcol-with-opensearch:latest +# container_name: otel-collector +# ports: +# - "4317:4317" +# - "13133:13133" +# volumes: +# - ./otelcol/otel-config.yml:/etc/otel-config.yml +# - /Users/schohn/dev/opensearch-migrations/TrafficCapture/containerLogs:/logs +# networks: +# - migrations +# depends_on: +# - opensearchanalytics +# command: tail -f /dev/null migration-console: image: 'migrations/migration_console:latest' diff --git a/TrafficCapture/dockerSolution/src/main/docker/otel-collector-config-demo.yaml b/TrafficCapture/dockerSolution/src/main/docker/otel-collector-config-demo.yaml new file mode 100644 index 000000000..92582f7e8 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/otel-collector-config-demo.yaml @@ -0,0 +1,50 @@ +receivers: + otlp: + protocols: + grpc: + +exporters: + prometheus: + endpoint: "0.0.0.0:8889" + namespace: capturereplay + const_labels: + label1: value1 + logging: + loglevel: debug + + zipkin: + endpoint: "http://zipkin-all-in-one:9411/api/v2/spans" + format: proto + + otlp/jaeger: + endpoint: jaeger-all-in-one:14250 + tls: + insecure: true + +# Alternatively, use jaeger_thrift_http with the settings below. In this case +# update the list of exporters on the traces pipeline. +# +# jaeger_thrift_http: +# url: http://jaeger-all-in-one:14268/api/traces + +processors: + batch: + +extensions: + health_check: + pprof: + endpoint: :1888 + zpages: + endpoint: :55679 + +service: + extensions: [pprof, zpages, health_check] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [logging, zipkin, otlp/jaeger] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [logging, prometheus] diff --git a/TrafficCapture/dockerSolution/src/main/docker/prometheus.yaml b/TrafficCapture/dockerSolution/src/main/docker/prometheus.yaml new file mode 100644 index 000000000..ddea76205 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/prometheus.yaml @@ -0,0 +1,6 @@ +scrape_configs: + - job_name: 'otel-collector' + scrape_interval: 2s + static_configs: + - targets: ['otel-collector:8889'] + - targets: ['otel-collector:8888'] diff --git a/TrafficCapture/nettyWireLogging/build.gradle b/TrafficCapture/nettyWireLogging/build.gradle index ade3744c9..1a3c2bb64 100644 --- a/TrafficCapture/nettyWireLogging/build.gradle +++ b/TrafficCapture/nettyWireLogging/build.gradle @@ -8,20 +8,28 @@ plugins { } dependencies { + implementation platform("io.opentelemetry:opentelemetry-bom:1.32.0") + implementation platform("io.netty:netty-bom:4.1.100.Final") + implementation project(':captureOffloader') implementation project(':coreUtilities') api group: 'io.netty', name: 'netty-all', version: '4.1.100.Final' + + implementation group: 'io.opentelemetry', name:'opentelemetry-api' implementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7' testImplementation project(':captureProtobufs') - testImplementation group: 'org.apache.httpcomponents.client5', name: 'httpclient5', version: '5.2.1' - testImplementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7' testImplementation group: 'com.google.guava', name: 'guava', version: '32.0.1-jre' testImplementation group: 'com.google.protobuf', name: 'protobuf-java', version:'3.22.2' - + testImplementation group: 'io.opentelemetry', name: 'opentelemetry-sdk' + testImplementation group: 'io.opentelemetry', name: 'opentelemetry-sdk-testing' + testImplementation group: 'org.apache.httpcomponents.client5', name: 'httpclient5', version: '5.2.1' testImplementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.20.0' testImplementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.20.0' testImplementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl', version: '2.20.0' + testImplementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7' testImplementation testFixtures(project(path: ':testUtilities')) + + } diff --git a/TrafficCapture/trafficCaptureProxyServer/build.gradle b/TrafficCapture/trafficCaptureProxyServer/build.gradle index f9f14abe4..05f00f8b8 100644 --- a/TrafficCapture/trafficCaptureProxyServer/build.gradle +++ b/TrafficCapture/trafficCaptureProxyServer/build.gradle @@ -38,9 +38,14 @@ dependencies { implementation group: 'com.beust', name: 'jcommander', version: '1.82' implementation 'com.google.protobuf:protobuf-java:3.22.2' + implementation group: 'io.opentelemetry', name:'opentelemetry-api', version: '1.30.0' + implementation group: 'io.opentelemetry', name: 'opentelemetry-sdk', version: '1.32.0' + testImplementation project(':captureProtobufs') testImplementation testFixtures(project(path: ':testUtilities')) testImplementation testFixtures(project(path: ':captureOffloader')) + + testImplementation group: 'io.opentelemetry', name: 'opentelemetry-sdk-testing', version: '1.32.0' } tasks.withType(Tar){ From 7fb8e2e9a2a2e5062fbf0c67ac48cef95b02f302 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Mon, 27 Nov 2023 17:06:30 -0500 Subject: [PATCH 04/20] WIP Signed-off-by: Greg Schohn --- .../captureKafkaOffloader/build.gradle | 2 +- .../kafkaoffloader/KafkaCaptureFactory.java | 89 +++++++++++++------ .../migrations/coreutils/MetricsLogger.java | 8 +- .../src/main/docker/docker-compose.yml | 52 +++++------ TrafficCapture/nettyWireLogging/build.gradle | 4 +- ...allyReliableLoggingHttpRequestHandler.java | 22 ++++- .../netty/LoggingHttpRequestHandler.java | 41 ++++++++- .../netty/LoggingHttpResponseHandler.java | 49 ++++++++-- ...ReliableLoggingHttpRequestHandlerTest.java | 22 ++++- .../netty/ProxyChannelInitializer.java | 11 ++- .../src/main/resources/logging.properties | 9 ++ 11 files changed, 233 insertions(+), 76 deletions(-) create mode 100644 TrafficCapture/trafficCaptureProxyServer/src/main/resources/logging.properties diff --git a/TrafficCapture/captureKafkaOffloader/build.gradle b/TrafficCapture/captureKafkaOffloader/build.gradle index 0b516f677..65e332890 100644 --- a/TrafficCapture/captureKafkaOffloader/build.gradle +++ b/TrafficCapture/captureKafkaOffloader/build.gradle @@ -9,12 +9,12 @@ repositories { } dependencies { - api 'io.netty:netty-buffer:4.1.100.Final' implementation platform("io.opentelemetry:opentelemetry-bom:1.32.0") implementation project(':captureOffloader') implementation project(':coreUtilities') implementation group: 'com.google.protobuf', name:'protobuf-java', version:'3.22.2' + api group:'io.netty', name:'netty-buffer', version: '4.1.100.Final' implementation group: 'io.opentelemetry', name:'opentelemetry-api', version: '1.30.0' implementation group: 'org.projectlombok', name:'lombok', version:'1.18.26' implementation group: 'org.apache.kafka', name:'kafka-clients', version:'3.6.0' diff --git a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java index acbe92c10..a3696fefe 100644 --- a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java +++ b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java @@ -2,11 +2,9 @@ import com.google.protobuf.CodedOutputStream; import io.opentelemetry.api.GlobalOpenTelemetry; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.metrics.LongCounter; import io.opentelemetry.api.trace.Span; import io.opentelemetry.context.Context; -import io.opentelemetry.context.Scope; +import io.opentelemetry.context.ContextKey; import lombok.AllArgsConstructor; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -28,19 +26,26 @@ import java.time.Duration; import java.time.Instant; import java.util.Arrays; +import java.util.Optional; import java.util.concurrent.CompletableFuture; @Slf4j public class KafkaCaptureFactory implements IConnectionCaptureFactory { + private static final ContextKey RECORD_ID_KEY = ContextKey.named("recordId"); + private static final ContextKey TOPIC_KEY = ContextKey.named("topic"); + private static final ContextKey RECORD_SIZE_KEY = ContextKey.named("recordSize"); + public static final String TELEMETRY_SCOPE_NAME = "KafkaCapture"; + public static final Optional METERING_CLOSURE_OP = + Optional.of(new MetricsLogger.SimpleMeteringClosure(TELEMETRY_SCOPE_NAME)); + private static final MetricsLogger metricsLogger = new MetricsLogger("BacksideHandler"); private static final String DEFAULT_TOPIC_NAME_FOR_TRAFFIC = "logging-traffic-topic"; // This value encapsulates overhead we should reserve for a given Producer record to account for record key bytes and // general Kafka message overhead public static final int KAFKA_MESSAGE_OVERHEAD_BYTES = 500; - public static final String TELEMETRY_SCOPE_NAME = "KafkaCaptureFactory"; private final String nodeId; // Potential future optimization here to use a direct buffer (e.g. nio) instead of byte array @@ -62,22 +67,24 @@ public KafkaCaptureFactory(String nodeId, Producer producer, int @Override public IChannelConnectionCaptureSerializer createOffloader(String connectionId) { - var tracer = GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME); - Span connectionSpan = tracer.spanBuilder("connection").startSpan(); - - try (var namedOnlyForAutoClose = Context.current().with(connectionSpan).makeCurrent()) { - var meter = GlobalOpenTelemetry.get().getMeter(TELEMETRY_SCOPE_NAME); - meter.counterBuilder("connection_created").build().add(1); - } - + var context = METERING_CLOSURE_OP.map(m->{ + Span offloaderSpan = GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME) + .spanBuilder("offloader").startSpan(); + offloaderSpan.setAttribute("offloaderConnectionId", connectionId); + var c = Context.current().with(offloaderSpan); + m.meterIncrementEvent(c, "offloader_created"); + m.meterDeltaEvent(c, "offloaders_active", 1); + return c; + }).orElse(null); return new StreamChannelConnectionCaptureSerializer<>(nodeId, connectionId, - new StreamManager(connectionSpan, connectionId)); + new StreamManager(context, connectionId)); } @AllArgsConstructor static class CodedOutputStreamWrapper implements CodedOutputStreamHolder { private final CodedOutputStream codedOutputStream; private final ByteBuffer byteBuffer; + final Context streamContext; @Override public @NonNull CodedOutputStream getOutputStream() { return codedOutputStream; @@ -85,32 +92,37 @@ static class CodedOutputStreamWrapper implements CodedOutputStreamHolder { } class StreamManager extends OrderedStreamLifecyleManager { - Span telemetrySpan; + Context telemetryContext; String connectionId; Instant startTime; - public StreamManager(Span telemetrySpan, String connectionId) { - this.telemetrySpan = telemetrySpan; + public StreamManager(Context incomingTelemetryContext, String connectionId) { + this.telemetryContext = incomingTelemetryContext; this.connectionId = connectionId; this.startTime = Instant.now(); } @Override public void close() throws IOException { - try (var namedOnlyForAutoClose = Context.current().with(telemetrySpan).makeCurrent()) { - var histogram = GlobalOpenTelemetry.get().getMeter(TELEMETRY_SCOPE_NAME) - .histogramBuilder("connection_lifetime").build(); - telemetrySpan.setAttribute("connectionId", connectionId); - histogram.record((double) Duration.between(startTime, Instant.now()).toMillis(), Attributes.empty(), - Context.current().with(telemetrySpan)); - telemetrySpan.end(); - } + METERING_CLOSURE_OP.ifPresent(m->{ + m.meterHistogramMillis(telemetryContext, "connection_lifetime", + Duration.between(startTime, Instant.now())); + m.meterDeltaEvent(telemetryContext, "offloaders_active", -1); + m.meterIncrementEvent(telemetryContext, "offloader_closed"); + }); + Span.fromContext(telemetryContext).end(); } @Override public CodedOutputStreamWrapper createStream() { + var newStreamCtx = METERING_CLOSURE_OP.map(m-> { + m.meterIncrementEvent(telemetryContext, "stream_created"); + try (var scope = telemetryContext.makeCurrent()) { + return Context.current().with(m.tracer.spanBuilder("recordStream").startSpan()); + } + }).orElse(null); ByteBuffer bb = ByteBuffer.allocate(bufferSize); - return new CodedOutputStreamWrapper(CodedOutputStream.newInstance(bb), bb); + return new CodedOutputStreamWrapper(CodedOutputStream.newInstance(bb), bb, newStreamCtx); } @Override @@ -131,8 +143,20 @@ public CodedOutputStreamWrapper createStream() { // Used to essentially wrap Future returned by Producer to CompletableFuture var cf = new CompletableFuture(); log.debug("Sending Kafka producer record: {} for topic: {}", recordId, topicNameForTraffic); + + var flushContext = METERING_CLOSURE_OP.map(m-> { + Span.fromContext(osh.streamContext).end(); + try (var scope = telemetryContext + .with(RECORD_ID_KEY, recordId) + .with(TOPIC_KEY, topicNameForTraffic) + .with(RECORD_SIZE_KEY, kafkaRecord.value().length).makeCurrent()) { + m.meterIncrementEvent(telemetryContext, "stream_flush_called"); + return Context.current().with(m.tracer.spanBuilder("flushRecord").startSpan()); + } + }).orElse(null); + // Async request to Kafka cluster - producer.send(kafkaRecord, handleProducerRecordSent(cf, recordId)); + producer.send(kafkaRecord, handleProducerRecordSent(cf, recordId, flushContext)); metricsLogger.atSuccess(MetricsEvent.RECORD_SENT_TO_KAFKA) .setAttribute(MetricsAttributeKey.CHANNEL_ID, connectionId) .setAttribute(MetricsAttributeKey.TOPIC_NAME, topicNameForTraffic) @@ -157,8 +181,18 @@ public CodedOutputStreamWrapper createStream() { * retried or not retried at all: https://kafka.apache.org/35/javadoc/org/apache/kafka/common/errors/RetriableException.html * as well as basic retry backoff */ - private Callback handleProducerRecordSent(CompletableFuture cf, String recordId) { + private Callback handleProducerRecordSent(CompletableFuture cf, String recordId, + Context flushContext) { return (metadata, exception) -> { + METERING_CLOSURE_OP.ifPresent(m-> { + m.meterIncrementEvent(telemetryContext, + exception==null ? "stream_flush_success" : "stream_flush_failure"); + m.meterIncrementEvent(telemetryContext, + exception==null ? "stream_flush_success_bytes" : "stream_flush_failure_bytes", + flushContext.get(RECORD_SIZE_KEY)); + Span.fromContext(flushContext).end(); + }); + if (exception != null) { log.error("Error sending producer record: {}", recordId, exception); cf.completeExceptionally(exception); @@ -170,5 +204,4 @@ private Callback handleProducerRecordSent(CompletableFuture cf, }; } } - } diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogger.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogger.java index 23f73aa29..ea116fc47 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogger.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogger.java @@ -129,7 +129,7 @@ public void meterHistogram(Context ctx, String eventName, double value) { * metricsLogger.atSuccess().addKeyValue("key", "value").setMessage("Task succeeded").log(); */ public MetricsLogBuilder atSuccess(MetricsEvent event) { - return new MetricsLogBuilder().atSuccess(event); + return new MetricsLogBuilder(logger).atSuccess(event); } /** @@ -142,7 +142,7 @@ public MetricsLogBuilder atError(MetricsEvent event, Throwable cause) { if (cause == null) { return atError(event); } - return new MetricsLogBuilder().atError(event) + return new MetricsLogBuilder(logger).atError(event) .setAttribute(MetricsAttributeKey.EXCEPTION_MESSAGE, cause.getMessage()) .setAttribute(MetricsAttributeKey.EXCEPTION_TYPE, cause.getClass().getName()); } @@ -153,10 +153,10 @@ public MetricsLogBuilder atError(MetricsEvent event, Throwable cause) { */ public MetricsLogBuilder atError(MetricsEvent event) { - return new MetricsLogBuilder().atError(event); + return new MetricsLogBuilder(logger).atError(event); } public MetricsLogBuilder atTrace(MetricsEvent event) { - return new MetricsLogBuilder().atTrace(event); + return new MetricsLogBuilder(logger).atTrace(event); } } diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index e220c357d..98625caa9 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -54,40 +54,40 @@ services: - zipkin-all-in-one # Run combined instance of Capture Proxy and Elasticsearch - capture-proxy-es: +# capture-proxy-es: +# image: 'migrations/capture_proxy:latest' +# networks: +# - migrations +# ports: +# - "9200:9200" +# - "19200:19200" +# volumes: +# - /Users/schohn/dev/opensearch-migrations/TrafficCapture/containerLogs:/logs +# environment: +# - http.port=19200 +# # Run processes for elasticsearch and capture proxy, and exit if either one ends +# command: /bin/sh -c '/usr/local/bin/docker-entrypoint.sh eswrapper & /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy --kafkaConnection kafka:9092 --destinationUri https://localhost:19200 --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml --otelCollectorEndpoint http://otel-collector:4317 & wait -n 1' +# depends_on: +# - kafka + +# Run separate instances of Capture Proxy and Elasticsearch + capture-proxy: image: 'migrations/capture_proxy:latest' networks: - migrations ports: - "9200:9200" - - "19200:19200" - volumes: - - /Users/schohn/dev/opensearch-migrations/TrafficCapture/containerLogs:/logs - environment: - - http.port=19200 - # Run processes for elasticsearch and capture proxy, and exit if either one ends - command: /bin/sh -c '/usr/local/bin/docker-entrypoint.sh eswrapper & /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy --kafkaConnection kafka:9092 --destinationUri https://localhost:19200 --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml --otelCollectorEndpoint http://otel-collector:4317 & wait -n 1' + command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy --kafkaConnection kafka:9092 --destinationUri https://elasticsearch:9200 --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml --otelCollectorEndpoint http://otel-collector:4317" depends_on: - kafka + - elasticsearch -# Run separate instances of Capture Proxy and Elasticsearch -# capture-proxy: -# image: 'migrations/capture_proxy:latest' -# networks: -# - migrations -# ports: -# - "9200:9200" -# command: /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy --kafkaConnection kafka:9092 --destinationUri https://elasticsearch:9200 --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml -# depends_on: -# - kafka -# - elasticsearch -# -# elasticsearch: -# image: 'migrations/elasticsearch_searchguard:latest' -# networks: -# - migrations -# ports: -# - '19200:9200' + elasticsearch: + image: 'migrations/elasticsearch_searchguard:latest' + networks: + - migrations + ports: + - '19200:9200' zookeeper: image: docker.io/bitnami/zookeeper:3.8 diff --git a/TrafficCapture/nettyWireLogging/build.gradle b/TrafficCapture/nettyWireLogging/build.gradle index 1a3c2bb64..523fe0078 100644 --- a/TrafficCapture/nettyWireLogging/build.gradle +++ b/TrafficCapture/nettyWireLogging/build.gradle @@ -13,7 +13,9 @@ dependencies { implementation project(':captureOffloader') implementation project(':coreUtilities') - api group: 'io.netty', name: 'netty-all', version: '4.1.100.Final' + api group: 'io.netty', name: 'netty-buffer' + api group: 'io.netty', name: 'netty-codec-http' + api group: 'io.netty', name: 'netty-handler' implementation group: 'io.opentelemetry', name:'opentelemetry-api' implementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7' diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java index 7b5161ed6..52e745230 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java @@ -3,6 +3,8 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.HttpRequest; import io.netty.util.ReferenceCountUtil; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.context.Context; import lombok.Lombok; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; @@ -13,9 +15,10 @@ public class ConditionallyReliableLoggingHttpRequestHandler extends LoggingHttpRequestHandler { private final Predicate shouldBlockPredicate; - public ConditionallyReliableLoggingHttpRequestHandler(IChannelConnectionCaptureSerializer trafficOffloader, + public ConditionallyReliableLoggingHttpRequestHandler(Context incomingContext, + IChannelConnectionCaptureSerializer trafficOffloader, Predicate headerPredicateForWhenToBlock) { - super(trafficOffloader); + super(incomingContext, trafficOffloader); this.shouldBlockPredicate = headerPredicateForWhenToBlock; } @@ -23,12 +26,22 @@ public ConditionallyReliableLoggingHttpRequestHandler(IChannelConnectionCaptureS protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Object msg, HttpRequest httpRequest) throws Exception { if (shouldBlockPredicate.test(httpRequest)) { + var blockingSpan = METERING_CLOSURE_OP.map(m->{ + m.meterIncrementEvent(telemetryContext, "blockingRequestUntilFlush"); + try (var namedOnlyForAutoClose = telemetryContext.makeCurrent()) { + return GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME) + .spanBuilder("blockedOnFlush").startSpan(); + }}).orElse(null); trafficOffloader.flushCommitAndResetStream(false).whenComplete((result, t) -> { + METERING_CLOSURE_OP.ifPresent(m->{ + blockingSpan.end(); + m.meterIncrementEvent(telemetryContext, t != null ? "blockedFlushFailure" : "blockedFlushSuccess"); + }); if (t != null) { // This is a spot where we would benefit from having a behavioral policy that different users // could set as needed. Some users may be fine with just logging a failed offloading of a request // where other users may want to stop entirely. JIRA here: https://opensearch.atlassian.net/browse/MIGRATIONS-1276 - log.warn("Got error: " + t.getMessage()); + log.warn("Dropping request - Got error: " + t.getMessage()); ReferenceCountUtil.release(msg); } else { try { @@ -39,6 +52,9 @@ protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Ob } }); } else { + METERING_CLOSURE_OP.ifPresent(m->{ + m.meterIncrementEvent(telemetryContext, "nonBlockingRequest"); + }); super.channelFinishedReadingAnHttpMessage(ctx, msg, httpRequest); } } diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java index de5b0256e..f47b7de8a 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java @@ -13,6 +13,9 @@ import io.netty.handler.codec.http.HttpRequestDecoder; import io.netty.handler.codec.http.HttpVersion; import io.netty.handler.codec.http.LastHttpContent; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; import lombok.Getter; import lombok.Lombok; import lombok.extern.slf4j.Slf4j; @@ -22,9 +25,13 @@ import org.opensearch.migrations.coreutils.MetricsLogger; import java.time.Instant; +import java.util.Optional; @Slf4j public class LoggingHttpRequestHandler extends ChannelInboundHandlerAdapter { + public static final String TELEMETRY_SCOPE_NAME = "LoggingHttpInboundHandler"; + public static final Optional METERING_CLOSURE_OP = + Optional.of(new MetricsLogger.SimpleMeteringClosure(TELEMETRY_SCOPE_NAME)); private static final MetricsLogger metricsLogger = new MetricsLogger("LoggingHttpRequestHandler"); static class SimpleHttpRequestDecoder extends HttpRequestDecoder { @@ -72,9 +79,21 @@ public HttpRequest resetCurrentRequest() { protected final EmbeddedChannel httpDecoderChannel; protected final SimpleHttpRequestDecoder requestDecoder; - - - public LoggingHttpRequestHandler(IChannelConnectionCaptureSerializer trafficOffloader) { + protected final Context telemetryContext; + private final Instant createdTime; + + + public LoggingHttpRequestHandler(Context incomingContext, IChannelConnectionCaptureSerializer trafficOffloader) { + this.createdTime = Instant.now(); + telemetryContext = METERING_CLOSURE_OP.map(m->{ + try (var scope = incomingContext.makeCurrent()) { + var span = GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME) + .spanBuilder("frontendConnection").startSpan(); + var ctx = incomingContext.with(span); + m.meterIncrementEvent(ctx, "requestStarted"); + return ctx; + } + }).orElse(null); this.trafficOffloader = trafficOffloader; requestDecoder = new SimpleHttpRequestDecoder(); // as a field for easier debugging httpDecoderChannel = new EmbeddedChannel( @@ -85,9 +104,12 @@ public LoggingHttpRequestHandler(IChannelConnectionCaptureSerializer trafficO private HttpProcessedState parseHttpMessageParts(ByteBuf msg) { httpDecoderChannel.writeInbound(msg); // Consume this outright, up to the caller to know what else to do - return getHandlerThatHoldsParsedHttpRequest().isDone ? + var state = getHandlerThatHoldsParsedHttpRequest().isDone ? HttpProcessedState.FULL_MESSAGE : HttpProcessedState.ONGOING; + METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, + state == HttpProcessedState.FULL_MESSAGE ? "requestFullyParsed" : "requestPartiallyParsed")); + return state; } private SimpleDecodedHttpRequestHandler getHandlerThatHoldsParsedHttpRequest() { @@ -97,6 +119,7 @@ private SimpleDecodedHttpRequestHandler getHandlerThatHoldsParsedHttpRequest() { @Override public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { trafficOffloader.addCloseEvent(Instant.now()); + METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, "unregistered")); trafficOffloader.flushCommitAndResetStream(true).whenComplete((result, t) -> { if (t != null) { log.warn("Got error: " + t.getMessage()); @@ -113,6 +136,10 @@ public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { @Override public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { + METERING_CLOSURE_OP.ifPresent(m->{ + m.meterIncrementEvent(telemetryContext, "handlerRemoved"); + Span.fromContext(telemetryContext).end(); + }); trafficOffloader.flushCommitAndResetStream(true).whenComplete((result, t) -> { if (t != null) { log.warn("Got error: " + t.getMessage()); @@ -128,6 +155,7 @@ public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Object msg, HttpRequest httpRequest) throws Exception { super.channelRead(ctx, msg); + METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, "requestReceived")); metricsLogger.atSuccess(MetricsEvent.RECEIVED_FULL_HTTP_REQUEST) .setAttribute(MetricsAttributeKey.CHANNEL_ID, ctx.channel().id().asLongText()) .setAttribute(MetricsAttributeKey.HTTP_METHOD, httpRequest.method().toString()) @@ -141,6 +169,10 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { var bb = ((ByteBuf) msg).retainedDuplicate(); trafficOffloader.addReadEvent(timestamp, bb); + METERING_CLOSURE_OP.ifPresent(m-> { + m.meterIncrementEvent(telemetryContext, "read"); + m.meterIncrementEvent(telemetryContext, "readBytes", bb.readableBytes()); + }); metricsLogger.atSuccess(MetricsEvent.RECEIVED_REQUEST_COMPONENT) .setAttribute(MetricsAttributeKey.CHANNEL_ID, ctx.channel().id().asLongText()).emit(); @@ -164,6 +196,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { trafficOffloader.addExceptionCaughtEvent(Instant.now(), cause); + METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, "exception")); httpDecoderChannel.close(); super.exceptionCaught(ctx, cause); } diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java index 66d8912a4..9bb33fd22 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java @@ -4,6 +4,9 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelOutboundHandlerAdapter; import io.netty.channel.ChannelPromise; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; @@ -11,66 +14,102 @@ import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import java.net.SocketAddress; +import java.time.Duration; import java.time.Instant; +import java.util.Optional; @Slf4j public class LoggingHttpResponseHandler extends ChannelOutboundHandlerAdapter { - - private final IChannelConnectionCaptureSerializer trafficOffloader; + public static final String TELEMETRY_SCOPE_NAME = "LoggingHttpOutboundHandler"; + public static final Optional METERING_CLOSURE_OP = + Optional.of(new MetricsLogger.SimpleMeteringClosure(TELEMETRY_SCOPE_NAME)); private static final MetricsLogger metricsLogger = new MetricsLogger("LoggingHttpResponseHandler"); + private final IChannelConnectionCaptureSerializer trafficOffloader; + private Context telemetryContext; + private Instant connectTime; - public LoggingHttpResponseHandler(IChannelConnectionCaptureSerializer trafficOffloader) { + public LoggingHttpResponseHandler(Context incomingContext, + IChannelConnectionCaptureSerializer trafficOffloader) { this.trafficOffloader = trafficOffloader; + this.telemetryContext = incomingContext; } @Override public void bind(ChannelHandlerContext ctx, SocketAddress localAddress, ChannelPromise promise) throws Exception { trafficOffloader.addBindEvent(Instant.now(), localAddress); + METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, "bind")); super.bind(ctx, localAddress, promise); } @Override public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, SocketAddress localAddress, ChannelPromise promise) throws Exception { trafficOffloader.addConnectEvent(Instant.now(), remoteAddress, localAddress); + + METERING_CLOSURE_OP.ifPresent(m->{ + var span = GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME) + .spanBuilder("backendConnection").startSpan(); + telemetryContext = telemetryContext.with(span); + connectTime = Instant.now(); + + m.meterIncrementEvent(telemetryContext, "connect"); + m.meterDeltaEvent(telemetryContext, "connections", 1); + }); + super.connect(ctx, remoteAddress, localAddress, promise); } @Override public void disconnect(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { trafficOffloader.addDisconnectEvent(Instant.now()); + METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, "disconnect")); super.disconnect(ctx, promise); } @Override public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { trafficOffloader.addCloseEvent(Instant.now()); - super.close(ctx, promise); + + METERING_CLOSURE_OP.ifPresent(m-> { + m.meterIncrementEvent(telemetryContext, "close"); + m.meterDeltaEvent(telemetryContext, "connections", -1); + m.meterHistogramMillis(telemetryContext, "connectionDuration", + Duration.between(connectTime, Instant.now())); + Span.fromContext(telemetryContext).end(); + }); } @Override public void deregister(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { trafficOffloader.addDeregisterEvent(Instant.now()); + METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, "deregister")); super.deregister(ctx, promise); } @Override public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { - trafficOffloader.addWriteEvent(Instant.now(), (ByteBuf) msg); + var bb = (ByteBuf) msg; + trafficOffloader.addWriteEvent(Instant.now(), bb); metricsLogger.atSuccess(MetricsEvent.RECEIVED_RESPONSE_COMPONENT) .setAttribute(MetricsAttributeKey.CHANNEL_ID, ctx.channel().id().asLongText()).emit(); + METERING_CLOSURE_OP.ifPresent(m->{ + m.meterIncrementEvent(telemetryContext, "write"); + m.meterIncrementEvent(telemetryContext, "writeBytes", bb.readableBytes()); + }); super.write(ctx, msg, promise); } @Override public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { flush(ctx); + METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, "removed")); super.handlerRemoved(ctx); } @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { trafficOffloader.addExceptionCaughtEvent(Instant.now(), cause); + METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, "exception")); super.exceptionCaught(ctx, cause); } diff --git a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java index 3110d0188..76a2cc762 100644 --- a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java +++ b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java @@ -3,10 +3,19 @@ import com.google.protobuf.CodedOutputStream; import io.netty.buffer.ByteBuf; import io.netty.channel.embedded.EmbeddedChannel; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; +import io.opentelemetry.sdk.trace.data.SpanData; import lombok.AllArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.opensearch.migrations.testutils.TestUtilities; @@ -33,6 +42,10 @@ @Slf4j public class ConditionallyReliableLoggingHttpRequestHandlerTest { + @RegisterExtension + static final OpenTelemetryExtension otelTesting = OpenTelemetryExtension.create(); + private final Tracer tracer = otelTesting.getOpenTelemetry().getTracer("test"); + private final Meter meter = otelTesting.getOpenTelemetry().getMeter("test"); @AllArgsConstructor static class StreamManager extends OrderedStreamLifecyleManager { @@ -74,8 +87,12 @@ private static void writeMessageAndVerify(byte[] fullTrafficBytes, Consumertrue)); // true: block every request + new ConditionallyReliableLoggingHttpRequestHandler(telemetryCtx, offloader, + x->true)); // true: block every request channelWriter.accept(channel); // we wrote the correct data to the downstream handler/channel @@ -98,6 +115,9 @@ private static void writeMessageAndVerify(byte[] fullTrafficBytes, Consumer extends ChannelInitializer { + static final ContextKey CONNECTION_ID_KEY = ContextKey.named("connectionId"); private final IConnectionCaptureFactory connectionCaptureFactory; private final Supplier sslEngineProvider; @@ -41,9 +44,11 @@ protected void initChannel(SocketChannel ch) throws IOException { ch.pipeline().addLast(new SslHandler(sslEngineProvider.get())); } - var offloader = connectionCaptureFactory.createOffloader(ch.id().asLongText()); - ch.pipeline().addLast(new LoggingHttpResponseHandler(offloader)); - ch.pipeline().addLast(new ConditionallyReliableLoggingHttpRequestHandler(offloader, + var connectionId = ch.id().asLongText(); + var offloader = connectionCaptureFactory.createOffloader(connectionId); + var ctx = Context.current().with(CONNECTION_ID_KEY, connectionId); + ch.pipeline().addLast(new LoggingHttpResponseHandler<>(ctx, offloader)); + ch.pipeline().addLast(new ConditionallyReliableLoggingHttpRequestHandler(ctx, offloader, this::shouldGuaranteeMessageOffloading)); ch.pipeline().addLast(new FrontsideHandler(backsideConnectionPool)); } diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/resources/logging.properties b/TrafficCapture/trafficCaptureProxyServer/src/main/resources/logging.properties new file mode 100644 index 000000000..42fe83cdd --- /dev/null +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/resources/logging.properties @@ -0,0 +1,9 @@ +# Set the global logging level for all loggers +.level=FINE + +# Configure the console handler (or other handlers if you use them) +handlers=java.util.logging.ConsoleHandler + +# Set the logging level for the console handler +java.util.logging.ConsoleHandler.level=FINE +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter \ No newline at end of file From a8ae3d12617cdc067795033c6cbb39666fcb8de6 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Mon, 27 Nov 2023 22:31:30 -0500 Subject: [PATCH 05/20] Restore the docker-compose single-node/multi-node split docker-compose config hierarchy. This was broken from the merge https://github.com/opensearch-project/opensearch-migrations/pull/376/files#diff-430f89dc33402ecf692b9a8372f66e585bb2f9215596433216580efc2a56795c. Signed-off-by: Greg Schohn --- .../src/main/docker/docker-compose-single.yml | 4 +-- .../src/main/docker/docker-compose.yml | 36 ------------------- 2 files changed, 2 insertions(+), 38 deletions(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose-single.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose-single.yml index febda3d42..0ae8ffbcb 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose-single.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose-single.yml @@ -14,7 +14,7 @@ services: - http.port=19200 - discovery.type=single-node # Run processes for elasticsearch and capture proxy, and exit if either one ends - command: /bin/sh -c '/usr/local/bin/docker-entrypoint.sh eswrapper & /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy --kafkaConnection kafka:9092 --destinationUri https://localhost:19200 --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml & wait -n 1' + command: /bin/sh -c '/usr/local/bin/docker-entrypoint.sh eswrapper & /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy --kafkaConnection kafka:9092 --destinationUri https://localhost:19200 --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml --otelCollectorEndpoint http://otel-collector:4317 & wait -n 1' depends_on: - kafka @@ -25,7 +25,7 @@ services: # - migrations # ports: # - "9200:9200" -# command: /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy --kafkaConnection kafka:9092 --destinationUri https://elasticsearch:9200 --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml +# command: /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy --kafkaConnection kafka:9092 --destinationUri https://elasticsearch:9200 --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml --otelCollectorEndpoint http://otel-collector:4317 --otelCollectorEndpoint http://otel-collector:4317 # depends_on: # - kafka # - elasticsearch diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 98625caa9..ef31558c7 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -53,42 +53,6 @@ services: - jaeger-all-in-one - zipkin-all-in-one - # Run combined instance of Capture Proxy and Elasticsearch -# capture-proxy-es: -# image: 'migrations/capture_proxy:latest' -# networks: -# - migrations -# ports: -# - "9200:9200" -# - "19200:19200" -# volumes: -# - /Users/schohn/dev/opensearch-migrations/TrafficCapture/containerLogs:/logs -# environment: -# - http.port=19200 -# # Run processes for elasticsearch and capture proxy, and exit if either one ends -# command: /bin/sh -c '/usr/local/bin/docker-entrypoint.sh eswrapper & /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy --kafkaConnection kafka:9092 --destinationUri https://localhost:19200 --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml --otelCollectorEndpoint http://otel-collector:4317 & wait -n 1' -# depends_on: -# - kafka - -# Run separate instances of Capture Proxy and Elasticsearch - capture-proxy: - image: 'migrations/capture_proxy:latest' - networks: - - migrations - ports: - - "9200:9200" - command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy --kafkaConnection kafka:9092 --destinationUri https://elasticsearch:9200 --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml --otelCollectorEndpoint http://otel-collector:4317" - depends_on: - - kafka - - elasticsearch - - elasticsearch: - image: 'migrations/elasticsearch_searchguard:latest' - networks: - - migrations - ports: - - '19200:9200' - zookeeper: image: docker.io/bitnami/zookeeper:3.8 networks: From da9d36bf8fe889320fe5bde3ea27b12af32b5904 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Mon, 27 Nov 2023 22:32:18 -0500 Subject: [PATCH 06/20] Add labels to each metric instrument so that multiple values can be plotted within the same graph in prometheus. Signed-off-by: Greg Schohn --- .../migrations/coreutils/MetricsLogger.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogger.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogger.java index ea116fc47..f516c9de3 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogger.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogger.java @@ -2,6 +2,7 @@ import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.Tracer; @@ -101,13 +102,19 @@ public void meterIncrementEvent(Context ctx, String eventName) { public void meterIncrementEvent(Context ctx, String eventName, long increment) { if (ctx == null) { return; } try (var namedOnlyForAutoClose = ctx.makeCurrent()) { - meter.counterBuilder(eventName).build().add(increment); + meter.counterBuilder(eventName) + .build().add(increment, Attributes.builder() + .put("labelName", eventName) + .build()); } } public void meterDeltaEvent(Context ctx, String eventName, long delta) { if (ctx == null) { return; } try (var namedOnlyForAutoClose = ctx.makeCurrent()) { - meter.upDownCounterBuilder(eventName).build().add(delta); + meter.upDownCounterBuilder(eventName) + .build().add(delta, Attributes.builder() + .put("labelName", eventName) + .build()); } } public void meterHistogramMillis(Context ctx, String eventName, Duration between) { @@ -116,7 +123,10 @@ public void meterHistogramMillis(Context ctx, String eventName, Duration between public void meterHistogram(Context ctx, String eventName, double value) { if (ctx == null) { return; } try (var namedOnlyForAutoClose = ctx.makeCurrent()) { - meter.histogramBuilder(eventName).build().record(value); + meter.histogramBuilder(eventName) + .build().record(value, Attributes.builder() + .put("labelName", eventName) + .build()); } } } From 06618caa8e31faa65acccf311f293978854cd7c3 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Mon, 27 Nov 2023 23:13:49 -0500 Subject: [PATCH 07/20] Move the MetricsClosure into its own class and stop stuffing the metrics into an optional. Dropping the optionals makes the code simpler and if we don't want to do logging, we can just not fill in the configuration for the SDK. Signed-off-by: Greg Schohn --- .../kafkaoffloader/KafkaCaptureFactory.java | 77 +++++------ .../migrations/coreutils/MetricsLogger.java | 90 ------------- .../coreutils/SimpleMeteringClosure.java | 123 ++++++++++++++++++ ...allyReliableLoggingHttpRequestHandler.java | 25 ++-- .../netty/LoggingHttpRequestHandler.java | 46 +++---- .../netty/LoggingHttpResponseHandler.java | 49 +++---- .../proxyserver/CaptureProxy.java | 4 +- .../migrations/replay/TrafficReplayer.java | 2 +- 8 files changed, 216 insertions(+), 200 deletions(-) create mode 100644 TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java diff --git a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java index a3696fefe..4a94d93d9 100644 --- a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java +++ b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java @@ -14,6 +14,7 @@ import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; import org.apache.kafka.clients.producer.RecordMetadata; +import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.trafficcapture.CodedOutputStreamHolder; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; @@ -26,7 +27,6 @@ import java.time.Duration; import java.time.Instant; import java.util.Arrays; -import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -37,8 +37,7 @@ public class KafkaCaptureFactory implements IConnectionCaptureFactory TOPIC_KEY = ContextKey.named("topic"); private static final ContextKey RECORD_SIZE_KEY = ContextKey.named("recordSize"); public static final String TELEMETRY_SCOPE_NAME = "KafkaCapture"; - public static final Optional METERING_CLOSURE_OP = - Optional.of(new MetricsLogger.SimpleMeteringClosure(TELEMETRY_SCOPE_NAME)); + public static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure(TELEMETRY_SCOPE_NAME); private static final MetricsLogger metricsLogger = new MetricsLogger("BacksideHandler"); @@ -67,15 +66,13 @@ public KafkaCaptureFactory(String nodeId, Producer producer, int @Override public IChannelConnectionCaptureSerializer createOffloader(String connectionId) { - var context = METERING_CLOSURE_OP.map(m->{ - Span offloaderSpan = GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME) - .spanBuilder("offloader").startSpan(); - offloaderSpan.setAttribute("offloaderConnectionId", connectionId); - var c = Context.current().with(offloaderSpan); - m.meterIncrementEvent(c, "offloader_created"); - m.meterDeltaEvent(c, "offloaders_active", 1); - return c; - }).orElse(null); + Span offloaderSpan = GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME) + .spanBuilder("offloader").startSpan(); + offloaderSpan.setAttribute("offloaderConnectionId", connectionId); + var context = Context.current().with(offloaderSpan); + METERING_CLOSURE.meterIncrementEvent(context, "offloader_created"); + METERING_CLOSURE.meterDeltaEvent(context, "offloaders_active", 1); + return new StreamChannelConnectionCaptureSerializer<>(nodeId, connectionId, new StreamManager(context, connectionId)); } @@ -104,23 +101,23 @@ public StreamManager(Context incomingTelemetryContext, String connectionId) { @Override public void close() throws IOException { - METERING_CLOSURE_OP.ifPresent(m->{ - m.meterHistogramMillis(telemetryContext, "connection_lifetime", + METERING_CLOSURE.meterHistogramMillis(telemetryContext, "connection_lifetime", Duration.between(startTime, Instant.now())); - m.meterDeltaEvent(telemetryContext, "offloaders_active", -1); - m.meterIncrementEvent(telemetryContext, "offloader_closed"); - }); + METERING_CLOSURE.meterDeltaEvent(telemetryContext, "offloaders_active", -1); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "offloader_closed"); + Span.fromContext(telemetryContext).end(); } @Override public CodedOutputStreamWrapper createStream() { - var newStreamCtx = METERING_CLOSURE_OP.map(m-> { - m.meterIncrementEvent(telemetryContext, "stream_created"); - try (var scope = telemetryContext.makeCurrent()) { - return Context.current().with(m.tracer.spanBuilder("recordStream").startSpan()); - } - }).orElse(null); + Context newStreamCtx; + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "stream_created"); + try (var scope = telemetryContext.makeCurrent()) { + newStreamCtx = Context.current() + .with(METERING_CLOSURE.tracer.spanBuilder("recordStream").startSpan()); + } + ByteBuffer bb = ByteBuffer.allocate(bufferSize); return new CodedOutputStreamWrapper(CodedOutputStream.newInstance(bb), bb, newStreamCtx); } @@ -144,16 +141,16 @@ public CodedOutputStreamWrapper createStream() { var cf = new CompletableFuture(); log.debug("Sending Kafka producer record: {} for topic: {}", recordId, topicNameForTraffic); - var flushContext = METERING_CLOSURE_OP.map(m-> { - Span.fromContext(osh.streamContext).end(); - try (var scope = telemetryContext - .with(RECORD_ID_KEY, recordId) - .with(TOPIC_KEY, topicNameForTraffic) - .with(RECORD_SIZE_KEY, kafkaRecord.value().length).makeCurrent()) { - m.meterIncrementEvent(telemetryContext, "stream_flush_called"); - return Context.current().with(m.tracer.spanBuilder("flushRecord").startSpan()); - } - }).orElse(null); + Context flushContext; + Span.fromContext(osh.streamContext).end(); + try (var scope = telemetryContext + .with(RECORD_ID_KEY, recordId) + .with(TOPIC_KEY, topicNameForTraffic) + .with(RECORD_SIZE_KEY, kafkaRecord.value().length).makeCurrent()) { + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "stream_flush_called"); + flushContext = Context.current() + .with(METERING_CLOSURE.tracer.spanBuilder("flushRecord").startSpan()); + } // Async request to Kafka cluster producer.send(kafkaRecord, handleProducerRecordSent(cf, recordId, flushContext)); @@ -184,14 +181,12 @@ public CodedOutputStreamWrapper createStream() { private Callback handleProducerRecordSent(CompletableFuture cf, String recordId, Context flushContext) { return (metadata, exception) -> { - METERING_CLOSURE_OP.ifPresent(m-> { - m.meterIncrementEvent(telemetryContext, - exception==null ? "stream_flush_success" : "stream_flush_failure"); - m.meterIncrementEvent(telemetryContext, - exception==null ? "stream_flush_success_bytes" : "stream_flush_failure_bytes", - flushContext.get(RECORD_SIZE_KEY)); - Span.fromContext(flushContext).end(); - }); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, + exception==null ? "stream_flush_success" : "stream_flush_failure"); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, + exception==null ? "stream_flush_success_bytes" : "stream_flush_failure_bytes", + flushContext.get(RECORD_SIZE_KEY)); + Span.fromContext(flushContext).end(); if (exception != null) { log.error("Error sending producer record: {}", recordId, exception); diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogger.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogger.java index f516c9de3..ef61af2d6 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogger.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/MetricsLogger.java @@ -4,7 +4,6 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.Meter; -import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Context; import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter; @@ -42,95 +41,6 @@ public MetricsLogger(String source) { logger = LoggerFactory.getLogger(String.format("MetricsLogger.%s", source)); } - public static void initializeOpenTelemetry(String serviceName, String collectorEndpoint) { - var serviceResource = Resource.getDefault().toBuilder() - .put(ResourceAttributes.SERVICE_NAME, serviceName) - .build(); - - OpenTelemetrySdk openTelemetrySdk = - OpenTelemetrySdk.builder() - .setLoggerProvider( - SdkLoggerProvider.builder() - .setResource(serviceResource) - .addLogRecordProcessor( - BatchLogRecordProcessor.builder( - OtlpGrpcLogRecordExporter.builder() - .setEndpoint(collectorEndpoint) - .build()) - .build()) - .build()) - .setTracerProvider( - SdkTracerProvider.builder() - .setResource(serviceResource) - .addSpanProcessor( - BatchSpanProcessor.builder( - OtlpGrpcSpanExporter.builder() - .setEndpoint(collectorEndpoint) - .setTimeout(2, TimeUnit.SECONDS) - .build()) - .setScheduleDelay(100, TimeUnit.MILLISECONDS) - .build()) - .build()) - .setMeterProvider( - SdkMeterProvider.builder() - .setResource(serviceResource) - .registerMetricReader( - PeriodicMetricReader.builder( - OtlpGrpcMetricExporter.builder() - .setEndpoint(collectorEndpoint) - .build()) - .setInterval(Duration.ofMillis(1000)) - .build()) - .build()) - .buildAndRegisterGlobal(); - - // Add hook to close SDK, which flushes logs - Runtime.getRuntime().addShutdownHook(new Thread(openTelemetrySdk::close)); - //OpenTelemetryAppender.install(GlobalOpenTelemetry.get()); - } - - public static class SimpleMeteringClosure { - public final Meter meter; - public final Tracer tracer; - public SimpleMeteringClosure(String scopeName) { - meter = GlobalOpenTelemetry.getMeter(scopeName); - tracer = GlobalOpenTelemetry.getTracer(scopeName); - } - public void meterIncrementEvent(Context ctx, String eventName) { - meterIncrementEvent(ctx, eventName, 1); - } - public void meterIncrementEvent(Context ctx, String eventName, long increment) { - if (ctx == null) { return; } - try (var namedOnlyForAutoClose = ctx.makeCurrent()) { - meter.counterBuilder(eventName) - .build().add(increment, Attributes.builder() - .put("labelName", eventName) - .build()); - } - } - public void meterDeltaEvent(Context ctx, String eventName, long delta) { - if (ctx == null) { return; } - try (var namedOnlyForAutoClose = ctx.makeCurrent()) { - meter.upDownCounterBuilder(eventName) - .build().add(delta, Attributes.builder() - .put("labelName", eventName) - .build()); - } - } - public void meterHistogramMillis(Context ctx, String eventName, Duration between) { - meterHistogram(ctx, eventName, (double) between.toMillis()); - } - public void meterHistogram(Context ctx, String eventName, double value) { - if (ctx == null) { return; } - try (var namedOnlyForAutoClose = ctx.makeCurrent()) { - meter.histogramBuilder(eventName) - .build().record(value, Attributes.builder() - .put("labelName", eventName) - .build()); - } - } - } - /** * To indicate a successful event (e.g. data received or data sent) that may be a helpful * metric, this method can be used to return a LoggingEventBuilder. The LoggingEventBuilder diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java new file mode 100644 index 000000000..3ef604072 --- /dev/null +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java @@ -0,0 +1,123 @@ +package org.opensearch.migrations.coreutils; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter; +import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.logs.SdkLoggerProvider; +import io.opentelemetry.sdk.logs.export.BatchLogRecordProcessor; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +public class SimpleMeteringClosure { + public final Meter meter; + public final Tracer tracer; + + public SimpleMeteringClosure(String scopeName) { + meter = GlobalOpenTelemetry.getMeter(scopeName); + tracer = GlobalOpenTelemetry.getTracer(scopeName); + } + + public static void initializeOpenTelemetry(String serviceName, String collectorEndpoint) { + var serviceResource = Resource.getDefault().toBuilder() + .put(ResourceAttributes.SERVICE_NAME, serviceName) + .build(); + + OpenTelemetrySdk openTelemetrySdk = + OpenTelemetrySdk.builder() + .setLoggerProvider( + SdkLoggerProvider.builder() + .setResource(serviceResource) + .addLogRecordProcessor( + BatchLogRecordProcessor.builder( + OtlpGrpcLogRecordExporter.builder() + .setEndpoint(collectorEndpoint) + .build()) + .build()) + .build()) + .setTracerProvider( + SdkTracerProvider.builder() + .setResource(serviceResource) + .addSpanProcessor( + BatchSpanProcessor.builder( + OtlpGrpcSpanExporter.builder() + .setEndpoint(collectorEndpoint) + .setTimeout(2, TimeUnit.SECONDS) + .build()) + .setScheduleDelay(100, TimeUnit.MILLISECONDS) + .build()) + .build()) + .setMeterProvider( + SdkMeterProvider.builder() + .setResource(serviceResource) + .registerMetricReader( + PeriodicMetricReader.builder( + OtlpGrpcMetricExporter.builder() + .setEndpoint(collectorEndpoint) + .build()) + .setInterval(Duration.ofMillis(1000)) + .build()) + .build()) + .buildAndRegisterGlobal(); + + // Add hook to close SDK, which flushes logs + Runtime.getRuntime().addShutdownHook(new Thread(openTelemetrySdk::close)); + //OpenTelemetryAppender.install(GlobalOpenTelemetry.get()); + } + + public void meterIncrementEvent(Context ctx, String eventName) { + meterIncrementEvent(ctx, eventName, 1); + } + + public void meterIncrementEvent(Context ctx, String eventName, long increment) { + if (ctx == null) { + return; + } + try (var namedOnlyForAutoClose = ctx.makeCurrent()) { + meter.counterBuilder(eventName) + .build().add(increment, Attributes.builder() + .put("labelName", eventName) + .build()); + } + } + + public void meterDeltaEvent(Context ctx, String eventName, long delta) { + if (ctx == null) { + return; + } + try (var namedOnlyForAutoClose = ctx.makeCurrent()) { + meter.upDownCounterBuilder(eventName) + .build().add(delta, Attributes.builder() + .put("labelName", eventName) + .build()); + } + } + + public void meterHistogramMillis(Context ctx, String eventName, Duration between) { + meterHistogram(ctx, eventName, (double) between.toMillis()); + } + + public void meterHistogram(Context ctx, String eventName, double value) { + if (ctx == null) { + return; + } + try (var namedOnlyForAutoClose = ctx.makeCurrent()) { + meter.histogramBuilder(eventName) + .build().record(value, Attributes.builder() + .put("labelName", eventName) + .build()); + } + } +} diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java index 52e745230..41ee43295 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java @@ -4,6 +4,7 @@ import io.netty.handler.codec.http.HttpRequest; import io.netty.util.ReferenceCountUtil; import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; import io.opentelemetry.context.Context; import lombok.Lombok; import lombok.extern.slf4j.Slf4j; @@ -26,17 +27,17 @@ public ConditionallyReliableLoggingHttpRequestHandler(Context incomingContext, protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Object msg, HttpRequest httpRequest) throws Exception { if (shouldBlockPredicate.test(httpRequest)) { - var blockingSpan = METERING_CLOSURE_OP.map(m->{ - m.meterIncrementEvent(telemetryContext, "blockingRequestUntilFlush"); - try (var namedOnlyForAutoClose = telemetryContext.makeCurrent()) { - return GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME) - .spanBuilder("blockedOnFlush").startSpan(); - }}).orElse(null); + Span blockingSpan; + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "blockingRequestUntilFlush"); + try (var namedOnlyForAutoClose = telemetryContext.makeCurrent()) { + blockingSpan = GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME) + .spanBuilder("blockedOnFlush").startSpan(); + } trafficOffloader.flushCommitAndResetStream(false).whenComplete((result, t) -> { - METERING_CLOSURE_OP.ifPresent(m->{ - blockingSpan.end(); - m.meterIncrementEvent(telemetryContext, t != null ? "blockedFlushFailure" : "blockedFlushSuccess"); - }); + blockingSpan.end(); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, + t != null ? "blockedFlushFailure" : "blockedFlushSuccess"); + if (t != null) { // This is a spot where we would benefit from having a behavioral policy that different users // could set as needed. Some users may be fine with just logging a failed offloading of a request @@ -52,9 +53,7 @@ protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Ob } }); } else { - METERING_CLOSURE_OP.ifPresent(m->{ - m.meterIncrementEvent(telemetryContext, "nonBlockingRequest"); - }); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "nonBlockingRequest"); super.channelFinishedReadingAnHttpMessage(ctx, msg, httpRequest); } } diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java index f47b7de8a..0d6086c4a 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java @@ -21,17 +21,16 @@ import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; +import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import org.opensearch.migrations.coreutils.MetricsLogger; import java.time.Instant; -import java.util.Optional; @Slf4j public class LoggingHttpRequestHandler extends ChannelInboundHandlerAdapter { public static final String TELEMETRY_SCOPE_NAME = "LoggingHttpInboundHandler"; - public static final Optional METERING_CLOSURE_OP = - Optional.of(new MetricsLogger.SimpleMeteringClosure(TELEMETRY_SCOPE_NAME)); + public static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure(TELEMETRY_SCOPE_NAME); private static final MetricsLogger metricsLogger = new MetricsLogger("LoggingHttpRequestHandler"); static class SimpleHttpRequestDecoder extends HttpRequestDecoder { @@ -85,15 +84,13 @@ public HttpRequest resetCurrentRequest() { public LoggingHttpRequestHandler(Context incomingContext, IChannelConnectionCaptureSerializer trafficOffloader) { this.createdTime = Instant.now(); - telemetryContext = METERING_CLOSURE_OP.map(m->{ - try (var scope = incomingContext.makeCurrent()) { - var span = GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME) - .spanBuilder("frontendConnection").startSpan(); - var ctx = incomingContext.with(span); - m.meterIncrementEvent(ctx, "requestStarted"); - return ctx; - } - }).orElse(null); + try (var scope = incomingContext.makeCurrent()) { + var span = GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME) + .spanBuilder("frontendConnection").startSpan(); + telemetryContext = incomingContext.with(span); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "requestStarted"); + } + this.trafficOffloader = trafficOffloader; requestDecoder = new SimpleHttpRequestDecoder(); // as a field for easier debugging httpDecoderChannel = new EmbeddedChannel( @@ -107,8 +104,8 @@ private HttpProcessedState parseHttpMessageParts(ByteBuf msg) { var state = getHandlerThatHoldsParsedHttpRequest().isDone ? HttpProcessedState.FULL_MESSAGE : HttpProcessedState.ONGOING; - METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, - state == HttpProcessedState.FULL_MESSAGE ? "requestFullyParsed" : "requestPartiallyParsed")); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, + state == HttpProcessedState.FULL_MESSAGE ? "requestFullyParsed" : "requestPartiallyParsed"); return state; } @@ -119,7 +116,7 @@ private SimpleDecodedHttpRequestHandler getHandlerThatHoldsParsedHttpRequest() { @Override public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { trafficOffloader.addCloseEvent(Instant.now()); - METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, "unregistered")); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "unregistered"); trafficOffloader.flushCommitAndResetStream(true).whenComplete((result, t) -> { if (t != null) { log.warn("Got error: " + t.getMessage()); @@ -136,10 +133,9 @@ public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { @Override public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { - METERING_CLOSURE_OP.ifPresent(m->{ - m.meterIncrementEvent(telemetryContext, "handlerRemoved"); - Span.fromContext(telemetryContext).end(); - }); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "handlerRemoved"); + Span.fromContext(telemetryContext).end(); + trafficOffloader.flushCommitAndResetStream(true).whenComplete((result, t) -> { if (t != null) { log.warn("Got error: " + t.getMessage()); @@ -155,7 +151,8 @@ public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Object msg, HttpRequest httpRequest) throws Exception { super.channelRead(ctx, msg); - METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, "requestReceived")); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "requestReceived"); + metricsLogger.atSuccess(MetricsEvent.RECEIVED_FULL_HTTP_REQUEST) .setAttribute(MetricsAttributeKey.CHANNEL_ID, ctx.channel().id().asLongText()) .setAttribute(MetricsAttributeKey.HTTP_METHOD, httpRequest.method().toString()) @@ -169,10 +166,9 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { var bb = ((ByteBuf) msg).retainedDuplicate(); trafficOffloader.addReadEvent(timestamp, bb); - METERING_CLOSURE_OP.ifPresent(m-> { - m.meterIncrementEvent(telemetryContext, "read"); - m.meterIncrementEvent(telemetryContext, "readBytes", bb.readableBytes()); - }); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "read"); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "readBytes", bb.readableBytes()); + metricsLogger.atSuccess(MetricsEvent.RECEIVED_REQUEST_COMPONENT) .setAttribute(MetricsAttributeKey.CHANNEL_ID, ctx.channel().id().asLongText()).emit(); @@ -196,7 +192,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { trafficOffloader.addExceptionCaughtEvent(Instant.now(), cause); - METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, "exception")); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "exception"); httpDecoderChannel.close(); super.exceptionCaught(ctx, cause); } diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java index 9bb33fd22..ed8a8d55c 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java @@ -11,18 +11,17 @@ import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; import org.opensearch.migrations.coreutils.MetricsLogger; +import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import java.net.SocketAddress; import java.time.Duration; import java.time.Instant; -import java.util.Optional; @Slf4j public class LoggingHttpResponseHandler extends ChannelOutboundHandlerAdapter { public static final String TELEMETRY_SCOPE_NAME = "LoggingHttpOutboundHandler"; - public static final Optional METERING_CLOSURE_OP = - Optional.of(new MetricsLogger.SimpleMeteringClosure(TELEMETRY_SCOPE_NAME)); + public static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure(TELEMETRY_SCOPE_NAME); private static final MetricsLogger metricsLogger = new MetricsLogger("LoggingHttpResponseHandler"); private final IChannelConnectionCaptureSerializer trafficOffloader; @@ -38,7 +37,7 @@ public LoggingHttpResponseHandler(Context incomingContext, @Override public void bind(ChannelHandlerContext ctx, SocketAddress localAddress, ChannelPromise promise) throws Exception { trafficOffloader.addBindEvent(Instant.now(), localAddress); - METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, "bind")); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "bind"); super.bind(ctx, localAddress, promise); } @@ -46,15 +45,12 @@ public void bind(ChannelHandlerContext ctx, SocketAddress localAddress, ChannelP public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, SocketAddress localAddress, ChannelPromise promise) throws Exception { trafficOffloader.addConnectEvent(Instant.now(), remoteAddress, localAddress); - METERING_CLOSURE_OP.ifPresent(m->{ - var span = GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME) - .spanBuilder("backendConnection").startSpan(); - telemetryContext = telemetryContext.with(span); - connectTime = Instant.now(); - - m.meterIncrementEvent(telemetryContext, "connect"); - m.meterDeltaEvent(telemetryContext, "connections", 1); - }); + var span = GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME) + .spanBuilder("backendConnection").startSpan(); + telemetryContext = telemetryContext.with(span); + connectTime = Instant.now(); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "connect"); + METERING_CLOSURE.meterDeltaEvent(telemetryContext, "connections", 1); super.connect(ctx, remoteAddress, localAddress, promise); } @@ -62,7 +58,7 @@ public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, Sock @Override public void disconnect(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { trafficOffloader.addDisconnectEvent(Instant.now()); - METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, "disconnect")); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "disconnect"); super.disconnect(ctx, promise); } @@ -70,19 +66,17 @@ public void disconnect(ChannelHandlerContext ctx, ChannelPromise promise) throws public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { trafficOffloader.addCloseEvent(Instant.now()); - METERING_CLOSURE_OP.ifPresent(m-> { - m.meterIncrementEvent(telemetryContext, "close"); - m.meterDeltaEvent(telemetryContext, "connections", -1); - m.meterHistogramMillis(telemetryContext, "connectionDuration", - Duration.between(connectTime, Instant.now())); - Span.fromContext(telemetryContext).end(); - }); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "close"); + METERING_CLOSURE.meterDeltaEvent(telemetryContext, "connections", -1); + METERING_CLOSURE.meterHistogramMillis(telemetryContext, "connectionDuration", + Duration.between(connectTime, Instant.now())); + Span.fromContext(telemetryContext).end(); } @Override public void deregister(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { trafficOffloader.addDeregisterEvent(Instant.now()); - METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, "deregister")); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "deregister"); super.deregister(ctx, promise); } @@ -92,24 +86,23 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) trafficOffloader.addWriteEvent(Instant.now(), bb); metricsLogger.atSuccess(MetricsEvent.RECEIVED_RESPONSE_COMPONENT) .setAttribute(MetricsAttributeKey.CHANNEL_ID, ctx.channel().id().asLongText()).emit(); - METERING_CLOSURE_OP.ifPresent(m->{ - m.meterIncrementEvent(telemetryContext, "write"); - m.meterIncrementEvent(telemetryContext, "writeBytes", bb.readableBytes()); - }); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "write"); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "writeBytes", bb.readableBytes()); + super.write(ctx, msg, promise); } @Override public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { flush(ctx); - METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, "removed")); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "removed"); super.handlerRemoved(ctx); } @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { trafficOffloader.addExceptionCaughtEvent(Instant.now(), cause); - METERING_CLOSURE_OP.ifPresent(m->m.meterIncrementEvent(telemetryContext, "exception")); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "exception"); super.exceptionCaught(ctx, cause); } diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java index 2d400d036..60de2bcff 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java @@ -17,7 +17,7 @@ import org.apache.kafka.common.config.SaslConfigs; import org.apache.logging.log4j.core.util.NullOutputStream; import org.opensearch.common.settings.Settings; -import org.opensearch.migrations.coreutils.MetricsLogger; +import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.trafficcapture.CodedOutputStreamHolder; import org.opensearch.migrations.trafficcapture.FileConnectionCaptureFactory; import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; @@ -284,7 +284,7 @@ public static void main(String[] args) throws InterruptedException, IOException var backsideUri = convertStringToUri(params.backsideUriString); if (params.otelCollectorEndpoint != null) { - MetricsLogger.initializeOpenTelemetry("capture-proxy", params.otelCollectorEndpoint); + SimpleMeteringClosure.initializeOpenTelemetry("capture-proxy", params.otelCollectorEndpoint); } var sksOp = Optional.ofNullable(params.sslConfigFilePath) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index c1cc0d8a7..d92f34b0d 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -67,7 +67,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.opensearch.migrations.coreutils.MetricsLogger.initializeOpenTelemetry; +import static org.opensearch.migrations.coreutils.SimpleMeteringClosure.initializeOpenTelemetry; @Slf4j public class TrafficReplayer { From aba1aab9878256b5c00a0634f7c7532e1f394f5c Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Tue, 28 Nov 2023 18:53:15 -0500 Subject: [PATCH 08/20] WIP - Cleanup + get Jaeger to work by switching the endpoint. Also introduce some more typesafe wrappers for contexts. Lots more to come. Signed-off-by: Greg Schohn --- .../captureKafkaOffloader/build.gradle | 2 +- .../kafkaoffloader/KafkaCaptureFactory.java | 19 ++++-- TrafficCapture/coreUtilities/build.gradle | 8 ++- .../coreutils/SimpleMeteringClosure.java | 10 ++- .../src/main/docker/docker-compose.yml | 12 ++-- .../docker/otel-collector-config-demo.yaml | 33 ++++++--- ...allyReliableLoggingHttpRequestHandler.java | 22 ++++-- .../netty/LoggingHttpRequestHandler.java | 3 +- .../replay/AddCompressionEncodingTest.java | 2 +- .../trafficCaptureProxyServer/build.gradle | 8 ++- .../proxyserver/CaptureProxy.java | 2 +- TrafficCapture/trafficReplayer/build.gradle | 9 ++- .../replay/AccumulationCallbacks.java | 14 ++-- ...edTrafficToHttpTransactionAccumulator.java | 67 ++++++++++++------- .../replay/ClientConnectionPool.java | 19 ++++-- ...acketToTransformingHttpHandlerFactory.java | 3 +- .../migrations/replay/ReplayEngine.java | 11 +-- .../replay/RequestSenderOrchestrator.java | 35 +++++----- .../migrations/replay/TrafficReplayer.java | 32 ++++----- .../NettyPacketToHttpConsumer.java | 57 +++++++++------- .../http/HttpJsonTransformingConsumer.java | 22 +++--- ...dHttpRequestPreliminaryConvertHandler.java | 12 ++-- .../http/RequestPipelineOrchestrator.java | 9 +-- .../replay/tracing/ConnectionContext.java | 34 ++++++++++ .../replay/tracing/RequestContext.java | 26 +++++++ .../replay/tracing/WithAttributes.java | 9 +++ .../replay/HeaderTransformerTest.java | 7 +- .../replay/RequestSenderOrchestratorTest.java | 12 ++-- ...afficToHttpTransactionAccumulatorTest.java | 13 +++- .../replay/TrafficReplayerTest.java | 25 +++++-- .../NettyPacketToHttpConsumerTest.java | 14 ++-- .../HttpJsonTransformingConsumerTest.java | 8 +-- .../migrations/replay/TestRequestKey.java | 7 +- .../migrations/replay/TestUtils.java | 2 +- 34 files changed, 375 insertions(+), 193 deletions(-) create mode 100644 TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ConnectionContext.java create mode 100644 TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java create mode 100644 TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/WithAttributes.java diff --git a/TrafficCapture/captureKafkaOffloader/build.gradle b/TrafficCapture/captureKafkaOffloader/build.gradle index 65e332890..ab99a0327 100644 --- a/TrafficCapture/captureKafkaOffloader/build.gradle +++ b/TrafficCapture/captureKafkaOffloader/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation project(':coreUtilities') implementation group: 'com.google.protobuf', name:'protobuf-java', version:'3.22.2' api group:'io.netty', name:'netty-buffer', version: '4.1.100.Final' - implementation group: 'io.opentelemetry', name:'opentelemetry-api', version: '1.30.0' + implementation group: 'io.opentelemetry', name:'opentelemetry-api' implementation group: 'org.projectlombok', name:'lombok', version:'1.18.26' implementation group: 'org.apache.kafka', name:'kafka-clients', version:'3.6.0' implementation group: 'org.slf4j', name:'slf4j-api', version:'2.0.7' diff --git a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java index 4a94d93d9..aaa694e55 100644 --- a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java +++ b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java @@ -36,6 +36,7 @@ public class KafkaCaptureFactory implements IConnectionCaptureFactory RECORD_ID_KEY = ContextKey.named("recordId"); private static final ContextKey TOPIC_KEY = ContextKey.named("topic"); private static final ContextKey RECORD_SIZE_KEY = ContextKey.named("recordSize"); + private static final ContextKey START_FLUSH_KEY = ContextKey.named("startKafkaSend"); public static final String TELEMETRY_SCOPE_NAME = "KafkaCapture"; public static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure(TELEMETRY_SCOPE_NAME); @@ -101,7 +102,8 @@ public StreamManager(Context incomingTelemetryContext, String connectionId) { @Override public void close() throws IOException { - METERING_CLOSURE.meterHistogramMillis(telemetryContext, "connection_lifetime", + log.atInfo().setMessage(()->"factory.close()").log(); + METERING_CLOSURE.meterHistogramMillis(telemetryContext, "offloader_stream_lifetime", Duration.between(startTime, Instant.now())); METERING_CLOSURE.meterDeltaEvent(telemetryContext, "offloaders_active", -1); METERING_CLOSURE.meterIncrementEvent(telemetryContext, "offloader_closed"); @@ -111,12 +113,9 @@ public void close() throws IOException { @Override public CodedOutputStreamWrapper createStream() { - Context newStreamCtx; METERING_CLOSURE.meterIncrementEvent(telemetryContext, "stream_created"); - try (var scope = telemetryContext.makeCurrent()) { - newStreamCtx = Context.current() - .with(METERING_CLOSURE.tracer.spanBuilder("recordStream").startSpan()); - } + var newStreamCtx = telemetryContext + .with(METERING_CLOSURE.tracer.spanBuilder("recordStream").startSpan()); ByteBuffer bb = ByteBuffer.allocate(bufferSize); return new CodedOutputStreamWrapper(CodedOutputStream.newInstance(bb), bb, newStreamCtx); @@ -146,7 +145,9 @@ public CodedOutputStreamWrapper createStream() { try (var scope = telemetryContext .with(RECORD_ID_KEY, recordId) .with(TOPIC_KEY, topicNameForTraffic) - .with(RECORD_SIZE_KEY, kafkaRecord.value().length).makeCurrent()) { + .with(RECORD_SIZE_KEY, kafkaRecord.value().length) + .with(START_FLUSH_KEY, Instant.now()) + .makeCurrent()) { METERING_CLOSURE.meterIncrementEvent(telemetryContext, "stream_flush_called"); flushContext = Context.current() .with(METERING_CLOSURE.tracer.spanBuilder("flushRecord").startSpan()); @@ -181,6 +182,10 @@ public CodedOutputStreamWrapper createStream() { private Callback handleProducerRecordSent(CompletableFuture cf, String recordId, Context flushContext) { return (metadata, exception) -> { + log.atInfo().setMessage(()->"kafka completed sending a record").log(); + METERING_CLOSURE.meterHistogramMicros(telemetryContext, + exception==null ? "stream_flush_success_ms" : "stream_flush_failure_ms", + Duration.between(flushContext.get(START_FLUSH_KEY), Instant.now())); METERING_CLOSURE.meterIncrementEvent(telemetryContext, exception==null ? "stream_flush_success" : "stream_flush_failure"); METERING_CLOSURE.meterIncrementEvent(telemetryContext, diff --git a/TrafficCapture/coreUtilities/build.gradle b/TrafficCapture/coreUtilities/build.gradle index da76d0f15..355e2ec38 100644 --- a/TrafficCapture/coreUtilities/build.gradle +++ b/TrafficCapture/coreUtilities/build.gradle @@ -40,6 +40,8 @@ repositories { } dependencies { + implementation platform("io.opentelemetry:opentelemetry-bom:1.32.0") + implementation project(':captureProtobufs') implementation "com.google.protobuf:protobuf-java:3.22.2" @@ -56,9 +58,9 @@ dependencies { implementation("org.apache.logging.log4j:log4j-slf4j2-impl:2.20.0") // OpenTelemetry core - implementation group: 'io.opentelemetry', name:'opentelemetry-api', version: '1.30.0' - implementation group: 'io.opentelemetry', name:'opentelemetry-exporter-otlp', version: '1.30.0' - implementation group: 'io.opentelemetry', name:'opentelemetry-sdk', version: '1.30.0' + implementation group: 'io.opentelemetry', name:'opentelemetry-api' + implementation group: 'io.opentelemetry', name:'opentelemetry-exporter-otlp' + implementation group: 'io.opentelemetry', name:'opentelemetry-sdk' implementation group: 'io.opentelemetry.instrumentation', name:'opentelemetry-log4j-appender-2.17', version: '1.30.0-alpha' implementation group: 'io.opentelemetry', name:'opentelemetry-semconv', version: '1.30.0-alpha' diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java index 3ef604072..1ff13ec47 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java @@ -106,15 +106,21 @@ public void meterDeltaEvent(Context ctx, String eventName, long delta) { } public void meterHistogramMillis(Context ctx, String eventName, Duration between) { - meterHistogram(ctx, eventName, (double) between.toMillis()); + meterHistogram(ctx, eventName, "ms", between.toMillis()); } - public void meterHistogram(Context ctx, String eventName, double value) { + public void meterHistogramMicros(Context ctx, String eventName, Duration between) { + meterHistogram(ctx, eventName, "us", between.toNanos()*1000); + } + + public void meterHistogram(Context ctx, String eventName, String units, long value) { if (ctx == null) { return; } try (var namedOnlyForAutoClose = ctx.makeCurrent()) { meter.histogramBuilder(eventName) + .ofLongs() + .setUnit(units) .build().record(value, Attributes.builder() .put("labelName", eventName) .build()); diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index ef31558c7..1085d203f 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -13,19 +13,19 @@ services: - "9090:9090" # Jaeger - jaeger-all-in-one: + jaeger: image: jaegertracing/all-in-one:latest networks: - migrations ports: - "16686:16686" - - "14268" - - "14250:14250" + - "4317" + - "4318" environment: - COLLECTOR_OTLP_ENABLED=true # Zipkin - zipkin-all-in-one: + zipkin: image: openzipkin/zipkin:latest networks: - migrations @@ -50,8 +50,8 @@ services: - "55679:55679" # zpages extension - "4317:4317" # otlp receiver depends_on: - - jaeger-all-in-one - - zipkin-all-in-one + - jaeger + - zipkin zookeeper: image: docker.io/bitnami/zookeeper:3.8 diff --git a/TrafficCapture/dockerSolution/src/main/docker/otel-collector-config-demo.yaml b/TrafficCapture/dockerSolution/src/main/docker/otel-collector-config-demo.yaml index 92582f7e8..99ac784a8 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/otel-collector-config-demo.yaml +++ b/TrafficCapture/dockerSolution/src/main/docker/otel-collector-config-demo.yaml @@ -13,22 +13,33 @@ exporters: loglevel: debug zipkin: - endpoint: "http://zipkin-all-in-one:9411/api/v2/spans" + endpoint: "http://zipkin:9411/api/v2/spans" format: proto - otlp/jaeger: - endpoint: jaeger-all-in-one:14250 + otlp/jaeger: # Jaeger supports OTLP directly. The default port for OTLP/gRPC is 4317 + endpoint: jaeger:4317 tls: insecure: true -# Alternatively, use jaeger_thrift_http with the settings below. In this case -# update the list of exporters on the traces pipeline. -# -# jaeger_thrift_http: -# url: http://jaeger-all-in-one:14268/api/traces processors: - batch: + # Processor to set the namespace based on the service name + attributes/nscapture: + actions: + - key: namespace + value: "capture" + action: insert + - key: service.name + value: "capture" + action: update + attributes/nsreplayer: + actions: + - key: namespace + value: "replay" + action: insert + - key: service.name + value: "replay" + action: update extensions: health_check: @@ -42,9 +53,9 @@ service: pipelines: traces: receivers: [otlp] - processors: [batch] + processors: [] exporters: [logging, zipkin, otlp/jaeger] metrics: receivers: [otlp] - processors: [batch] + processors: [attributes/nscapture, attributes/nsreplayer] exporters: [logging, prometheus] diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java index 41ee43295..13b6edf87 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java @@ -6,14 +6,18 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.Span; import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextKey; import lombok.Lombok; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; +import java.time.Duration; +import java.time.Instant; import java.util.function.Predicate; @Slf4j public class ConditionallyReliableLoggingHttpRequestHandler extends LoggingHttpRequestHandler { + private ContextKey START_FLUSH_KEY = ContextKey.named("startTime"); private final Predicate shouldBlockPredicate; public ConditionallyReliableLoggingHttpRequestHandler(Context incomingContext, @@ -27,16 +31,22 @@ public ConditionallyReliableLoggingHttpRequestHandler(Context incomingContext, protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Object msg, HttpRequest httpRequest) throws Exception { if (shouldBlockPredicate.test(httpRequest)) { - Span blockingSpan; METERING_CLOSURE.meterIncrementEvent(telemetryContext, "blockingRequestUntilFlush"); - try (var namedOnlyForAutoClose = telemetryContext.makeCurrent()) { - blockingSpan = GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME) - .spanBuilder("blockedOnFlush").startSpan(); + Context flushContext; + try (var namedOnlyForAutoClose = telemetryContext + .with(START_FLUSH_KEY, Instant.now()) + .makeCurrent()) { + flushContext = Context.current() + .with(METERING_CLOSURE.tracer.spanBuilder("blockedForFlush").startSpan()); } trafficOffloader.flushCommitAndResetStream(false).whenComplete((result, t) -> { - blockingSpan.end(); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, + log.atInfo().setMessage(()->"Done flushing").log(); + METERING_CLOSURE.meterIncrementEvent(flushContext, t != null ? "blockedFlushFailure" : "blockedFlushSuccess"); + METERING_CLOSURE.meterHistogramMicros(flushContext, + t==null ? "blockedFlushFailure_micro" : "stream_flush_failure_micro", + Duration.between(flushContext.get(START_FLUSH_KEY), Instant.now())); + Span.fromContext(flushContext).end(); if (t != null) { // This is a spot where we would benefit from having a behavioral policy that different users diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java index 0d6086c4a..0d300eb57 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java @@ -85,8 +85,7 @@ public HttpRequest resetCurrentRequest() { public LoggingHttpRequestHandler(Context incomingContext, IChannelConnectionCaptureSerializer trafficOffloader) { this.createdTime = Instant.now(); try (var scope = incomingContext.makeCurrent()) { - var span = GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME) - .spanBuilder("frontendConnection").startSpan(); + var span = METERING_CLOSURE.tracer.spanBuilder("frontendConnection").startSpan(); telemetryContext = incomingContext.with(span); METERING_CLOSURE.meterIncrementEvent(telemetryContext, "requestStarted"); } diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java index 44da58363..33817c884 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java @@ -34,7 +34,7 @@ public void addingCompressionRequestHeaderCompressesPayload() throws ExecutionEx JsonJoltTransformer.newBuilder() .addCannedOperation(JsonJoltTransformBuilder.CANNED_OPERATION.ADD_GZIP) .build(), null, testPacketCapture, "TEST", - TestRequestKey.getTestConnectionRequestId(0)); + TestRequestKey.getTestConnectionRequestContext(0)); final var payloadPartSize = 511; final var numParts = 1025; diff --git a/TrafficCapture/trafficCaptureProxyServer/build.gradle b/TrafficCapture/trafficCaptureProxyServer/build.gradle index 05f00f8b8..81d2281dd 100644 --- a/TrafficCapture/trafficCaptureProxyServer/build.gradle +++ b/TrafficCapture/trafficCaptureProxyServer/build.gradle @@ -13,6 +13,8 @@ configurations { } dependencies { + implementation platform("io.opentelemetry:opentelemetry-bom:1.32.0") + implementation 'org.opensearch.plugin:opensearch-security:2.6.0.0' opensearchSecurityPlugin 'org.opensearch.plugin:opensearch-security:2.6.0.0' implementation files(zipTree("$configurations.opensearchSecurityPlugin.singleFile").matching { @@ -38,14 +40,14 @@ dependencies { implementation group: 'com.beust', name: 'jcommander', version: '1.82' implementation 'com.google.protobuf:protobuf-java:3.22.2' - implementation group: 'io.opentelemetry', name:'opentelemetry-api', version: '1.30.0' - implementation group: 'io.opentelemetry', name: 'opentelemetry-sdk', version: '1.32.0' + implementation group: 'io.opentelemetry', name:'opentelemetry-api' + implementation group: 'io.opentelemetry', name: 'opentelemetry-sdk' testImplementation project(':captureProtobufs') testImplementation testFixtures(project(path: ':testUtilities')) testImplementation testFixtures(project(path: ':captureOffloader')) - testImplementation group: 'io.opentelemetry', name: 'opentelemetry-sdk-testing', version: '1.32.0' + testImplementation group: 'io.opentelemetry', name: 'opentelemetry-sdk-testing' } tasks.withType(Tar){ diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java index 60de2bcff..97b2a8293 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java @@ -284,7 +284,7 @@ public static void main(String[] args) throws InterruptedException, IOException var backsideUri = convertStringToUri(params.backsideUriString); if (params.otelCollectorEndpoint != null) { - SimpleMeteringClosure.initializeOpenTelemetry("capture-proxy", params.otelCollectorEndpoint); + SimpleMeteringClosure.initializeOpenTelemetry("capture", params.otelCollectorEndpoint); } var sksOp = Optional.ofNullable(params.sslConfigFilePath) diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle index 7ca67b9d0..7cd49c99d 100644 --- a/TrafficCapture/trafficReplayer/build.gradle +++ b/TrafficCapture/trafficReplayer/build.gradle @@ -35,6 +35,7 @@ repositories { dependencies { //spotbugs 'com.github.spotbugs:spotbugs:4.7.3' def resilience4jVersion = "1.7.0"; + implementation platform("io.opentelemetry:opentelemetry-bom:1.32.0") implementation project(':captureProtobufs') implementation project(':coreUtilities') @@ -50,6 +51,8 @@ dependencies { implementation group: 'io.github.resilience4j', name: 'resilience4j-ratelimiter', version:"${resilience4jVersion}" implementation group: 'io.github.resilience4j', name: 'resilience4j-retry', version:"${resilience4jVersion}" implementation group: 'io.netty', name: 'netty-all', version: '4.1.100.Final' + implementation group: 'io.opentelemetry', name:'opentelemetry-api' + implementation group: 'io.opentelemetry', name: 'opentelemetry-sdk' implementation group: 'org.apache.kafka', name: 'kafka-clients', version: '3.6.0' implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.20.0' implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.20.0' @@ -65,11 +68,14 @@ dependencies { testFixturesImplementation project(':replayerPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') testFixturesImplementation testFixtures(project(path: ':testUtilities')) + testFixturesImplementation platform("io.opentelemetry:opentelemetry-bom:1.32.0") testFixturesImplementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7' testFixturesImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.0' testFixturesImplementation group: 'io.netty', name: 'netty-all', version: '4.1.100.Final' testFixturesImplementation group: 'org.junit.jupiter', name:'junit-jupiter-api', version:'5.9.3' - + testFixturesImplementation group: 'io.opentelemetry', name:'opentelemetry-api' + testFixturesImplementation group: 'io.opentelemetry', name: 'opentelemetry-sdk' + testFixturesImplementation group: 'io.opentelemetry', name: 'opentelemetry-sdk-testing' testImplementation project(':captureOffloader') testImplementation testFixtures(project(path: ':captureOffloader')) @@ -78,6 +84,7 @@ dependencies { testImplementation project(':replayerPlugins:jsonMessageTransformers:jsonJoltMessageTransformerProvider') testImplementation project(':replayerPlugins:jsonMessageTransformers:openSearch23PlusTargetTransformerProvider') + testImplementation group: 'io.opentelemetry', name: 'opentelemetry-sdk-testing' testImplementation group: 'org.apache.httpcomponents.client5', name: 'httpclient5', version: '5.2.1' testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-api', version:'5.x.x' testImplementation group: 'org.testcontainers', name: 'junit-jupiter', version: '1.19.0' diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java index c581ffa0c..3f2504862 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java @@ -4,17 +4,21 @@ import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.replay.tracing.ConnectionContext; +import org.opensearch.migrations.replay.tracing.RequestContext; import java.time.Instant; import java.util.List; public interface AccumulationCallbacks { - void onRequestReceived(@NonNull UniqueReplayerRequestKey key, @NonNull HttpMessageAndTimestamp request); - void onFullDataReceived(@NonNull UniqueReplayerRequestKey key, @NonNull RequestResponsePacketPair rrpp); - void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, + void onRequestReceived(@NonNull UniqueReplayerRequestKey key, RequestContext ctx, + @NonNull HttpMessageAndTimestamp request); + void onFullDataReceived(@NonNull UniqueReplayerRequestKey key, RequestContext ctx, + @NonNull RequestResponsePacketPair rrpp); + void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, ConnectionContext ctx, @NonNull List trafficStreamKeysBeingHeld); - void onConnectionClose(@NonNull ISourceTrafficChannelKey key, int channelInteractionNumber, + void onConnectionClose(@NonNull ISourceTrafficChannelKey key, int channelInteractionNumber, ConnectionContext ctx, RequestResponsePacketPair.ReconstructionStatus status, @NonNull Instant when, @NonNull List trafficStreamKeysBeingHeld); - void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk); + void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk, ConnectionContext ctx); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java index dc5aca221..29fc83126 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java @@ -1,11 +1,16 @@ package org.opensearch.migrations.replay; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextKey; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; import org.opensearch.migrations.coreutils.MetricsLogger; +import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; +import org.opensearch.migrations.replay.tracing.ConnectionContext; +import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.traffic.expiration.BehavioralPolicy; import org.opensearch.migrations.replay.traffic.expiration.ExpiringTrafficStreamMap; import org.opensearch.migrations.replay.traffic.source.ITrafficStreamWithKey; @@ -49,6 +54,8 @@ */ @Slf4j public class CapturedTrafficToHttpTransactionAccumulator { + public static final String TELEMETRY_SCOPE_NAME = "Accumulator"; + public static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure(TELEMETRY_SCOPE_NAME); public static final Duration EXPIRATION_GRANULARITY = Duration.ofSeconds(1); private final ExpiringTrafficStreamMap liveStreams; @@ -129,9 +136,10 @@ public void accept(ITrafficStreamWithKey trafficStreamAndKey) { var tsk = trafficStreamAndKey.getKey(); var accum = liveStreams.getOrCreateWithoutExpiration(tsk, k->createInitialAccumulation(trafficStreamAndKey)); var trafficStream = trafficStreamAndKey.getStream(); + var ctx = new ConnectionContext(tsk); for (int i=0; i"Connection terminated: removing " + partitionId + ":" + connectionId + " from liveStreams map").log(); @@ -145,7 +153,7 @@ public void accept(ITrafficStreamWithKey trafficStreamAndKey) { assert accum.state == Accumulation.State.WAITING_FOR_NEXT_READ_CHUNK || accum.state == Accumulation.State.IGNORING_LAST_REQUEST || trafficStream.getSubStreamCount() == 0; - listener.onTrafficStreamIgnored(tsk); + listener.onTrafficStreamIgnored(tsk, ctx); } } @@ -175,15 +183,16 @@ private enum CONNECTION_STATUS { public CONNECTION_STATUS addObservationToAccumulation(@NonNull Accumulation accum, @NonNull ITrafficStreamKey trafficStreamKey, + ConnectionContext ctx, TrafficObservation observation) { log.atTrace().setMessage(()->"Adding observation: "+observation).log(); var timestamp = TrafficStreamUtils.instantFromProtoTimestamp(observation.getTs()); liveStreams.expireOldEntries(trafficStreamKey, accum, timestamp); - return handleCloseObservationThatAffectEveryState(accum, observation, trafficStreamKey, timestamp) + return handleCloseObservationThatAffectEveryState(accum, observation, trafficStreamKey, ctx, timestamp) .or(() -> handleObservationForSkipState(accum, observation)) - .or(() -> handleObservationForReadState(accum, observation, trafficStreamKey, timestamp)) - .or(() -> handleObservationForWriteState(accum, observation, trafficStreamKey, timestamp)) + .or(() -> handleObservationForReadState(accum, ctx, observation, trafficStreamKey, timestamp)) + .or(() -> handleObservationForWriteState(accum, ctx, observation, trafficStreamKey, timestamp)) .orElseGet(() -> { log.atWarn().setMessage(()->"unaccounted for observation type " + observation).log(); return CONNECTION_STATUS.ALIVE; @@ -216,17 +225,19 @@ private static List getTrafficStreamsHeldByAccum(Accumulation handleCloseObservationThatAffectEveryState(Accumulation accum, TrafficObservation observation, @NonNull ITrafficStreamKey trafficStreamKey, + ConnectionContext ctx, Instant timestamp) { if (observation.hasClose()) { accum.getOrCreateTransactionPair(trafficStreamKey).holdTrafficStream(trafficStreamKey); - rotateAccumulationIfNecessary(trafficStreamKey.getConnectionId(), accum); + rotateAccumulationIfNecessary(trafficStreamKey.getConnectionId(), accum, ctx); closedConnectionCounter.incrementAndGet(); - listener.onConnectionClose(accum.trafficChannelKey, accum.getIndexOfCurrentRequest(), - RequestResponsePacketPair.ReconstructionStatus.COMPLETE, timestamp, getTrafficStreamsHeldByAccum(accum)); + listener.onConnectionClose(accum.trafficChannelKey, accum.getIndexOfCurrentRequest(), ctx, + RequestResponsePacketPair.ReconstructionStatus.COMPLETE, + timestamp, getTrafficStreamsHeldByAccum(accum)); return Optional.of(CONNECTION_STATUS.CLOSED); } else if (observation.hasConnectionException()) { accum.getOrCreateTransactionPair(trafficStreamKey).holdTrafficStream(trafficStreamKey); - rotateAccumulationIfNecessary(trafficStreamKey.getConnectionId(), accum); + rotateAccumulationIfNecessary(trafficStreamKey.getConnectionId(), accum, ctx); exceptionConnectionCounter.incrementAndGet(); accum.resetForNextRequest(); log.atDebug().setMessage(()->"Removing accumulated traffic pair due to " + @@ -238,6 +249,7 @@ private static List getTrafficStreamsHeldByAccum(Accumulation } private Optional handleObservationForReadState(@NonNull Accumulation accum, + ConnectionContext ctx, TrafficObservation observation, @NonNull ITrafficStreamKey trafficStreamKey, Instant timestamp) { @@ -256,7 +268,7 @@ private Optional handleObservationForReadState(@NonNull Accum log.atTrace().setMessage(() -> "Added request data for accum[" + connectionId + "]=" + accum).log(); } else if (observation.hasEndOfMessageIndicator()) { assert accum.hasRrPair(); - handleEndOfRequest(accum); + handleEndOfRequest(accum, ctx); } else if (observation.hasReadSegment()) { log.atTrace().setMessage(()->"Adding request segment for accum[" + connectionId + "]=" + accum).log(); var rrPair = accum.getOrCreateTransactionPair(trafficStreamKey); @@ -274,7 +286,7 @@ private Optional handleObservationForReadState(@NonNull Accum return Optional.of(CONNECTION_STATUS.ALIVE); } - private Optional handleObservationForWriteState(Accumulation accum, + private Optional handleObservationForWriteState(Accumulation accum, ConnectionContext ctx, TrafficObservation observation, @NonNull ITrafficStreamKey trafficStreamKey, Instant timestamp) { @@ -301,8 +313,8 @@ private Optional handleObservationForWriteState(Accumulation assert rrPair.responseData.hasInProgressSegment(); rrPair.responseData.finalizeRequestSegments(timestamp); } else if (observation.hasRead() || observation.hasReadSegment()) { - rotateAccumulationOnReadIfNecessary(connectionId, accum); - return handleObservationForReadState(accum, observation, trafficStreamKey, timestamp); + rotateAccumulationOnReadIfNecessary(connectionId, accum, ctx); + return handleObservationForReadState(accum, ctx, observation, trafficStreamKey, timestamp); } return Optional.of(CONNECTION_STATUS.ALIVE); @@ -311,19 +323,20 @@ private Optional handleObservationForWriteState(Accumulation // This function manages the transition case when an observation comes in that would terminate // any previous HTTP transaction for the connection. It returns true if there WAS a previous // transaction that has been reset and false otherwise - private boolean rotateAccumulationIfNecessary(String connectionId, Accumulation accum) { + private boolean rotateAccumulationIfNecessary(String connectionId, Accumulation accum, ConnectionContext ctx) { // If this was brand new, we don't need to care about triggering the callback. // We only need to worry about this if we have yet to send the RESPONSE. if (accum.state == Accumulation.State.ACCUMULATING_WRITES) { log.atDebug().setMessage(()->"Resetting accum[" + connectionId + "]=" + accum).log(); - handleEndOfResponse(accum, RequestResponsePacketPair.ReconstructionStatus.COMPLETE); + handleEndOfResponse(accum, ctx, RequestResponsePacketPair.ReconstructionStatus.COMPLETE); return true; } return false; } - private boolean rotateAccumulationOnReadIfNecessary(String connectionId, Accumulation accum) { - if (rotateAccumulationIfNecessary(connectionId, accum)) { + private boolean rotateAccumulationOnReadIfNecessary(String connectionId, Accumulation accum, + ConnectionContext ctx) { + if (rotateAccumulationIfNecessary(connectionId, accum, ctx)) { reusedKeepAliveCounter.incrementAndGet(); return true; } else { @@ -334,7 +347,7 @@ private boolean rotateAccumulationOnReadIfNecessary(String connectionId, Accumu /** * @return True if something was sent to the callback, false if nothing had been accumulated */ - private boolean handleEndOfRequest(Accumulation accumulation) { + private boolean handleEndOfRequest(Accumulation accumulation, ConnectionContext ctx) { assert accumulation.state == Accumulation.State.ACCUMULATING_READS : "state == " + accumulation.state; var requestPacketBytes = accumulation.getRrPair().requestData; metricsLogger.atSuccess(MetricsEvent.ACCUMULATED_FULL_CAPTURED_SOURCE_RESPONSE) @@ -342,12 +355,13 @@ private boolean handleEndOfRequest(Accumulation accumulation) { .setAttribute(MetricsAttributeKey.CONNECTION_ID, accumulation.getRequestKey().getTrafficStreamKey().getConnectionId()).emit(); assert (requestPacketBytes != null); assert (!requestPacketBytes.hasInProgressSegment()); - listener.onRequestReceived(accumulation.getRequestKey(), requestPacketBytes); + var requestContext = new RequestContext(ctx, accumulation.getRequestKey()); + listener.onRequestReceived(accumulation.getRequestKey(), requestContext, requestPacketBytes); accumulation.state = Accumulation.State.ACCUMULATING_WRITES; return true; } - private void handleEndOfResponse(Accumulation accumulation, + private void handleEndOfResponse(Accumulation accumulation, ConnectionContext ctx, RequestResponsePacketPair.ReconstructionStatus status) { assert accumulation.state == Accumulation.State.ACCUMULATING_WRITES; metricsLogger.atSuccess(MetricsEvent.ACCUMULATED_FULL_CAPTURED_SOURCE_RESPONSE) @@ -355,20 +369,23 @@ private void handleEndOfResponse(Accumulation accumulation, .setAttribute(MetricsAttributeKey.CONNECTION_ID, accumulation.getRequestKey().getTrafficStreamKey().getConnectionId()).emit(); var rrPair = accumulation.getRrPair(); rrPair.completionStatus = status; - listener.onFullDataReceived(accumulation.getRequestKey(), rrPair); + var requestContext = new RequestContext(ctx, accumulation.getRequestKey()); + listener.onFullDataReceived(accumulation.getRequestKey(), requestContext, rrPair); accumulation.resetForNextRequest(); } public void close() { liveStreams.values().forEach(accum -> { requestsTerminatedUponAccumulatorCloseCounter.incrementAndGet(); - fireAccumulationsCallbacksAndClose(accum, RequestResponsePacketPair.ReconstructionStatus.CLOSED_PREMATURELY); + fireAccumulationsCallbacksAndClose(accum, + RequestResponsePacketPair.ReconstructionStatus.CLOSED_PREMATURELY); }); liveStreams.clear(); } private void fireAccumulationsCallbacksAndClose(Accumulation accumulation, RequestResponsePacketPair.ReconstructionStatus status) { + ConnectionContext ctx = new ConnectionContext(accumulation.trafficChannelKey); try { switch (accumulation.state) { case ACCUMULATING_READS: @@ -383,12 +400,12 @@ private void fireAccumulationsCallbacksAndClose(Accumulation accumulation, log.warn("Terminating a TrafficStream reconstruction w/out an accumulated value, " + "assuming an empty server interaction and NOT reproducing this to the target cluster."); if (accumulation.hasRrPair()) { - listener.onTrafficStreamsExpired(status, + listener.onTrafficStreamsExpired(status, ctx, Collections.unmodifiableList(accumulation.getRrPair().trafficStreamKeysBeingHeld)); } return; case ACCUMULATING_WRITES: - handleEndOfResponse(accumulation, status); + handleEndOfResponse(accumulation, ctx, status); break; case WAITING_FOR_NEXT_READ_CHUNK: case IGNORING_LAST_REQUEST: @@ -399,7 +416,7 @@ private void fireAccumulationsCallbacksAndClose(Accumulation accumulation, } finally { if (accumulation.hasSignaledRequests()) { listener.onConnectionClose(accumulation.trafficChannelKey, accumulation.getIndexOfCurrentRequest(), - status, accumulation.getLastTimestamp(), getTrafficStreamsHeldByAccum(accumulation)); + ctx, status, accumulation.getLastTimestamp(), getTrafficStreamsHeldByAccum(accumulation)); } } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java index d647abb34..35ae238b8 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java @@ -11,21 +11,29 @@ import io.netty.util.concurrent.DefaultPromise; import io.netty.util.concurrent.DefaultThreadFactory; import io.netty.util.concurrent.Future; +import io.opentelemetry.context.ContextKey; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.replay.datahandlers.NettyPacketToHttpConsumer; import org.opensearch.migrations.replay.datatypes.ConnectionReplaySession; import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; +import org.opensearch.migrations.replay.tracing.ConnectionContext; +import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture; import java.net.URI; +import java.time.Instant; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @Slf4j public class ClientConnectionPool { + private static final ContextKey RECORD_ID_KEY = ContextKey.named("recordId"); + public static final String TELEMETRY_SCOPE_NAME = "ClientConnectionPool"; + public static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure(TELEMETRY_SCOPE_NAME); public static final String TARGET_CONNECTION_POOL_NAME = "targetConnectionPool"; private final URI serverUri; @@ -61,17 +69,17 @@ public ConnectionReplaySession load(final String s) { } private DiagnosticTrackableCompletableFuture - getResilientClientChannelProducer(EventLoop eventLoop, String diagnosticLabel) { + getResilientClientChannelProducer(EventLoop eventLoop, ConnectionContext connectionContext) { return new AdaptiveRateLimiter() .get(() -> { var clientConnectionChannelCreatedFuture = new StringTrackableCompletableFuture(new CompletableFuture<>(), () -> "waiting for createClientConnection to finish"); var channelFuture = NettyPacketToHttpConsumer.createClientConnection(eventLoop, - sslContext, serverUri, diagnosticLabel); + sslContext, serverUri, connectionContext); channelFuture.addListener(f -> { log.atInfo().setMessage(()-> - "New network connection result for " + diagnosticLabel + "=" + f.isSuccess()).log(); + "New network connection result for " + connectionContext + "=" + f.isSuccess()).log(); if (f.isSuccess()) { clientConnectionChannelCreatedFuture.future.complete(channelFuture); } else { @@ -135,7 +143,7 @@ public void closeConnection(String connId) { } public Future - submitEventualSessionGet(ISourceTrafficChannelKey channelKey, boolean ignoreIfNotPresent) { + submitEventualSessionGet(ISourceTrafficChannelKey channelKey, boolean ignoreIfNotPresent, ConnectionContext ctx) { ConnectionReplaySession channelFutureAndSchedule = getCachedSession(channelKey, ignoreIfNotPresent); if (channelFutureAndSchedule == null) { @@ -146,8 +154,7 @@ public void closeConnection(String connId) { return channelFutureAndSchedule.eventLoop.submit(() -> { if (channelFutureAndSchedule.getChannelFutureFuture() == null) { channelFutureAndSchedule.setChannelFutureFuture( - getResilientClientChannelProducer(channelFutureAndSchedule.eventLoop, - channelKey.getConnectionId())); + getResilientClientChannelProducer(channelFutureAndSchedule.eventLoop, ctx)); } return channelFutureAndSchedule; }); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java index 29eff701b..5d03d7ddd 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java @@ -7,6 +7,7 @@ import org.opensearch.migrations.replay.datatypes.TransformedOutputAndResult; import org.opensearch.migrations.replay.datatypes.TransformedPackets; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.transform.IAuthTransformerFactory; import org.opensearch.migrations.transform.IJsonTransformer; @@ -28,6 +29,6 @@ public PacketToTransformingHttpHandlerFactory(IJsonTransformer jsonTransformer, create(UniqueReplayerRequestKey requestKey) { log.trace("creating HttpJsonTransformingConsumer"); return new HttpJsonTransformingConsumer<>(jsonTransformer, authTransformerFactory, - new TransformedPacketReceiver(), requestKey.toString(), requestKey); + new TransformedPacketReceiver(), requestKey.toString(), new RequestContext(requestKey)); } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java index 542243d56..eaecdcca6 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java @@ -9,6 +9,8 @@ import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.IndexedChannelInteraction; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.replay.tracing.ConnectionContext; +import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.traffic.source.BufferedFlowController; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; @@ -134,7 +136,7 @@ private static void logStartOfWork(Object stringableKey, long newCount, Instant } public DiagnosticTrackableCompletableFuture - scheduleRequest(UniqueReplayerRequestKey requestKey, Instant originalStart, Instant originalEnd, + scheduleRequest(UniqueReplayerRequestKey requestKey, RequestContext ctx, Instant originalStart, Instant originalEnd, int numPackets, Stream packets) { var newCount = totalCountOfScheduledTasksOutstanding.incrementAndGet(); final String label = "request"; @@ -147,16 +149,17 @@ private static void logStartOfWork(Object stringableKey, long newCount, Instant .setAttribute(MetricsAttributeKey.CONNECTION_ID, requestKey.getTrafficStreamKey().getConnectionId()) .setAttribute(MetricsAttributeKey.DELAY_FROM_ORIGINAL_TO_SCHEDULED_START, Duration.between(originalStart, start).toMillis()) .setAttribute(MetricsAttributeKey.SCHEDULED_SEND_TIME, start.toString()).emit(); - var sendResult = networkSendOrchestrator.scheduleRequest(requestKey, start, interval, packets); + var sendResult = networkSendOrchestrator.scheduleRequest(requestKey, ctx, start, interval, packets); return hookWorkFinishingUpdates(sendResult, originalStart, requestKey, label); } - public void closeConnection(ISourceTrafficChannelKey channelKey, int channelInteractionNum, Instant timestamp) { + public void closeConnection(ISourceTrafficChannelKey channelKey, int channelInteractionNum, + ConnectionContext ctx, Instant timestamp) { var newCount = totalCountOfScheduledTasksOutstanding.incrementAndGet(); final String label = "close"; var atTime = timeShifter.transformSourceTimeToRealTime(timestamp); logStartOfWork(new IndexedChannelInteraction(channelKey, channelInteractionNum), newCount, atTime, label); - var future = networkSendOrchestrator.scheduleClose(channelKey, channelInteractionNum, atTime); + var future = networkSendOrchestrator.scheduleClose(channelKey, channelInteractionNum, ctx, atTime); hookWorkFinishingUpdates(future, timestamp, channelKey, label); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java index df6851652..94387dc3f 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java @@ -9,6 +9,8 @@ import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.IndexedChannelInteraction; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.replay.tracing.ConnectionContext; +import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture; @@ -55,30 +57,32 @@ public RequestSenderOrchestrator(ClientConnectionPool clientConnectionPool) { } public DiagnosticTrackableCompletableFuture - scheduleRequest(UniqueReplayerRequestKey requestKey, Instant start, Duration interval, Stream packets) { + scheduleRequest(UniqueReplayerRequestKey requestKey, RequestContext ctx, + Instant start, Duration interval, Stream packets) { var finalTunneledResponse = new StringTrackableCompletableFuture(new CompletableFuture<>(), ()->"waiting for final aggregated response"); log.atDebug().setMessage(()->"Scheduling request for "+requestKey+" at start time "+start).log(); return asynchronouslyInvokeRunnableToSetupFuture(requestKey.getTrafficStreamKey(), - requestKey.getReplayerRequestIndex(), - false, finalTunneledResponse, - channelFutureAndRequestSchedule-> scheduleSendOnConnectionReplaySession(requestKey, + requestKey.getReplayerRequestIndex(), ctx, false, finalTunneledResponse, + channelFutureAndRequestSchedule-> scheduleSendOnConnectionReplaySession(requestKey, ctx, channelFutureAndRequestSchedule, finalTunneledResponse, start, interval, packets)); } public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChannelKey channelKey, - int channelInteractionNum, Instant timestamp) { + int channelInteractionNum, + ConnectionContext ctx, + Instant timestamp) { var channelInteraction = new IndexedChannelInteraction(channelKey, channelInteractionNum); var finalTunneledResponse = new StringTrackableCompletableFuture(new CompletableFuture<>(), ()->"waiting for final signal to confirm close has finished"); log.atDebug().setMessage(()->"Scheduling CLOSE for "+channelInteraction+" at time "+timestamp).log(); - asynchronouslyInvokeRunnableToSetupFuture(channelKey, channelInteractionNum, true, + asynchronouslyInvokeRunnableToSetupFuture(channelKey, channelInteractionNum, ctx,true, finalTunneledResponse, channelFutureAndRequestSchedule-> - scheduleOnConnectionReplaySession(channelKey, channelInteractionNum, channelFutureAndRequestSchedule, - finalTunneledResponse, timestamp, "close", () -> { + scheduleOnConnectionReplaySession(channelKey, channelInteractionNum, ctx, + channelFutureAndRequestSchedule, finalTunneledResponse, timestamp, "close", () -> { log.trace("Closing client connection " + channelInteraction); clientConnectionPool.closeConnection(channelKey.getConnectionId()); finalTunneledResponse.future.complete(null); @@ -89,11 +93,11 @@ public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChanne private DiagnosticTrackableCompletableFuture asynchronouslyInvokeRunnableToSetupFuture(ISourceTrafficChannelKey channelKey, int channelInteractionNumber, - boolean ignoreIfChannelNotPresent, + ConnectionContext ctx, boolean ignoreIfChannelNotPresent, DiagnosticTrackableCompletableFuture finalTunneledResponse, Consumer successFn) { var channelFutureAndScheduleFuture = - clientConnectionPool.submitEventualSessionGet(channelKey, ignoreIfChannelNotPresent); + clientConnectionPool.submitEventualSessionGet(channelKey, ignoreIfChannelNotPresent, ctx); channelFutureAndScheduleFuture.addListener(submitFuture->{ if (!submitFuture.isSuccess()) { log.atError().setCause(submitFuture.cause()) @@ -141,6 +145,7 @@ public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChanne } private void scheduleOnConnectionReplaySession(ISourceTrafficChannelKey channelKey, int channelInteractionIdx, + ConnectionContext ctx, ConnectionReplaySession channelFutureAndRequestSchedule, StringTrackableCompletableFuture futureToBeCompletedByTask, Instant atTime, String activityNameForLogging, Runnable task) { @@ -186,17 +191,17 @@ private void scheduleOnConnectionReplaySession(ISourceTrafficChannelKey chan "... " + schedule).log(); } - private void scheduleSendOnConnectionReplaySession(UniqueReplayerRequestKey requestKey, + private void scheduleSendOnConnectionReplaySession(UniqueReplayerRequestKey requestKey, RequestContext ctx, ConnectionReplaySession channelFutureAndRequestSchedule, StringTrackableCompletableFuture responseFuture, Instant start, Duration interval, Stream packets) { var eventLoop = channelFutureAndRequestSchedule.eventLoop; var packetReceiverRef = new AtomicReference(); Runnable packetSender = () -> sendNextPartAndContinue(() -> - getPacketReceiver(requestKey, channelFutureAndRequestSchedule.getInnerChannelFuture(), + getPacketReceiver(ctx, channelFutureAndRequestSchedule.getInnerChannelFuture(), packetReceiverRef), eventLoop, packets.iterator(), start, interval, new AtomicInteger(), responseFuture); - scheduleOnConnectionReplaySession(requestKey.trafficStreamKey, requestKey.getSourceRequestIndex(), + scheduleOnConnectionReplaySession(requestKey.trafficStreamKey, requestKey.getSourceRequestIndex(), ctx, channelFutureAndRequestSchedule, responseFuture, start, "send", packetSender); } @@ -224,10 +229,10 @@ private long getDelayFromNowMs(Instant to) { } private static NettyPacketToHttpConsumer - getPacketReceiver(UniqueReplayerRequestKey requestKey, ChannelFuture channelFuture, + getPacketReceiver(RequestContext requestContext, ChannelFuture channelFuture, AtomicReference packetReceiver) { if (packetReceiver.get() == null) { - packetReceiver.set(new NettyPacketToHttpConsumer(channelFuture, requestKey.toString(), requestKey)); + packetReceiver.set(new NettyPacketToHttpConsumer(channelFuture, requestContext)); } return packetReceiver.get(); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index d92f34b0d..47e3ea27f 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -14,7 +14,10 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsLogger; +import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer; +import org.opensearch.migrations.replay.tracing.ConnectionContext; +import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.transform.IHttpMessage; import org.opensearch.migrations.replay.datatypes.HttpRequestTransformationStatus; import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; @@ -67,8 +70,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.opensearch.migrations.coreutils.SimpleMeteringClosure.initializeOpenTelemetry; - @Slf4j public class TrafficReplayer { @@ -384,7 +385,7 @@ public static void main(String[] args) return; } if (params.otelCollectorEndpoint != null) { - initializeOpenTelemetry("traffic-replayer", params.otelCollectorEndpoint); + SimpleMeteringClosure.initializeOpenTelemetry("replay", params.otelCollectorEndpoint); } try (var blockingTrafficSource = TrafficCaptureSourceFactory.createTrafficCaptureSource(params, @@ -601,11 +602,12 @@ class TrafficReplayerAccumulationCallbacks implements AccumulationCallbacks { private ITrafficCaptureSource trafficCaptureSource; @Override - public void onRequestReceived(UniqueReplayerRequestKey requestKey, HttpMessageAndTimestamp request) { + public void onRequestReceived(UniqueReplayerRequestKey requestKey, RequestContext ctx, + HttpMessageAndTimestamp request) { replayEngine.setFirstTimestamp(request.getFirstPacketTimestamp()); liveTrafficStreamLimiter.addWork(1); - var requestPushFuture = transformAndSendRequest(replayEngine, request, requestKey); + var requestPushFuture = transformAndSendRequest(replayEngine, request, requestKey, ctx); requestFutureMap.put(requestKey, requestPushFuture); liveRequests.put(requestKey, true); requestPushFuture.map(f->f.whenComplete((v,t)->{ @@ -618,7 +620,7 @@ public void onRequestReceived(UniqueReplayerRequestKey requestKey, HttpMessageAn } @Override - public void onFullDataReceived(@NonNull UniqueReplayerRequestKey requestKey, + public void onFullDataReceived(@NonNull UniqueReplayerRequestKey requestKey, RequestContext ctx, @NonNull RequestResponsePacketPair rrPair) { log.atInfo().setMessage(()->"Done receiving captured stream for " + requestKey + ":" + rrPair.requestData).log(); @@ -674,7 +676,7 @@ Void handleCompletedTransaction(@NonNull UniqueReplayerRequestKey requestKey, Re @Override public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, - List trafficStreamKeysBeingHeld) { + ConnectionContext ctx, List trafficStreamKeysBeingHeld) { commitTrafficStreams(trafficStreamKeysBeingHeld, status); } @@ -696,15 +698,15 @@ private void commitTrafficStreams(List trafficStreamKeysBeing @Override public void onConnectionClose(ISourceTrafficChannelKey channelKey, int channelInteractionNum, - RequestResponsePacketPair.ReconstructionStatus status, Instant timestamp, - List trafficStreamKeysBeingHeld) { + ConnectionContext ctx, RequestResponsePacketPair.ReconstructionStatus status, + Instant timestamp, List trafficStreamKeysBeingHeld) { replayEngine.setFirstTimestamp(timestamp); - replayEngine.closeConnection(channelKey, channelInteractionNum, timestamp); + replayEngine.closeConnection(channelKey, channelInteractionNum, ctx, timestamp); commitTrafficStreams(trafficStreamKeysBeingHeld, status); } @Override - public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk) { + public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk, ConnectionContext ctx) { commitTrafficStreams(List.of(tsk), true); } @@ -858,15 +860,15 @@ private static String formatWorkItem(DiagnosticTrackableCompletableFuture transformAndSendRequest(ReplayEngine replayEngine, HttpMessageAndTimestamp request, - UniqueReplayerRequestKey requestKey) { - return transformAndSendRequest(inputRequestTransformerFactory, replayEngine, + UniqueReplayerRequestKey requestKey, RequestContext ctx) { + return transformAndSendRequest(inputRequestTransformerFactory, replayEngine, ctx, request.getFirstPacketTimestamp(), request.getLastPacketTimestamp(), requestKey, request.packetBytes::stream); } public static DiagnosticTrackableCompletableFuture transformAndSendRequest(PacketToTransformingHttpHandlerFactory inputRequestTransformerFactory, - ReplayEngine replayEngine, + ReplayEngine replayEngine, RequestContext ctx, @NonNull Instant start, @NonNull Instant end, UniqueReplayerRequestKey requestKey, Supplier> packetsSupplier) @@ -880,7 +882,7 @@ private static String formatWorkItem(DiagnosticTrackableCompletableFuture - replayEngine.scheduleRequest(requestKey, start, end, + replayEngine.scheduleRequest(requestKey, ctx, start, end, transformedResult.transformedOutput.size(), transformedResult.transformedOutput.streamRetained()) .map(future->future.thenApply(t-> diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java index 47ea9ad0a..321557815 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java @@ -18,14 +18,17 @@ import io.netty.handler.logging.LoggingHandler; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; +import io.opentelemetry.context.ContextKey; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; import org.opensearch.migrations.coreutils.MetricsLogger; +import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.replay.AggregatedRawResponse; -import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; import org.opensearch.migrations.replay.netty.BacksideHttpWatcherHandler; import org.opensearch.migrations.replay.netty.BacksideSnifferHandler; +import org.opensearch.migrations.replay.tracing.ConnectionContext; +import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture; @@ -36,6 +39,11 @@ @Slf4j public class NettyPacketToHttpConsumer implements IPacketFinalizingConsumer { + private static final ContextKey START_OF_REQUEST_KEY = ContextKey.named("startOfRequest"); + private static final ContextKey START_OF_WRITE_KEY = ContextKey.named("startOfWrite"); + public static final String TELEMETRY_SCOPE_NAME = "HttpSender"; + public static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure(TELEMETRY_SCOPE_NAME); + /** * Set this to of(LogLevel.ERROR) or whatever level you'd like to get logging between each handler. * Set this to Optional.empty() to disable intra-handler logging. @@ -52,18 +60,15 @@ public class NettyPacketToHttpConsumer implements IPacketFinalizingConsumer activeChannelFuture; private final Channel channel; AggregatedRawResponse.Builder responseBuilder; - final String diagnosticLabel; - private UniqueReplayerRequestKey uniqueRequestKeyForMetricsLogging; + RequestContext tracingContext; public NettyPacketToHttpConsumer(NioEventLoopGroup eventLoopGroup, URI serverUri, SslContext sslContext, - String diagnosticLabel, UniqueReplayerRequestKey uniqueRequestKeyForMetricsLogging) { - this(createClientConnection(eventLoopGroup, sslContext, serverUri, diagnosticLabel), - diagnosticLabel, uniqueRequestKeyForMetricsLogging); + RequestContext requestContext) { + this(createClientConnection(eventLoopGroup, sslContext, serverUri, requestContext), requestContext); } - public NettyPacketToHttpConsumer(ChannelFuture clientConnection, String diagnosticLabel, UniqueReplayerRequestKey uniqueRequestKeyForMetricsLogging) { - this.diagnosticLabel = "[" + diagnosticLabel + "] "; - this.uniqueRequestKeyForMetricsLogging = uniqueRequestKeyForMetricsLogging; + public NettyPacketToHttpConsumer(ChannelFuture clientConnection, RequestContext ctx) { + this.tracingContext = ctx; responseBuilder = AggregatedRawResponse.builder(Instant.now()); DiagnosticTrackableCompletableFuture initialFuture = new StringTrackableCompletableFuture<>(new CompletableFuture<>(), @@ -87,7 +92,7 @@ public NettyPacketToHttpConsumer(ChannelFuture clientConnection, String diagnost } public static ChannelFuture createClientConnection(EventLoopGroup eventLoopGroup, SslContext sslContext, - URI serverUri, String diagnosticLabel) { + URI serverUri, ConnectionContext connectionContext) { String host = serverUri.getHost(); int port = serverUri.getPort(); log.atTrace().setMessage(()->"Active - setting up backend connection to " + host + ":" + port).log(); @@ -105,8 +110,8 @@ public static ChannelFuture createClientConnection(EventLoopGroup eventLoopGroup if (connectFuture.isSuccess()) { var pipeline = connectFuture.channel().pipeline(); pipeline.removeFirst(); - log.atTrace() - .setMessage(()->diagnosticLabel + " Done setting up client channel & it was successful").log(); + log.atTrace().setMessage(()->connectionContext.getChannelKey() + + " Done setting up client channel & it was successful").log(); if (sslContext != null) { var sslEngine = sslContext.newEngine(connectFuture.channel().alloc()); sslEngine.setUseClientMode(true); @@ -126,7 +131,7 @@ public static ChannelFuture createClientConnection(EventLoopGroup eventLoopGroup } else { // Close the connection if the connection attempt has failed. log.atWarn().setCause(connectFuture.cause()) - .setMessage(() -> diagnosticLabel + " CONNECT future was not successful, " + + .setMessage(() -> connectionContext.getChannelKey() + " CONNECT future was not successful, " + "so setting the channel future's result to an exception").log(); rval.setFailure(connectFuture.cause()); } @@ -190,8 +195,8 @@ public DiagnosticTrackableCompletableFuture consumeBytes(ByteBuf pa System.identityHashCode(packetData) + ")").log(); return writePacketAndUpdateFuture(packetData); } else { - log.atWarn().setMessage(()->diagnosticLabel + "outbound channel was not set up successfully, " + - "NOT writing bytes hash=" + System.identityHashCode(packetData)).log(); + log.atWarn().setMessage(()->tracingContext.getRequestKey() + "outbound channel was not set up " + + "successfully, NOT writing bytes hash=" + System.identityHashCode(packetData)).log(); channel.close(); return DiagnosticTrackableCompletableFuture.Factory.failedFuture(channelInitException, ()->""); } @@ -206,13 +211,14 @@ public DiagnosticTrackableCompletableFuture consumeBytes(ByteBuf pa final var completableFuture = new DiagnosticTrackableCompletableFuture(new CompletableFuture<>(), ()->"CompletableFuture that will wait for the netty future to fill in the completion value"); final int readableBytes = packetData.readableBytes(); + METERING_CLOSURE.meterIncrementEvent(tracingContext.context, "readBytes", packetData.readableBytes()); channel.writeAndFlush(packetData) .addListener((ChannelFutureListener) future -> { Throwable cause = null; try { if (!future.isSuccess()) { - log.atWarn().setMessage(()->diagnosticLabel + "closing outbound channel because WRITE " + - "future was not successful " + future.cause() + " hash=" + + log.atWarn().setMessage(()->tracingContext.getRequestKey() + "closing outbound channel " + + "because WRITE future was not successful " + future.cause() + " hash=" + System.identityHashCode(packetData) + " will be sending the exception to " + completableFuture).log(); future.channel().close(); // close the backside @@ -222,17 +228,17 @@ public DiagnosticTrackableCompletableFuture consumeBytes(ByteBuf pa cause = e; } if (cause == null) { - log.atTrace().setMessage(()->"Signaling previously returned CompletableFuture packet write was successful: " - + packetData + " hash=" + System.identityHashCode(packetData)).log(); + log.atTrace().setMessage(()->"Previously returned CompletableFuture packet write was " + + "successful: " + packetData + " hash=" + System.identityHashCode(packetData)).log(); completableFuture.future.complete(null); } else { - log.atInfo().setMessage(()->"Signaling previously returned CompletableFuture packet write had an exception : " - + packetData + " hash=" + System.identityHashCode(packetData)).log(); + log.atInfo().setMessage(()->"Previously returned CompletableFuture packet write had " + + " an exception :" + packetData + " hash=" + System.identityHashCode(packetData)).log(); metricsLogger.atError(MetricsEvent.WRITING_REQUEST_COMPONENT_FAILED, cause) .setAttribute(MetricsAttributeKey.CHANNEL_ID, channel.id().asLongText()) - .setAttribute(MetricsAttributeKey.REQUEST_ID, uniqueRequestKeyForMetricsLogging) + .setAttribute(MetricsAttributeKey.REQUEST_ID, tracingContext.getRequestKey().toString()) .setAttribute(MetricsAttributeKey.CONNECTION_ID, - uniqueRequestKeyForMetricsLogging.getTrafficStreamKey().getConnectionId()).emit(); + tracingContext.getRequestKey().getTrafficStreamKey().getConnectionId()).emit(); completableFuture.future.completeExceptionally(cause); channel.close(); } @@ -241,8 +247,9 @@ public DiagnosticTrackableCompletableFuture consumeBytes(ByteBuf pa ". Created future for writing data="+completableFuture).log(); metricsLogger.atSuccess(MetricsEvent.WROTE_REQUEST_COMPONENT) .setAttribute(MetricsAttributeKey.CHANNEL_ID, channel.id().asLongText()) - .setAttribute(MetricsAttributeKey.REQUEST_ID, uniqueRequestKeyForMetricsLogging) - .setAttribute(MetricsAttributeKey.CONNECTION_ID, uniqueRequestKeyForMetricsLogging.getTrafficStreamKey().getConnectionId()) + .setAttribute(MetricsAttributeKey.REQUEST_ID, tracingContext.getRequestKey()) + .setAttribute(MetricsAttributeKey.CONNECTION_ID, + tracingContext.getRequestKey().getTrafficStreamKey().getConnectionId()) .setAttribute(MetricsAttributeKey.SIZE_IN_BYTES, readableBytes).emit(); return completableFuture; } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java index 8d17c8187..3019e0c0b 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java @@ -11,7 +11,7 @@ import org.opensearch.migrations.replay.datatypes.TransformedOutputAndResult; import org.opensearch.migrations.replay.Utils; import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer; -import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture; import org.opensearch.migrations.transform.IAuthTransformerFactory; @@ -50,7 +50,7 @@ public class HttpJsonTransformingConsumer implements IPacketFinalizingConsume private final RequestPipelineOrchestrator pipelineOrchestrator; private final EmbeddedChannel channel; private static final MetricsLogger metricsLogger = new MetricsLogger("HttpJsonTransformingConsumer"); - private UniqueReplayerRequestKey requestKeyForMetricsLogging; + private RequestContext requestContext; /** * Roughly try to keep track of how big each data chunk was that came into the transformer. These values @@ -68,15 +68,15 @@ public HttpJsonTransformingConsumer(IJsonTransformer transformer, IAuthTransformerFactory authTransformerFactory, IPacketFinalizingConsumer transformedPacketReceiver, String diagnosticLabel, - UniqueReplayerRequestKey requestKeyForMetricsLogging) { + RequestContext requestContext) { chunkSizes = new ArrayList<>(HTTP_MESSAGE_NUM_SEGMENTS); chunkSizes.add(new ArrayList<>(EXPECTED_PACKET_COUNT_GUESS_FOR_HEADERS)); chunks = new ArrayList<>(HTTP_MESSAGE_NUM_SEGMENTS + EXPECTED_PACKET_COUNT_GUESS_FOR_HEADERS); channel = new EmbeddedChannel(); + this.requestContext = requestContext; pipelineOrchestrator = new RequestPipelineOrchestrator<>(chunkSizes, transformedPacketReceiver, - authTransformerFactory, diagnosticLabel, requestKeyForMetricsLogging); + authTransformerFactory, diagnosticLabel, requestContext); pipelineOrchestrator.addInitialHandlers(channel.pipeline(), transformer); - this.requestKeyForMetricsLogging = requestKeyForMetricsLogging; } private NettySendByteBufsToPacketHandlerHandler getOffloadingHandler() { @@ -142,15 +142,15 @@ public DiagnosticTrackableCompletableFuture"transformedHttpMessageValue"); } @@ -180,8 +180,8 @@ private static Throwable unwindPossibleCompletionException(Throwable t) { consumptionChainedFuture.thenCompose(v -> packetConsumer.finalizeRequest(), ()->"HttpJsonTransformingConsumer.redriveWithoutTransformation.compose()"); metricsLogger.atError(MetricsEvent.REQUEST_REDRIVEN_WITHOUT_TRANSFORMATION, reason) - .setAttribute(MetricsAttributeKey.REQUEST_ID, requestKeyForMetricsLogging) - .setAttribute(MetricsAttributeKey.CONNECTION_ID, requestKeyForMetricsLogging.getTrafficStreamKey().getConnectionId()) + .setAttribute(MetricsAttributeKey.REQUEST_ID, requestContext) + .setAttribute(MetricsAttributeKey.CONNECTION_ID, requestContext.getChannelKey().getConnectionId()) .setAttribute(MetricsAttributeKey.CHANNEL_ID, channel.id().asLongText()).emit(); return finalizedFuture.map(f->f.thenApply(r->reason == null ? new TransformedOutputAndResult(r, HttpRequestTransformationStatus.SKIPPED, null) : diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java index 8debf6c1e..771076d52 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java @@ -10,7 +10,7 @@ import org.opensearch.migrations.coreutils.MetricsLogger; import org.opensearch.migrations.replay.datahandlers.PayloadAccessFaultingMap; import org.opensearch.migrations.replay.datahandlers.PayloadNotLoadedException; -import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.transform.IAuthTransformer; import org.opensearch.migrations.transform.IJsonTransformer; @@ -27,19 +27,19 @@ public class NettyDecodedHttpRequestPreliminaryConvertHandler extends Channel final IJsonTransformer transformer; final List> chunkSizes; final String diagnosticLabel; - private UniqueReplayerRequestKey requestKeyForMetricsLogging; + private RequestContext requestContext; static final MetricsLogger metricsLogger = new MetricsLogger("NettyDecodedHttpRequestPreliminaryConvertHandler"); public NettyDecodedHttpRequestPreliminaryConvertHandler(IJsonTransformer transformer, List> chunkSizes, RequestPipelineOrchestrator requestPipelineOrchestrator, String diagnosticLabel, - UniqueReplayerRequestKey requestKeyForMetricsLogging) { + RequestContext requestContext) { this.transformer = transformer; this.chunkSizes = chunkSizes; this.requestPipelineOrchestrator = requestPipelineOrchestrator; this.diagnosticLabel = "[" + diagnosticLabel + "] "; - this.requestKeyForMetricsLogging = requestKeyForMetricsLogging; + this.requestContext = requestContext; } @Override @@ -55,8 +55,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception .append(request.protocolVersion().text()) .toString()); metricsLogger.atSuccess(MetricsEvent.CAPTURED_REQUEST_PARSED_TO_HTTP) - .setAttribute(MetricsAttributeKey.REQUEST_ID, requestKeyForMetricsLogging) - .setAttribute(MetricsAttributeKey.CONNECTION_ID, requestKeyForMetricsLogging.getTrafficStreamKey().getConnectionId()) + .setAttribute(MetricsAttributeKey.REQUEST_ID, requestContext) + .setAttribute(MetricsAttributeKey.CONNECTION_ID, requestContext.getChannelKey().getConnectionId()) .setAttribute(MetricsAttributeKey.HTTP_METHOD, request.method()) .setAttribute(MetricsAttributeKey.HTTP_ENDPOINT, request.uri()).emit(); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/RequestPipelineOrchestrator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/RequestPipelineOrchestrator.java index e434efce9..5e285847d 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/RequestPipelineOrchestrator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/RequestPipelineOrchestrator.java @@ -11,6 +11,7 @@ import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.transform.IAuthTransformer; import org.opensearch.migrations.transform.IAuthTransformerFactory; import org.opensearch.migrations.transform.IJsonTransformer; @@ -43,7 +44,7 @@ public class RequestPipelineOrchestrator { private final List> chunkSizes; final IPacketFinalizingConsumer packetReceiver; final String diagnosticLabel; - private UniqueReplayerRequestKey requestKeyForMetricsLogging; + private RequestContext requestContext; @Getter final IAuthTransformerFactory authTransfomerFactory; @@ -51,13 +52,13 @@ public RequestPipelineOrchestrator(List> chunkSizes, IPacketFinalizingConsumer packetReceiver, IAuthTransformerFactory incomingAuthTransformerFactory, String diagnosticLabel, - UniqueReplayerRequestKey requestKeyForMetricsLogging) { + RequestContext requestContext) { this.chunkSizes = chunkSizes; this.packetReceiver = packetReceiver; this.authTransfomerFactory = incomingAuthTransformerFactory != null ? incomingAuthTransformerFactory : IAuthTransformerFactory.NullAuthTransformerFactory.instance; this.diagnosticLabel = diagnosticLabel; - this.requestKeyForMetricsLogging = requestKeyForMetricsLogging; + this.requestContext = requestContext; } static void removeThisAndPreviousHandlers(ChannelPipeline pipeline, ChannelHandler targetHandler) { @@ -100,7 +101,7 @@ void addInitialHandlers(ChannelPipeline pipeline, IJsonTransformer transformer) // HttpRequestDecoder when the HttpRequestDecoder is removed from the pipeline BEFORE the // NettyDecodedHttpRequestHandler is removed. pipeline.addLast(new NettyDecodedHttpRequestPreliminaryConvertHandler(transformer, chunkSizes, this, - diagnosticLabel, requestKeyForMetricsLogging)); + diagnosticLabel, requestContext)); addLoggingHandler(pipeline, "B"); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ConnectionContext.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ConnectionContext.java new file mode 100644 index 000000000..0312d5140 --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ConnectionContext.java @@ -0,0 +1,34 @@ +package org.opensearch.migrations.replay.tracing; + +import io.netty.util.AttributeKey; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextKey; +import lombok.NonNull; +import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; + +import java.util.stream.Stream; + +public class ConnectionContext implements WithAttributes { + protected static final ContextKey CHANNEL_KEY_CONTEXT_KEY = ContextKey.named("channelKey"); + protected static final AttributeKey CHANNEL_ATTR = AttributeKey.newInstance("channelKey"); + + public final Context context; + + public ConnectionContext(ISourceTrafficChannelKey tsk) { + this(Context.current().with(CHANNEL_KEY_CONTEXT_KEY, tsk)); + } + + public ConnectionContext(Context c) { + assert c.get(CHANNEL_KEY_CONTEXT_KEY) != null; + context = c; + } + + public @NonNull ISourceTrafficChannelKey getChannelKey() { + return context.get(CHANNEL_KEY_CONTEXT_KEY); + } + + @Override + public Stream getAttributeKeys() { + return Stream.of(CHANNEL_ATTR); + } +} diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java new file mode 100644 index 000000000..bfc0f5407 --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java @@ -0,0 +1,26 @@ +package org.opensearch.migrations.replay.tracing; + +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextKey; +import lombok.NonNull; +import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; + +public class RequestContext extends ConnectionContext { + private static final ContextKey UNIQUE_REQUEST_KEY = ContextKey.named("requestId"); + + public RequestContext(UniqueReplayerRequestKey requestKey) { + this(Context.current(), requestKey); + } + + public RequestContext(ConnectionContext ctx, UniqueReplayerRequestKey requestKey) { + this(ctx.context, requestKey); + } + + public RequestContext(Context context, UniqueReplayerRequestKey requestKey) { + super(context.with(UNIQUE_REQUEST_KEY, requestKey).with(CHANNEL_KEY_CONTEXT_KEY, requestKey.trafficStreamKey)); + } + + public @NonNull UniqueReplayerRequestKey getRequestKey() { + return context.get(UNIQUE_REQUEST_KEY); + } +} diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/WithAttributes.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/WithAttributes.java new file mode 100644 index 000000000..1988c7ae3 --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/WithAttributes.java @@ -0,0 +1,9 @@ +package org.opensearch.migrations.replay.tracing; + +import io.netty.util.AttributeKey; + +import java.util.stream.Stream; + +public interface WithAttributes { + Stream getAttributeKeys(); +} diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java index c84987938..ea1bb6eb9 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java @@ -15,7 +15,6 @@ import java.util.Random; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; import java.util.function.IntFunction; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -35,7 +34,7 @@ public void testTransformer() throws Exception { var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse); var transformer = new TransformationLoader().getTransformerFactoryLoader(SILLY_TARGET_CLUSTER_NAME); var transformingHandler = new HttpJsonTransformingConsumer(transformer, null, testPacketCapture, - "TEST", TestRequestKey.getTestConnectionRequestId(0)); + "TEST", TestRequestKey.getTestConnectionRequestContext(0)); runRandomPayloadWithTransformer(transformingHandler, dummyAggregatedResponse, testPacketCapture, contentLength -> "GET / HTTP/1.1\r\n" + "HoSt: " + SOURCE_CLUSTER_NAME + "\r\n" + @@ -88,7 +87,7 @@ public void testMalformedPayloadIsPassedThrough() throws Exception { var transformingHandler = new HttpJsonTransformingConsumer( new TransformationLoader().getTransformerFactoryLoader(SILLY_TARGET_CLUSTER_NAME), httpBasicAuthTransformer, testPacketCapture, "TEST", - TestRequestKey.getTestConnectionRequestId(0)); + TestRequestKey.getTestConnectionRequestContext(0)); runRandomPayloadWithTransformer(transformingHandler, dummyAggregatedResponse, testPacketCapture, contentLength -> "GET / HTTP/1.1\r\n" + @@ -114,7 +113,7 @@ public void testMalformedPayload_andTypeMappingUri_IsPassedThrough() throws Exce var transformingHandler = new HttpJsonTransformingConsumer( new TransformationLoader().getTransformerFactoryLoader(SILLY_TARGET_CLUSTER_NAME, null, "[{\"JsonTransformerForOpenSearch23PlusTargetTransformerProvider\":\"\"}]"), - null, testPacketCapture, "TEST", TestRequestKey.getTestConnectionRequestId(0)); + null, testPacketCapture, "TEST", TestRequestKey.getTestConnectionRequestContext(0)); Random r = new Random(2); var stringParts = IntStream.range(0, 1).mapToObj(i-> TestUtils.makeRandomString(r, 10)).map(o->(String)o) diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/RequestSenderOrchestratorTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/RequestSenderOrchestratorTest.java index 657254c16..e7c4b71e6 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/RequestSenderOrchestratorTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/RequestSenderOrchestratorTest.java @@ -5,7 +5,6 @@ import io.netty.buffer.Unpooled; import io.netty.handler.codec.http.FullHttpResponse; import lombok.extern.slf4j.Slf4j; -import org.junit.Assume; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -40,20 +39,21 @@ public void testThatSchedulingWorks() throws Exception { Instant lastEndTime = baseTime; var scheduledItems = new ArrayList>(); for (int i = 0; i trafficStreamKeysBeingHeld) {} @Override public void onConnectionClose(ISourceTrafficChannelKey key, int channelInteractionNumber, + ConnectionContext ctx, RequestResponsePacketPair.ReconstructionStatus status, Instant when, List trafficStreamKeysBeingHeld) { } - @Override public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk) {} + @Override public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk, + ConnectionContext ctx) {} }); var tsList = trafficStreams.collect(Collectors.toList()); trafficStreams = tsList.stream(); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java index 089880081..f52770fb9 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java @@ -9,6 +9,8 @@ import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.replay.tracing.ConnectionContext; +import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.traffic.source.InputStreamOfTraffic; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; import org.opensearch.migrations.trafficcapture.protos.CloseObservation; @@ -154,30 +156,35 @@ public void testReader() throws Exception { new CapturedTrafficToHttpTransactionAccumulator(Duration.ofSeconds(30), null, new AccumulationCallbacks() { @Override - public void onRequestReceived(UniqueReplayerRequestKey id, HttpMessageAndTimestamp request) { + public void onRequestReceived(UniqueReplayerRequestKey id, RequestContext ctx, + HttpMessageAndTimestamp request) { var bytesList = request.stream().collect(Collectors.toList()); byteArrays.add(bytesList); Assertions.assertEquals(FAKE_READ_PACKET_DATA, collectBytesToUtf8String(bytesList)); } @Override - public void onFullDataReceived(UniqueReplayerRequestKey key, RequestResponsePacketPair fullPair) { + public void onFullDataReceived(UniqueReplayerRequestKey key, RequestContext ctx, + RequestResponsePacketPair fullPair) { var responseBytes = fullPair.responseData.packetBytes.stream().collect(Collectors.toList()); Assertions.assertEquals(FAKE_READ_PACKET_DATA, collectBytesToUtf8String(responseBytes)); } @Override public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, + ConnectionContext ctx, List trafficStreamKeysBeingHeld) {} @Override public void onConnectionClose(ISourceTrafficChannelKey key, int channelInteractionNumber, + ConnectionContext ctx, RequestResponsePacketPair.ReconstructionStatus status, Instant when, List trafficStreamKeysBeingHeld) { } - @Override public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk) {} + @Override public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk, + ConnectionContext ctx) {} }); var bytes = synthesizeTrafficStreamsIntoByteArray(Instant.now(), 1); @@ -201,29 +208,33 @@ public void testCapturedReadsAfterCloseAreHandledAsNew() throws Exception { "CapturedTrafficToHttpTransactionAccumulator that's being used in this unit test!", new AccumulationCallbacks() { @Override - public void onRequestReceived(UniqueReplayerRequestKey id, HttpMessageAndTimestamp request) { + public void onRequestReceived(UniqueReplayerRequestKey id, RequestContext ctx, + HttpMessageAndTimestamp request) { var bytesList = request.stream().collect(Collectors.toList()); byteArrays.add(bytesList); Assertions.assertEquals(FAKE_READ_PACKET_DATA, collectBytesToUtf8String(bytesList)); } @Override - public void onFullDataReceived(UniqueReplayerRequestKey key, RequestResponsePacketPair fullPair) { + public void onFullDataReceived(UniqueReplayerRequestKey key, RequestContext ctx, + RequestResponsePacketPair fullPair) { var responseBytes = fullPair.responseData.packetBytes.stream().collect(Collectors.toList()); Assertions.assertEquals(FAKE_READ_PACKET_DATA, collectBytesToUtf8String(responseBytes)); } @Override public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, + ConnectionContext ctx, List trafficStreamKeysBeingHeld) {} @Override public void onConnectionClose(ISourceTrafficChannelKey key, int channelInteractionNumber, - RequestResponsePacketPair.ReconstructionStatus status, + ConnectionContext ctx, RequestResponsePacketPair.ReconstructionStatus status, Instant when, List trafficStreamKeysBeingHeld) { } - @Override public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk) {} + @Override public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk, + ConnectionContext ctx) {} } ); byte[] serializedChunks; diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java index e60578199..5927410f0 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java @@ -22,6 +22,7 @@ import org.opensearch.migrations.replay.TransformationLoader; import org.opensearch.migrations.replay.datatypes.PojoTrafficStreamKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.traffic.source.BufferedFlowController; import org.opensearch.migrations.testutils.HttpFirstLine; import org.opensearch.migrations.testutils.PortFinder; @@ -128,9 +129,11 @@ public void testHttpResponseIsSuccessfullyCaptured(boolean useTls) throws Except var testServer = testServers.get(useTls); var sslContext = !testServer.localhostEndpoint().getScheme().toLowerCase().equals("https") ? null : SslContextBuilder.forClient().trustManager(InsecureTrustManagerFactory.INSTANCE).build(); - var nphc = new NettyPacketToHttpConsumer(new NioEventLoopGroup(4, new DefaultThreadFactory("test")), - testServer.localhostEndpoint(), sslContext, "unitTest"+i, - TestRequestKey.getTestConnectionRequestId(0)); + var nphc = new NettyPacketToHttpConsumer( + new NioEventLoopGroup(4, new DefaultThreadFactory("test")), + testServer.localhostEndpoint(), + sslContext, + TestRequestKey.getTestConnectionRequestContext(0)); nphc.consumeBytes((EXPECTED_REQUEST_STRING).getBytes(StandardCharsets.UTF_8)); var aggregatedResponse = nphc.finalizeRequest().get(); var responseBytePackets = aggregatedResponse.getCopyOfPackets(); @@ -162,9 +165,10 @@ public void testThatConnectionsAreKeptAliveAndShared(boolean useTls) for (int i = 0; i < 2; ++i) { String connId = "TEST_" + j; var trafficStreamKey = new PojoTrafficStreamKey("testNodeId", connId, 0); + var requestKey = new UniqueReplayerRequestKey(trafficStreamKey, 0, i); + var ctx = new RequestContext(requestKey); var requestFinishFuture = TrafficReplayer.transformAndSendRequest(transformingHttpHandlerFactory, - sendingFactory, Instant.now(), Instant.now(), - new UniqueReplayerRequestKey(trafficStreamKey, 0, i), + sendingFactory, ctx, Instant.now(), Instant.now(), requestKey, ()->Stream.of(EXPECTED_REQUEST_STRING.getBytes(StandardCharsets.UTF_8))); log.info("requestFinishFuture="+requestFinishFuture); var aggregatedResponse = requestFinishFuture.get(); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java index ba6ab87f0..f4c7031ff 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java @@ -28,7 +28,7 @@ public void testPassThroughSinglePacketPost() throws Exception { new HttpJsonTransformingConsumer(new TransformationLoader() .getTransformerFactoryLoader(null), null, testPacketCapture, "TEST", - TestRequestKey.getTestConnectionRequestId(0)); + TestRequestKey.getTestConnectionRequestContext(0)); byte[] testBytes; try (var sampleStream = HttpJsonTransformingConsumer.class.getResourceAsStream( "/requests/raw/post_formUrlEncoded_withFixedLength.txt")) { @@ -49,7 +49,7 @@ public void testPassThroughSinglePacketWithoutBodyTransformationPost() throws Ex new HttpJsonTransformingConsumer( new TransformationLoader().getTransformerFactoryLoader("test.domain"), null, testPacketCapture, "TEST", - TestRequestKey.getTestConnectionRequestId(0)); + TestRequestKey.getTestConnectionRequestContext(0)); byte[] testBytes; try (var sampleStream = HttpJsonTransformingConsumer.class.getResourceAsStream( "/requests/raw/post_formUrlEncoded_withFixedLength.txt")) { @@ -75,7 +75,7 @@ public void testRemoveAuthHeadersWorks() throws Exception { new TransformationLoader().getTransformerFactoryLoader("test.domain"), RemovingAuthTransformerFactory.instance, testPacketCapture, "TEST", - TestRequestKey.getTestConnectionRequestId(0)); + TestRequestKey.getTestConnectionRequestContext(0)); byte[] testBytes; try (var sampleStream = HttpJsonTransformingConsumer.class.getResourceAsStream( "/requests/raw/get_withAuthHeader.txt")) { @@ -114,7 +114,7 @@ private void walkMaps(Object o) { }); var transformingHandler = new HttpJsonTransformingConsumer(complexTransformer, null, - testPacketCapture, "TEST", TestRequestKey.getTestConnectionRequestId(0)); + testPacketCapture, "TEST", TestRequestKey.getTestConnectionRequestContext(0)); byte[] testBytes; try (var sampleStream = HttpJsonTransformingConsumer.class.getResourceAsStream( "/requests/raw/post_formUrlEncoded_withFixedLength.txt")) { diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java index 5751cb4aa..2d94b11a2 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java @@ -2,14 +2,17 @@ import org.opensearch.migrations.replay.datatypes.PojoTrafficStreamKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.replay.tracing.ConnectionContext; +import org.opensearch.migrations.replay.tracing.RequestContext; public class TestRequestKey { private TestRequestKey() {} - public static final UniqueReplayerRequestKey getTestConnectionRequestId(int replayerIdx) { - return new UniqueReplayerRequestKey( + public static final RequestContext getTestConnectionRequestContext(int replayerIdx) { + var rk = new UniqueReplayerRequestKey( new PojoTrafficStreamKey("testNodeId", "testConnectionId", 0), 0, replayerIdx); + return new RequestContext(new UniqueReplayerRequestKey(rk.trafficStreamKey, 1, 1)); } } diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java index 77322ffa6..59d4576f2 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java @@ -141,7 +141,7 @@ static void runPipelineAndValidate(IJsonTransformer transformer, var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), new AggregatedRawResponse(-1, Duration.ZERO, new ArrayList<>(), null)); var transformingHandler = new HttpJsonTransformingConsumer<>(transformer, authTransformer, testPacketCapture, - "TEST", TestRequestKey.getTestConnectionRequestId(0)); + "TEST", TestRequestKey.getTestConnectionRequestContext(0)); var contentLength = stringParts.stream().mapToInt(String::length).sum(); var headerString = "GET / HTTP/1.1\r\n" + From 900bc6dae0f6d2a5b9d9c2611133f1634bac93f7 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Wed, 29 Nov 2023 16:56:00 -0500 Subject: [PATCH 09/20] Start moving away from ThreadLocal and 'current contexts' and toward explicitly passing strongly typed context objects. Signed-off-by: Greg Schohn --- .../kafkaoffloader/KafkaCaptureFactory.java | 130 ++++++++---------- .../tracing/KafkaRecordContext.java | 49 +++++++ .../KafkaCaptureFactoryTest.java | 12 +- TrafficCapture/captureOffloader/build.gradle | 7 +- .../FileConnectionCaptureFactory.java | 3 +- .../IConnectionCaptureFactory.java | 4 +- .../tracing/ConnectionContext.java | 33 +++++ .../InMemoryConnectionCaptureFactory.java | 6 +- TrafficCapture/coreUtilities/build.gradle | 6 +- .../coreutils/SimpleMeteringClosure.java | 66 +++++---- .../migrations/tracing/ContextWithSpan.java | 10 ++ .../migrations/tracing/EmptyContext.java | 25 ++++ .../tracing/IConnectionContext.java | 21 +++ .../tracing/IReplayerRequestContext.java | 16 +++ .../migrations/tracing/IRequestContext.java | 16 +++ .../migrations/tracing/IWithAttributes.java | 30 ++++ .../migrations/tracing/IWithStartTime.java | 7 + ...allyReliableLoggingHttpRequestHandler.java | 26 ++-- .../netty/LoggingHttpRequestHandler.java | 36 ++--- .../netty/LoggingHttpResponseHandler.java | 15 +- ...ReliableLoggingHttpRequestHandlerTest.java | 14 +- .../proxyserver/CaptureProxy.java | 2 +- .../netty/ProxyChannelInitializer.java | 12 +- .../migrations/replay/Accumulation.java | 21 ++- .../replay/AccumulationCallbacks.java | 8 +- ...edTrafficToHttpTransactionAccumulator.java | 66 ++++----- .../replay/ClientConnectionPool.java | 8 +- .../replay/PacketConsumerFactory.java | 3 +- ...acketToTransformingHttpHandlerFactory.java | 6 +- .../migrations/replay/ReplayEngine.java | 4 +- .../replay/RequestResponsePacketPair.java | 6 +- .../replay/RequestSenderOrchestrator.java | 15 +- .../migrations/replay/TrafficReplayer.java | 10 +- .../NettyPacketToHttpConsumer.java | 27 ++-- .../http/HttpJsonTransformingConsumer.java | 54 +++++--- ...dHttpRequestPreliminaryConvertHandler.java | 2 +- .../http/RequestPipelineOrchestrator.java | 1 - .../replay/tracing/ChannelKeyContext.java | 25 ++++ .../replay/tracing/ConnectionContext.java | 34 ----- .../replay/tracing/RequestContext.java | 54 ++++++-- .../replay/tracing/WithAttributes.java | 9 -- .../replay/RequestSenderOrchestratorTest.java | 4 +- .../replay/ResultsToLogsConsumerTest.java | 6 +- ...afficToHttpTransactionAccumulatorTest.java | 14 +- .../replay/TrafficReplayerTest.java | 14 +- .../NettyPacketToHttpConsumerTest.java | 4 +- .../migrations/replay/TestRequestKey.java | 5 +- 47 files changed, 591 insertions(+), 355 deletions(-) create mode 100644 TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/tracing/KafkaRecordContext.java create mode 100644 TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/tracing/ConnectionContext.java create mode 100644 TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/ContextWithSpan.java create mode 100644 TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/EmptyContext.java create mode 100644 TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IConnectionContext.java create mode 100644 TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IReplayerRequestContext.java create mode 100644 TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IRequestContext.java create mode 100644 TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IWithAttributes.java create mode 100644 TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IWithStartTime.java create mode 100644 TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelKeyContext.java delete mode 100644 TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ConnectionContext.java delete mode 100644 TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/WithAttributes.java diff --git a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java index aaa694e55..29c27f425 100644 --- a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java +++ b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java @@ -1,7 +1,6 @@ package org.opensearch.migrations.trafficcapture.kafkaoffloader; import com.google.protobuf.CodedOutputStream; -import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.Span; import io.opentelemetry.context.Context; import io.opentelemetry.context.ContextKey; @@ -15,12 +14,15 @@ import org.opensearch.migrations.coreutils.MetricsEvent; import org.apache.kafka.clients.producer.RecordMetadata; import org.opensearch.migrations.coreutils.SimpleMeteringClosure; +import org.opensearch.migrations.tracing.EmptyContext; import org.opensearch.migrations.trafficcapture.CodedOutputStreamHolder; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; import org.opensearch.migrations.trafficcapture.OrderedStreamLifecyleManager; import org.opensearch.migrations.trafficcapture.StreamChannelConnectionCaptureSerializer; import org.opensearch.migrations.coreutils.MetricsLogger; +import org.opensearch.migrations.trafficcapture.kafkaoffloader.tracing.KafkaRecordContext; +import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; import java.io.IOException; import java.nio.ByteBuffer; @@ -33,12 +35,8 @@ @Slf4j public class KafkaCaptureFactory implements IConnectionCaptureFactory { - private static final ContextKey RECORD_ID_KEY = ContextKey.named("recordId"); - private static final ContextKey TOPIC_KEY = ContextKey.named("topic"); - private static final ContextKey RECORD_SIZE_KEY = ContextKey.named("recordSize"); - private static final ContextKey START_FLUSH_KEY = ContextKey.named("startKafkaSend"); - public static final String TELEMETRY_SCOPE_NAME = "KafkaCapture"; - public static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure(TELEMETRY_SCOPE_NAME); + private static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure("KafkaCapture"); + private static final MetricsLogger metricsLogger = new MetricsLogger("BacksideHandler"); @@ -66,23 +64,20 @@ public KafkaCaptureFactory(String nodeId, Producer producer, int } @Override - public IChannelConnectionCaptureSerializer createOffloader(String connectionId) { - Span offloaderSpan = GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME) - .spanBuilder("offloader").startSpan(); - offloaderSpan.setAttribute("offloaderConnectionId", connectionId); - var context = Context.current().with(offloaderSpan); - METERING_CLOSURE.meterIncrementEvent(context, "offloader_created"); - METERING_CLOSURE.meterDeltaEvent(context, "offloaders_active", 1); + public IChannelConnectionCaptureSerializer createOffloader(ConnectionContext ctx, + String connectionId) { + METERING_CLOSURE.meterIncrementEvent(ctx, "offloader_created"); + METERING_CLOSURE.meterDeltaEvent(ctx, "offloaders_active", 1); return new StreamChannelConnectionCaptureSerializer<>(nodeId, connectionId, - new StreamManager(context, connectionId)); + new StreamManager(ctx, connectionId)); } @AllArgsConstructor static class CodedOutputStreamWrapper implements CodedOutputStreamHolder { private final CodedOutputStream codedOutputStream; private final ByteBuffer byteBuffer; - final Context streamContext; + final ConnectionContext streamContext; @Override public @NonNull CodedOutputStream getOutputStream() { return codedOutputStream; @@ -90,32 +85,33 @@ static class CodedOutputStreamWrapper implements CodedOutputStreamHolder { } class StreamManager extends OrderedStreamLifecyleManager { - Context telemetryContext; + ConnectionContext telemetryContext; String connectionId; Instant startTime; - public StreamManager(Context incomingTelemetryContext, String connectionId) { - this.telemetryContext = incomingTelemetryContext; + public StreamManager(ConnectionContext incomingTelemetryContext, String connectionId) { + this.telemetryContext = new ConnectionContext(incomingTelemetryContext, + METERING_CLOSURE.makeSpan(incomingTelemetryContext, "offloaderLifetime")); this.connectionId = connectionId; this.startTime = Instant.now(); } @Override public void close() throws IOException { - log.atInfo().setMessage(()->"factory.close()").log(); + log.atInfo().setMessage(() -> "factory.close()").log(); METERING_CLOSURE.meterHistogramMillis(telemetryContext, "offloader_stream_lifetime", - Duration.between(startTime, Instant.now())); + Duration.between(startTime, Instant.now())); METERING_CLOSURE.meterDeltaEvent(telemetryContext, "offloaders_active", -1); METERING_CLOSURE.meterIncrementEvent(telemetryContext, "offloader_closed"); - Span.fromContext(telemetryContext).end(); + telemetryContext.currentSpan.end(); } @Override public CodedOutputStreamWrapper createStream() { METERING_CLOSURE.meterIncrementEvent(telemetryContext, "stream_created"); - var newStreamCtx = telemetryContext - .with(METERING_CLOSURE.tracer.spanBuilder("recordStream").startSpan()); + var newStreamCtx = new ConnectionContext(telemetryContext, + METERING_CLOSURE.makeSpan(telemetryContext, "recordStream")); ByteBuffer bb = ByteBuffer.allocate(bufferSize); return new CodedOutputStreamWrapper(CodedOutputStream.newInstance(bb), bb, newStreamCtx); @@ -140,18 +136,10 @@ public CodedOutputStreamWrapper createStream() { var cf = new CompletableFuture(); log.debug("Sending Kafka producer record: {} for topic: {}", recordId, topicNameForTraffic); - Context flushContext; - Span.fromContext(osh.streamContext).end(); - try (var scope = telemetryContext - .with(RECORD_ID_KEY, recordId) - .with(TOPIC_KEY, topicNameForTraffic) - .with(RECORD_SIZE_KEY, kafkaRecord.value().length) - .with(START_FLUSH_KEY, Instant.now()) - .makeCurrent()) { - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "stream_flush_called"); - flushContext = Context.current() - .with(METERING_CLOSURE.tracer.spanBuilder("flushRecord").startSpan()); - } + var flushContext = new KafkaRecordContext(telemetryContext, + METERING_CLOSURE.makeSpan(telemetryContext, "flushRecord"), + topicNameForTraffic, recordId, kafkaRecord.value().length); + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "stream_flush_called"); // Async request to Kafka cluster producer.send(kafkaRecord, handleProducerRecordSent(cf, recordId, flushContext)); @@ -168,40 +156,42 @@ public CodedOutputStreamWrapper createStream() { throw e; } } + } - /** - * The default KafkaProducer comes with built-in retry and error-handling logic that suits many cases. From the - * documentation here for retry: https://kafka.apache.org/35/javadoc/org/apache/kafka/clients/producer/KafkaProducer.html - * "If the request fails, the producer can automatically retry. The retries setting defaults to Integer.MAX_VALUE, - * and it's recommended to use delivery.timeout.ms to control retry behavior, instead of retries." - *

- * Apart from this the KafkaProducer has logic for deciding whether an error is transient and should be - * retried or not retried at all: https://kafka.apache.org/35/javadoc/org/apache/kafka/common/errors/RetriableException.html - * as well as basic retry backoff - */ - private Callback handleProducerRecordSent(CompletableFuture cf, String recordId, - Context flushContext) { - return (metadata, exception) -> { - log.atInfo().setMessage(()->"kafka completed sending a record").log(); - METERING_CLOSURE.meterHistogramMicros(telemetryContext, - exception==null ? "stream_flush_success_ms" : "stream_flush_failure_ms", - Duration.between(flushContext.get(START_FLUSH_KEY), Instant.now())); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, - exception==null ? "stream_flush_success" : "stream_flush_failure"); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, - exception==null ? "stream_flush_success_bytes" : "stream_flush_failure_bytes", - flushContext.get(RECORD_SIZE_KEY)); - Span.fromContext(flushContext).end(); - - if (exception != null) { - log.error("Error sending producer record: {}", recordId, exception); - cf.completeExceptionally(exception); - } else { - log.debug("Kafka producer record: {} has finished sending for topic: {} and partition {}", - recordId, metadata.topic(), metadata.partition()); - cf.complete(metadata); - } - }; - } + /** + * The default KafkaProducer comes with built-in retry and error-handling logic that suits many cases. From the + * documentation here for retry: https://kafka.apache.org/35/javadoc/org/apache/kafka/clients/producer/KafkaProducer.html + * "If the request fails, the producer can automatically retry. The retries setting defaults to Integer.MAX_VALUE, + * and it's recommended to use delivery.timeout.ms to control retry behavior, instead of retries." + *

+ * Apart from this the KafkaProducer has logic for deciding whether an error is transient and should be + * retried or not retried at all: https://kafka.apache.org/35/javadoc/org/apache/kafka/common/errors/RetriableException.html + * as well as basic retry backoff + */ + private Callback handleProducerRecordSent(CompletableFuture cf, String recordId, + KafkaRecordContext flushContext) { + // Keep this out of the inner class because it is more unsafe to include it within + // the inner class since the inner class has context that shouldn't be used. This keeps + // that field out of scope. + return (metadata, exception) -> { + log.atInfo().setMessage(()->"kafka completed sending a record").log(); + METERING_CLOSURE.meterHistogramMicros(flushContext, + exception==null ? "stream_flush_success_ms" : "stream_flush_failure_ms"); + METERING_CLOSURE.meterIncrementEvent(flushContext, + exception==null ? "stream_flush_success" : "stream_flush_failure"); + METERING_CLOSURE.meterIncrementEvent(flushContext, + exception==null ? "stream_flush_success_bytes" : "stream_flush_failure_bytes", + flushContext.getRecordSize()); + flushContext.currentSpan.end(); + + if (exception != null) { + log.error("Error sending producer record: {}", recordId, exception); + cf.completeExceptionally(exception); + } else { + log.debug("Kafka producer record: {} has finished sending for topic: {} and partition {}", + recordId, metadata.topic(), metadata.partition()); + cf.complete(metadata); + } + }; } } diff --git a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/tracing/KafkaRecordContext.java b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/tracing/KafkaRecordContext.java new file mode 100644 index 000000000..006112753 --- /dev/null +++ b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/tracing/KafkaRecordContext.java @@ -0,0 +1,49 @@ +package org.opensearch.migrations.trafficcapture.kafkaoffloader.tracing; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.trace.Span; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.opensearch.migrations.tracing.IConnectionContext; +import org.opensearch.migrations.tracing.IWithAttributes; +import org.opensearch.migrations.tracing.IWithStartTime; + +import java.time.Instant; + +@AllArgsConstructor +public class KafkaRecordContext implements IWithAttributes, IWithStartTime { + static final AttributeKey TOPIC_ATTR = AttributeKey.stringKey("topic"); + static final AttributeKey RECORD_ID_ATTR = AttributeKey.stringKey("recordId"); + static final AttributeKey RECORD_SIZE_ATTR = AttributeKey.longKey("recordSize"); + + @Getter + public final IConnectionContext enclosingScope; + @Getter + public final Span currentSpan; + @Getter + public final Instant startTime; + @Getter + public final String topic; + @Getter + public final String recordId; + @Getter + public final int recordSize; + + public KafkaRecordContext(IConnectionContext enclosingScope, Span currentSpan, + String topic, String recordId, int recordSize) { + this.enclosingScope = enclosingScope; + this.currentSpan = currentSpan; + this.topic = topic; + this.recordId = recordId; + this.recordSize = recordSize; + this.startTime = Instant.now(); + } + + @Override + public AttributesBuilder fillAttributes(AttributesBuilder builder) { + return builder.put(TOPIC_ATTR, getTopic()) + .put(RECORD_ID_ATTR, getRecordId()) + .put(RECORD_SIZE_ATTR, getRecordSize()); + } +} diff --git a/TrafficCapture/captureKafkaOffloader/src/test/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactoryTest.java b/TrafficCapture/captureKafkaOffloader/src/test/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactoryTest.java index 732cc7df6..7099f6317 100644 --- a/TrafficCapture/captureKafkaOffloader/src/test/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactoryTest.java +++ b/TrafficCapture/captureKafkaOffloader/src/test/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactoryTest.java @@ -1,6 +1,7 @@ package org.opensearch.migrations.trafficcapture.kafkaoffloader; import io.netty.buffer.Unpooled; +import io.opentelemetry.api.GlobalOpenTelemetry; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.ApiVersions; import org.apache.kafka.clients.producer.Callback; @@ -18,7 +19,9 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.migrations.tracing.EmptyContext; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; +import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -51,7 +54,7 @@ public void testLargeRequestIsWithinKafkaMessageSizeLimit() throws IOException, MockProducer producer = new MockProducer<>(true, new StringSerializer(), new ByteArraySerializer()); KafkaCaptureFactory kafkaCaptureFactory = new KafkaCaptureFactory(TEST_NODE_ID_STRING, producer, maxAllowableMessageSize); - IChannelConnectionCaptureSerializer serializer = kafkaCaptureFactory.createOffloader(connectionId); + IChannelConnectionCaptureSerializer serializer = kafkaCaptureFactory.createOffloader(createCtx(), connectionId); StringBuilder sb = new StringBuilder(); for (int i = 0; i < 15000; i++) { @@ -73,6 +76,11 @@ public void testLargeRequestIsWithinKafkaMessageSizeLimit() throws IOException, producer.close(); } + private static ConnectionContext createCtx() { + return new ConnectionContext("test", "test", + GlobalOpenTelemetry.getTracer("test").spanBuilder("test").startSpan()); + } + /** * This size calculation is based off the KafkaProducer client request size validation check done when Producer * records are sent. This validation appears to be consistent for several versions now, here is a reference to @@ -97,7 +105,7 @@ private int calculateRecordSize(ProducerRecord record, String re public void testLinearOffloadingIsSuccessful() throws IOException { KafkaCaptureFactory kafkaCaptureFactory = new KafkaCaptureFactory(TEST_NODE_ID_STRING, mockProducer, 1024*1024); - IChannelConnectionCaptureSerializer offloader = kafkaCaptureFactory.createOffloader(connectionId); + IChannelConnectionCaptureSerializer offloader = kafkaCaptureFactory.createOffloader(createCtx(), connectionId); List recordSentCallbacks = new ArrayList<>(3); when(mockProducer.send(any(), any())).thenAnswer(invocation -> { diff --git a/TrafficCapture/captureOffloader/build.gradle b/TrafficCapture/captureOffloader/build.gradle index c375d39a2..9c183b4a0 100644 --- a/TrafficCapture/captureOffloader/build.gradle +++ b/TrafficCapture/captureOffloader/build.gradle @@ -20,11 +20,14 @@ sourceSets { } } dependencies { + implementation platform("io.opentelemetry:opentelemetry-bom:1.32.0") api group: 'io.netty', name: 'netty-buffer', version: '4.1.100.Final' implementation project(':captureProtobufs') - implementation "com.google.protobuf:protobuf-java:3.22.2" - implementation 'org.projectlombok:lombok:1.18.26' + implementation project(':coreUtilities') + implementation group: 'io.opentelemetry', name:'opentelemetry-api' + implementation group: 'com.google.protobuf', name: 'protobuf-java', version: '3.22.2' + implementation group: 'org.projectlombok', name: 'lombok', version: '1.18.26' implementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7' testImplementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.20.0' diff --git a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java index 751d39f63..b7ce9c029 100644 --- a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java +++ b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java @@ -3,6 +3,7 @@ import lombok.AllArgsConstructor; import lombok.Lombok; import lombok.extern.slf4j.Slf4j; +import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -83,7 +84,7 @@ public CodedOutputStreamAndByteBufferWrapper createStream() { } @Override - public IChannelConnectionCaptureSerializer createOffloader(String connectionId) { + public IChannelConnectionCaptureSerializer createOffloader(ConnectionContext ctx, String connectionId) { return new StreamChannelConnectionCaptureSerializer(nodeId, connectionId, new StreamManager(connectionId)); } } diff --git a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/IConnectionCaptureFactory.java b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/IConnectionCaptureFactory.java index 9f5ec26c0..c5c5270e5 100644 --- a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/IConnectionCaptureFactory.java +++ b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/IConnectionCaptureFactory.java @@ -1,7 +1,9 @@ package org.opensearch.migrations.trafficcapture; +import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; + import java.io.IOException; public interface IConnectionCaptureFactory { - IChannelConnectionCaptureSerializer createOffloader(String connectionId) throws IOException; + IChannelConnectionCaptureSerializer createOffloader(ConnectionContext ctx, String connectionId) throws IOException; } diff --git a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/tracing/ConnectionContext.java b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/tracing/ConnectionContext.java new file mode 100644 index 000000000..6df11b03d --- /dev/null +++ b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/tracing/ConnectionContext.java @@ -0,0 +1,33 @@ +package org.opensearch.migrations.trafficcapture.tracing; + +import io.opentelemetry.api.trace.Span; +import lombok.Getter; +import org.opensearch.migrations.tracing.IConnectionContext; +import org.opensearch.migrations.tracing.IWithStartTime; + +import java.time.Instant; + +public class ConnectionContext implements IConnectionContext, IWithStartTime { + @Getter + public final String connectionId; + @Getter + public final String nodeId; + @Getter + public final Span currentSpan; + @Getter + private final Instant startTime; + + public ConnectionContext(ConnectionContext oldContext, Span currentSpan) { + this.connectionId = oldContext.getConnectionId(); + this.nodeId = oldContext.getNodeId(); + this.currentSpan = currentSpan; + this.startTime = Instant.now(); + } + + public ConnectionContext(String connectionId, String nodeId, Span currentSpan) { + this.connectionId = connectionId; + this.nodeId = nodeId; + this.currentSpan = currentSpan; + this.startTime = Instant.now(); + } +} diff --git a/TrafficCapture/captureOffloader/src/testFixtures/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java b/TrafficCapture/captureOffloader/src/testFixtures/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java index b63ef52af..3c899e2eb 100644 --- a/TrafficCapture/captureOffloader/src/testFixtures/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java +++ b/TrafficCapture/captureOffloader/src/testFixtures/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java @@ -1,14 +1,12 @@ package org.opensearch.migrations.trafficcapture; -import com.google.protobuf.CodedOutputStream; import com.google.protobuf.InvalidProtocolBufferException; import lombok.AllArgsConstructor; import lombok.Getter; -import lombok.NonNull; import org.opensearch.migrations.trafficcapture.protos.TrafficStream; +import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Arrays; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; @@ -62,7 +60,7 @@ protected CompletableFuture kickoffCloseStream(CodedOutputStreamHolder out } @Override - public IChannelConnectionCaptureSerializer createOffloader(String connectionId) throws IOException { + public IChannelConnectionCaptureSerializer createOffloader(ConnectionContext ctx, String connectionId) throws IOException { // This array is only an indirection to work around Java's constraint that lambda values are final return new StreamChannelConnectionCaptureSerializer<>(nodeId, connectionId, new StreamManager()); } diff --git a/TrafficCapture/coreUtilities/build.gradle b/TrafficCapture/coreUtilities/build.gradle index 355e2ec38..fbe9a0307 100644 --- a/TrafficCapture/coreUtilities/build.gradle +++ b/TrafficCapture/coreUtilities/build.gradle @@ -53,9 +53,9 @@ dependencies { // Log4j implementation(platform("org.apache.logging.log4j:log4j-bom:2.21.1")) - implementation("org.apache.logging.log4j:log4j-api") - implementation("org.apache.logging.log4j:log4j-core") - implementation("org.apache.logging.log4j:log4j-slf4j2-impl:2.20.0") + implementation group: 'org.apache.logging.log4j', name: 'log4j-api' + implementation group: 'org.apache.logging.log4j', name :'log4j-core' + implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl', version: '2.20.0' // OpenTelemetry core implementation group: 'io.opentelemetry', name:'opentelemetry-api' diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java index 1ff13ec47..f890264f2 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java @@ -1,10 +1,9 @@ package org.opensearch.migrations.coreutils; import io.opentelemetry.api.GlobalOpenTelemetry; -import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.context.Context; import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter; import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; @@ -17,8 +16,11 @@ import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; +import org.opensearch.migrations.tracing.IWithAttributes; +import org.opensearch.migrations.tracing.IWithStartTime; import java.time.Duration; +import java.time.Instant; import java.util.concurrent.TimeUnit; public class SimpleMeteringClosure { @@ -77,53 +79,61 @@ public static void initializeOpenTelemetry(String serviceName, String collectorE //OpenTelemetryAppender.install(GlobalOpenTelemetry.get()); } - public void meterIncrementEvent(Context ctx, String eventName) { + public void meterIncrementEvent(IWithAttributes ctx, String eventName) { meterIncrementEvent(ctx, eventName, 1); } - public void meterIncrementEvent(Context ctx, String eventName, long increment) { + public void meterIncrementEvent(IWithAttributes ctx, String eventName, long increment) { if (ctx == null) { return; } - try (var namedOnlyForAutoClose = ctx.makeCurrent()) { - meter.counterBuilder(eventName) - .build().add(increment, Attributes.builder() - .put("labelName", eventName) - .build()); - } + meter.counterBuilder(eventName) + .build().add(increment, ctx.getPopulatedAttributesBuilder() + .put("labelName", eventName) + .build()); } - public void meterDeltaEvent(Context ctx, String eventName, long delta) { + public void meterDeltaEvent(IWithAttributes ctx, String eventName, long delta) { if (ctx == null) { return; } - try (var namedOnlyForAutoClose = ctx.makeCurrent()) { - meter.upDownCounterBuilder(eventName) - .build().add(delta, Attributes.builder() - .put("labelName", eventName) - .build()); - } + meter.upDownCounterBuilder(eventName) + .build().add(delta, ctx.getPopulatedAttributesBuilder() + .put("labelName", eventName) + .build()); } - public void meterHistogramMillis(Context ctx, String eventName, Duration between) { + public void meterHistogramMillis(T ctx, String eventName) { + meterHistogram(ctx, eventName, "ms", Duration.between(ctx.getStartTime(), Instant.now()).toMillis()); + } + + public void meterHistogramMicros(T ctx, String eventName) { + meterHistogram(ctx, eventName, "us", Duration.between(ctx.getStartTime(), Instant.now()).toNanos()*1000); + } + + public void meterHistogramMillis(IWithAttributes ctx, String eventName, Duration between) { meterHistogram(ctx, eventName, "ms", between.toMillis()); } - public void meterHistogramMicros(Context ctx, String eventName, Duration between) { + public void meterHistogramMicros(IWithAttributes ctx, String eventName, Duration between) { meterHistogram(ctx, eventName, "us", between.toNanos()*1000); } - public void meterHistogram(Context ctx, String eventName, String units, long value) { + public void meterHistogram(IWithAttributes ctx, String eventName, String units, long value) { if (ctx == null) { return; } - try (var namedOnlyForAutoClose = ctx.makeCurrent()) { - meter.histogramBuilder(eventName) - .ofLongs() - .setUnit(units) - .build().record(value, Attributes.builder() - .put("labelName", eventName) - .build()); - } + meter.histogramBuilder(eventName) + .ofLongs() + .setUnit(units) + .build().record(value, ctx.getPopulatedAttributesBuilder() + .put("labelName", eventName) + .build()); + } + + public Span makeSpan(IWithAttributes ctx, String spanName) { + var span = tracer.spanBuilder(spanName).startSpan(); + span.setAllAttributes(ctx.getPopulatedAttributesBuilder().build()); + return span; } } diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/ContextWithSpan.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/ContextWithSpan.java new file mode 100644 index 000000000..80d419891 --- /dev/null +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/ContextWithSpan.java @@ -0,0 +1,10 @@ +package org.opensearch.migrations.tracing; + +import io.opentelemetry.api.trace.Span; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class ContextWithSpan> { + public final T context; + public final Span span; +} diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/EmptyContext.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/EmptyContext.java new file mode 100644 index 000000000..d44a356c3 --- /dev/null +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/EmptyContext.java @@ -0,0 +1,25 @@ +package org.opensearch.migrations.tracing; + +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.trace.Span; + +public class EmptyContext implements IWithAttributes { + public static final EmptyContext singleton = new EmptyContext(); + + private EmptyContext() {} + + @Override + public Span getCurrentSpan() { + throw new IllegalStateException("This class doesn't track spans"); + } + + @Override + public IWithAttributes getEnclosingScope() { + return null; + } + + @Override + public AttributesBuilder fillAttributes(AttributesBuilder builder) { + return builder; // nothing more to do + } +} diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IConnectionContext.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IConnectionContext.java new file mode 100644 index 000000000..075ba18f1 --- /dev/null +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IConnectionContext.java @@ -0,0 +1,21 @@ +package org.opensearch.migrations.tracing; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.AttributesBuilder; + +public interface IConnectionContext extends IWithAttributes { + static final AttributeKey CONNECTION_ID_ATTR = AttributeKey.stringKey("connectionId"); + static final AttributeKey NODE_ID_ATTR = AttributeKey.stringKey("nodeId"); + + String getConnectionId(); + String getNodeId(); + + @Override + default EmptyContext getEnclosingScope() { return EmptyContext.singleton; } + + @Override + default AttributesBuilder fillAttributes(AttributesBuilder builder) { + return builder.put(CONNECTION_ID_ATTR, getConnectionId()) + .put(NODE_ID_ATTR, getNodeId()); + } +} diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IReplayerRequestContext.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IReplayerRequestContext.java new file mode 100644 index 000000000..5bb61fce0 --- /dev/null +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IReplayerRequestContext.java @@ -0,0 +1,16 @@ +package org.opensearch.migrations.tracing; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.AttributesBuilder; + +public interface IReplayerRequestContext extends IRequestContext { + static final AttributeKey REPLAYER_REQUEST_INDEX_KEY = AttributeKey.longKey("replayerRequestIndex"); + + long replayerRequestIndex(); + + @Override + default AttributesBuilder fillAttributes(AttributesBuilder builder) { + return IRequestContext.super.fillAttributes( + builder.put(REPLAYER_REQUEST_INDEX_KEY, replayerRequestIndex())); + } +} diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IRequestContext.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IRequestContext.java new file mode 100644 index 000000000..67744ff5f --- /dev/null +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IRequestContext.java @@ -0,0 +1,16 @@ +package org.opensearch.migrations.tracing; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.AttributesBuilder; + +public interface IRequestContext extends IConnectionContext { + static final AttributeKey SOURCE_REQUEST_INDEX_KEY = AttributeKey.longKey("sourceRequestIndex"); + + long sourceRequestIndex(); + + @Override + default AttributesBuilder fillAttributes(AttributesBuilder builder) { + return IConnectionContext.super.fillAttributes( + builder.put(SOURCE_REQUEST_INDEX_KEY, sourceRequestIndex())); + } +} diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IWithAttributes.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IWithAttributes.java new file mode 100644 index 000000000..75d443d2d --- /dev/null +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IWithAttributes.java @@ -0,0 +1,30 @@ +package org.opensearch.migrations.tracing; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.trace.Span; + +import java.util.ArrayList; + +public interface IWithAttributes { + T getEnclosingScope(); + + Span getCurrentSpan(); + + AttributesBuilder fillAttributes(AttributesBuilder builder); + + default AttributesBuilder getPopulatedAttributesBuilder() { + var currentObj = this; + var stack = new ArrayList(); + var builder = Attributes.builder(); + while (currentObj != null) { + stack.add(currentObj); + currentObj = currentObj.getEnclosingScope(); + } + // reverse the order so that the lowest attribute scopes will overwrite the upper ones if there were conflicts + for (int i=stack.size()-1; i>=0; --i) { + builder = stack.get(i).fillAttributes(builder); + } + return builder; + } +} diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IWithStartTime.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IWithStartTime.java new file mode 100644 index 000000000..b8e362ddb --- /dev/null +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IWithStartTime.java @@ -0,0 +1,7 @@ +package org.opensearch.migrations.tracing; + +import java.time.Instant; + +public interface IWithStartTime { + Instant getStartTime(); +} diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java index 13b6edf87..de3391fbe 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java @@ -3,15 +3,12 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.HttpRequest; import io.netty.util.ReferenceCountUtil; -import io.opentelemetry.api.GlobalOpenTelemetry; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.context.Context; import io.opentelemetry.context.ContextKey; import lombok.Lombok; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; +import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; -import java.time.Duration; import java.time.Instant; import java.util.function.Predicate; @@ -20,7 +17,7 @@ public class ConditionallyReliableLoggingHttpRequestHandler extends LoggingHt private ContextKey START_FLUSH_KEY = ContextKey.named("startTime"); private final Predicate shouldBlockPredicate; - public ConditionallyReliableLoggingHttpRequestHandler(Context incomingContext, + public ConditionallyReliableLoggingHttpRequestHandler(ConnectionContext incomingContext, IChannelConnectionCaptureSerializer trafficOffloader, Predicate headerPredicateForWhenToBlock) { super(incomingContext, trafficOffloader); @@ -31,22 +28,17 @@ public ConditionallyReliableLoggingHttpRequestHandler(Context incomingContext, protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Object msg, HttpRequest httpRequest) throws Exception { if (shouldBlockPredicate.test(httpRequest)) { - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "blockingRequestUntilFlush"); - Context flushContext; - try (var namedOnlyForAutoClose = telemetryContext - .with(START_FLUSH_KEY, Instant.now()) - .makeCurrent()) { - flushContext = Context.current() - .with(METERING_CLOSURE.tracer.spanBuilder("blockedForFlush").startSpan()); - } + METERING_CLOSURE.meterIncrementEvent(connectionContext, "blockingRequestUntilFlush"); + var flushContext = new ConnectionContext(connectionContext, + METERING_CLOSURE.tracer.spanBuilder("blockedForFlush").startSpan()); + trafficOffloader.flushCommitAndResetStream(false).whenComplete((result, t) -> { log.atInfo().setMessage(()->"Done flushing").log(); METERING_CLOSURE.meterIncrementEvent(flushContext, t != null ? "blockedFlushFailure" : "blockedFlushSuccess"); METERING_CLOSURE.meterHistogramMicros(flushContext, - t==null ? "blockedFlushFailure_micro" : "stream_flush_failure_micro", - Duration.between(flushContext.get(START_FLUSH_KEY), Instant.now())); - Span.fromContext(flushContext).end(); + t==null ? "blockedFlushFailure_micro" : "stream_flush_failure_micro"); + flushContext.currentSpan.end(); if (t != null) { // This is a spot where we would benefit from having a behavioral policy that different users @@ -63,7 +55,7 @@ protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Ob } }); } else { - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "nonBlockingRequest"); + METERING_CLOSURE.meterIncrementEvent(connectionContext, "nonBlockingRequest"); super.channelFinishedReadingAnHttpMessage(ctx, msg, httpRequest); } } diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java index 0d300eb57..7f3191dc0 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java @@ -13,9 +13,6 @@ import io.netty.handler.codec.http.HttpRequestDecoder; import io.netty.handler.codec.http.HttpVersion; import io.netty.handler.codec.http.LastHttpContent; -import io.opentelemetry.api.GlobalOpenTelemetry; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.context.Context; import lombok.Getter; import lombok.Lombok; import lombok.extern.slf4j.Slf4j; @@ -24,6 +21,7 @@ import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import org.opensearch.migrations.coreutils.MetricsLogger; +import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; import java.time.Instant; @@ -78,17 +76,13 @@ public HttpRequest resetCurrentRequest() { protected final EmbeddedChannel httpDecoderChannel; protected final SimpleHttpRequestDecoder requestDecoder; - protected final Context telemetryContext; - private final Instant createdTime; + protected final ConnectionContext connectionContext; - - public LoggingHttpRequestHandler(Context incomingContext, IChannelConnectionCaptureSerializer trafficOffloader) { - this.createdTime = Instant.now(); - try (var scope = incomingContext.makeCurrent()) { - var span = METERING_CLOSURE.tracer.spanBuilder("frontendConnection").startSpan(); - telemetryContext = incomingContext.with(span); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "requestStarted"); - } + public LoggingHttpRequestHandler(ConnectionContext incomingContext, + IChannelConnectionCaptureSerializer trafficOffloader) { + this.connectionContext = incomingContext; + var span = METERING_CLOSURE.makeSpan(incomingContext, "frontendConnection"); + METERING_CLOSURE.meterIncrementEvent(incomingContext, "requestStarted"); this.trafficOffloader = trafficOffloader; requestDecoder = new SimpleHttpRequestDecoder(); // as a field for easier debugging @@ -103,7 +97,7 @@ private HttpProcessedState parseHttpMessageParts(ByteBuf msg) { var state = getHandlerThatHoldsParsedHttpRequest().isDone ? HttpProcessedState.FULL_MESSAGE : HttpProcessedState.ONGOING; - METERING_CLOSURE.meterIncrementEvent(telemetryContext, + METERING_CLOSURE.meterIncrementEvent(connectionContext, state == HttpProcessedState.FULL_MESSAGE ? "requestFullyParsed" : "requestPartiallyParsed"); return state; } @@ -115,7 +109,7 @@ private SimpleDecodedHttpRequestHandler getHandlerThatHoldsParsedHttpRequest() { @Override public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { trafficOffloader.addCloseEvent(Instant.now()); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "unregistered"); + METERING_CLOSURE.meterIncrementEvent(connectionContext, "unregistered"); trafficOffloader.flushCommitAndResetStream(true).whenComplete((result, t) -> { if (t != null) { log.warn("Got error: " + t.getMessage()); @@ -132,8 +126,8 @@ public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { @Override public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "handlerRemoved"); - Span.fromContext(telemetryContext).end(); + METERING_CLOSURE.meterIncrementEvent(connectionContext, "handlerRemoved"); + connectionContext.getCurrentSpan().end(); trafficOffloader.flushCommitAndResetStream(true).whenComplete((result, t) -> { if (t != null) { @@ -150,7 +144,7 @@ public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Object msg, HttpRequest httpRequest) throws Exception { super.channelRead(ctx, msg); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "requestReceived"); + METERING_CLOSURE.meterIncrementEvent(connectionContext, "requestReceived"); metricsLogger.atSuccess(MetricsEvent.RECEIVED_FULL_HTTP_REQUEST) .setAttribute(MetricsAttributeKey.CHANNEL_ID, ctx.channel().id().asLongText()) @@ -165,8 +159,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { var bb = ((ByteBuf) msg).retainedDuplicate(); trafficOffloader.addReadEvent(timestamp, bb); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "read"); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "readBytes", bb.readableBytes()); + METERING_CLOSURE.meterIncrementEvent(connectionContext, "read"); + METERING_CLOSURE.meterIncrementEvent(connectionContext, "readBytes", bb.readableBytes()); metricsLogger.atSuccess(MetricsEvent.RECEIVED_REQUEST_COMPONENT) .setAttribute(MetricsAttributeKey.CHANNEL_ID, ctx.channel().id().asLongText()).emit(); @@ -191,7 +185,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { trafficOffloader.addExceptionCaughtEvent(Instant.now(), cause); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "exception"); + METERING_CLOSURE.meterIncrementEvent(connectionContext, "exception"); httpDecoderChannel.close(); super.exceptionCaught(ctx, cause); } diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java index ed8a8d55c..94e93d021 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java @@ -4,15 +4,13 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelOutboundHandlerAdapter; import io.netty.channel.ChannelPromise; -import io.opentelemetry.api.GlobalOpenTelemetry; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.context.Context; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; import org.opensearch.migrations.coreutils.MetricsLogger; import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; +import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; import java.net.SocketAddress; import java.time.Duration; @@ -25,10 +23,10 @@ public class LoggingHttpResponseHandler extends ChannelOutboundHandlerAdapter private static final MetricsLogger metricsLogger = new MetricsLogger("LoggingHttpResponseHandler"); private final IChannelConnectionCaptureSerializer trafficOffloader; - private Context telemetryContext; + private ConnectionContext telemetryContext; private Instant connectTime; - public LoggingHttpResponseHandler(Context incomingContext, + public LoggingHttpResponseHandler(ConnectionContext incomingContext, IChannelConnectionCaptureSerializer trafficOffloader) { this.trafficOffloader = trafficOffloader; this.telemetryContext = incomingContext; @@ -45,9 +43,8 @@ public void bind(ChannelHandlerContext ctx, SocketAddress localAddress, ChannelP public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, SocketAddress localAddress, ChannelPromise promise) throws Exception { trafficOffloader.addConnectEvent(Instant.now(), remoteAddress, localAddress); - var span = GlobalOpenTelemetry.get().getTracer(TELEMETRY_SCOPE_NAME) - .spanBuilder("backendConnection").startSpan(); - telemetryContext = telemetryContext.with(span); + var span = METERING_CLOSURE.makeSpan(telemetryContext,"backendConnection"); + telemetryContext = new ConnectionContext(telemetryContext, span); connectTime = Instant.now(); METERING_CLOSURE.meterIncrementEvent(telemetryContext, "connect"); METERING_CLOSURE.meterDeltaEvent(telemetryContext, "connections", 1); @@ -70,7 +67,7 @@ public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exce METERING_CLOSURE.meterDeltaEvent(telemetryContext, "connections", -1); METERING_CLOSURE.meterHistogramMillis(telemetryContext, "connectionDuration", Duration.between(connectTime, Instant.now())); - Span.fromContext(telemetryContext).end(); + telemetryContext.currentSpan.end(); } @Override diff --git a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java index 76a2cc762..86cd36ea5 100644 --- a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java +++ b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java @@ -6,15 +6,11 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.context.Context; -import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; -import io.opentelemetry.sdk.trace.data.SpanData; import lombok.AllArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -24,10 +20,10 @@ import org.opensearch.migrations.trafficcapture.CodedOutputStreamHolder; import org.opensearch.migrations.trafficcapture.OrderedStreamLifecyleManager; import org.opensearch.migrations.trafficcapture.StreamChannelConnectionCaptureSerializer; +import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; import org.opensearch.migrations.trafficcapture.protos.TrafficStream; import java.io.ByteArrayInputStream; -import java.io.FileOutputStream; import java.io.IOException; import java.io.SequenceInputStream; import java.nio.ByteBuffer; @@ -87,12 +83,10 @@ private static void writeMessageAndVerify(byte[] fullTrafficBytes, Consumertrue)); // true: block every request + new ConditionallyReliableLoggingHttpRequestHandler(ctx, offloader, x->true)); // true: block every request channelWriter.accept(channel); // we wrote the correct data to the downstream handler/channel diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java index 97b2a8293..c4736c75f 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java @@ -174,7 +174,7 @@ private static Settings getSettings(@NonNull String configFile) { private static IConnectionCaptureFactory getNullConnectionCaptureFactory() { System.err.println("No trace log directory specified. Logging to /dev/null"); - return connectionId -> new StreamChannelConnectionCaptureSerializer<>(null, connectionId, + return (ctx, connectionId) -> new StreamChannelConnectionCaptureSerializer<>(null, connectionId, new StreamLifecycleManager<>() { @Override public void close() {} diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java index 2ec2eb3e3..2e48c9765 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java @@ -5,18 +5,19 @@ import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.ssl.SslHandler; -import io.opentelemetry.context.Context; -import io.opentelemetry.context.ContextKey; +import org.opensearch.migrations.coreutils.SimpleMeteringClosure; +import org.opensearch.migrations.tracing.EmptyContext; import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; import org.opensearch.migrations.trafficcapture.netty.ConditionallyReliableLoggingHttpRequestHandler; import org.opensearch.migrations.trafficcapture.netty.LoggingHttpResponseHandler; +import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; import javax.net.ssl.SSLEngine; import java.io.IOException; import java.util.function.Supplier; public class ProxyChannelInitializer extends ChannelInitializer { - static final ContextKey CONNECTION_ID_KEY = ContextKey.named("connectionId"); + static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure("FrontendConnection"); private final IConnectionCaptureFactory connectionCaptureFactory; private final Supplier sslEngineProvider; @@ -45,8 +46,9 @@ protected void initChannel(SocketChannel ch) throws IOException { } var connectionId = ch.id().asLongText(); - var offloader = connectionCaptureFactory.createOffloader(connectionId); - var ctx = Context.current().with(CONNECTION_ID_KEY, connectionId); + var ctx = new ConnectionContext(connectionId, "", + METERING_CLOSURE.makeSpan(EmptyContext.singleton, "connectionLifetime")); + var offloader = connectionCaptureFactory.createOffloader(ctx, connectionId); ch.pipeline().addLast(new LoggingHttpResponseHandler<>(ctx, offloader)); ch.pipeline().addLast(new ConditionallyReliableLoggingHttpRequestHandler(ctx, offloader, this::shouldGuaranteeMessageOffloading)); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java index b3f65daf2..fa474ac4e 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java @@ -1,15 +1,21 @@ package org.opensearch.migrations.replay; import lombok.NonNull; +import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.replay.tracing.ChannelKeyContext; +import org.opensearch.migrations.replay.tracing.RequestContext; +import org.opensearch.migrations.tracing.EmptyContext; import java.time.Instant; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; public class Accumulation { + public static final String TELEMETRY_SCOPE_NAME = "Accumulator"; + public static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure(TELEMETRY_SCOPE_NAME); enum State { // Ignore all initial READs, the first EOM & the following WRITEs (if they or EOMs exist) @@ -22,12 +28,14 @@ enum State { } public final ISourceTrafficChannelKey trafficChannelKey; + public final ChannelKeyContext channelContext; private RequestResponsePacketPair rrPair; AtomicLong newestPacketTimestampInMillis; State state; AtomicInteger numberOfResets; final int startingSourceRequestIndex; + public Accumulation(@NonNull ITrafficStreamKey trafficChannelKey, int startingSourceRequestIndex) { this(trafficChannelKey, startingSourceRequestIndex, false); } @@ -40,19 +48,26 @@ public Accumulation(@NonNull ITrafficStreamKey trafficChannelKey, this.startingSourceRequestIndex = startingSourceRequestIndex; this.state = dropObservationsLeftoverFromPrevious ? State.IGNORING_LAST_REQUEST : State.WAITING_FOR_NEXT_READ_CHUNK; + channelContext = new ChannelKeyContext(trafficChannelKey, + METERING_CLOSURE.makeSpan(EmptyContext.singleton, "accumulatingChannel")); } public RequestResponsePacketPair getOrCreateTransactionPair(ITrafficStreamKey forTrafficStreamKey) { if (rrPair != null) { return rrPair; } - rrPair = new RequestResponsePacketPair(forTrafficStreamKey); + this.rrPair = new RequestResponsePacketPair(forTrafficStreamKey, + new RequestContext(getRequestKey(forTrafficStreamKey), + METERING_CLOSURE.makeSpan(channelContext, "accumulatingRequest"))); return rrPair; } public UniqueReplayerRequestKey getRequestKey() { - return new UniqueReplayerRequestKey(getRrPair().getBeginningTrafficStreamKey(), - startingSourceRequestIndex, getIndexOfCurrentRequest()); + return getRequestKey(getRrPair().getBeginningTrafficStreamKey()); + } + + private UniqueReplayerRequestKey getRequestKey(@NonNull ITrafficStreamKey tsk) { + return new UniqueReplayerRequestKey(tsk, startingSourceRequestIndex, getIndexOfCurrentRequest()); } public boolean hasSignaledRequests() { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java index 3f2504862..28d988f3d 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java @@ -4,7 +4,7 @@ import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; -import org.opensearch.migrations.replay.tracing.ConnectionContext; +import org.opensearch.migrations.replay.tracing.ChannelKeyContext; import org.opensearch.migrations.replay.tracing.RequestContext; import java.time.Instant; @@ -15,10 +15,10 @@ void onRequestReceived(@NonNull UniqueReplayerRequestKey key, RequestContext ctx @NonNull HttpMessageAndTimestamp request); void onFullDataReceived(@NonNull UniqueReplayerRequestKey key, RequestContext ctx, @NonNull RequestResponsePacketPair rrpp); - void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, ConnectionContext ctx, + void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, ChannelKeyContext ctx, @NonNull List trafficStreamKeysBeingHeld); - void onConnectionClose(@NonNull ISourceTrafficChannelKey key, int channelInteractionNumber, ConnectionContext ctx, + void onConnectionClose(@NonNull ISourceTrafficChannelKey key, int channelInteractionNumber, ChannelKeyContext ctx, RequestResponsePacketPair.ReconstructionStatus status, @NonNull Instant when, @NonNull List trafficStreamKeysBeingHeld); - void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk, ConnectionContext ctx); + void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk, ChannelKeyContext ctx); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java index 29fc83126..35cde47fd 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java @@ -1,16 +1,11 @@ package org.opensearch.migrations.replay; -import io.opentelemetry.context.Context; -import io.opentelemetry.context.ContextKey; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; import org.opensearch.migrations.coreutils.MetricsLogger; -import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; -import org.opensearch.migrations.replay.tracing.ConnectionContext; -import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.traffic.expiration.BehavioralPolicy; import org.opensearch.migrations.replay.traffic.expiration.ExpiringTrafficStreamMap; import org.opensearch.migrations.replay.traffic.source.ITrafficStreamWithKey; @@ -54,8 +49,6 @@ */ @Slf4j public class CapturedTrafficToHttpTransactionAccumulator { - public static final String TELEMETRY_SCOPE_NAME = "Accumulator"; - public static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure(TELEMETRY_SCOPE_NAME); public static final Duration EXPIRATION_GRANULARITY = Duration.ofSeconds(1); private final ExpiringTrafficStreamMap liveStreams; @@ -136,10 +129,9 @@ public void accept(ITrafficStreamWithKey trafficStreamAndKey) { var tsk = trafficStreamAndKey.getKey(); var accum = liveStreams.getOrCreateWithoutExpiration(tsk, k->createInitialAccumulation(trafficStreamAndKey)); var trafficStream = trafficStreamAndKey.getStream(); - var ctx = new ConnectionContext(tsk); for (int i=0; i"Connection terminated: removing " + partitionId + ":" + connectionId + " from liveStreams map").log(); @@ -153,7 +145,7 @@ public void accept(ITrafficStreamWithKey trafficStreamAndKey) { assert accum.state == Accumulation.State.WAITING_FOR_NEXT_READ_CHUNK || accum.state == Accumulation.State.IGNORING_LAST_REQUEST || trafficStream.getSubStreamCount() == 0; - listener.onTrafficStreamIgnored(tsk, ctx); + listener.onTrafficStreamIgnored(tsk, accum.channelContext); } } @@ -183,16 +175,15 @@ private enum CONNECTION_STATUS { public CONNECTION_STATUS addObservationToAccumulation(@NonNull Accumulation accum, @NonNull ITrafficStreamKey trafficStreamKey, - ConnectionContext ctx, TrafficObservation observation) { log.atTrace().setMessage(()->"Adding observation: "+observation).log(); var timestamp = TrafficStreamUtils.instantFromProtoTimestamp(observation.getTs()); liveStreams.expireOldEntries(trafficStreamKey, accum, timestamp); - return handleCloseObservationThatAffectEveryState(accum, observation, trafficStreamKey, ctx, timestamp) + return handleCloseObservationThatAffectEveryState(accum, observation, trafficStreamKey, timestamp) .or(() -> handleObservationForSkipState(accum, observation)) - .or(() -> handleObservationForReadState(accum, ctx, observation, trafficStreamKey, timestamp)) - .or(() -> handleObservationForWriteState(accum, ctx, observation, trafficStreamKey, timestamp)) + .or(() -> handleObservationForReadState(accum, observation, trafficStreamKey, timestamp)) + .or(() -> handleObservationForWriteState(accum, observation, trafficStreamKey, timestamp)) .orElseGet(() -> { log.atWarn().setMessage(()->"unaccounted for observation type " + observation).log(); return CONNECTION_STATUS.ALIVE; @@ -225,19 +216,18 @@ private static List getTrafficStreamsHeldByAccum(Accumulation handleCloseObservationThatAffectEveryState(Accumulation accum, TrafficObservation observation, @NonNull ITrafficStreamKey trafficStreamKey, - ConnectionContext ctx, Instant timestamp) { if (observation.hasClose()) { accum.getOrCreateTransactionPair(trafficStreamKey).holdTrafficStream(trafficStreamKey); - rotateAccumulationIfNecessary(trafficStreamKey.getConnectionId(), accum, ctx); + rotateAccumulationIfNecessary(trafficStreamKey.getConnectionId(), accum); closedConnectionCounter.incrementAndGet(); - listener.onConnectionClose(accum.trafficChannelKey, accum.getIndexOfCurrentRequest(), ctx, + listener.onConnectionClose(accum.trafficChannelKey, accum.getIndexOfCurrentRequest(), accum.channelContext, RequestResponsePacketPair.ReconstructionStatus.COMPLETE, timestamp, getTrafficStreamsHeldByAccum(accum)); return Optional.of(CONNECTION_STATUS.CLOSED); } else if (observation.hasConnectionException()) { accum.getOrCreateTransactionPair(trafficStreamKey).holdTrafficStream(trafficStreamKey); - rotateAccumulationIfNecessary(trafficStreamKey.getConnectionId(), accum, ctx); + rotateAccumulationIfNecessary(trafficStreamKey.getConnectionId(), accum); exceptionConnectionCounter.incrementAndGet(); accum.resetForNextRequest(); log.atDebug().setMessage(()->"Removing accumulated traffic pair due to " + @@ -249,7 +239,6 @@ private static List getTrafficStreamsHeldByAccum(Accumulation } private Optional handleObservationForReadState(@NonNull Accumulation accum, - ConnectionContext ctx, TrafficObservation observation, @NonNull ITrafficStreamKey trafficStreamKey, Instant timestamp) { @@ -268,7 +257,7 @@ private Optional handleObservationForReadState(@NonNull Accum log.atTrace().setMessage(() -> "Added request data for accum[" + connectionId + "]=" + accum).log(); } else if (observation.hasEndOfMessageIndicator()) { assert accum.hasRrPair(); - handleEndOfRequest(accum, ctx); + handleEndOfRequest(accum); } else if (observation.hasReadSegment()) { log.atTrace().setMessage(()->"Adding request segment for accum[" + connectionId + "]=" + accum).log(); var rrPair = accum.getOrCreateTransactionPair(trafficStreamKey); @@ -286,7 +275,7 @@ private Optional handleObservationForReadState(@NonNull Accum return Optional.of(CONNECTION_STATUS.ALIVE); } - private Optional handleObservationForWriteState(Accumulation accum, ConnectionContext ctx, + private Optional handleObservationForWriteState(Accumulation accum, TrafficObservation observation, @NonNull ITrafficStreamKey trafficStreamKey, Instant timestamp) { @@ -313,8 +302,8 @@ private Optional handleObservationForWriteState(Accumulation assert rrPair.responseData.hasInProgressSegment(); rrPair.responseData.finalizeRequestSegments(timestamp); } else if (observation.hasRead() || observation.hasReadSegment()) { - rotateAccumulationOnReadIfNecessary(connectionId, accum, ctx); - return handleObservationForReadState(accum, ctx, observation, trafficStreamKey, timestamp); + rotateAccumulationOnReadIfNecessary(connectionId, accum); + return handleObservationForReadState(accum, observation, trafficStreamKey, timestamp); } return Optional.of(CONNECTION_STATUS.ALIVE); @@ -323,20 +312,19 @@ private Optional handleObservationForWriteState(Accumulation // This function manages the transition case when an observation comes in that would terminate // any previous HTTP transaction for the connection. It returns true if there WAS a previous // transaction that has been reset and false otherwise - private boolean rotateAccumulationIfNecessary(String connectionId, Accumulation accum, ConnectionContext ctx) { + private boolean rotateAccumulationIfNecessary(String connectionId, Accumulation accum) { // If this was brand new, we don't need to care about triggering the callback. // We only need to worry about this if we have yet to send the RESPONSE. if (accum.state == Accumulation.State.ACCUMULATING_WRITES) { log.atDebug().setMessage(()->"Resetting accum[" + connectionId + "]=" + accum).log(); - handleEndOfResponse(accum, ctx, RequestResponsePacketPair.ReconstructionStatus.COMPLETE); + handleEndOfResponse(accum, RequestResponsePacketPair.ReconstructionStatus.COMPLETE); return true; } return false; } - private boolean rotateAccumulationOnReadIfNecessary(String connectionId, Accumulation accum, - ConnectionContext ctx) { - if (rotateAccumulationIfNecessary(connectionId, accum, ctx)) { + private boolean rotateAccumulationOnReadIfNecessary(String connectionId, Accumulation accum) { + if (rotateAccumulationIfNecessary(connectionId, accum)) { reusedKeepAliveCounter.incrementAndGet(); return true; } else { @@ -347,30 +335,28 @@ private boolean rotateAccumulationOnReadIfNecessary(String connectionId, Accumul /** * @return True if something was sent to the callback, false if nothing had been accumulated */ - private boolean handleEndOfRequest(Accumulation accumulation, ConnectionContext ctx) { + private boolean handleEndOfRequest(Accumulation accumulation) { assert accumulation.state == Accumulation.State.ACCUMULATING_READS : "state == " + accumulation.state; - var requestPacketBytes = accumulation.getRrPair().requestData; + var rrPair = accumulation.getRrPair(); + var requestPacketBytes = rrPair.requestData; metricsLogger.atSuccess(MetricsEvent.ACCUMULATED_FULL_CAPTURED_SOURCE_RESPONSE) .setAttribute(MetricsAttributeKey.REQUEST_ID, accumulation.getRequestKey().toString()) .setAttribute(MetricsAttributeKey.CONNECTION_ID, accumulation.getRequestKey().getTrafficStreamKey().getConnectionId()).emit(); assert (requestPacketBytes != null); assert (!requestPacketBytes.hasInProgressSegment()); - var requestContext = new RequestContext(ctx, accumulation.getRequestKey()); - listener.onRequestReceived(accumulation.getRequestKey(), requestContext, requestPacketBytes); + listener.onRequestReceived(accumulation.getRequestKey(), rrPair.requestContext, requestPacketBytes); accumulation.state = Accumulation.State.ACCUMULATING_WRITES; return true; } - private void handleEndOfResponse(Accumulation accumulation, ConnectionContext ctx, - RequestResponsePacketPair.ReconstructionStatus status) { + private void handleEndOfResponse(Accumulation accumulation, RequestResponsePacketPair.ReconstructionStatus status) { assert accumulation.state == Accumulation.State.ACCUMULATING_WRITES; metricsLogger.atSuccess(MetricsEvent.ACCUMULATED_FULL_CAPTURED_SOURCE_RESPONSE) .setAttribute(MetricsAttributeKey.REQUEST_ID, accumulation.getRequestKey().toString()) .setAttribute(MetricsAttributeKey.CONNECTION_ID, accumulation.getRequestKey().getTrafficStreamKey().getConnectionId()).emit(); var rrPair = accumulation.getRrPair(); rrPair.completionStatus = status; - var requestContext = new RequestContext(ctx, accumulation.getRequestKey()); - listener.onFullDataReceived(accumulation.getRequestKey(), requestContext, rrPair); + listener.onFullDataReceived(accumulation.getRequestKey(), rrPair.requestContext, rrPair); accumulation.resetForNextRequest(); } @@ -385,7 +371,6 @@ public void close() { private void fireAccumulationsCallbacksAndClose(Accumulation accumulation, RequestResponsePacketPair.ReconstructionStatus status) { - ConnectionContext ctx = new ConnectionContext(accumulation.trafficChannelKey); try { switch (accumulation.state) { case ACCUMULATING_READS: @@ -400,12 +385,12 @@ private void fireAccumulationsCallbacksAndClose(Accumulation accumulation, log.warn("Terminating a TrafficStream reconstruction w/out an accumulated value, " + "assuming an empty server interaction and NOT reproducing this to the target cluster."); if (accumulation.hasRrPair()) { - listener.onTrafficStreamsExpired(status, ctx, + listener.onTrafficStreamsExpired(status, accumulation.channelContext, Collections.unmodifiableList(accumulation.getRrPair().trafficStreamKeysBeingHeld)); } return; case ACCUMULATING_WRITES: - handleEndOfResponse(accumulation, ctx, status); + handleEndOfResponse(accumulation, status); break; case WAITING_FOR_NEXT_READ_CHUNK: case IGNORING_LAST_REQUEST: @@ -416,7 +401,8 @@ private void fireAccumulationsCallbacksAndClose(Accumulation accumulation, } finally { if (accumulation.hasSignaledRequests()) { listener.onConnectionClose(accumulation.trafficChannelKey, accumulation.getIndexOfCurrentRequest(), - ctx, status, accumulation.getLastTimestamp(), getTrafficStreamsHeldByAccum(accumulation)); + accumulation.channelContext, status, accumulation.getLastTimestamp(), + getTrafficStreamsHeldByAccum(accumulation)); } } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java index 35ae238b8..e04d45e87 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java @@ -18,13 +18,11 @@ import org.opensearch.migrations.replay.datahandlers.NettyPacketToHttpConsumer; import org.opensearch.migrations.replay.datatypes.ConnectionReplaySession; import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; -import org.opensearch.migrations.replay.tracing.ConnectionContext; -import org.opensearch.migrations.replay.tracing.RequestContext; +import org.opensearch.migrations.replay.tracing.ChannelKeyContext; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture; import java.net.URI; -import java.time.Instant; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -69,7 +67,7 @@ public ConnectionReplaySession load(final String s) { } private DiagnosticTrackableCompletableFuture - getResilientClientChannelProducer(EventLoop eventLoop, ConnectionContext connectionContext) { + getResilientClientChannelProducer(EventLoop eventLoop, ChannelKeyContext connectionContext) { return new AdaptiveRateLimiter() .get(() -> { var clientConnectionChannelCreatedFuture = @@ -143,7 +141,7 @@ public void closeConnection(String connId) { } public Future - submitEventualSessionGet(ISourceTrafficChannelKey channelKey, boolean ignoreIfNotPresent, ConnectionContext ctx) { + submitEventualSessionGet(ISourceTrafficChannelKey channelKey, boolean ignoreIfNotPresent, ChannelKeyContext ctx) { ConnectionReplaySession channelFutureAndSchedule = getCachedSession(channelKey, ignoreIfNotPresent); if (channelFutureAndSchedule == null) { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketConsumerFactory.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketConsumerFactory.java index 3bb96fdee..e87ac3492 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketConsumerFactory.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketConsumerFactory.java @@ -2,7 +2,8 @@ import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.replay.tracing.RequestContext; public interface PacketConsumerFactory { - IPacketFinalizingConsumer create(UniqueReplayerRequestKey requestKey); + IPacketFinalizingConsumer create(UniqueReplayerRequestKey requestKey, RequestContext context); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java index 5d03d7ddd..07b8c7647 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java @@ -1,6 +1,7 @@ package org.opensearch.migrations.replay; import lombok.extern.slf4j.Slf4j; +import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer; import org.opensearch.migrations.replay.datahandlers.TransformedPacketReceiver; import org.opensearch.migrations.replay.datahandlers.http.HttpJsonTransformingConsumer; @@ -14,6 +15,7 @@ @Slf4j public class PacketToTransformingHttpHandlerFactory implements PacketConsumerFactory> { + private final IJsonTransformer jsonTransformer; private final IAuthTransformerFactory authTransformerFactory; @@ -26,9 +28,9 @@ public PacketToTransformingHttpHandlerFactory(IJsonTransformer jsonTransformer, @Override public IPacketFinalizingConsumer> - create(UniqueReplayerRequestKey requestKey) { + create(UniqueReplayerRequestKey requestKey, RequestContext requestContext) { log.trace("creating HttpJsonTransformingConsumer"); return new HttpJsonTransformingConsumer<>(jsonTransformer, authTransformerFactory, - new TransformedPacketReceiver(), requestKey.toString(), new RequestContext(requestKey)); + new TransformedPacketReceiver(), requestKey.toString(), requestContext); } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java index eaecdcca6..f75b92331 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java @@ -9,7 +9,7 @@ import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.IndexedChannelInteraction; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; -import org.opensearch.migrations.replay.tracing.ConnectionContext; +import org.opensearch.migrations.replay.tracing.ChannelKeyContext; import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.traffic.source.BufferedFlowController; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; @@ -154,7 +154,7 @@ private static void logStartOfWork(Object stringableKey, long newCount, Instant } public void closeConnection(ISourceTrafficChannelKey channelKey, int channelInteractionNum, - ConnectionContext ctx, Instant timestamp) { + ChannelKeyContext ctx, Instant timestamp) { var newCount = totalCountOfScheduledTasksOutstanding.incrementAndGet(); final String label = "close"; var atTime = timeShifter.transformSourceTimeToRealTime(timestamp); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestResponsePacketPair.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestResponsePacketPair.java index 9a2f085fe..45f686eb7 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestResponsePacketPair.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestResponsePacketPair.java @@ -4,6 +4,8 @@ import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; +import org.opensearch.migrations.replay.tracing.RequestContext; +import org.opensearch.migrations.tracing.IRequestContext; import java.nio.charset.StandardCharsets; import java.time.Instant; @@ -25,10 +27,12 @@ public enum ReconstructionStatus { HttpMessageAndTimestamp responseData; List trafficStreamKeysBeingHeld; ReconstructionStatus completionStatus; + RequestContext requestContext; - public RequestResponsePacketPair(ITrafficStreamKey startingAtTrafficStreamKey) { + public RequestResponsePacketPair(ITrafficStreamKey startingAtTrafficStreamKey, RequestContext requestContext) { this.trafficStreamKeysBeingHeld = new ArrayList<>(); this.trafficStreamKeysBeingHeld.add(startingAtTrafficStreamKey); + this.requestContext = requestContext; } @NonNull ITrafficStreamKey getBeginningTrafficStreamKey() { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java index 94387dc3f..107c21bae 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java @@ -9,7 +9,7 @@ import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.IndexedChannelInteraction; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; -import org.opensearch.migrations.replay.tracing.ConnectionContext; +import org.opensearch.migrations.replay.tracing.ChannelKeyContext; import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture; @@ -64,14 +64,14 @@ public RequestSenderOrchestrator(ClientConnectionPool clientConnectionPool) { ()->"waiting for final aggregated response"); log.atDebug().setMessage(()->"Scheduling request for "+requestKey+" at start time "+start).log(); return asynchronouslyInvokeRunnableToSetupFuture(requestKey.getTrafficStreamKey(), - requestKey.getReplayerRequestIndex(), ctx, false, finalTunneledResponse, + requestKey.getReplayerRequestIndex(), ctx.getChannelKeyContext(), false, finalTunneledResponse, channelFutureAndRequestSchedule-> scheduleSendOnConnectionReplaySession(requestKey, ctx, channelFutureAndRequestSchedule, finalTunneledResponse, start, interval, packets)); } public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChannelKey channelKey, int channelInteractionNum, - ConnectionContext ctx, + ChannelKeyContext ctx, Instant timestamp) { var channelInteraction = new IndexedChannelInteraction(channelKey, channelInteractionNum); var finalTunneledResponse = @@ -93,7 +93,7 @@ public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChanne private DiagnosticTrackableCompletableFuture asynchronouslyInvokeRunnableToSetupFuture(ISourceTrafficChannelKey channelKey, int channelInteractionNumber, - ConnectionContext ctx, boolean ignoreIfChannelNotPresent, + ChannelKeyContext ctx, boolean ignoreIfChannelNotPresent, DiagnosticTrackableCompletableFuture finalTunneledResponse, Consumer successFn) { var channelFutureAndScheduleFuture = @@ -145,7 +145,7 @@ public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChanne } private void scheduleOnConnectionReplaySession(ISourceTrafficChannelKey channelKey, int channelInteractionIdx, - ConnectionContext ctx, + ChannelKeyContext ctx, ConnectionReplaySession channelFutureAndRequestSchedule, StringTrackableCompletableFuture futureToBeCompletedByTask, Instant atTime, String activityNameForLogging, Runnable task) { @@ -201,8 +201,9 @@ private void scheduleSendOnConnectionReplaySession(UniqueReplayerRequestKey requ getPacketReceiver(ctx, channelFutureAndRequestSchedule.getInnerChannelFuture(), packetReceiverRef), eventLoop, packets.iterator(), start, interval, new AtomicInteger(), responseFuture); - scheduleOnConnectionReplaySession(requestKey.trafficStreamKey, requestKey.getSourceRequestIndex(), ctx, - channelFutureAndRequestSchedule, responseFuture, start, "send", packetSender); + scheduleOnConnectionReplaySession(requestKey.trafficStreamKey, requestKey.getSourceRequestIndex(), + ctx.getChannelKeyContext(), channelFutureAndRequestSchedule, responseFuture, start, + "send", packetSender); } private void runAfterChannelSetup(ConnectionReplaySession channelFutureAndItsFutureRequests, diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index 47e3ea27f..086cb14e2 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -16,7 +16,7 @@ import org.opensearch.migrations.coreutils.MetricsLogger; import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer; -import org.opensearch.migrations.replay.tracing.ConnectionContext; +import org.opensearch.migrations.replay.tracing.ChannelKeyContext; import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.transform.IHttpMessage; import org.opensearch.migrations.replay.datatypes.HttpRequestTransformationStatus; @@ -676,7 +676,7 @@ Void handleCompletedTransaction(@NonNull UniqueReplayerRequestKey requestKey, Re @Override public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, - ConnectionContext ctx, List trafficStreamKeysBeingHeld) { + ChannelKeyContext ctx, List trafficStreamKeysBeingHeld) { commitTrafficStreams(trafficStreamKeysBeingHeld, status); } @@ -698,7 +698,7 @@ private void commitTrafficStreams(List trafficStreamKeysBeing @Override public void onConnectionClose(ISourceTrafficChannelKey channelKey, int channelInteractionNum, - ConnectionContext ctx, RequestResponsePacketPair.ReconstructionStatus status, + ChannelKeyContext ctx, RequestResponsePacketPair.ReconstructionStatus status, Instant timestamp, List trafficStreamKeysBeingHeld) { replayEngine.setFirstTimestamp(timestamp); replayEngine.closeConnection(channelKey, channelInteractionNum, ctx, timestamp); @@ -706,7 +706,7 @@ public void onConnectionClose(ISourceTrafficChannelKey channelKey, int channelIn } @Override - public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk, ConnectionContext ctx) { + public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk, ChannelKeyContext ctx) { commitTrafficStreams(List.of(tsk), true); } @@ -875,7 +875,7 @@ private static String formatWorkItem(DiagnosticTrackableCompletableFuture - transformAllData(inputRequestTransformerFactory.create(requestKey), packetsSupplier)); + transformAllData(inputRequestTransformerFactory.create(requestKey, ctx), packetsSupplier)); log.atDebug().setMessage(()->"finalizeRequest future for transformation of " + requestKey + " = " + transformationCompleteFuture).log(); // It might be safer to chain this work directly inside the scheduleWork call above so that the diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java index 321557815..bf85d026f 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java @@ -27,7 +27,7 @@ import org.opensearch.migrations.replay.AggregatedRawResponse; import org.opensearch.migrations.replay.netty.BacksideHttpWatcherHandler; import org.opensearch.migrations.replay.netty.BacksideSnifferHandler; -import org.opensearch.migrations.replay.tracing.ConnectionContext; +import org.opensearch.migrations.replay.tracing.ChannelKeyContext; import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture; @@ -64,7 +64,8 @@ public class NettyPacketToHttpConsumer implements IPacketFinalizingConsumer"Active - setting up backend connection to " + host + ":" + port).log(); @@ -110,7 +111,7 @@ public static ChannelFuture createClientConnection(EventLoopGroup eventLoopGroup if (connectFuture.isSuccess()) { var pipeline = connectFuture.channel().pipeline(); pipeline.removeFirst(); - log.atTrace().setMessage(()->connectionContext.getChannelKey() + + log.atTrace().setMessage(()-> channelKeyContext.getChannelKey() + " Done setting up client channel & it was successful").log(); if (sslContext != null) { var sslEngine = sslContext.newEngine(connectFuture.channel().alloc()); @@ -131,7 +132,7 @@ public static ChannelFuture createClientConnection(EventLoopGroup eventLoopGroup } else { // Close the connection if the connection attempt has failed. log.atWarn().setCause(connectFuture.cause()) - .setMessage(() -> connectionContext.getChannelKey() + " CONNECT future was not successful, " + + .setMessage(() -> channelKeyContext.getChannelKey() + " CONNECT future was not successful, " + "so setting the channel future's result to an exception").log(); rval.setFailure(connectFuture.cause()); } @@ -195,8 +196,8 @@ public DiagnosticTrackableCompletableFuture consumeBytes(ByteBuf pa System.identityHashCode(packetData) + ")").log(); return writePacketAndUpdateFuture(packetData); } else { - log.atWarn().setMessage(()->tracingContext.getRequestKey() + "outbound channel was not set up " + - "successfully, NOT writing bytes hash=" + System.identityHashCode(packetData)).log(); + log.atWarn().setMessage(()->tracingContext.getReplayerRequestKey() + "outbound channel was not set " + + "up successfully, NOT writing bytes hash=" + System.identityHashCode(packetData)).log(); channel.close(); return DiagnosticTrackableCompletableFuture.Factory.failedFuture(channelInitException, ()->""); } @@ -211,13 +212,13 @@ public DiagnosticTrackableCompletableFuture consumeBytes(ByteBuf pa final var completableFuture = new DiagnosticTrackableCompletableFuture(new CompletableFuture<>(), ()->"CompletableFuture that will wait for the netty future to fill in the completion value"); final int readableBytes = packetData.readableBytes(); - METERING_CLOSURE.meterIncrementEvent(tracingContext.context, "readBytes", packetData.readableBytes()); + METERING_CLOSURE.meterIncrementEvent(tracingContext, "readBytes", packetData.readableBytes()); channel.writeAndFlush(packetData) .addListener((ChannelFutureListener) future -> { Throwable cause = null; try { if (!future.isSuccess()) { - log.atWarn().setMessage(()->tracingContext.getRequestKey() + "closing outbound channel " + + log.atWarn().setMessage(()->tracingContext.getReplayerRequestKey() + "closing outbound channel " + "because WRITE future was not successful " + future.cause() + " hash=" + System.identityHashCode(packetData) + " will be sending the exception to " + completableFuture).log(); @@ -236,9 +237,9 @@ public DiagnosticTrackableCompletableFuture consumeBytes(ByteBuf pa " an exception :" + packetData + " hash=" + System.identityHashCode(packetData)).log(); metricsLogger.atError(MetricsEvent.WRITING_REQUEST_COMPONENT_FAILED, cause) .setAttribute(MetricsAttributeKey.CHANNEL_ID, channel.id().asLongText()) - .setAttribute(MetricsAttributeKey.REQUEST_ID, tracingContext.getRequestKey().toString()) + .setAttribute(MetricsAttributeKey.REQUEST_ID, tracingContext.getReplayerRequestKey().toString()) .setAttribute(MetricsAttributeKey.CONNECTION_ID, - tracingContext.getRequestKey().getTrafficStreamKey().getConnectionId()).emit(); + tracingContext.getReplayerRequestKey().getTrafficStreamKey().getConnectionId()).emit(); completableFuture.future.completeExceptionally(cause); channel.close(); } @@ -247,9 +248,9 @@ public DiagnosticTrackableCompletableFuture consumeBytes(ByteBuf pa ". Created future for writing data="+completableFuture).log(); metricsLogger.atSuccess(MetricsEvent.WROTE_REQUEST_COMPONENT) .setAttribute(MetricsAttributeKey.CHANNEL_ID, channel.id().asLongText()) - .setAttribute(MetricsAttributeKey.REQUEST_ID, tracingContext.getRequestKey()) + .setAttribute(MetricsAttributeKey.REQUEST_ID, tracingContext.getReplayerRequestKey()) .setAttribute(MetricsAttributeKey.CONNECTION_ID, - tracingContext.getRequestKey().getTrafficStreamKey().getConnectionId()) + tracingContext.getConnectionId()) .setAttribute(MetricsAttributeKey.SIZE_IN_BYTES, readableBytes).emit(); return completableFuture; } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java index 3019e0c0b..f82014543 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java @@ -7,6 +7,7 @@ import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; import org.opensearch.migrations.coreutils.MetricsLogger; +import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.replay.datatypes.HttpRequestTransformationStatus; import org.opensearch.migrations.replay.datatypes.TransformedOutputAndResult; import org.opensearch.migrations.replay.Utils; @@ -45,6 +46,9 @@ */ @Slf4j public class HttpJsonTransformingConsumer implements IPacketFinalizingConsumer> { + public static final String TELEMETRY_SCOPE_NAME = "HttpTransformer"; + public static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure(TELEMETRY_SCOPE_NAME); + public static final int HTTP_MESSAGE_NUM_SEGMENTS = 2; public static final int EXPECTED_PACKET_COUNT_GUESS_FOR_HEADERS = 4; private final RequestPipelineOrchestrator pipelineOrchestrator; @@ -69,6 +73,8 @@ public HttpJsonTransformingConsumer(IJsonTransformer transformer, IPacketFinalizingConsumer transformedPacketReceiver, String diagnosticLabel, RequestContext requestContext) { + this.requestContext = new RequestContext(requestContext.getReplayerRequestKey(), + METERING_CLOSURE.makeSpan(requestContext, "httpRequestTransformation")); chunkSizes = new ArrayList<>(HTTP_MESSAGE_NUM_SEGMENTS); chunkSizes.add(new ArrayList<>(EXPECTED_PACKET_COUNT_GUESS_FOR_HEADERS)); chunks = new ArrayList<>(HTTP_MESSAGE_NUM_SEGMENTS + EXPECTED_PACKET_COUNT_GUESS_FOR_HEADERS); @@ -125,7 +131,7 @@ public DiagnosticTrackableCompletableFuture { - if (t != null) { - t = unwindPossibleCompletionException(t); - if (t instanceof NoContentException) { - return redriveWithoutTransformation(offloadingHandler.packetReceiver, t); - } else { - metricsLogger.atError(MetricsEvent.TRANSFORMING_REQUEST_FAILED, t) - .setAttribute(MetricsAttributeKey.REQUEST_ID, requestContext.toString()) - .setAttribute(MetricsAttributeKey.CONNECTION_ID, requestContext.getChannelKey().getConnectionId()) - .setAttribute(MetricsAttributeKey.CHANNEL_ID, channel.id().asLongText()).emit(); - throw new CompletionException(t); - } - } else { - metricsLogger.atSuccess(MetricsEvent.REQUEST_WAS_TRANSFORMED) - .setAttribute(MetricsAttributeKey.REQUEST_ID, requestContext) - .setAttribute(MetricsAttributeKey.CONNECTION_ID, requestContext.getChannelKey().getConnectionId()) - .setAttribute(MetricsAttributeKey.CHANNEL_ID, channel.id().asLongText()).emit(); - return StringTrackableCompletableFuture.completedFuture(v, ()->"transformedHttpMessageValue"); - } - }, ()->"HttpJsonTransformingConsumer.finalizeRequest() is waiting to handle"); + (v, t) -> { + requestContext.getCurrentSpan().end(); + METERING_CLOSURE.meterIncrementEvent(requestContext, + t != null ? "transformRequestFailed" : "transformRequestSuccess"); + METERING_CLOSURE.meterHistogramMicros(requestContext, "transformationDuration"); + if (t != null) { + t = unwindPossibleCompletionException(t); + if (t instanceof NoContentException) { + return redriveWithoutTransformation(offloadingHandler.packetReceiver, t); + } else { + metricsLogger.atError(MetricsEvent.TRANSFORMING_REQUEST_FAILED, t) + .setAttribute(MetricsAttributeKey.REQUEST_ID, requestContext.toString()) + .setAttribute(MetricsAttributeKey.CONNECTION_ID, requestContext.getConnectionId()) + .setAttribute(MetricsAttributeKey.CHANNEL_ID, channel.id().asLongText()).emit(); + throw new CompletionException(t); + } + } else { + metricsLogger.atSuccess(MetricsEvent.REQUEST_WAS_TRANSFORMED) + .setAttribute(MetricsAttributeKey.REQUEST_ID, requestContext) + .setAttribute(MetricsAttributeKey.CONNECTION_ID, requestContext.getConnectionId()) + .setAttribute(MetricsAttributeKey.CHANNEL_ID, channel.id().asLongText()).emit(); + return StringTrackableCompletableFuture.completedFuture(v, ()->"transformedHttpMessageValue"); + } + }, ()->"HttpJsonTransformingConsumer.finalizeRequest() is waiting to handle"); } private static Throwable unwindPossibleCompletionException(Throwable t) { @@ -181,7 +191,7 @@ private static Throwable unwindPossibleCompletionException(Throwable t) { ()->"HttpJsonTransformingConsumer.redriveWithoutTransformation.compose()"); metricsLogger.atError(MetricsEvent.REQUEST_REDRIVEN_WITHOUT_TRANSFORMATION, reason) .setAttribute(MetricsAttributeKey.REQUEST_ID, requestContext) - .setAttribute(MetricsAttributeKey.CONNECTION_ID, requestContext.getChannelKey().getConnectionId()) + .setAttribute(MetricsAttributeKey.CONNECTION_ID, requestContext.getConnectionId()) .setAttribute(MetricsAttributeKey.CHANNEL_ID, channel.id().asLongText()).emit(); return finalizedFuture.map(f->f.thenApply(r->reason == null ? new TransformedOutputAndResult(r, HttpRequestTransformationStatus.SKIPPED, null) : diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java index 771076d52..5db40bb9a 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java @@ -56,7 +56,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception .toString()); metricsLogger.atSuccess(MetricsEvent.CAPTURED_REQUEST_PARSED_TO_HTTP) .setAttribute(MetricsAttributeKey.REQUEST_ID, requestContext) - .setAttribute(MetricsAttributeKey.CONNECTION_ID, requestContext.getChannelKey().getConnectionId()) + .setAttribute(MetricsAttributeKey.CONNECTION_ID, requestContext.getConnectionId()) .setAttribute(MetricsAttributeKey.HTTP_METHOD, request.method()) .setAttribute(MetricsAttributeKey.HTTP_ENDPOINT, request.uri()).emit(); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/RequestPipelineOrchestrator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/RequestPipelineOrchestrator.java index 5e285847d..650c5e003 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/RequestPipelineOrchestrator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/RequestPipelineOrchestrator.java @@ -10,7 +10,6 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer; -import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.transform.IAuthTransformer; import org.opensearch.migrations.transform.IAuthTransformerFactory; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelKeyContext.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelKeyContext.java new file mode 100644 index 000000000..9b8f81896 --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelKeyContext.java @@ -0,0 +1,25 @@ +package org.opensearch.migrations.replay.tracing; + +import io.opentelemetry.api.trace.Span; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; +import org.opensearch.migrations.tracing.IConnectionContext; + +@AllArgsConstructor +public class ChannelKeyContext implements IConnectionContext { + @Getter + final ISourceTrafficChannelKey channelKey; + @Getter + final Span currentSpan; + + @Override + public String getConnectionId() { + return channelKey.getConnectionId(); + } + + @Override + public String getNodeId() { + return channelKey.getNodeId(); + } +} diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ConnectionContext.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ConnectionContext.java deleted file mode 100644 index 0312d5140..000000000 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ConnectionContext.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.opensearch.migrations.replay.tracing; - -import io.netty.util.AttributeKey; -import io.opentelemetry.context.Context; -import io.opentelemetry.context.ContextKey; -import lombok.NonNull; -import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; - -import java.util.stream.Stream; - -public class ConnectionContext implements WithAttributes { - protected static final ContextKey CHANNEL_KEY_CONTEXT_KEY = ContextKey.named("channelKey"); - protected static final AttributeKey CHANNEL_ATTR = AttributeKey.newInstance("channelKey"); - - public final Context context; - - public ConnectionContext(ISourceTrafficChannelKey tsk) { - this(Context.current().with(CHANNEL_KEY_CONTEXT_KEY, tsk)); - } - - public ConnectionContext(Context c) { - assert c.get(CHANNEL_KEY_CONTEXT_KEY) != null; - context = c; - } - - public @NonNull ISourceTrafficChannelKey getChannelKey() { - return context.get(CHANNEL_KEY_CONTEXT_KEY); - } - - @Override - public Stream getAttributeKeys() { - return Stream.of(CHANNEL_ATTR); - } -} diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java index bfc0f5407..34feabad4 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java @@ -1,26 +1,54 @@ package org.opensearch.migrations.replay.tracing; -import io.opentelemetry.context.Context; -import io.opentelemetry.context.ContextKey; -import lombok.NonNull; +import io.opentelemetry.api.trace.Span; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.tracing.IReplayerRequestContext; +import org.opensearch.migrations.tracing.IWithStartTime; -public class RequestContext extends ConnectionContext { - private static final ContextKey UNIQUE_REQUEST_KEY = ContextKey.named("requestId"); +import java.time.Instant; - public RequestContext(UniqueReplayerRequestKey requestKey) { - this(Context.current(), requestKey); +public class RequestContext implements IReplayerRequestContext, IWithStartTime { + @Getter + final UniqueReplayerRequestKey replayerRequestKey; + @Getter + final Instant startTime; + @Getter + final Span currentSpan; + + public RequestContext(UniqueReplayerRequestKey replayerRequestKey, Span currentSpan) { + this.replayerRequestKey = replayerRequestKey; + this.currentSpan = currentSpan; + this.startTime = Instant.now(); + } + + public ChannelKeyContext getChannelKeyContext() { + return new ChannelKeyContext(replayerRequestKey.trafficStreamKey, currentSpan); + } + + @Override + public String getConnectionId() { + return replayerRequestKey.trafficStreamKey.getConnectionId(); + } + + @Override + public String getNodeId() { + return replayerRequestKey.trafficStreamKey.getNodeId(); } - public RequestContext(ConnectionContext ctx, UniqueReplayerRequestKey requestKey) { - this(ctx.context, requestKey); + @Override + public long sourceRequestIndex() { + return replayerRequestKey.getSourceRequestIndex(); } - public RequestContext(Context context, UniqueReplayerRequestKey requestKey) { - super(context.with(UNIQUE_REQUEST_KEY, requestKey).with(CHANNEL_KEY_CONTEXT_KEY, requestKey.trafficStreamKey)); + @Override + public long replayerRequestIndex() { + return replayerRequestKey.getReplayerRequestIndex(); } - public @NonNull UniqueReplayerRequestKey getRequestKey() { - return context.get(UNIQUE_REQUEST_KEY); + public ISourceTrafficChannelKey getChannelKey() { + return replayerRequestKey.trafficStreamKey; } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/WithAttributes.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/WithAttributes.java deleted file mode 100644 index 1988c7ae3..000000000 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/WithAttributes.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.opensearch.migrations.replay.tracing; - -import io.netty.util.AttributeKey; - -import java.util.stream.Stream; - -public interface WithAttributes { - Stream getAttributeKeys(); -} diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/RequestSenderOrchestratorTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/RequestSenderOrchestratorTest.java index e7c4b71e6..5fda5f7cf 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/RequestSenderOrchestratorTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/RequestSenderOrchestratorTest.java @@ -44,7 +44,7 @@ public void testThatSchedulingWorks() throws Exception { var perPacketShift = Duration.ofMillis(10*i/NUM_REPEATS); var startTimeForThisRequest = baseTime.plus(perPacketShift); var requestPackets = makeRequest(i/NUM_REPEATS); - var arr = senderOrchestrator.scheduleRequest(requestContext.getRequestKey(), requestContext, + var arr = senderOrchestrator.scheduleRequest(requestContext.getReplayerRequestKey(), requestContext, startTimeForThisRequest, Duration.ofMillis(1), requestPackets.stream()); log.info("Scheduled item to run at " + startTimeForThisRequest); scheduledItems.add(arr); @@ -52,7 +52,7 @@ public void testThatSchedulingWorks() throws Exception { } var connectionCtx = TestRequestKey.getTestConnectionRequestContext(NUM_REQUESTS_TO_SCHEDULE); var closeFuture = senderOrchestrator.scheduleClose( - connectionCtx.getChannelKey(), NUM_REQUESTS_TO_SCHEDULE, connectionCtx, + connectionCtx.getChannelKey(), NUM_REQUESTS_TO_SCHEDULE, connectionCtx.getChannelKeyContext(), lastEndTime.plus(Duration.ofMillis(100))); Assertions.assertEquals(NUM_REQUESTS_TO_SCHEDULE, scheduledItems.size()); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ResultsToLogsConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ResultsToLogsConsumerTest.java index 90ee03710..400407dec 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ResultsToLogsConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ResultsToLogsConsumerTest.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.netty.buffer.Unpooled; +import io.opentelemetry.api.GlobalOpenTelemetry; import lombok.extern.slf4j.Slf4j; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -19,7 +20,9 @@ import org.opensearch.migrations.replay.datatypes.PojoUniqueSourceRequestKey; import org.opensearch.migrations.replay.datatypes.TransformedPackets; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; +import org.opensearch.migrations.tracing.EmptyContext; import java.io.IOException; import java.io.InputStream; @@ -229,7 +232,8 @@ public void testOutputterForPost() throws IOException { @Test private void testOutputterForRequest(String requestResourceName, String expected) throws IOException { var trafficStreamKey = new PojoTrafficStreamKey(NODE_ID,"c",0); - var sourcePair = new RequestResponsePacketPair(trafficStreamKey); + var sourcePair = new RequestResponsePacketPair(trafficStreamKey, + TestRequestKey.getTestConnectionRequestContext(0)); var rawRequestData = loadResourceAsBytes("/requests/raw/" + requestResourceName); sourcePair.addRequestData(Instant.EPOCH, rawRequestData); var rawResponseData = NettyPacketToHttpConsumerTest.EXPECTED_RESPONSE_STRING.getBytes(StandardCharsets.UTF_8); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java index c6959c581..ca9e268a6 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java @@ -2,6 +2,7 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import io.opentelemetry.api.GlobalOpenTelemetry; import io.vavr.Tuple2; import lombok.AllArgsConstructor; import lombok.NonNull; @@ -14,12 +15,13 @@ import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.datatypes.RawPackets; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; -import org.opensearch.migrations.replay.tracing.ConnectionContext; +import org.opensearch.migrations.replay.tracing.ChannelKeyContext; import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.traffic.source.TrafficStreamWithEmbeddedKey; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import org.opensearch.migrations.trafficcapture.InMemoryConnectionCaptureFactory; import org.opensearch.migrations.trafficcapture.protos.TrafficStream; +import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; import java.io.IOException; import java.time.Duration; @@ -97,7 +99,9 @@ static ByteBuf makeSequentialByteBuf(int offset, int size) { static TrafficStream[] makeTrafficStreams(int bufferSize, int interactionOffset, List directives) throws Exception { var connectionFactory = buildSerializerFactory(bufferSize, ()->{}); - var offloader = connectionFactory.createOffloader("TEST_"+uniqueIdCounter.incrementAndGet()); + var offloader = connectionFactory.createOffloader(new ConnectionContext("test", "test", + GlobalOpenTelemetry.getTracer("test").spanBuilder("test").startSpan()), + "TEST_"+uniqueIdCounter.incrementAndGet()); for (var directive : directives) { serializeEvent(offloader, interactionOffset++, directive); } @@ -219,19 +223,19 @@ public void onFullDataReceived(UniqueReplayerRequestKey requestKey, RequestConte @Override public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, - ConnectionContext ctx, + ChannelKeyContext ctx, List trafficStreamKeysBeingHeld) {} @Override public void onConnectionClose(ISourceTrafficChannelKey key, int channelInteractionNumber, - ConnectionContext ctx, + ChannelKeyContext ctx, RequestResponsePacketPair.ReconstructionStatus status, Instant when, List trafficStreamKeysBeingHeld) { } @Override public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk, - ConnectionContext ctx) {} + ChannelKeyContext ctx) {} }); var tsList = trafficStreams.collect(Collectors.toList()); trafficStreams = tsList.stream(); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java index f52770fb9..a7fdfda40 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java @@ -9,7 +9,7 @@ import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; -import org.opensearch.migrations.replay.tracing.ConnectionContext; +import org.opensearch.migrations.replay.tracing.ChannelKeyContext; import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.traffic.source.InputStreamOfTraffic; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; @@ -172,19 +172,19 @@ public void onFullDataReceived(UniqueReplayerRequestKey key, RequestContext ctx, @Override public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, - ConnectionContext ctx, + ChannelKeyContext ctx, List trafficStreamKeysBeingHeld) {} @Override public void onConnectionClose(ISourceTrafficChannelKey key, int channelInteractionNumber, - ConnectionContext ctx, + ChannelKeyContext ctx, RequestResponsePacketPair.ReconstructionStatus status, Instant when, List trafficStreamKeysBeingHeld) { } @Override public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk, - ConnectionContext ctx) {} + ChannelKeyContext ctx) {} }); var bytes = synthesizeTrafficStreamsIntoByteArray(Instant.now(), 1); @@ -224,17 +224,17 @@ public void onFullDataReceived(UniqueReplayerRequestKey key, RequestContext ctx, @Override public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, - ConnectionContext ctx, + ChannelKeyContext ctx, List trafficStreamKeysBeingHeld) {} @Override public void onConnectionClose(ISourceTrafficChannelKey key, int channelInteractionNumber, - ConnectionContext ctx, RequestResponsePacketPair.ReconstructionStatus status, + ChannelKeyContext ctx, RequestResponsePacketPair.ReconstructionStatus status, Instant when, List trafficStreamKeysBeingHeld) { } @Override public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk, - ConnectionContext ctx) {} + ChannelKeyContext ctx) {} } ); byte[] serializedChunks; diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java index 5927410f0..29b8cc72d 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java @@ -4,6 +4,7 @@ import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.util.InsecureTrustManagerFactory; import io.netty.util.concurrent.DefaultThreadFactory; +import io.opentelemetry.api.GlobalOpenTelemetry; import lombok.Lombok; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.AfterAll; @@ -166,7 +167,8 @@ public void testThatConnectionsAreKeptAliveAndShared(boolean useTls) String connId = "TEST_" + j; var trafficStreamKey = new PojoTrafficStreamKey("testNodeId", connId, 0); var requestKey = new UniqueReplayerRequestKey(trafficStreamKey, 0, i); - var ctx = new RequestContext(requestKey); + var ctx = new RequestContext(requestKey, + GlobalOpenTelemetry.getTracer("test").spanBuilder("test").startSpan()); var requestFinishFuture = TrafficReplayer.transformAndSendRequest(transformingHttpHandlerFactory, sendingFactory, ctx, Instant.now(), Instant.now(), requestKey, ()->Stream.of(EXPECTED_REQUEST_STRING.getBytes(StandardCharsets.UTF_8))); diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java index 2d94b11a2..b0695c5de 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java @@ -1,8 +1,8 @@ package org.opensearch.migrations.replay; +import io.opentelemetry.api.GlobalOpenTelemetry; import org.opensearch.migrations.replay.datatypes.PojoTrafficStreamKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; -import org.opensearch.migrations.replay.tracing.ConnectionContext; import org.opensearch.migrations.replay.tracing.RequestContext; public class TestRequestKey { @@ -13,6 +13,7 @@ public static final RequestContext getTestConnectionRequestContext(int replayerI var rk = new UniqueReplayerRequestKey( new PojoTrafficStreamKey("testNodeId", "testConnectionId", 0), 0, replayerIdx); - return new RequestContext(new UniqueReplayerRequestKey(rk.trafficStreamKey, 1, 1)); + return new RequestContext(new UniqueReplayerRequestKey(rk.trafficStreamKey, 1, 1), + GlobalOpenTelemetry.getTracer("test").spanBuilder("test").startSpan()); } } From 3746a8ed62fd69f65709bd2dc20a88b11de111bd Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Wed, 29 Nov 2023 23:49:36 -0500 Subject: [PATCH 10/20] Get span parenting to work. Signed-off-by: Greg Schohn --- .../kafkaoffloader/KafkaCaptureFactory.java | 11 ++---- .../tracing/KafkaRecordContext.java | 6 ++- .../KafkaCaptureFactoryTest.java | 2 +- .../tracing/ConnectionContext.java | 10 +++-- .../coreutils/SimpleMeteringClosure.java | 22 +++++++++-- .../migrations/coreutils/SpanGenerator.java | 8 ++++ .../coreutils/SpanWithParentGenerator.java | 9 +++++ .../migrations/tracing/IRequestContext.java | 5 +-- .../migrations/tracing/IWithAttributes.java | 8 +++- .../tracing/IWithStartTimeAndAttributes.java | 4 ++ ...allyReliableLoggingHttpRequestHandler.java | 2 +- .../netty/LoggingHttpRequestHandler.java | 1 - .../netty/LoggingHttpResponseHandler.java | 4 +- ...ReliableLoggingHttpRequestHandlerTest.java | 2 +- .../netty/ProxyChannelInitializer.java | 2 +- TrafficCapture/trafficReplayer/build.gradle | 1 + .../migrations/replay/Accumulation.java | 7 ++-- .../http/HttpJsonTransformingConsumer.java | 37 ++++++++++++------- .../replay/tracing/ChannelKeyContext.java | 9 ++++- .../replay/tracing/RequestContext.java | 26 +++++++++---- ...afficToHttpTransactionAccumulatorTest.java | 2 +- .../NettyPacketToHttpConsumerTest.java | 3 +- .../migrations/replay/TestRequestKey.java | 11 +++++- 23 files changed, 132 insertions(+), 60 deletions(-) create mode 100644 TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SpanGenerator.java create mode 100644 TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SpanWithParentGenerator.java create mode 100644 TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IWithStartTimeAndAttributes.java diff --git a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java index 29c27f425..b2df34d69 100644 --- a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java +++ b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java @@ -1,9 +1,6 @@ package org.opensearch.migrations.trafficcapture.kafkaoffloader; import com.google.protobuf.CodedOutputStream; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.context.Context; -import io.opentelemetry.context.ContextKey; import lombok.AllArgsConstructor; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -14,7 +11,6 @@ import org.opensearch.migrations.coreutils.MetricsEvent; import org.apache.kafka.clients.producer.RecordMetadata; import org.opensearch.migrations.coreutils.SimpleMeteringClosure; -import org.opensearch.migrations.tracing.EmptyContext; import org.opensearch.migrations.trafficcapture.CodedOutputStreamHolder; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; @@ -91,7 +87,7 @@ class StreamManager extends OrderedStreamLifecyleManager { public StreamManager(ConnectionContext incomingTelemetryContext, String connectionId) { this.telemetryContext = new ConnectionContext(incomingTelemetryContext, - METERING_CLOSURE.makeSpan(incomingTelemetryContext, "offloaderLifetime")); + METERING_CLOSURE.makeSpanContinuation("offloaderLifetime")); this.connectionId = connectionId; this.startTime = Instant.now(); } @@ -111,7 +107,7 @@ public void close() throws IOException { public CodedOutputStreamWrapper createStream() { METERING_CLOSURE.meterIncrementEvent(telemetryContext, "stream_created"); var newStreamCtx = new ConnectionContext(telemetryContext, - METERING_CLOSURE.makeSpan(telemetryContext, "recordStream")); + METERING_CLOSURE.makeSpanContinuation("recordStream")); ByteBuffer bb = ByteBuffer.allocate(bufferSize); return new CodedOutputStreamWrapper(CodedOutputStream.newInstance(bb), bb, newStreamCtx); @@ -125,6 +121,7 @@ public CodedOutputStreamWrapper createStream() { outputStreamHolder); } var osh = (CodedOutputStreamWrapper) outputStreamHolder; + osh.streamContext.currentSpan.end(); // Structured context for MetricsLogger try { @@ -137,7 +134,7 @@ public CodedOutputStreamWrapper createStream() { log.debug("Sending Kafka producer record: {} for topic: {}", recordId, topicNameForTraffic); var flushContext = new KafkaRecordContext(telemetryContext, - METERING_CLOSURE.makeSpan(telemetryContext, "flushRecord"), + METERING_CLOSURE.makeSpanContinuation("flushRecord"), topicNameForTraffic, recordId, kafkaRecord.value().length); METERING_CLOSURE.meterIncrementEvent(telemetryContext, "stream_flush_called"); diff --git a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/tracing/KafkaRecordContext.java b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/tracing/KafkaRecordContext.java index 006112753..1e179a21e 100644 --- a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/tracing/KafkaRecordContext.java +++ b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/tracing/KafkaRecordContext.java @@ -5,6 +5,8 @@ import io.opentelemetry.api.trace.Span; import lombok.AllArgsConstructor; import lombok.Getter; +import org.opensearch.migrations.coreutils.SpanGenerator; +import org.opensearch.migrations.coreutils.SpanWithParentGenerator; import org.opensearch.migrations.tracing.IConnectionContext; import org.opensearch.migrations.tracing.IWithAttributes; import org.opensearch.migrations.tracing.IWithStartTime; @@ -30,14 +32,14 @@ public class KafkaRecordContext implements IWithAttributes, @Getter public final int recordSize; - public KafkaRecordContext(IConnectionContext enclosingScope, Span currentSpan, + public KafkaRecordContext(IConnectionContext enclosingScope, SpanWithParentGenerator incomingSpan, String topic, String recordId, int recordSize) { this.enclosingScope = enclosingScope; - this.currentSpan = currentSpan; this.topic = topic; this.recordId = recordId; this.recordSize = recordSize; this.startTime = Instant.now(); + currentSpan = incomingSpan.apply(this.getPopulatedAttributes(), enclosingScope.getCurrentSpan()); } @Override diff --git a/TrafficCapture/captureKafkaOffloader/src/test/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactoryTest.java b/TrafficCapture/captureKafkaOffloader/src/test/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactoryTest.java index 7099f6317..403bf9ef1 100644 --- a/TrafficCapture/captureKafkaOffloader/src/test/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactoryTest.java +++ b/TrafficCapture/captureKafkaOffloader/src/test/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactoryTest.java @@ -78,7 +78,7 @@ public void testLargeRequestIsWithinKafkaMessageSizeLimit() throws IOException, private static ConnectionContext createCtx() { return new ConnectionContext("test", "test", - GlobalOpenTelemetry.getTracer("test").spanBuilder("test").startSpan()); + x->GlobalOpenTelemetry.getTracer("test").spanBuilder("test").startSpan()); } /** diff --git a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/tracing/ConnectionContext.java b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/tracing/ConnectionContext.java index 6df11b03d..d661377ea 100644 --- a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/tracing/ConnectionContext.java +++ b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/tracing/ConnectionContext.java @@ -2,6 +2,8 @@ import io.opentelemetry.api.trace.Span; import lombok.Getter; +import org.opensearch.migrations.coreutils.SpanGenerator; +import org.opensearch.migrations.coreutils.SpanWithParentGenerator; import org.opensearch.migrations.tracing.IConnectionContext; import org.opensearch.migrations.tracing.IWithStartTime; @@ -17,17 +19,17 @@ public class ConnectionContext implements IConnectionContext, IWithStartTime { @Getter private final Instant startTime; - public ConnectionContext(ConnectionContext oldContext, Span currentSpan) { + public ConnectionContext(ConnectionContext oldContext, SpanWithParentGenerator spanGenerator) { this.connectionId = oldContext.getConnectionId(); this.nodeId = oldContext.getNodeId(); - this.currentSpan = currentSpan; this.startTime = Instant.now(); + this.currentSpan = spanGenerator.apply(getPopulatedAttributes(), oldContext.getCurrentSpan()); } - public ConnectionContext(String connectionId, String nodeId, Span currentSpan) { + public ConnectionContext(String connectionId, String nodeId, SpanGenerator spanGenerator) { this.connectionId = connectionId; this.nodeId = nodeId; - this.currentSpan = currentSpan; + this.currentSpan = spanGenerator.apply(getPopulatedAttributes()); this.startTime = Instant.now(); } } diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java index f890264f2..ad6b484f0 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java @@ -1,9 +1,12 @@ package org.opensearch.migrations.coreutils; import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter; import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; @@ -21,6 +24,7 @@ import java.time.Duration; import java.time.Instant; +import java.util.Optional; import java.util.concurrent.TimeUnit; public class SimpleMeteringClosure { @@ -131,9 +135,19 @@ public void meterHistogram(IWithAttributes ctx, String eventName, String units, .build()); } - public Span makeSpan(IWithAttributes ctx, String spanName) { - var span = tracer.spanBuilder(spanName).startSpan(); - span.setAllAttributes(ctx.getPopulatedAttributesBuilder().build()); - return span; + public SpanGenerator makeSpanContinuation(String spanName, Span parentSpan) { + var builder = tracer.spanBuilder(spanName); + return (attrs) -> getSpanWithParent(builder, attrs, parentSpan); + } + + public static Span getSpanWithParent(SpanBuilder builder, Attributes attrs, Span parentSpan) { + return Optional.ofNullable(parentSpan).map(p -> builder.setParent(Context.current().with(p))) + .orElseGet(builder::setNoParent) + .startSpan().setAllAttributes(attrs); + } + + public SpanWithParentGenerator makeSpanContinuation(String spanName) { + var builder = tracer.spanBuilder(spanName); + return (attrs,parentSpan) -> getSpanWithParent(builder, attrs, parentSpan); } } diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SpanGenerator.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SpanGenerator.java new file mode 100644 index 000000000..188f04099 --- /dev/null +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SpanGenerator.java @@ -0,0 +1,8 @@ +package org.opensearch.migrations.coreutils; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; + +import java.util.function.Function; + +public interface SpanGenerator extends Function { } diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SpanWithParentGenerator.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SpanWithParentGenerator.java new file mode 100644 index 000000000..c404e46b9 --- /dev/null +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SpanWithParentGenerator.java @@ -0,0 +1,9 @@ +package org.opensearch.migrations.coreutils; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; + +import java.util.function.BiFunction; + +public interface SpanWithParentGenerator extends BiFunction { +} diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IRequestContext.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IRequestContext.java index 67744ff5f..c199c7aa8 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IRequestContext.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IRequestContext.java @@ -3,14 +3,13 @@ import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.AttributesBuilder; -public interface IRequestContext extends IConnectionContext { +public interface IRequestContext extends IWithAttributes { static final AttributeKey SOURCE_REQUEST_INDEX_KEY = AttributeKey.longKey("sourceRequestIndex"); long sourceRequestIndex(); @Override default AttributesBuilder fillAttributes(AttributesBuilder builder) { - return IConnectionContext.super.fillAttributes( - builder.put(SOURCE_REQUEST_INDEX_KEY, sourceRequestIndex())); + return builder.put(SOURCE_REQUEST_INDEX_KEY, sourceRequestIndex()); } } diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IWithAttributes.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IWithAttributes.java index 75d443d2d..6f4f244b0 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IWithAttributes.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IWithAttributes.java @@ -11,7 +11,13 @@ public interface IWithAttributes { Span getCurrentSpan(); - AttributesBuilder fillAttributes(AttributesBuilder builder); + default AttributesBuilder fillAttributes(AttributesBuilder builder) { + return builder; + } + + default Attributes getPopulatedAttributes() { + return getPopulatedAttributesBuilder().build(); + } default AttributesBuilder getPopulatedAttributesBuilder() { var currentObj = this; diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IWithStartTimeAndAttributes.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IWithStartTimeAndAttributes.java new file mode 100644 index 000000000..31be5b08c --- /dev/null +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IWithStartTimeAndAttributes.java @@ -0,0 +1,4 @@ +package org.opensearch.migrations.tracing; + +public interface IWithStartTimeAndAttributes extends IWithStartTime, IWithAttributes { +} diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java index de3391fbe..b01a1dcb3 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java @@ -30,7 +30,7 @@ protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Ob if (shouldBlockPredicate.test(httpRequest)) { METERING_CLOSURE.meterIncrementEvent(connectionContext, "blockingRequestUntilFlush"); var flushContext = new ConnectionContext(connectionContext, - METERING_CLOSURE.tracer.spanBuilder("blockedForFlush").startSpan()); + METERING_CLOSURE.makeSpanContinuation("blockedForFlush")); trafficOffloader.flushCommitAndResetStream(false).whenComplete((result, t) -> { log.atInfo().setMessage(()->"Done flushing").log(); diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java index 7f3191dc0..c1814be03 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java @@ -81,7 +81,6 @@ public HttpRequest resetCurrentRequest() { public LoggingHttpRequestHandler(ConnectionContext incomingContext, IChannelConnectionCaptureSerializer trafficOffloader) { this.connectionContext = incomingContext; - var span = METERING_CLOSURE.makeSpan(incomingContext, "frontendConnection"); METERING_CLOSURE.meterIncrementEvent(incomingContext, "requestStarted"); this.trafficOffloader = trafficOffloader; diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java index 94e93d021..fce1be7e2 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java @@ -43,8 +43,8 @@ public void bind(ChannelHandlerContext ctx, SocketAddress localAddress, ChannelP public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, SocketAddress localAddress, ChannelPromise promise) throws Exception { trafficOffloader.addConnectEvent(Instant.now(), remoteAddress, localAddress); - var span = METERING_CLOSURE.makeSpan(telemetryContext,"backendConnection"); - telemetryContext = new ConnectionContext(telemetryContext, span); + telemetryContext = new ConnectionContext(telemetryContext, + METERING_CLOSURE.makeSpanContinuation("backendConnection")); connectTime = Instant.now(); METERING_CLOSURE.meterIncrementEvent(telemetryContext, "connect"); METERING_CLOSURE.meterDeltaEvent(telemetryContext, "connections", 1); diff --git a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java index 86cd36ea5..d6fe17004 100644 --- a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java +++ b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java @@ -84,7 +84,7 @@ private static void writeMessageAndVerify(byte[] fullTrafficBytes, ConsumerGlobalOpenTelemetry.getTracer("test").spanBuilder("test").startSpan()); EmbeddedChannel channel = new EmbeddedChannel( new ConditionallyReliableLoggingHttpRequestHandler(ctx, offloader, x->true)); // true: block every request channelWriter.accept(channel); diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java index 2e48c9765..feddb9f91 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java @@ -47,7 +47,7 @@ protected void initChannel(SocketChannel ch) throws IOException { var connectionId = ch.id().asLongText(); var ctx = new ConnectionContext(connectionId, "", - METERING_CLOSURE.makeSpan(EmptyContext.singleton, "connectionLifetime")); + METERING_CLOSURE.makeSpanContinuation("connectionLifetime", null)); var offloader = connectionCaptureFactory.createOffloader(ctx, connectionId); ch.pipeline().addLast(new LoggingHttpResponseHandler<>(ctx, offloader)); ch.pipeline().addLast(new ConditionallyReliableLoggingHttpRequestHandler(ctx, offloader, diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle index 7cd49c99d..c50a37d27 100644 --- a/TrafficCapture/trafficReplayer/build.gradle +++ b/TrafficCapture/trafficReplayer/build.gradle @@ -66,6 +66,7 @@ dependencies { implementation 'org.apache.commons:commons-compress:1.24.0' testFixturesImplementation project(':replayerPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') + testFixturesImplementation project(':coreUtilities') testFixturesImplementation testFixtures(project(path: ':testUtilities')) testFixturesImplementation platform("io.opentelemetry:opentelemetry-bom:1.32.0") diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java index fa474ac4e..7b76a000b 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java @@ -7,7 +7,6 @@ import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; import org.opensearch.migrations.replay.tracing.ChannelKeyContext; import org.opensearch.migrations.replay.tracing.RequestContext; -import org.opensearch.migrations.tracing.EmptyContext; import java.time.Instant; import java.util.concurrent.atomic.AtomicInteger; @@ -49,7 +48,7 @@ public Accumulation(@NonNull ITrafficStreamKey trafficChannelKey, this.state = dropObservationsLeftoverFromPrevious ? State.IGNORING_LAST_REQUEST : State.WAITING_FOR_NEXT_READ_CHUNK; channelContext = new ChannelKeyContext(trafficChannelKey, - METERING_CLOSURE.makeSpan(EmptyContext.singleton, "accumulatingChannel")); + METERING_CLOSURE.makeSpanContinuation("accumulatingChannel", null)); } public RequestResponsePacketPair getOrCreateTransactionPair(ITrafficStreamKey forTrafficStreamKey) { @@ -57,8 +56,8 @@ public RequestResponsePacketPair getOrCreateTransactionPair(ITrafficStreamKey fo return rrPair; } this.rrPair = new RequestResponsePacketPair(forTrafficStreamKey, - new RequestContext(getRequestKey(forTrafficStreamKey), - METERING_CLOSURE.makeSpan(channelContext, "accumulatingRequest"))); + new RequestContext(channelContext, getRequestKey(forTrafficStreamKey), + METERING_CLOSURE.makeSpanContinuation("accumulatingRequest"))); return rrPair; } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java index f82014543..ae86fd5a9 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java @@ -3,6 +3,9 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.http.HttpRequestDecoder; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.trace.Span; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; @@ -15,11 +18,14 @@ import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture; +import org.opensearch.migrations.tracing.IWithAttributes; +import org.opensearch.migrations.tracing.IWithStartTimeAndAttributes; import org.opensearch.migrations.transform.IAuthTransformerFactory; import org.opensearch.migrations.transform.IJsonTransformer; import org.slf4j.event.Level; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -54,7 +60,7 @@ public class HttpJsonTransformingConsumer implements IPacketFinalizingConsume private final RequestPipelineOrchestrator pipelineOrchestrator; private final EmbeddedChannel channel; private static final MetricsLogger metricsLogger = new MetricsLogger("HttpJsonTransformingConsumer"); - private RequestContext requestContext; + private IWithStartTimeAndAttributes transformationContext; /** * Roughly try to keep track of how big each data chunk was that came into the transformer. These values @@ -73,13 +79,18 @@ public HttpJsonTransformingConsumer(IJsonTransformer transformer, IPacketFinalizingConsumer transformedPacketReceiver, String diagnosticLabel, RequestContext requestContext) { - this.requestContext = new RequestContext(requestContext.getReplayerRequestKey(), - METERING_CLOSURE.makeSpan(requestContext, "httpRequestTransformation")); + this.transformationContext = new IWithStartTimeAndAttributes<>() { + @Getter + Span currentSpan = METERING_CLOSURE.makeSpanContinuation("httpRequestTransformation") + .apply(requestContext.getPopulatedAttributes(), requestContext.getCurrentSpan()); + @Getter + Instant startTime = Instant.now(); + @Override public RequestContext getEnclosingScope() { return requestContext; } + }; chunkSizes = new ArrayList<>(HTTP_MESSAGE_NUM_SEGMENTS); chunkSizes.add(new ArrayList<>(EXPECTED_PACKET_COUNT_GUESS_FOR_HEADERS)); chunks = new ArrayList<>(HTTP_MESSAGE_NUM_SEGMENTS + EXPECTED_PACKET_COUNT_GUESS_FOR_HEADERS); channel = new EmbeddedChannel(); - this.requestContext = requestContext; pipelineOrchestrator = new RequestPipelineOrchestrator<>(chunkSizes, transformedPacketReceiver, authTransformerFactory, diagnosticLabel, requestContext); pipelineOrchestrator.addInitialHandlers(channel.pipeline(), transformer); @@ -142,25 +153,25 @@ public DiagnosticTrackableCompletableFuture { - requestContext.getCurrentSpan().end(); - METERING_CLOSURE.meterIncrementEvent(requestContext, + transformationContext.getCurrentSpan().end(); + METERING_CLOSURE.meterIncrementEvent(transformationContext, t != null ? "transformRequestFailed" : "transformRequestSuccess"); - METERING_CLOSURE.meterHistogramMicros(requestContext, "transformationDuration"); + METERING_CLOSURE.meterHistogramMicros(transformationContext, "transformationDuration"); if (t != null) { t = unwindPossibleCompletionException(t); if (t instanceof NoContentException) { return redriveWithoutTransformation(offloadingHandler.packetReceiver, t); } else { metricsLogger.atError(MetricsEvent.TRANSFORMING_REQUEST_FAILED, t) - .setAttribute(MetricsAttributeKey.REQUEST_ID, requestContext.toString()) - .setAttribute(MetricsAttributeKey.CONNECTION_ID, requestContext.getConnectionId()) + .setAttribute(MetricsAttributeKey.REQUEST_ID, transformationContext.toString()) + .setAttribute(MetricsAttributeKey.CONNECTION_ID, transformationContext.getEnclosingScope().getConnectionId()) .setAttribute(MetricsAttributeKey.CHANNEL_ID, channel.id().asLongText()).emit(); throw new CompletionException(t); } } else { metricsLogger.atSuccess(MetricsEvent.REQUEST_WAS_TRANSFORMED) - .setAttribute(MetricsAttributeKey.REQUEST_ID, requestContext) - .setAttribute(MetricsAttributeKey.CONNECTION_ID, requestContext.getConnectionId()) + .setAttribute(MetricsAttributeKey.REQUEST_ID, transformationContext) + .setAttribute(MetricsAttributeKey.CONNECTION_ID, transformationContext.getEnclosingScope().getConnectionId()) .setAttribute(MetricsAttributeKey.CHANNEL_ID, channel.id().asLongText()).emit(); return StringTrackableCompletableFuture.completedFuture(v, ()->"transformedHttpMessageValue"); } @@ -190,8 +201,8 @@ private static Throwable unwindPossibleCompletionException(Throwable t) { consumptionChainedFuture.thenCompose(v -> packetConsumer.finalizeRequest(), ()->"HttpJsonTransformingConsumer.redriveWithoutTransformation.compose()"); metricsLogger.atError(MetricsEvent.REQUEST_REDRIVEN_WITHOUT_TRANSFORMATION, reason) - .setAttribute(MetricsAttributeKey.REQUEST_ID, requestContext) - .setAttribute(MetricsAttributeKey.CONNECTION_ID, requestContext.getConnectionId()) + .setAttribute(MetricsAttributeKey.REQUEST_ID, transformationContext) + .setAttribute(MetricsAttributeKey.CONNECTION_ID, transformationContext.getEnclosingScope().getConnectionId()) .setAttribute(MetricsAttributeKey.CHANNEL_ID, channel.id().asLongText()).emit(); return finalizedFuture.map(f->f.thenApply(r->reason == null ? new TransformedOutputAndResult(r, HttpRequestTransformationStatus.SKIPPED, null) : diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelKeyContext.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelKeyContext.java index 9b8f81896..075b49f08 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelKeyContext.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelKeyContext.java @@ -1,18 +1,23 @@ package org.opensearch.migrations.replay.tracing; import io.opentelemetry.api.trace.Span; -import lombok.AllArgsConstructor; import lombok.Getter; +import org.opensearch.migrations.coreutils.SpanGenerator; +import org.opensearch.migrations.coreutils.SpanWithParentGenerator; import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.tracing.IConnectionContext; -@AllArgsConstructor public class ChannelKeyContext implements IConnectionContext { @Getter final ISourceTrafficChannelKey channelKey; @Getter final Span currentSpan; + public ChannelKeyContext(ISourceTrafficChannelKey channelKey, SpanGenerator spanGenerator) { + this.channelKey = channelKey; + this.currentSpan = spanGenerator.apply(getPopulatedAttributes()); + } + @Override public String getConnectionId() { return channelKey.getConnectionId(); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java index 34feabad4..947ba19fb 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java @@ -1,16 +1,23 @@ package org.opensearch.migrations.replay.tracing; import io.opentelemetry.api.trace.Span; -import lombok.AllArgsConstructor; import lombok.Getter; +import org.opensearch.migrations.coreutils.SpanGenerator; +import org.opensearch.migrations.coreutils.SpanWithParentGenerator; import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.tracing.EmptyContext; +import org.opensearch.migrations.tracing.IConnectionContext; import org.opensearch.migrations.tracing.IReplayerRequestContext; +import org.opensearch.migrations.tracing.IRequestContext; +import org.opensearch.migrations.tracing.IWithAttributes; import org.opensearch.migrations.tracing.IWithStartTime; import java.time.Instant; public class RequestContext implements IReplayerRequestContext, IWithStartTime { + @Getter + IConnectionContext enclosingScope; @Getter final UniqueReplayerRequestKey replayerRequestKey; @Getter @@ -18,24 +25,27 @@ public class RequestContext implements IReplayerRequestContext, IWithStartTime { @Getter final Span currentSpan; - public RequestContext(UniqueReplayerRequestKey replayerRequestKey, Span currentSpan) { + IWithAttributes> foo; + + public RequestContext(ChannelKeyContext enclosingScope, UniqueReplayerRequestKey replayerRequestKey, + SpanWithParentGenerator spanGenerator) { + this.enclosingScope = enclosingScope; this.replayerRequestKey = replayerRequestKey; - this.currentSpan = currentSpan; this.startTime = Instant.now(); + this.currentSpan = spanGenerator.apply(getPopulatedAttributes(), enclosingScope.getCurrentSpan()); } public ChannelKeyContext getChannelKeyContext() { - return new ChannelKeyContext(replayerRequestKey.trafficStreamKey, currentSpan); + return new ChannelKeyContext(replayerRequestKey.trafficStreamKey, + innerAttributesToIgnore_LeavingOriginalAttributesInPlace->currentSpan); } - @Override public String getConnectionId() { - return replayerRequestKey.trafficStreamKey.getConnectionId(); + return enclosingScope.getConnectionId(); } - @Override public String getNodeId() { - return replayerRequestKey.trafficStreamKey.getNodeId(); + return enclosingScope.getNodeId(); } @Override diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java index ca9e268a6..eb4746335 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java @@ -100,7 +100,7 @@ static TrafficStream[] makeTrafficStreams(int bufferSize, int interactionOffset, List directives) throws Exception { var connectionFactory = buildSerializerFactory(bufferSize, ()->{}); var offloader = connectionFactory.createOffloader(new ConnectionContext("test", "test", - GlobalOpenTelemetry.getTracer("test").spanBuilder("test").startSpan()), + x->GlobalOpenTelemetry.getTracer("test").spanBuilder("test").startSpan()), "TEST_"+uniqueIdCounter.incrementAndGet()); for (var directive : directives) { serializeEvent(offloader, interactionOffset++, directive); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java index 29b8cc72d..e6f833c7b 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java @@ -167,8 +167,7 @@ public void testThatConnectionsAreKeptAliveAndShared(boolean useTls) String connId = "TEST_" + j; var trafficStreamKey = new PojoTrafficStreamKey("testNodeId", connId, 0); var requestKey = new UniqueReplayerRequestKey(trafficStreamKey, 0, i); - var ctx = new RequestContext(requestKey, - GlobalOpenTelemetry.getTracer("test").spanBuilder("test").startSpan()); + var ctx = TestRequestKey.getTestConnectionRequestContext(0); var requestFinishFuture = TrafficReplayer.transformAndSendRequest(transformingHttpHandlerFactory, sendingFactory, ctx, Instant.now(), Instant.now(), requestKey, ()->Stream.of(EXPECTED_REQUEST_STRING.getBytes(StandardCharsets.UTF_8))); diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java index b0695c5de..ff11ff423 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java @@ -1,9 +1,13 @@ package org.opensearch.migrations.replay; import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.replay.datatypes.PojoTrafficStreamKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.replay.tracing.ChannelKeyContext; import org.opensearch.migrations.replay.tracing.RequestContext; +import org.opensearch.migrations.tracing.EmptyContext; public class TestRequestKey { @@ -13,7 +17,10 @@ public static final RequestContext getTestConnectionRequestContext(int replayerI var rk = new UniqueReplayerRequestKey( new PojoTrafficStreamKey("testNodeId", "testConnectionId", 0), 0, replayerIdx); - return new RequestContext(new UniqueReplayerRequestKey(rk.trafficStreamKey, 1, 1), - GlobalOpenTelemetry.getTracer("test").spanBuilder("test").startSpan()); + var smc = new SimpleMeteringClosure("test"); + var channelKeyContext = new ChannelKeyContext(rk.trafficStreamKey, smc.makeSpanContinuation("test", null)); + return new RequestContext(channelKeyContext, + new UniqueReplayerRequestKey(rk.trafficStreamKey, 1, 1), + smc.makeSpanContinuation("test2")); } } From 4b432622f72cf4778529d20725fc76ce4fbdb743 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Thu, 30 Nov 2023 09:31:14 -0500 Subject: [PATCH 11/20] Attempt to fix a failing unit test. Make sure that the context is using the right requestKey, which also will have the appropriate indices as per the test context. Signed-off-by: Greg Schohn --- .../java/org/opensearch/migrations/replay/TestRequestKey.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java index ff11ff423..a47c7b98f 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java @@ -19,8 +19,6 @@ public static final RequestContext getTestConnectionRequestContext(int replayerI 0, replayerIdx); var smc = new SimpleMeteringClosure("test"); var channelKeyContext = new ChannelKeyContext(rk.trafficStreamKey, smc.makeSpanContinuation("test", null)); - return new RequestContext(channelKeyContext, - new UniqueReplayerRequestKey(rk.trafficStreamKey, 1, 1), - smc.makeSpanContinuation("test2")); + return new RequestContext(channelKeyContext, rk, smc.makeSpanContinuation("test2")); } } From 322e12fc368d69c8ca21d1a842884b3786cab809 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Thu, 30 Nov 2023 15:26:15 -0500 Subject: [PATCH 12/20] Refactor. Couple name changes, class package changes, and moved IReplayerRequestContext to the replayer Signed-off-by: Greg Schohn --- .../kafkaoffloader/KafkaCaptureFactory.java | 2 +- .../kafkaoffloader/tracing/KafkaRecordContext.java | 7 +++---- .../trafficcapture/tracing/ConnectionContext.java | 10 +++++----- .../migrations/tracing/ContextWithSpan.java | 10 ---------- .../ISpanGenerator.java} | 4 ++-- .../ISpanWithParentGenerator.java} | 4 ++-- .../{coreutils => tracing}/SimpleMeteringClosure.java | 8 +++----- .../{ => commoncontexts}/IConnectionContext.java | 4 +++- .../tracing/{ => commoncontexts}/IRequestContext.java | 7 ++++--- .../netty/LoggingHttpRequestHandler.java | 2 +- .../netty/LoggingHttpResponseHandler.java | 2 +- .../trafficcapture/proxyserver/CaptureProxy.java | 2 +- .../proxyserver/netty/FrontsideHandler.java | 4 ++++ .../proxyserver/netty/ProxyChannelInitializer.java | 3 +-- .../opensearch/migrations/replay/Accumulation.java | 2 +- .../migrations/replay/ClientConnectionPool.java | 2 +- .../PacketToTransformingHttpHandlerFactory.java | 1 - .../migrations/replay/RequestResponsePacketPair.java | 2 -- .../opensearch/migrations/replay/TrafficReplayer.java | 2 +- .../datahandlers/NettyPacketToHttpConsumer.java | 2 +- .../http/HttpJsonTransformingConsumer.java | 4 +--- .../migrations/replay/tracing/ChannelKeyContext.java | 7 +++---- .../replay}/tracing/IReplayerRequestContext.java | 3 ++- .../migrations/replay/tracing/RequestContext.java | 11 ++++------- .../opensearch/migrations/replay/TestRequestKey.java | 5 +---- 25 files changed, 46 insertions(+), 64 deletions(-) delete mode 100644 TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/ContextWithSpan.java rename TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/{coreutils/SpanGenerator.java => tracing/ISpanGenerator.java} (52%) rename TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/{coreutils/SpanWithParentGenerator.java => tracing/ISpanWithParentGenerator.java} (50%) rename TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/{coreutils => tracing}/SimpleMeteringClosure.java (95%) rename TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/{ => commoncontexts}/IConnectionContext.java (81%) rename TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/{ => commoncontexts}/IRequestContext.java (62%) rename TrafficCapture/{coreUtilities/src/main/java/org/opensearch/migrations => trafficReplayer/src/main/java/org/opensearch/migrations/replay}/tracing/IReplayerRequestContext.java (81%) diff --git a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java index b2df34d69..25c1c967b 100644 --- a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java +++ b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java @@ -10,7 +10,7 @@ import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; import org.apache.kafka.clients.producer.RecordMetadata; -import org.opensearch.migrations.coreutils.SimpleMeteringClosure; +import org.opensearch.migrations.tracing.SimpleMeteringClosure; import org.opensearch.migrations.trafficcapture.CodedOutputStreamHolder; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; diff --git a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/tracing/KafkaRecordContext.java b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/tracing/KafkaRecordContext.java index 1e179a21e..140cfb601 100644 --- a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/tracing/KafkaRecordContext.java +++ b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/tracing/KafkaRecordContext.java @@ -5,9 +5,8 @@ import io.opentelemetry.api.trace.Span; import lombok.AllArgsConstructor; import lombok.Getter; -import org.opensearch.migrations.coreutils.SpanGenerator; -import org.opensearch.migrations.coreutils.SpanWithParentGenerator; -import org.opensearch.migrations.tracing.IConnectionContext; +import org.opensearch.migrations.tracing.ISpanWithParentGenerator; +import org.opensearch.migrations.tracing.commoncontexts.IConnectionContext; import org.opensearch.migrations.tracing.IWithAttributes; import org.opensearch.migrations.tracing.IWithStartTime; @@ -32,7 +31,7 @@ public class KafkaRecordContext implements IWithAttributes, @Getter public final int recordSize; - public KafkaRecordContext(IConnectionContext enclosingScope, SpanWithParentGenerator incomingSpan, + public KafkaRecordContext(IConnectionContext enclosingScope, ISpanWithParentGenerator incomingSpan, String topic, String recordId, int recordSize) { this.enclosingScope = enclosingScope; this.topic = topic; diff --git a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/tracing/ConnectionContext.java b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/tracing/ConnectionContext.java index d661377ea..fa43248cb 100644 --- a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/tracing/ConnectionContext.java +++ b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/tracing/ConnectionContext.java @@ -2,9 +2,9 @@ import io.opentelemetry.api.trace.Span; import lombok.Getter; -import org.opensearch.migrations.coreutils.SpanGenerator; -import org.opensearch.migrations.coreutils.SpanWithParentGenerator; -import org.opensearch.migrations.tracing.IConnectionContext; +import org.opensearch.migrations.tracing.ISpanGenerator; +import org.opensearch.migrations.tracing.ISpanWithParentGenerator; +import org.opensearch.migrations.tracing.commoncontexts.IConnectionContext; import org.opensearch.migrations.tracing.IWithStartTime; import java.time.Instant; @@ -19,14 +19,14 @@ public class ConnectionContext implements IConnectionContext, IWithStartTime { @Getter private final Instant startTime; - public ConnectionContext(ConnectionContext oldContext, SpanWithParentGenerator spanGenerator) { + public ConnectionContext(ConnectionContext oldContext, ISpanWithParentGenerator spanGenerator) { this.connectionId = oldContext.getConnectionId(); this.nodeId = oldContext.getNodeId(); this.startTime = Instant.now(); this.currentSpan = spanGenerator.apply(getPopulatedAttributes(), oldContext.getCurrentSpan()); } - public ConnectionContext(String connectionId, String nodeId, SpanGenerator spanGenerator) { + public ConnectionContext(String connectionId, String nodeId, ISpanGenerator spanGenerator) { this.connectionId = connectionId; this.nodeId = nodeId; this.currentSpan = spanGenerator.apply(getPopulatedAttributes()); diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/ContextWithSpan.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/ContextWithSpan.java deleted file mode 100644 index 80d419891..000000000 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/ContextWithSpan.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.opensearch.migrations.tracing; - -import io.opentelemetry.api.trace.Span; -import lombok.AllArgsConstructor; - -@AllArgsConstructor -public class ContextWithSpan> { - public final T context; - public final Span span; -} diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SpanGenerator.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/ISpanGenerator.java similarity index 52% rename from TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SpanGenerator.java rename to TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/ISpanGenerator.java index 188f04099..84eb59192 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SpanGenerator.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/ISpanGenerator.java @@ -1,8 +1,8 @@ -package org.opensearch.migrations.coreutils; +package org.opensearch.migrations.tracing; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Span; import java.util.function.Function; -public interface SpanGenerator extends Function { } +public interface ISpanGenerator extends Function { } diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SpanWithParentGenerator.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/ISpanWithParentGenerator.java similarity index 50% rename from TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SpanWithParentGenerator.java rename to TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/ISpanWithParentGenerator.java index c404e46b9..bdd4dc066 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SpanWithParentGenerator.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/ISpanWithParentGenerator.java @@ -1,9 +1,9 @@ -package org.opensearch.migrations.coreutils; +package org.opensearch.migrations.tracing; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Span; import java.util.function.BiFunction; -public interface SpanWithParentGenerator extends BiFunction { +public interface ISpanWithParentGenerator extends BiFunction { } diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/SimpleMeteringClosure.java similarity index 95% rename from TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java rename to TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/SimpleMeteringClosure.java index ad6b484f0..3714739c7 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/coreutils/SimpleMeteringClosure.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/SimpleMeteringClosure.java @@ -1,4 +1,4 @@ -package org.opensearch.migrations.coreutils; +package org.opensearch.migrations.tracing; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.Attributes; @@ -19,8 +19,6 @@ import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; -import org.opensearch.migrations.tracing.IWithAttributes; -import org.opensearch.migrations.tracing.IWithStartTime; import java.time.Duration; import java.time.Instant; @@ -135,7 +133,7 @@ public void meterHistogram(IWithAttributes ctx, String eventName, String units, .build()); } - public SpanGenerator makeSpanContinuation(String spanName, Span parentSpan) { + public ISpanGenerator makeSpanContinuation(String spanName, Span parentSpan) { var builder = tracer.spanBuilder(spanName); return (attrs) -> getSpanWithParent(builder, attrs, parentSpan); } @@ -146,7 +144,7 @@ public static Span getSpanWithParent(SpanBuilder builder, Attributes attrs, Span .startSpan().setAllAttributes(attrs); } - public SpanWithParentGenerator makeSpanContinuation(String spanName) { + public ISpanWithParentGenerator makeSpanContinuation(String spanName) { var builder = tracer.spanBuilder(spanName); return (attrs,parentSpan) -> getSpanWithParent(builder, attrs, parentSpan); } diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IConnectionContext.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/commoncontexts/IConnectionContext.java similarity index 81% rename from TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IConnectionContext.java rename to TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/commoncontexts/IConnectionContext.java index 075ba18f1..83476b9fa 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IConnectionContext.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/commoncontexts/IConnectionContext.java @@ -1,7 +1,9 @@ -package org.opensearch.migrations.tracing; +package org.opensearch.migrations.tracing.commoncontexts; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.AttributesBuilder; +import org.opensearch.migrations.tracing.EmptyContext; +import org.opensearch.migrations.tracing.IWithAttributes; public interface IConnectionContext extends IWithAttributes { static final AttributeKey CONNECTION_ID_ATTR = AttributeKey.stringKey("connectionId"); diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IRequestContext.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/commoncontexts/IRequestContext.java similarity index 62% rename from TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IRequestContext.java rename to TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/commoncontexts/IRequestContext.java index c199c7aa8..c6b932551 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IRequestContext.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/commoncontexts/IRequestContext.java @@ -1,15 +1,16 @@ -package org.opensearch.migrations.tracing; +package org.opensearch.migrations.tracing.commoncontexts; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.AttributesBuilder; +import org.opensearch.migrations.tracing.IWithAttributes; public interface IRequestContext extends IWithAttributes { static final AttributeKey SOURCE_REQUEST_INDEX_KEY = AttributeKey.longKey("sourceRequestIndex"); - long sourceRequestIndex(); + long getSourceRequestIndex(); @Override default AttributesBuilder fillAttributes(AttributesBuilder builder) { - return builder.put(SOURCE_REQUEST_INDEX_KEY, sourceRequestIndex()); + return builder.put(SOURCE_REQUEST_INDEX_KEY, getSourceRequestIndex()); } } diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java index c1814be03..4dcb35ea0 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java @@ -18,7 +18,7 @@ import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; -import org.opensearch.migrations.coreutils.SimpleMeteringClosure; +import org.opensearch.migrations.tracing.SimpleMeteringClosure; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import org.opensearch.migrations.coreutils.MetricsLogger; import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java index fce1be7e2..c82c1b27e 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java @@ -8,7 +8,7 @@ import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; import org.opensearch.migrations.coreutils.MetricsLogger; -import org.opensearch.migrations.coreutils.SimpleMeteringClosure; +import org.opensearch.migrations.tracing.SimpleMeteringClosure; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java index c4736c75f..7350e1f4f 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java @@ -17,7 +17,7 @@ import org.apache.kafka.common.config.SaslConfigs; import org.apache.logging.log4j.core.util.NullOutputStream; import org.opensearch.common.settings.Settings; -import org.opensearch.migrations.coreutils.SimpleMeteringClosure; +import org.opensearch.migrations.tracing.SimpleMeteringClosure; import org.opensearch.migrations.trafficcapture.CodedOutputStreamHolder; import org.opensearch.migrations.trafficcapture.FileConnectionCaptureFactory; import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/FrontsideHandler.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/FrontsideHandler.java index 14416838b..d3347d8ce 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/FrontsideHandler.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/FrontsideHandler.java @@ -8,6 +8,10 @@ import io.netty.util.ReferenceCountUtil; import lombok.extern.slf4j.Slf4j; +/** + * TODO - this should be renamed ForwardingHandler. It's the last handler of the front, + * but since it isn't the front of the frontside handlers, this name seems misleading. + */ @Slf4j public class FrontsideHandler extends ChannelInboundHandlerAdapter { diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java index feddb9f91..f1e86ff2a 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java @@ -5,8 +5,7 @@ import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.ssl.SslHandler; -import org.opensearch.migrations.coreutils.SimpleMeteringClosure; -import org.opensearch.migrations.tracing.EmptyContext; +import org.opensearch.migrations.tracing.SimpleMeteringClosure; import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; import org.opensearch.migrations.trafficcapture.netty.ConditionallyReliableLoggingHttpRequestHandler; import org.opensearch.migrations.trafficcapture.netty.LoggingHttpResponseHandler; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java index 7b76a000b..8d657613a 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java @@ -1,7 +1,7 @@ package org.opensearch.migrations.replay; import lombok.NonNull; -import org.opensearch.migrations.coreutils.SimpleMeteringClosure; +import org.opensearch.migrations.tracing.SimpleMeteringClosure; import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java index e04d45e87..a40cbb01a 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java @@ -14,7 +14,7 @@ import io.opentelemetry.context.ContextKey; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.opensearch.migrations.coreutils.SimpleMeteringClosure; +import org.opensearch.migrations.tracing.SimpleMeteringClosure; import org.opensearch.migrations.replay.datahandlers.NettyPacketToHttpConsumer; import org.opensearch.migrations.replay.datatypes.ConnectionReplaySession; import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java index 07b8c7647..d827f28bf 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java @@ -1,7 +1,6 @@ package org.opensearch.migrations.replay; import lombok.extern.slf4j.Slf4j; -import org.opensearch.migrations.coreutils.SimpleMeteringClosure; import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer; import org.opensearch.migrations.replay.datahandlers.TransformedPacketReceiver; import org.opensearch.migrations.replay.datahandlers.http.HttpJsonTransformingConsumer; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestResponsePacketPair.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestResponsePacketPair.java index 45f686eb7..f6cf9dbde 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestResponsePacketPair.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestResponsePacketPair.java @@ -5,14 +5,12 @@ import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.tracing.RequestContext; -import org.opensearch.migrations.tracing.IRequestContext; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Optional; @Slf4j public class RequestResponsePacketPair { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index 086cb14e2..85b90d822 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -14,7 +14,7 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsLogger; -import org.opensearch.migrations.coreutils.SimpleMeteringClosure; +import org.opensearch.migrations.tracing.SimpleMeteringClosure; import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer; import org.opensearch.migrations.replay.tracing.ChannelKeyContext; import org.opensearch.migrations.replay.tracing.RequestContext; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java index bf85d026f..35615fe44 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java @@ -23,7 +23,7 @@ import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; import org.opensearch.migrations.coreutils.MetricsLogger; -import org.opensearch.migrations.coreutils.SimpleMeteringClosure; +import org.opensearch.migrations.tracing.SimpleMeteringClosure; import org.opensearch.migrations.replay.AggregatedRawResponse; import org.opensearch.migrations.replay.netty.BacksideHttpWatcherHandler; import org.opensearch.migrations.replay.netty.BacksideSnifferHandler; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java index ae86fd5a9..4df9735dc 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java @@ -3,14 +3,13 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.http.HttpRequestDecoder; -import io.opentelemetry.api.common.AttributesBuilder; import io.opentelemetry.api.trace.Span; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; import org.opensearch.migrations.coreutils.MetricsLogger; -import org.opensearch.migrations.coreutils.SimpleMeteringClosure; +import org.opensearch.migrations.tracing.SimpleMeteringClosure; import org.opensearch.migrations.replay.datatypes.HttpRequestTransformationStatus; import org.opensearch.migrations.replay.datatypes.TransformedOutputAndResult; import org.opensearch.migrations.replay.Utils; @@ -18,7 +17,6 @@ import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture; -import org.opensearch.migrations.tracing.IWithAttributes; import org.opensearch.migrations.tracing.IWithStartTimeAndAttributes; import org.opensearch.migrations.transform.IAuthTransformerFactory; import org.opensearch.migrations.transform.IJsonTransformer; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelKeyContext.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelKeyContext.java index 075b49f08..3a8780fbe 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelKeyContext.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelKeyContext.java @@ -2,10 +2,9 @@ import io.opentelemetry.api.trace.Span; import lombok.Getter; -import org.opensearch.migrations.coreutils.SpanGenerator; -import org.opensearch.migrations.coreutils.SpanWithParentGenerator; +import org.opensearch.migrations.tracing.ISpanGenerator; import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; -import org.opensearch.migrations.tracing.IConnectionContext; +import org.opensearch.migrations.tracing.commoncontexts.IConnectionContext; public class ChannelKeyContext implements IConnectionContext { @Getter @@ -13,7 +12,7 @@ public class ChannelKeyContext implements IConnectionContext { @Getter final Span currentSpan; - public ChannelKeyContext(ISourceTrafficChannelKey channelKey, SpanGenerator spanGenerator) { + public ChannelKeyContext(ISourceTrafficChannelKey channelKey, ISpanGenerator spanGenerator) { this.channelKey = channelKey; this.currentSpan = spanGenerator.apply(getPopulatedAttributes()); } diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IReplayerRequestContext.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/IReplayerRequestContext.java similarity index 81% rename from TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IReplayerRequestContext.java rename to TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/IReplayerRequestContext.java index 5bb61fce0..3de2e3dc1 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/tracing/IReplayerRequestContext.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/IReplayerRequestContext.java @@ -1,7 +1,8 @@ -package org.opensearch.migrations.tracing; +package org.opensearch.migrations.replay.tracing; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.AttributesBuilder; +import org.opensearch.migrations.tracing.commoncontexts.IRequestContext; public interface IReplayerRequestContext extends IRequestContext { static final AttributeKey REPLAYER_REQUEST_INDEX_KEY = AttributeKey.longKey("replayerRequestIndex"); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java index 947ba19fb..2fd747f32 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java @@ -2,14 +2,11 @@ import io.opentelemetry.api.trace.Span; import lombok.Getter; -import org.opensearch.migrations.coreutils.SpanGenerator; -import org.opensearch.migrations.coreutils.SpanWithParentGenerator; +import org.opensearch.migrations.tracing.ISpanWithParentGenerator; import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; import org.opensearch.migrations.tracing.EmptyContext; -import org.opensearch.migrations.tracing.IConnectionContext; -import org.opensearch.migrations.tracing.IReplayerRequestContext; -import org.opensearch.migrations.tracing.IRequestContext; +import org.opensearch.migrations.tracing.commoncontexts.IConnectionContext; import org.opensearch.migrations.tracing.IWithAttributes; import org.opensearch.migrations.tracing.IWithStartTime; @@ -28,7 +25,7 @@ public class RequestContext implements IReplayerRequestContext, IWithStartTime { IWithAttributes> foo; public RequestContext(ChannelKeyContext enclosingScope, UniqueReplayerRequestKey replayerRequestKey, - SpanWithParentGenerator spanGenerator) { + ISpanWithParentGenerator spanGenerator) { this.enclosingScope = enclosingScope; this.replayerRequestKey = replayerRequestKey; this.startTime = Instant.now(); @@ -49,7 +46,7 @@ public String getNodeId() { } @Override - public long sourceRequestIndex() { + public long getSourceRequestIndex() { return replayerRequestKey.getSourceRequestIndex(); } diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java index a47c7b98f..5089e6d7f 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java @@ -1,13 +1,10 @@ package org.opensearch.migrations.replay; -import io.opentelemetry.api.GlobalOpenTelemetry; -import io.opentelemetry.api.common.Attributes; -import org.opensearch.migrations.coreutils.SimpleMeteringClosure; +import org.opensearch.migrations.tracing.SimpleMeteringClosure; import org.opensearch.migrations.replay.datatypes.PojoTrafficStreamKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; import org.opensearch.migrations.replay.tracing.ChannelKeyContext; import org.opensearch.migrations.replay.tracing.RequestContext; -import org.opensearch.migrations.tracing.EmptyContext; public class TestRequestKey { From 723bf778f1a505a0c2ef59d5c96a8d9f6126c9f7 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Fri, 1 Dec 2023 12:40:57 -0500 Subject: [PATCH 13/20] Bundle all of the offloader spans with the netty handler spans. Signed-off-by: Greg Schohn --- .../kafkaoffloader/KafkaCaptureFactory.java | 18 ++- TrafficCapture/captureOffloader/build.gradle | 4 +- .../FileConnectionCaptureFactory.java | 5 +- .../IConnectionCaptureFactory.java | 3 +- .../tracing/ConnectionContext.java | 2 +- .../InMemoryConnectionCaptureFactory.java | 3 +- .../src/main/docker/docker-compose.yml | 1 + ...allyReliableLoggingHttpRequestHandler.java | 31 +++-- .../netty/LoggingHttpRequestHandler.java | 81 ++++++++---- .../netty/LoggingHttpResponseHandler.java | 118 +++++++----------- .../netty/RequestContextStateMachine.java | 24 ++++ .../netty/tracing/HttpMessageContext.java | 39 ++++++ ...ReliableLoggingHttpRequestHandlerTest.java | 10 +- .../proxyserver/CaptureProxy.java | 2 +- .../netty/ProxyChannelInitializer.java | 10 +- .../http/HttpJsonTransformingConsumer.java | 6 +- .../replay/tracing/RequestContext.java | 4 - ...afficToHttpTransactionAccumulatorTest.java | 5 +- .../src/test/resources/log4j2.properties | 5 + 19 files changed, 227 insertions(+), 144 deletions(-) create mode 100644 TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/RequestContextStateMachine.java create mode 100644 TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/tracing/HttpMessageContext.java diff --git a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java index 25c1c967b..c1c84ad81 100644 --- a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java +++ b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java @@ -11,6 +11,7 @@ import org.opensearch.migrations.coreutils.MetricsEvent; import org.apache.kafka.clients.producer.RecordMetadata; import org.opensearch.migrations.tracing.SimpleMeteringClosure; +import org.opensearch.migrations.tracing.commoncontexts.IConnectionContext; import org.opensearch.migrations.trafficcapture.CodedOutputStreamHolder; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; @@ -60,11 +61,8 @@ public KafkaCaptureFactory(String nodeId, Producer producer, int } @Override - public IChannelConnectionCaptureSerializer createOffloader(ConnectionContext ctx, + public IChannelConnectionCaptureSerializer createOffloader(IConnectionContext ctx, String connectionId) { - METERING_CLOSURE.meterIncrementEvent(ctx, "offloader_created"); - METERING_CLOSURE.meterDeltaEvent(ctx, "offloaders_active", 1); - return new StreamChannelConnectionCaptureSerializer<>(nodeId, connectionId, new StreamManager(ctx, connectionId)); } @@ -81,13 +79,15 @@ static class CodedOutputStreamWrapper implements CodedOutputStreamHolder { } class StreamManager extends OrderedStreamLifecyleManager { - ConnectionContext telemetryContext; + IConnectionContext telemetryContext; String connectionId; Instant startTime; - public StreamManager(ConnectionContext incomingTelemetryContext, String connectionId) { - this.telemetryContext = new ConnectionContext(incomingTelemetryContext, - METERING_CLOSURE.makeSpanContinuation("offloaderLifetime")); + public StreamManager(IConnectionContext ctx, String connectionId) { + this.telemetryContext = ctx; + METERING_CLOSURE.meterIncrementEvent(telemetryContext, "offloader_created"); + METERING_CLOSURE.meterDeltaEvent(telemetryContext, "offloaders_active", 1); + this.connectionId = connectionId; this.startTime = Instant.now(); } @@ -99,8 +99,6 @@ public void close() throws IOException { Duration.between(startTime, Instant.now())); METERING_CLOSURE.meterDeltaEvent(telemetryContext, "offloaders_active", -1); METERING_CLOSURE.meterIncrementEvent(telemetryContext, "offloader_closed"); - - telemetryContext.currentSpan.end(); } @Override diff --git a/TrafficCapture/captureOffloader/build.gradle b/TrafficCapture/captureOffloader/build.gradle index 9c183b4a0..d4b9848dd 100644 --- a/TrafficCapture/captureOffloader/build.gradle +++ b/TrafficCapture/captureOffloader/build.gradle @@ -35,7 +35,7 @@ dependencies { testImplementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl', version: '2.20.0' testImplementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7' - testFixturesImplementation "com.google.protobuf:protobuf-java:3.22.2" testFixturesImplementation project(':captureProtobufs') - + testFixturesImplementation project(':coreUtilities') + testFixturesImplementation group: 'com.google.protobuf', name: 'protobuf-java', version: '3.22.2' } diff --git a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java index b7ce9c029..a566b9e9c 100644 --- a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java +++ b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java @@ -3,6 +3,7 @@ import lombok.AllArgsConstructor; import lombok.Lombok; import lombok.extern.slf4j.Slf4j; +import org.opensearch.migrations.tracing.commoncontexts.IConnectionContext; import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; import java.io.FileNotFoundException; @@ -84,7 +85,7 @@ public CodedOutputStreamAndByteBufferWrapper createStream() { } @Override - public IChannelConnectionCaptureSerializer createOffloader(ConnectionContext ctx, String connectionId) { - return new StreamChannelConnectionCaptureSerializer(nodeId, connectionId, new StreamManager(connectionId)); + public IChannelConnectionCaptureSerializer createOffloader(IConnectionContext ctx, String connectionId) { + return new StreamChannelConnectionCaptureSerializer<>(nodeId, connectionId, new StreamManager(connectionId)); } } diff --git a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/IConnectionCaptureFactory.java b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/IConnectionCaptureFactory.java index c5c5270e5..1b8def0e1 100644 --- a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/IConnectionCaptureFactory.java +++ b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/IConnectionCaptureFactory.java @@ -1,9 +1,10 @@ package org.opensearch.migrations.trafficcapture; +import org.opensearch.migrations.tracing.commoncontexts.IConnectionContext; import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; import java.io.IOException; public interface IConnectionCaptureFactory { - IChannelConnectionCaptureSerializer createOffloader(ConnectionContext ctx, String connectionId) throws IOException; + IChannelConnectionCaptureSerializer createOffloader(IConnectionContext ctx, String connectionId) throws IOException; } diff --git a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/tracing/ConnectionContext.java b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/tracing/ConnectionContext.java index fa43248cb..c6d3bc5f0 100644 --- a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/tracing/ConnectionContext.java +++ b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/tracing/ConnectionContext.java @@ -19,7 +19,7 @@ public class ConnectionContext implements IConnectionContext, IWithStartTime { @Getter private final Instant startTime; - public ConnectionContext(ConnectionContext oldContext, ISpanWithParentGenerator spanGenerator) { + public ConnectionContext(IConnectionContext oldContext, ISpanWithParentGenerator spanGenerator) { this.connectionId = oldContext.getConnectionId(); this.nodeId = oldContext.getNodeId(); this.startTime = Instant.now(); diff --git a/TrafficCapture/captureOffloader/src/testFixtures/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java b/TrafficCapture/captureOffloader/src/testFixtures/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java index 3c899e2eb..24c7718dd 100644 --- a/TrafficCapture/captureOffloader/src/testFixtures/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java +++ b/TrafficCapture/captureOffloader/src/testFixtures/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java @@ -3,6 +3,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import lombok.AllArgsConstructor; import lombok.Getter; +import org.opensearch.migrations.tracing.commoncontexts.IConnectionContext; import org.opensearch.migrations.trafficcapture.protos.TrafficStream; import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; @@ -60,7 +61,7 @@ protected CompletableFuture kickoffCloseStream(CodedOutputStreamHolder out } @Override - public IChannelConnectionCaptureSerializer createOffloader(ConnectionContext ctx, String connectionId) throws IOException { + public IChannelConnectionCaptureSerializer createOffloader(IConnectionContext ctx, String connectionId) throws IOException { // This array is only an indirection to work around Java's constraint that lambda values are final return new StreamChannelConnectionCaptureSerializer<>(nodeId, connectionId, new StreamManager()); } diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 1085d203f..18a2c9f89 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -153,6 +153,7 @@ services: - sharedReplayerOutput:/shared-replayer-output environment: - MIGRATION_KAFKA_BROKER_ENDPOINTS=kafka:9092 +# command: ./runTestBenchmarks.sh volumes: zookeeper_data: diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java index b01a1dcb3..29df56e56 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java @@ -3,24 +3,31 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.HttpRequest; import io.netty.util.ReferenceCountUtil; -import io.opentelemetry.context.ContextKey; +import io.opentelemetry.api.trace.Span; +import lombok.Getter; import lombok.Lombok; import lombok.extern.slf4j.Slf4j; +import org.opensearch.migrations.tracing.IWithStartTimeAndAttributes; +import org.opensearch.migrations.tracing.commoncontexts.IConnectionContext; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; +import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; +import org.opensearch.migrations.trafficcapture.netty.tracing.HttpMessageContext; import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; +import java.io.IOException; import java.time.Instant; +import java.util.function.Function; import java.util.function.Predicate; @Slf4j public class ConditionallyReliableLoggingHttpRequestHandler extends LoggingHttpRequestHandler { - private ContextKey START_FLUSH_KEY = ContextKey.named("startTime"); private final Predicate shouldBlockPredicate; - public ConditionallyReliableLoggingHttpRequestHandler(ConnectionContext incomingContext, - IChannelConnectionCaptureSerializer trafficOffloader, - Predicate headerPredicateForWhenToBlock) { - super(incomingContext, trafficOffloader); + public ConditionallyReliableLoggingHttpRequestHandler(String nodeId, String connectionId, + IConnectionCaptureFactory trafficOffloaderFactory, + Predicate headerPredicateForWhenToBlock) + throws IOException { + super(nodeId, connectionId, trafficOffloaderFactory); this.shouldBlockPredicate = headerPredicateForWhenToBlock; } @@ -28,9 +35,13 @@ public ConditionallyReliableLoggingHttpRequestHandler(ConnectionContext incoming protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Object msg, HttpRequest httpRequest) throws Exception { if (shouldBlockPredicate.test(httpRequest)) { - METERING_CLOSURE.meterIncrementEvent(connectionContext, "blockingRequestUntilFlush"); - var flushContext = new ConnectionContext(connectionContext, - METERING_CLOSURE.makeSpanContinuation("blockedForFlush")); + METERING_CLOSURE.meterIncrementEvent(messageContext, "blockingRequestUntilFlush"); + var flushContext = new IWithStartTimeAndAttributes<>() { + @Getter Span currentSpan = METERING_CLOSURE.makeSpanContinuation("blockedForFlush") + .apply(messageContext.getPopulatedAttributes(), messageContext.getCurrentSpan()); + @Getter Instant startTime = Instant.now(); + @Override public HttpMessageContext getEnclosingScope() { return messageContext; } + }; trafficOffloader.flushCommitAndResetStream(false).whenComplete((result, t) -> { log.atInfo().setMessage(()->"Done flushing").log(); @@ -55,7 +66,7 @@ protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Ob } }); } else { - METERING_CLOSURE.meterIncrementEvent(connectionContext, "nonBlockingRequest"); + METERING_CLOSURE.meterIncrementEvent(messageContext, "nonBlockingRequest"); super.channelFinishedReadingAnHttpMessage(ctx, msg, httpRequest); } } diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java index 4dcb35ea0..aa4c5ef77 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java @@ -1,8 +1,10 @@ package org.opensearch.migrations.trafficcapture.netty; import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelDuplexHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.ChannelPromise; import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.http.DefaultHttpRequest; import io.netty.handler.codec.http.HttpContent; @@ -18,18 +20,24 @@ import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; +import org.opensearch.migrations.tracing.IWithAttributes; import org.opensearch.migrations.tracing.SimpleMeteringClosure; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import org.opensearch.migrations.coreutils.MetricsLogger; +import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; +import org.opensearch.migrations.trafficcapture.netty.tracing.HttpMessageContext; import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; +import java.io.IOException; import java.time.Instant; @Slf4j -public class LoggingHttpRequestHandler extends ChannelInboundHandlerAdapter { - public static final String TELEMETRY_SCOPE_NAME = "LoggingHttpInboundHandler"; +public class LoggingHttpRequestHandler extends ChannelDuplexHandler { + public static final String TELEMETRY_SCOPE_NAME = "CapturingHttpHandler"; public static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure(TELEMETRY_SCOPE_NAME); private static final MetricsLogger metricsLogger = new MetricsLogger("LoggingHttpRequestHandler"); + public static final String GATHERING_REQUEST = "gatheringRequest"; + public static final String GATHERING_RESPONSE = "gatheringResponse"; static class SimpleHttpRequestDecoder extends HttpRequestDecoder { /** @@ -75,20 +83,32 @@ public HttpRequest resetCurrentRequest() { protected final IChannelConnectionCaptureSerializer trafficOffloader; protected final EmbeddedChannel httpDecoderChannel; - protected final SimpleHttpRequestDecoder requestDecoder; - protected final ConnectionContext connectionContext; - public LoggingHttpRequestHandler(ConnectionContext incomingContext, - IChannelConnectionCaptureSerializer trafficOffloader) { - this.connectionContext = incomingContext; - METERING_CLOSURE.meterIncrementEvent(incomingContext, "requestStarted"); + protected HttpMessageContext messageContext; - this.trafficOffloader = trafficOffloader; - requestDecoder = new SimpleHttpRequestDecoder(); // as a field for easier debugging + public LoggingHttpRequestHandler(String nodeId, String channelKey, + IConnectionCaptureFactory trafficOffloaderFactory) + throws IOException { + var parentContext = new ConnectionContext(channelKey, nodeId, + METERING_CLOSURE.makeSpanContinuation("connectionLifetime", null)); + + this.messageContext = new HttpMessageContext(parentContext, 0, HttpMessageContext.Direction.REQUEST, + METERING_CLOSURE.makeSpanContinuation(GATHERING_REQUEST)); + METERING_CLOSURE.meterIncrementEvent(messageContext, "requestStarted"); + + this.trafficOffloader = trafficOffloaderFactory.createOffloader(parentContext, channelKey); httpDecoderChannel = new EmbeddedChannel( - requestDecoder, - new SimpleDecodedHttpRequestHandler() - ); + new SimpleHttpRequestDecoder(), + new SimpleDecodedHttpRequestHandler()); + } + + public void rotateNextMessageContext() { + messageContext.getCurrentSpan().end(); + final var wasResponse = messageContext.getDirection() == HttpMessageContext.Direction.RESPONSE; + messageContext = new HttpMessageContext(messageContext.getEnclosingScope(), + (wasResponse ? 1 : 0) + messageContext.getSourceRequestIndex(), + (wasResponse ? HttpMessageContext.Direction.REQUEST : HttpMessageContext.Direction.RESPONSE), + METERING_CLOSURE.makeSpanContinuation(wasResponse ? GATHERING_REQUEST : GATHERING_RESPONSE)); } private HttpProcessedState parseHttpMessageParts(ByteBuf msg) { @@ -96,7 +116,7 @@ private HttpProcessedState parseHttpMessageParts(ByteBuf msg) { var state = getHandlerThatHoldsParsedHttpRequest().isDone ? HttpProcessedState.FULL_MESSAGE : HttpProcessedState.ONGOING; - METERING_CLOSURE.meterIncrementEvent(connectionContext, + METERING_CLOSURE.meterIncrementEvent(messageContext, state == HttpProcessedState.FULL_MESSAGE ? "requestFullyParsed" : "requestPartiallyParsed"); return state; } @@ -108,7 +128,7 @@ private SimpleDecodedHttpRequestHandler getHandlerThatHoldsParsedHttpRequest() { @Override public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { trafficOffloader.addCloseEvent(Instant.now()); - METERING_CLOSURE.meterIncrementEvent(connectionContext, "unregistered"); + METERING_CLOSURE.meterIncrementEvent(messageContext, "unregistered"); trafficOffloader.flushCommitAndResetStream(true).whenComplete((result, t) -> { if (t != null) { log.warn("Got error: " + t.getMessage()); @@ -125,8 +145,9 @@ public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { @Override public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { - METERING_CLOSURE.meterIncrementEvent(connectionContext, "handlerRemoved"); - connectionContext.getCurrentSpan().end(); + METERING_CLOSURE.meterIncrementEvent(messageContext, "handlerRemoved"); + messageContext.getCurrentSpan().end(); + messageContext.getEnclosingScope().currentSpan.end(); trafficOffloader.flushCommitAndResetStream(true).whenComplete((result, t) -> { if (t != null) { @@ -143,7 +164,7 @@ public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Object msg, HttpRequest httpRequest) throws Exception { super.channelRead(ctx, msg); - METERING_CLOSURE.meterIncrementEvent(connectionContext, "requestReceived"); + METERING_CLOSURE.meterIncrementEvent(messageContext, "requestReceived"); metricsLogger.atSuccess(MetricsEvent.RECEIVED_FULL_HTTP_REQUEST) .setAttribute(MetricsAttributeKey.CHANNEL_ID, ctx.channel().id().asLongText()) @@ -153,13 +174,16 @@ protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Ob @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + if (messageContext.getDirection() == HttpMessageContext.Direction.RESPONSE) { + rotateNextMessageContext(); + } var timestamp = Instant.now(); HttpProcessedState httpProcessedState; { var bb = ((ByteBuf) msg).retainedDuplicate(); trafficOffloader.addReadEvent(timestamp, bb); - METERING_CLOSURE.meterIncrementEvent(connectionContext, "read"); - METERING_CLOSURE.meterIncrementEvent(connectionContext, "readBytes", bb.readableBytes()); + METERING_CLOSURE.meterIncrementEvent(messageContext, "read"); + METERING_CLOSURE.meterIncrementEvent(messageContext, "readBytes", bb.readableBytes()); metricsLogger.atSuccess(MetricsEvent.RECEIVED_REQUEST_COMPONENT) .setAttribute(MetricsAttributeKey.CHANNEL_ID, ctx.channel().id().asLongText()).emit(); @@ -181,10 +205,25 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception } } + @Override + public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { + if (messageContext.getDirection() == HttpMessageContext.Direction.REQUEST) { + rotateNextMessageContext(); + } + var bb = (ByteBuf) msg; + trafficOffloader.addWriteEvent(Instant.now(), bb); + metricsLogger.atSuccess(MetricsEvent.RECEIVED_RESPONSE_COMPONENT) + .setAttribute(MetricsAttributeKey.CHANNEL_ID, ctx.channel().id().asLongText()).emit(); + METERING_CLOSURE.meterIncrementEvent(messageContext, "write"); + METERING_CLOSURE.meterIncrementEvent(messageContext, "writeBytes", bb.readableBytes()); + + super.write(ctx, msg, promise); + } + @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { trafficOffloader.addExceptionCaughtEvent(Instant.now(), cause); - METERING_CLOSURE.meterIncrementEvent(connectionContext, "exception"); + METERING_CLOSURE.meterIncrementEvent(messageContext, "exception"); httpDecoderChannel.close(); super.exceptionCaught(ctx, cause); } diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java index c82c1b27e..50d263550 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java @@ -12,8 +12,6 @@ import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; -import java.net.SocketAddress; -import java.time.Duration; import java.time.Instant; @Slf4j @@ -24,83 +22,55 @@ public class LoggingHttpResponseHandler extends ChannelOutboundHandlerAdapter private final IChannelConnectionCaptureSerializer trafficOffloader; private ConnectionContext telemetryContext; - private Instant connectTime; public LoggingHttpResponseHandler(ConnectionContext incomingContext, IChannelConnectionCaptureSerializer trafficOffloader) { this.trafficOffloader = trafficOffloader; this.telemetryContext = incomingContext; } - - @Override - public void bind(ChannelHandlerContext ctx, SocketAddress localAddress, ChannelPromise promise) throws Exception { - trafficOffloader.addBindEvent(Instant.now(), localAddress); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "bind"); - super.bind(ctx, localAddress, promise); - } - - @Override - public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, SocketAddress localAddress, ChannelPromise promise) throws Exception { - trafficOffloader.addConnectEvent(Instant.now(), remoteAddress, localAddress); - - telemetryContext = new ConnectionContext(telemetryContext, - METERING_CLOSURE.makeSpanContinuation("backendConnection")); - connectTime = Instant.now(); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "connect"); - METERING_CLOSURE.meterDeltaEvent(telemetryContext, "connections", 1); - - super.connect(ctx, remoteAddress, localAddress, promise); - } - - @Override - public void disconnect(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { - trafficOffloader.addDisconnectEvent(Instant.now()); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "disconnect"); - super.disconnect(ctx, promise); - } - - @Override - public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { - trafficOffloader.addCloseEvent(Instant.now()); - - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "close"); - METERING_CLOSURE.meterDeltaEvent(telemetryContext, "connections", -1); - METERING_CLOSURE.meterHistogramMillis(telemetryContext, "connectionDuration", - Duration.between(connectTime, Instant.now())); - telemetryContext.currentSpan.end(); - } - - @Override - public void deregister(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { - trafficOffloader.addDeregisterEvent(Instant.now()); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "deregister"); - super.deregister(ctx, promise); - } - - @Override - public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { - var bb = (ByteBuf) msg; - trafficOffloader.addWriteEvent(Instant.now(), bb); - metricsLogger.atSuccess(MetricsEvent.RECEIVED_RESPONSE_COMPONENT) - .setAttribute(MetricsAttributeKey.CHANNEL_ID, ctx.channel().id().asLongText()).emit(); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "write"); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "writeBytes", bb.readableBytes()); - - super.write(ctx, msg, promise); - } - - @Override - public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { - flush(ctx); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "removed"); - super.handlerRemoved(ctx); - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - trafficOffloader.addExceptionCaughtEvent(Instant.now(), cause); - METERING_CLOSURE.meterIncrementEvent(telemetryContext, "exception"); - super.exceptionCaught(ctx, cause); - } +// +// @Override +// public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, SocketAddress localAddress, ChannelPromise promise) throws Exception { +// trafficOffloader.addConnectEvent(Instant.now(), remoteAddress, localAddress); +// +// telemetryContext = new ConnectionContext(telemetryContext, +// METERING_CLOSURE.makeSpanContinuation("backendConnection")); +// METERING_CLOSURE.meterIncrementEvent(telemetryContext, "connect"); +// METERING_CLOSURE.meterDeltaEvent(telemetryContext, "connections", 1); +// +// super.connect(ctx, remoteAddress, localAddress, promise); +// } + +// @Override +// public void disconnect(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { +// trafficOffloader.addDisconnectEvent(Instant.now()); +// METERING_CLOSURE.meterIncrementEvent(telemetryContext, "disconnect"); +// super.disconnect(ctx, promise); +// } + +// @Override +// public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { +// trafficOffloader.addCloseEvent(Instant.now()); +// +// METERING_CLOSURE.meterIncrementEvent(telemetryContext, "close"); +// METERING_CLOSURE.meterDeltaEvent(telemetryContext, "connections", -1); +// METERING_CLOSURE.meterHistogramMillis(telemetryContext, "connectionDuration"); +// telemetryContext.currentSpan.end(); +// } + +// @Override +// public void deregister(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { +// trafficOffloader.addDeregisterEvent(Instant.now()); +// METERING_CLOSURE.meterIncrementEvent(telemetryContext, "deregister"); +// super.deregister(ctx, promise); +// } + +// +// @Override +// public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { +// trafficOffloader.addExceptionCaughtEvent(Instant.now(), cause); +// METERING_CLOSURE.meterIncrementEvent(telemetryContext, "exception"); +// super.exceptionCaught(ctx, cause); +// } } diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/RequestContextStateMachine.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/RequestContextStateMachine.java new file mode 100644 index 000000000..4a670e8d0 --- /dev/null +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/RequestContextStateMachine.java @@ -0,0 +1,24 @@ +package org.opensearch.migrations.trafficcapture.netty; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.opensearch.migrations.trafficcapture.netty.tracing.HttpMessageContext; +import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; + +/** + * This is a helper class so that we can emit metrics and traces for when we're + * accumulating a request vs waiting for the next response, then repeating indefinitely. + * + * TODO - this may be a performance bottleneck and we should carefully evaluate it's utility. + */ +@Slf4j +public class RequestContextStateMachine { + @Getter + public final ConnectionContext connectionContext; + @Getter + HttpMessageContext currentRequestContext; + + public RequestContextStateMachine(ConnectionContext incoming) { + connectionContext = incoming; + } +} diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/tracing/HttpMessageContext.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/tracing/HttpMessageContext.java new file mode 100644 index 000000000..db93967f7 --- /dev/null +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/tracing/HttpMessageContext.java @@ -0,0 +1,39 @@ +package org.opensearch.migrations.trafficcapture.netty.tracing; + +import io.opentelemetry.api.trace.Span; +import lombok.Getter; +import org.opensearch.migrations.tracing.ISpanWithParentGenerator; +import org.opensearch.migrations.tracing.IWithStartTimeAndAttributes; +import org.opensearch.migrations.tracing.commoncontexts.IConnectionContext; +import org.opensearch.migrations.tracing.commoncontexts.IRequestContext; +import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; + +import java.time.Instant; + +public class HttpMessageContext implements IRequestContext, IWithStartTimeAndAttributes { + + public enum Direction { + REQUEST, + RESPONSE + } + + @Getter + final long sourceRequestIndex; + @Getter + final ConnectionContext enclosingScope; + @Getter + final Instant startTime; + @Getter + final Direction direction; + @Getter + final Span currentSpan; + + public HttpMessageContext(ConnectionContext enclosingScope, long sourceRequestIndex, Direction direction, + ISpanWithParentGenerator spanGenerator) { + this.sourceRequestIndex = sourceRequestIndex; + this.enclosingScope = enclosingScope; + this.startTime = Instant.now(); + this.direction = direction; + this.currentSpan = spanGenerator.apply(getPopulatedAttributes(), enclosingScope.getCurrentSpan()); + } +} diff --git a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java index 8dfd72c79..0cc960ea7 100644 --- a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java +++ b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java @@ -80,13 +80,13 @@ private static void writeMessageAndVerify(byte[] fullTrafficBytes, Consumer outputByteBuffer = new AtomicReference<>(); AtomicInteger flushCount = new AtomicInteger(); - var offloader = new StreamChannelConnectionCaptureSerializer("Test", "connection", - new StreamManager(outputByteBuffer, flushCount)); + ; - var ctx = new ConnectionContext("c", "n", - x->GlobalOpenTelemetry.getTracer("test").spanBuilder("test").startSpan()); EmbeddedChannel channel = new EmbeddedChannel( - new ConditionallyReliableLoggingHttpRequestHandler(ctx, offloader, x->true)); // true: block every request + new ConditionallyReliableLoggingHttpRequestHandler("n", "c", + (ctx, connectionId) -> new StreamChannelConnectionCaptureSerializer("Test", connectionId, + new StreamManager(outputByteBuffer, flushCount)), + x->true)); // true: block every request channelWriter.accept(channel); // we wrote the correct data to the downstream handler/channel diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java index 7350e1f4f..d2466a3d9 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java @@ -174,7 +174,7 @@ private static Settings getSettings(@NonNull String configFile) { private static IConnectionCaptureFactory getNullConnectionCaptureFactory() { System.err.println("No trace log directory specified. Logging to /dev/null"); - return (ctx, connectionId) -> new StreamChannelConnectionCaptureSerializer<>(null, connectionId, + return (ctx,connectionId) -> new StreamChannelConnectionCaptureSerializer<>(null, connectionId, new StreamLifecycleManager<>() { @Override public void close() {} diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java index f1e86ff2a..5d492db34 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java @@ -5,6 +5,7 @@ import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.ssl.SslHandler; +import org.opensearch.migrations.tracing.IWithAttributes; import org.opensearch.migrations.tracing.SimpleMeteringClosure; import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; import org.opensearch.migrations.trafficcapture.netty.ConditionallyReliableLoggingHttpRequestHandler; @@ -45,12 +46,9 @@ protected void initChannel(SocketChannel ch) throws IOException { } var connectionId = ch.id().asLongText(); - var ctx = new ConnectionContext(connectionId, "", - METERING_CLOSURE.makeSpanContinuation("connectionLifetime", null)); - var offloader = connectionCaptureFactory.createOffloader(ctx, connectionId); - ch.pipeline().addLast(new LoggingHttpResponseHandler<>(ctx, offloader)); - ch.pipeline().addLast(new ConditionallyReliableLoggingHttpRequestHandler(ctx, offloader, - this::shouldGuaranteeMessageOffloading)); + var capturingHandler = new ConditionallyReliableLoggingHttpRequestHandler<>(null, connectionId, + connectionCaptureFactory, this::shouldGuaranteeMessageOffloading); + ch.pipeline().addLast(capturingHandler); ch.pipeline().addLast(new FrontsideHandler(backsideConnectionPool)); } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java index 4df9735dc..dae5e2d05 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java @@ -78,11 +78,9 @@ public HttpJsonTransformingConsumer(IJsonTransformer transformer, String diagnosticLabel, RequestContext requestContext) { this.transformationContext = new IWithStartTimeAndAttributes<>() { - @Getter - Span currentSpan = METERING_CLOSURE.makeSpanContinuation("httpRequestTransformation") + @Getter Span currentSpan = METERING_CLOSURE.makeSpanContinuation("httpRequestTransformation") .apply(requestContext.getPopulatedAttributes(), requestContext.getCurrentSpan()); - @Getter - Instant startTime = Instant.now(); + @Getter Instant startTime = Instant.now(); @Override public RequestContext getEnclosingScope() { return requestContext; } }; chunkSizes = new ArrayList<>(HTTP_MESSAGE_NUM_SEGMENTS); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java index 2fd747f32..a3d509105 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java @@ -41,10 +41,6 @@ public String getConnectionId() { return enclosingScope.getConnectionId(); } - public String getNodeId() { - return enclosingScope.getNodeId(); - } - @Override public long getSourceRequestIndex() { return replayerRequestKey.getSourceRequestIndex(); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java index f666bd9e7..16a1286e7 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java @@ -18,6 +18,7 @@ import org.opensearch.migrations.replay.tracing.ChannelKeyContext; import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.traffic.source.TrafficStreamWithEmbeddedKey; +import org.opensearch.migrations.tracing.SimpleMeteringClosure; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import org.opensearch.migrations.trafficcapture.InMemoryConnectionCaptureFactory; import org.opensearch.migrations.trafficcapture.protos.TrafficStream; @@ -99,8 +100,8 @@ static ByteBuf makeSequentialByteBuf(int offset, int size) { static TrafficStream[] makeTrafficStreams(int bufferSize, int interactionOffset, List directives) throws Exception { var connectionFactory = buildSerializerFactory(bufferSize, ()->{}); - var offloader = connectionFactory.createOffloader(new ConnectionContext("test", "test", - x->GlobalOpenTelemetry.getTracer("test").spanBuilder("test").startSpan()), + var offloader = connectionFactory.createOffloader(new ConnectionContext("n", "test", + new SimpleMeteringClosure("test").makeSpanContinuation("test", null)), "TEST_"+uniqueIdCounter.incrementAndGet()); for (var directive : directives) { serializeEvent(offloader, interactionOffset++, directive); diff --git a/TrafficCapture/trafficReplayer/src/test/resources/log4j2.properties b/TrafficCapture/trafficReplayer/src/test/resources/log4j2.properties index 9098da413..43e08b306 100644 --- a/TrafficCapture/trafficReplayer/src/test/resources/log4j2.properties +++ b/TrafficCapture/trafficReplayer/src/test/resources/log4j2.properties @@ -18,3 +18,8 @@ logger.OutputTupleJsonLogger.level = OFF logger.KPC.name = org.opensearch.migrations.replay.kafka.KafkaProtobufConsumer logger.KPC.level = DEBUG logger.KPC.appenderRef.stdout.ref = Console + +logger.RSO.name = org.opensearch.migrations.replay.RequestSenderOrchestrator +logger.RSO.level = TRACE +logger.RSO.additivity = false +logger.RSO.appenderRef.RSO.ref = Console From 15a1705c35143b5fa168e4f2ade79bc38db824b9 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Fri, 1 Dec 2023 20:02:24 -0500 Subject: [PATCH 14/20] Improve the tracing story for the capture proxy. Don't bother showing the Kakfa offloader just buffering (was called recordStream). Now the offloader span is a child span of the connection span from the handler, so we can see the handler gathering the request/response (or waiting for the response). Signed-off-by: Greg Schohn --- .../kafkaoffloader/KafkaCaptureFactory.java | 6 +-- .../netty/LoggingHttpRequestHandler.java | 37 +++++++++++++------ .../netty/tracing/HttpMessageContext.java | 11 +++--- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java index c1c84ad81..bf502596c 100644 --- a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java +++ b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java @@ -71,7 +71,6 @@ public IChannelConnectionCaptureSerializer createOffloader(IConn static class CodedOutputStreamWrapper implements CodedOutputStreamHolder { private final CodedOutputStream codedOutputStream; private final ByteBuffer byteBuffer; - final ConnectionContext streamContext; @Override public @NonNull CodedOutputStream getOutputStream() { return codedOutputStream; @@ -104,11 +103,9 @@ public void close() throws IOException { @Override public CodedOutputStreamWrapper createStream() { METERING_CLOSURE.meterIncrementEvent(telemetryContext, "stream_created"); - var newStreamCtx = new ConnectionContext(telemetryContext, - METERING_CLOSURE.makeSpanContinuation("recordStream")); ByteBuffer bb = ByteBuffer.allocate(bufferSize); - return new CodedOutputStreamWrapper(CodedOutputStream.newInstance(bb), bb, newStreamCtx); + return new CodedOutputStreamWrapper(CodedOutputStream.newInstance(bb), bb); } @Override @@ -119,7 +116,6 @@ public CodedOutputStreamWrapper createStream() { outputStreamHolder); } var osh = (CodedOutputStreamWrapper) outputStreamHolder; - osh.streamContext.currentSpan.end(); // Structured context for MetricsLogger try { diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java index aa4c5ef77..84d34ab35 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java @@ -20,7 +20,6 @@ import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsAttributeKey; import org.opensearch.migrations.coreutils.MetricsEvent; -import org.opensearch.migrations.tracing.IWithAttributes; import org.opensearch.migrations.tracing.SimpleMeteringClosure; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import org.opensearch.migrations.coreutils.MetricsLogger; @@ -37,6 +36,7 @@ public class LoggingHttpRequestHandler extends ChannelDuplexHandler { public static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure(TELEMETRY_SCOPE_NAME); private static final MetricsLogger metricsLogger = new MetricsLogger("LoggingHttpRequestHandler"); public static final String GATHERING_REQUEST = "gatheringRequest"; + public static final String WAITING_FOR_RESPONSE = "waitingForResponse"; public static final String GATHERING_RESPONSE = "gatheringResponse"; static class SimpleHttpRequestDecoder extends HttpRequestDecoder { @@ -92,7 +92,7 @@ public LoggingHttpRequestHandler(String nodeId, String channelKey, var parentContext = new ConnectionContext(channelKey, nodeId, METERING_CLOSURE.makeSpanContinuation("connectionLifetime", null)); - this.messageContext = new HttpMessageContext(parentContext, 0, HttpMessageContext.Direction.REQUEST, + this.messageContext = new HttpMessageContext(parentContext, 0, HttpMessageContext.HttpTransactionState.REQUEST, METERING_CLOSURE.makeSpanContinuation(GATHERING_REQUEST)); METERING_CLOSURE.meterIncrementEvent(messageContext, "requestStarted"); @@ -102,13 +102,27 @@ public LoggingHttpRequestHandler(String nodeId, String channelKey, new SimpleDecodedHttpRequestHandler()); } - public void rotateNextMessageContext() { + static String getSpanLabelForState(HttpMessageContext.HttpTransactionState state) { + switch (state) { + case REQUEST: + return GATHERING_REQUEST; + case WAITING: + return WAITING_FOR_RESPONSE; + case RESPONSE: + return GATHERING_RESPONSE; + default: + throw new IllegalStateException("Unknown enum value: "+state); + } + } + + public void rotateNextMessageContext(HttpMessageContext.HttpTransactionState nextState) { messageContext.getCurrentSpan().end(); - final var wasResponse = messageContext.getDirection() == HttpMessageContext.Direction.RESPONSE; + final var wasResponse = HttpMessageContext.HttpTransactionState.RESPONSE.equals(messageContext.getState()); messageContext = new HttpMessageContext(messageContext.getEnclosingScope(), - (wasResponse ? 1 : 0) + messageContext.getSourceRequestIndex(), - (wasResponse ? HttpMessageContext.Direction.REQUEST : HttpMessageContext.Direction.RESPONSE), - METERING_CLOSURE.makeSpanContinuation(wasResponse ? GATHERING_REQUEST : GATHERING_RESPONSE)); + (nextState== HttpMessageContext.HttpTransactionState.REQUEST ? 1 : 0) + + messageContext.getSourceRequestIndex(), + nextState, + METERING_CLOSURE.makeSpanContinuation(getSpanLabelForState(nextState))); } private HttpProcessedState parseHttpMessageParts(ByteBuf msg) { @@ -163,6 +177,7 @@ public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { } protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Object msg, HttpRequest httpRequest) throws Exception { + rotateNextMessageContext(HttpMessageContext.HttpTransactionState.WAITING); super.channelRead(ctx, msg); METERING_CLOSURE.meterIncrementEvent(messageContext, "requestReceived"); @@ -174,8 +189,8 @@ protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Ob @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - if (messageContext.getDirection() == HttpMessageContext.Direction.RESPONSE) { - rotateNextMessageContext(); + if (messageContext.getState() == HttpMessageContext.HttpTransactionState.RESPONSE) { + rotateNextMessageContext(HttpMessageContext.HttpTransactionState.REQUEST); } var timestamp = Instant.now(); HttpProcessedState httpProcessedState; @@ -207,8 +222,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception @Override public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { - if (messageContext.getDirection() == HttpMessageContext.Direction.REQUEST) { - rotateNextMessageContext(); + if (messageContext.getState() != HttpMessageContext.HttpTransactionState.RESPONSE) { + rotateNextMessageContext(HttpMessageContext.HttpTransactionState.RESPONSE); } var bb = (ByteBuf) msg; trafficOffloader.addWriteEvent(Instant.now(), bb); diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/tracing/HttpMessageContext.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/tracing/HttpMessageContext.java index db93967f7..f0e4a48fc 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/tracing/HttpMessageContext.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/tracing/HttpMessageContext.java @@ -1,6 +1,7 @@ package org.opensearch.migrations.trafficcapture.netty.tracing; import io.opentelemetry.api.trace.Span; +import lombok.EqualsAndHashCode; import lombok.Getter; import org.opensearch.migrations.tracing.ISpanWithParentGenerator; import org.opensearch.migrations.tracing.IWithStartTimeAndAttributes; @@ -11,9 +12,9 @@ import java.time.Instant; public class HttpMessageContext implements IRequestContext, IWithStartTimeAndAttributes { - - public enum Direction { + public enum HttpTransactionState { REQUEST, + WAITING, RESPONSE } @@ -24,16 +25,16 @@ public enum Direction { @Getter final Instant startTime; @Getter - final Direction direction; + final HttpTransactionState state; @Getter final Span currentSpan; - public HttpMessageContext(ConnectionContext enclosingScope, long sourceRequestIndex, Direction direction, + public HttpMessageContext(ConnectionContext enclosingScope, long sourceRequestIndex, HttpTransactionState state, ISpanWithParentGenerator spanGenerator) { this.sourceRequestIndex = sourceRequestIndex; this.enclosingScope = enclosingScope; this.startTime = Instant.now(); - this.direction = direction; + this.state = state; this.currentSpan = spanGenerator.apply(getPopulatedAttributes(), enclosingScope.getCurrentSpan()); } } From 8a6f52af7752e9299b753d8be1a45241da42bd0d Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Sat, 2 Dec 2023 08:27:20 -0500 Subject: [PATCH 15/20] Tracing change: Flatten the flush span and just record it as 'blocked'. That makes it a separate state for the logging handler superclass. Signed-off-by: Greg Schohn --- ...ditionallyReliableLoggingHttpRequestHandler.java | 13 ++++--------- .../netty/LoggingHttpRequestHandler.java | 9 +++++++-- .../netty/tracing/HttpMessageContext.java | 1 + ...onallyReliableLoggingHttpRequestHandlerTest.java | 1 - 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java index 29df56e56..8f5d583fa 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java @@ -36,20 +36,15 @@ protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Ob throws Exception { if (shouldBlockPredicate.test(httpRequest)) { METERING_CLOSURE.meterIncrementEvent(messageContext, "blockingRequestUntilFlush"); - var flushContext = new IWithStartTimeAndAttributes<>() { - @Getter Span currentSpan = METERING_CLOSURE.makeSpanContinuation("blockedForFlush") - .apply(messageContext.getPopulatedAttributes(), messageContext.getCurrentSpan()); - @Getter Instant startTime = Instant.now(); - @Override public HttpMessageContext getEnclosingScope() { return messageContext; } - }; + rotateNextMessageContext(HttpMessageContext.HttpTransactionState.INTERNALLY_BLOCKED); trafficOffloader.flushCommitAndResetStream(false).whenComplete((result, t) -> { log.atInfo().setMessage(()->"Done flushing").log(); - METERING_CLOSURE.meterIncrementEvent(flushContext, + METERING_CLOSURE.meterIncrementEvent(messageContext, t != null ? "blockedFlushFailure" : "blockedFlushSuccess"); - METERING_CLOSURE.meterHistogramMicros(flushContext, + METERING_CLOSURE.meterHistogramMicros(messageContext, t==null ? "blockedFlushFailure_micro" : "stream_flush_failure_micro"); - flushContext.currentSpan.end(); + messageContext.getCurrentSpan().end(); if (t != null) { // This is a spot where we would benefit from having a behavioral policy that different users diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java index 84d34ab35..d7ffa3fb9 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java @@ -38,6 +38,7 @@ public class LoggingHttpRequestHandler extends ChannelDuplexHandler { public static final String GATHERING_REQUEST = "gatheringRequest"; public static final String WAITING_FOR_RESPONSE = "waitingForResponse"; public static final String GATHERING_RESPONSE = "gatheringResponse"; + public static final String BLOCKED = "blocked"; static class SimpleHttpRequestDecoder extends HttpRequestDecoder { /** @@ -106,6 +107,8 @@ static String getSpanLabelForState(HttpMessageContext.HttpTransactionState state switch (state) { case REQUEST: return GATHERING_REQUEST; + case INTERNALLY_BLOCKED: + return BLOCKED; case WAITING: return WAITING_FOR_RESPONSE; case RESPONSE: @@ -115,8 +118,7 @@ static String getSpanLabelForState(HttpMessageContext.HttpTransactionState state } } - public void rotateNextMessageContext(HttpMessageContext.HttpTransactionState nextState) { - messageContext.getCurrentSpan().end(); + protected void rotateNextMessageContext(HttpMessageContext.HttpTransactionState nextState) { final var wasResponse = HttpMessageContext.HttpTransactionState.RESPONSE.equals(messageContext.getState()); messageContext = new HttpMessageContext(messageContext.getEnclosingScope(), (nextState== HttpMessageContext.HttpTransactionState.REQUEST ? 1 : 0) @@ -190,6 +192,7 @@ protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Ob @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { if (messageContext.getState() == HttpMessageContext.HttpTransactionState.RESPONSE) { + messageContext.getCurrentSpan().end(); rotateNextMessageContext(HttpMessageContext.HttpTransactionState.REQUEST); } var timestamp = Instant.now(); @@ -206,6 +209,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception httpProcessedState = parseHttpMessageParts(bb); // bb is consumed/release by this method } if (httpProcessedState == HttpProcessedState.FULL_MESSAGE) { + messageContext.getCurrentSpan().end(); var httpRequest = getHandlerThatHoldsParsedHttpRequest().resetCurrentRequest(); var decoderResultLoose = httpRequest.decoderResult(); if (decoderResultLoose instanceof HttpMessageDecoderResult) { @@ -223,6 +227,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception @Override public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { if (messageContext.getState() != HttpMessageContext.HttpTransactionState.RESPONSE) { + messageContext.getCurrentSpan().end(); rotateNextMessageContext(HttpMessageContext.HttpTransactionState.RESPONSE); } var bb = (ByteBuf) msg; diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/tracing/HttpMessageContext.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/tracing/HttpMessageContext.java index f0e4a48fc..e406c4225 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/tracing/HttpMessageContext.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/tracing/HttpMessageContext.java @@ -14,6 +14,7 @@ public class HttpMessageContext implements IRequestContext, IWithStartTimeAndAttributes { public enum HttpTransactionState { REQUEST, + INTERNALLY_BLOCKED, WAITING, RESPONSE } diff --git a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java index 0cc960ea7..185dcd0b5 100644 --- a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java +++ b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java @@ -80,7 +80,6 @@ private static void writeMessageAndVerify(byte[] fullTrafficBytes, Consumer outputByteBuffer = new AtomicReference<>(); AtomicInteger flushCount = new AtomicInteger(); - ; EmbeddedChannel channel = new EmbeddedChannel( new ConditionallyReliableLoggingHttpRequestHandler("n", "c", From c50e01d228ef0c2a07d8cb5517b5395d074e6b25 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Mon, 4 Dec 2023 16:45:37 -0500 Subject: [PATCH 16/20] Minor cleanup - stop setting the namespace or trying to change in a processor. Prometheus metrics already have an export_name that is unique, the processors weren't doing anything useful, & the namespace was appending EVERYTHING from one of the two services. Signed-off-by: Greg Schohn --- .../docker/otel-collector-config-demo.yaml | 22 ------------------- .../netty/LoggingHttpRequestHandler.java | 1 - 2 files changed, 23 deletions(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/otel-collector-config-demo.yaml b/TrafficCapture/dockerSolution/src/main/docker/otel-collector-config-demo.yaml index 99ac784a8..ac9a2a6d5 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/otel-collector-config-demo.yaml +++ b/TrafficCapture/dockerSolution/src/main/docker/otel-collector-config-demo.yaml @@ -6,7 +6,6 @@ receivers: exporters: prometheus: endpoint: "0.0.0.0:8889" - namespace: capturereplay const_labels: label1: value1 logging: @@ -21,26 +20,6 @@ exporters: tls: insecure: true - -processors: - # Processor to set the namespace based on the service name - attributes/nscapture: - actions: - - key: namespace - value: "capture" - action: insert - - key: service.name - value: "capture" - action: update - attributes/nsreplayer: - actions: - - key: namespace - value: "replay" - action: insert - - key: service.name - value: "replay" - action: update - extensions: health_check: pprof: @@ -57,5 +36,4 @@ service: exporters: [logging, zipkin, otlp/jaeger] metrics: receivers: [otlp] - processors: [attributes/nscapture, attributes/nsreplayer] exporters: [logging, prometheus] diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java index d7ffa3fb9..8c8fc5135 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java @@ -119,7 +119,6 @@ static String getSpanLabelForState(HttpMessageContext.HttpTransactionState state } protected void rotateNextMessageContext(HttpMessageContext.HttpTransactionState nextState) { - final var wasResponse = HttpMessageContext.HttpTransactionState.RESPONSE.equals(messageContext.getState()); messageContext = new HttpMessageContext(messageContext.getEnclosingScope(), (nextState== HttpMessageContext.HttpTransactionState.REQUEST ? 1 : 0) + messageContext.getSourceRequestIndex(), From 17c517dddf49ce915ba80c363398629a237351fb Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Mon, 4 Dec 2023 16:47:43 -0500 Subject: [PATCH 17/20] Start instrumenting the replayer with more contexts so that traces and (less so for now) metrics can be exported across more of the lifetime of a request/connection. Signed-off-by: Greg Schohn --- .../migrations/replay/Accumulation.java | 10 +++- ...edTrafficToHttpTransactionAccumulator.java | 4 ++ .../replay/ClientConnectionPool.java | 9 ++- .../migrations/replay/ReplayEngine.java | 10 ++-- .../replay/RequestSenderOrchestrator.java | 55 +++++++++---------- .../migrations/replay/TrafficReplayer.java | 2 +- .../NettyPacketToHttpConsumer.java | 4 +- .../datatypes/ConnectionReplaySession.java | 5 +- .../replay/tracing/RequestContext.java | 9 +-- .../replay/RequestSenderOrchestratorTest.java | 2 +- 10 files changed, 56 insertions(+), 54 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java index 8d657613a..1fcb4ff47 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java @@ -48,7 +48,7 @@ public Accumulation(@NonNull ITrafficStreamKey trafficChannelKey, this.state = dropObservationsLeftoverFromPrevious ? State.IGNORING_LAST_REQUEST : State.WAITING_FOR_NEXT_READ_CHUNK; channelContext = new ChannelKeyContext(trafficChannelKey, - METERING_CLOSURE.makeSpanContinuation("accumulatingChannel", null)); + METERING_CLOSURE.makeSpanContinuation("processingChannel", null)); } public RequestResponsePacketPair getOrCreateTransactionPair(ITrafficStreamKey forTrafficStreamKey) { @@ -87,6 +87,14 @@ public boolean hasRrPair() { return rrPair; } + public void rotateRequestGatheringToResponse() { + var ctx = rrPair.requestContext; + ctx.getCurrentSpan().end(); + rrPair.requestContext = new RequestContext(ctx.getEnclosingScope(), + ctx.getReplayerRequestKey(), + METERING_CLOSURE.makeSpanContinuation("accumulatingResponse")); + } + public Instant getLastTimestamp() { return Instant.ofEpochMilli(newestPacketTimestampInMillis.get()); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java index 35cde47fd..28d1584ca 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java @@ -221,6 +221,7 @@ private static List getTrafficStreamsHeldByAccum(Accumulation accum.getOrCreateTransactionPair(trafficStreamKey).holdTrafficStream(trafficStreamKey); rotateAccumulationIfNecessary(trafficStreamKey.getConnectionId(), accum); closedConnectionCounter.incrementAndGet(); + accum.channelContext.getCurrentSpan().end(); listener.onConnectionClose(accum.trafficChannelKey, accum.getIndexOfCurrentRequest(), accum.channelContext, RequestResponsePacketPair.ReconstructionStatus.COMPLETE, timestamp, getTrafficStreamsHeldByAccum(accum)); @@ -344,6 +345,7 @@ private boolean handleEndOfRequest(Accumulation accumulation) { .setAttribute(MetricsAttributeKey.CONNECTION_ID, accumulation.getRequestKey().getTrafficStreamKey().getConnectionId()).emit(); assert (requestPacketBytes != null); assert (!requestPacketBytes.hasInProgressSegment()); + accumulation.rotateRequestGatheringToResponse(); listener.onRequestReceived(accumulation.getRequestKey(), rrPair.requestContext, requestPacketBytes); accumulation.state = Accumulation.State.ACCUMULATING_WRITES; return true; @@ -356,6 +358,7 @@ private void handleEndOfResponse(Accumulation accumulation, RequestResponsePacke .setAttribute(MetricsAttributeKey.CONNECTION_ID, accumulation.getRequestKey().getTrafficStreamKey().getConnectionId()).emit(); var rrPair = accumulation.getRrPair(); rrPair.completionStatus = status; + rrPair.requestContext.getCurrentSpan().end(); listener.onFullDataReceived(accumulation.getRequestKey(), rrPair.requestContext, rrPair); accumulation.resetForNextRequest(); } @@ -400,6 +403,7 @@ private void fireAccumulationsCallbacksAndClose(Accumulation accumulation, } } finally { if (accumulation.hasSignaledRequests()) { + accumulation.channelContext.getCurrentSpan().end(); listener.onConnectionClose(accumulation.trafficChannelKey, accumulation.getIndexOfCurrentRequest(), accumulation.channelContext, status, accumulation.getLastTimestamp(), getTrafficStreamsHeldByAccum(accumulation)); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java index a40cbb01a..f0a41143a 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java @@ -17,7 +17,6 @@ import org.opensearch.migrations.tracing.SimpleMeteringClosure; import org.opensearch.migrations.replay.datahandlers.NettyPacketToHttpConsumer; import org.opensearch.migrations.replay.datatypes.ConnectionReplaySession; -import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.tracing.ChannelKeyContext; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture; @@ -141,7 +140,7 @@ public void closeConnection(String connId) { } public Future - submitEventualSessionGet(ISourceTrafficChannelKey channelKey, boolean ignoreIfNotPresent, ChannelKeyContext ctx) { + submitEventualSessionGet(ChannelKeyContext channelKey, boolean ignoreIfNotPresent, ChannelKeyContext ctx) { ConnectionReplaySession channelFutureAndSchedule = getCachedSession(channelKey, ignoreIfNotPresent); if (channelFutureAndSchedule == null) { @@ -159,11 +158,11 @@ public void closeConnection(String connId) { } @SneakyThrows - public ConnectionReplaySession getCachedSession(ISourceTrafficChannelKey channelKey, boolean dontCreate) { + public ConnectionReplaySession getCachedSession(ChannelKeyContext channelKey, boolean dontCreate) { var crs = dontCreate ? connectionId2ChannelCache.getIfPresent(channelKey.getConnectionId()) : connectionId2ChannelCache.get(channelKey.getConnectionId()); if (crs != null) { - crs.setChannelId(channelKey); + crs.setChannelContext(channelKey); } return crs; } @@ -188,7 +187,7 @@ public ConnectionReplaySession getCachedSession(ISourceTrafficChannelKey channel if (channelAndFutureWork.hasWorkRemaining()) { log.atWarn().setMessage(()->"Work items are still remaining for this connection session" + "(last associated with connection=" + - channelAndFutureWork.getChannelId() + + channelAndFutureWork.getChannelContext() + "). " + channelAndFutureWork.calculateSizeSlowly() + " requests that were enqueued won't be run").log(); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java index f75b92331..c80fdabfa 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java @@ -124,15 +124,15 @@ private static void logStartOfWork(Object stringableKey, long newCount, Instant } public DiagnosticTrackableCompletableFuture - scheduleTransformationWork(UniqueReplayerRequestKey requestKey, Instant originalStart, + scheduleTransformationWork(RequestContext requestCtx, Instant originalStart, Supplier> task) { var newCount = totalCountOfScheduledTasksOutstanding.incrementAndGet(); final String label = "processing"; var start = timeShifter.transformSourceTimeToRealTime(originalStart); - logStartOfWork(requestKey, newCount, start, label); - var result = networkSendOrchestrator.scheduleWork(requestKey.trafficStreamKey, + logStartOfWork(requestCtx, newCount, start, label); + var result = networkSendOrchestrator.scheduleWork(requestCtx.getEnclosingScope(), start.minus(EXPECTED_TRANSFORMATION_DURATION), task); - return hookWorkFinishingUpdates(result, originalStart, requestKey, label); + return hookWorkFinishingUpdates(result, originalStart, requestCtx, label); } public DiagnosticTrackableCompletableFuture @@ -159,7 +159,7 @@ public void closeConnection(ISourceTrafficChannelKey channelKey, int channelInte final String label = "close"; var atTime = timeShifter.transformSourceTimeToRealTime(timestamp); logStartOfWork(new IndexedChannelInteraction(channelKey, channelInteractionNum), newCount, atTime, label); - var future = networkSendOrchestrator.scheduleClose(channelKey, channelInteractionNum, ctx, atTime); + var future = networkSendOrchestrator.scheduleClose(ctx, channelInteractionNum, atTime); hookWorkFinishingUpdates(future, timestamp, channelKey, label); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java index 107c21bae..45bbaf521 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java @@ -36,13 +36,13 @@ public RequestSenderOrchestrator(ClientConnectionPool clientConnectionPool) { } public DiagnosticTrackableCompletableFuture - scheduleWork(ISourceTrafficChannelKey channelKey, Instant timestamp, + scheduleWork(ChannelKeyContext ctx, Instant timestamp, Supplier> task) { - var connectionSession = clientConnectionPool.getCachedSession(channelKey, false); + var connectionSession = clientConnectionPool.getCachedSession(ctx, false); var finalTunneledResponse = new StringTrackableCompletableFuture(new CompletableFuture<>(), ()->"waiting for final signal to confirm processing work has finished"); - log.atDebug().setMessage(()->"Scheduling work for "+channelKey+" at time "+timestamp).log(); + log.atDebug().setMessage(()->"Scheduling work for "+ctx.getConnectionId()+" at time "+timestamp).log(); connectionSession.eventLoop.schedule(()-> task.get().map(f->f.whenComplete((v,t) -> { if (t!=null) { @@ -63,25 +63,24 @@ public RequestSenderOrchestrator(ClientConnectionPool clientConnectionPool) { new StringTrackableCompletableFuture(new CompletableFuture<>(), ()->"waiting for final aggregated response"); log.atDebug().setMessage(()->"Scheduling request for "+requestKey+" at start time "+start).log(); - return asynchronouslyInvokeRunnableToSetupFuture(requestKey.getTrafficStreamKey(), - requestKey.getReplayerRequestIndex(), ctx.getChannelKeyContext(), false, finalTunneledResponse, - channelFutureAndRequestSchedule-> scheduleSendOnConnectionReplaySession(requestKey, ctx, + return asynchronouslyInvokeRunnableToSetupFuture( + ctx.getEnclosingScope(), requestKey.getReplayerRequestIndex(), false, finalTunneledResponse, + channelFutureAndRequestSchedule-> scheduleSendOnConnectionReplaySession(ctx, channelFutureAndRequestSchedule, finalTunneledResponse, start, interval, packets)); } - public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChannelKey channelKey, - int channelInteractionNum, - ChannelKeyContext ctx, + public StringTrackableCompletableFuture scheduleClose(ChannelKeyContext ctx, int channelInteractionNum, Instant timestamp) { + var channelKey = ctx.getChannelKey(); var channelInteraction = new IndexedChannelInteraction(channelKey, channelInteractionNum); var finalTunneledResponse = new StringTrackableCompletableFuture(new CompletableFuture<>(), ()->"waiting for final signal to confirm close has finished"); log.atDebug().setMessage(()->"Scheduling CLOSE for "+channelInteraction+" at time "+timestamp).log(); - asynchronouslyInvokeRunnableToSetupFuture(channelKey, channelInteractionNum, ctx,true, + asynchronouslyInvokeRunnableToSetupFuture(ctx, channelInteractionNum, true, finalTunneledResponse, channelFutureAndRequestSchedule-> - scheduleOnConnectionReplaySession(channelKey, channelInteractionNum, ctx, + scheduleOnConnectionReplaySession(ctx, channelInteractionNum, channelFutureAndRequestSchedule, finalTunneledResponse, timestamp, "close", () -> { log.trace("Closing client connection " + channelInteraction); clientConnectionPool.closeConnection(channelKey.getConnectionId()); @@ -92,21 +91,20 @@ public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChanne } private DiagnosticTrackableCompletableFuture - asynchronouslyInvokeRunnableToSetupFuture(ISourceTrafficChannelKey channelKey, int channelInteractionNumber, - ChannelKeyContext ctx, boolean ignoreIfChannelNotPresent, + asynchronouslyInvokeRunnableToSetupFuture(ChannelKeyContext ctx, int channelInteractionNumber, + boolean ignoreIfChannelNotPresent, DiagnosticTrackableCompletableFuture finalTunneledResponse, Consumer successFn) { var channelFutureAndScheduleFuture = - clientConnectionPool.submitEventualSessionGet(channelKey, ignoreIfChannelNotPresent, ctx); + clientConnectionPool.submitEventualSessionGet(ctx, ignoreIfChannelNotPresent, ctx); channelFutureAndScheduleFuture.addListener(submitFuture->{ if (!submitFuture.isSuccess()) { log.atError().setCause(submitFuture.cause()) - .setMessage(()->channelKey.toString() + " unexpected issue found from a scheduled task") - .log(); + .setMessage(()->ctx + " unexpected issue found from a scheduled task").log(); finalTunneledResponse.future.completeExceptionally(submitFuture.cause()); } else { - log.atTrace().setMessage(()->channelKey.toString() + - " on the channel's thread... getting a ConnectionReplaySession for it").log(); + log.atTrace().setMessage(()->ctx + " on the channel's thread... " + + "getting a ConnectionReplaySession for it").log(); var channelFutureAndRequestSchedule = ((ConnectionReplaySession) submitFuture.get()); if (channelFutureAndRequestSchedule == null) { finalTunneledResponse.future.complete(null); @@ -115,8 +113,8 @@ public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChanne channelFutureAndRequestSchedule.getChannelFutureFuture() .map(channelFutureGetAttemptFuture->channelFutureGetAttemptFuture .thenAccept(v->{ - log.atTrace().setMessage(()->channelKey.toString() + " in submitFuture(success) and scheduling the task" + - " for " + finalTunneledResponse.toString()).log(); + log.atTrace().setMessage(()->ctx + " in submitFuture(success) and " + + "scheduling the task for " + finalTunneledResponse.toString()).log(); assert v.channel() == channelFutureAndRequestSchedule.getChannelFutureFuture().future .getNow(null).channel(); @@ -127,13 +125,13 @@ public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChanne () -> successFn.accept(channelFutureAndRequestSchedule), x -> x.run()); if (cffr.scheduleSequencer.hasPending()) { - log.atDebug().setMessage(()->"Sequencer for "+channelKey+ + log.atDebug().setMessage(()->"Sequencer for "+ctx+ " = "+cffr.scheduleSequencer).log(); } }); }) .exceptionally(t->{ - log.atTrace().setCause(t).setMessage(()->channelKey.toString() + + log.atTrace().setCause(t).setMessage(()->ctx + " ChannelFuture creation threw an exception").log(); finalTunneledResponse.future.completeExceptionally(t); return null; @@ -144,12 +142,11 @@ public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChanne return finalTunneledResponse; } - private void scheduleOnConnectionReplaySession(ISourceTrafficChannelKey channelKey, int channelInteractionIdx, - ChannelKeyContext ctx, + private void scheduleOnConnectionReplaySession(ChannelKeyContext ctx, int channelInteractionIdx, ConnectionReplaySession channelFutureAndRequestSchedule, StringTrackableCompletableFuture futureToBeCompletedByTask, Instant atTime, String activityNameForLogging, Runnable task) { - var channelInteraction = new IndexedChannelInteraction(channelKey, channelInteractionIdx); + var channelInteraction = new IndexedChannelInteraction(ctx.getChannelKey(), channelInteractionIdx); log.atInfo().setMessage(()->channelInteraction + " scheduling " + activityNameForLogging + " at " + atTime).log(); var schedule = channelFutureAndRequestSchedule.schedule; @@ -191,7 +188,7 @@ private void scheduleOnConnectionReplaySession(ISourceTrafficChannelKey chan "... " + schedule).log(); } - private void scheduleSendOnConnectionReplaySession(UniqueReplayerRequestKey requestKey, RequestContext ctx, + private void scheduleSendOnConnectionReplaySession(RequestContext ctx, ConnectionReplaySession channelFutureAndRequestSchedule, StringTrackableCompletableFuture responseFuture, Instant start, Duration interval, Stream packets) { @@ -201,9 +198,9 @@ private void scheduleSendOnConnectionReplaySession(UniqueReplayerRequestKey requ getPacketReceiver(ctx, channelFutureAndRequestSchedule.getInnerChannelFuture(), packetReceiverRef), eventLoop, packets.iterator(), start, interval, new AtomicInteger(), responseFuture); - scheduleOnConnectionReplaySession(requestKey.trafficStreamKey, requestKey.getSourceRequestIndex(), - ctx.getChannelKeyContext(), channelFutureAndRequestSchedule, responseFuture, start, - "send", packetSender); + scheduleOnConnectionReplaySession(ctx.getEnclosingScope(), + ctx.getReplayerRequestKey().getSourceRequestIndex(), + channelFutureAndRequestSchedule, responseFuture, start, "send", packetSender); } private void runAfterChannelSetup(ConnectionReplaySession channelFutureAndItsFutureRequests, diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index 85b90d822..850be007b 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -874,7 +874,7 @@ private static String formatWorkItem(DiagnosticTrackableCompletableFuture> packetsSupplier) { try { - var transformationCompleteFuture = replayEngine.scheduleTransformationWork(requestKey, start, ()-> + var transformationCompleteFuture = replayEngine.scheduleTransformationWork(ctx, start, ()-> transformAllData(inputRequestTransformerFactory.create(requestKey, ctx), packetsSupplier)); log.atDebug().setMessage(()->"finalizeRequest future for transformation of " + requestKey + " = " + transformationCompleteFuture).log(); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java index 35615fe44..361688ced 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java @@ -39,8 +39,6 @@ @Slf4j public class NettyPacketToHttpConsumer implements IPacketFinalizingConsumer { - private static final ContextKey START_OF_REQUEST_KEY = ContextKey.named("startOfRequest"); - private static final ContextKey START_OF_WRITE_KEY = ContextKey.named("startOfWrite"); public static final String TELEMETRY_SCOPE_NAME = "HttpSender"; public static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure(TELEMETRY_SCOPE_NAME); @@ -64,7 +62,7 @@ public class NettyPacketToHttpConsumer implements IPacketFinalizingConsumer> foo; - public RequestContext(ChannelKeyContext enclosingScope, UniqueReplayerRequestKey replayerRequestKey, ISpanWithParentGenerator spanGenerator) { this.enclosingScope = enclosingScope; @@ -32,11 +30,6 @@ public RequestContext(ChannelKeyContext enclosingScope, UniqueReplayerRequestKey this.currentSpan = spanGenerator.apply(getPopulatedAttributes(), enclosingScope.getCurrentSpan()); } - public ChannelKeyContext getChannelKeyContext() { - return new ChannelKeyContext(replayerRequestKey.trafficStreamKey, - innerAttributesToIgnore_LeavingOriginalAttributesInPlace->currentSpan); - } - public String getConnectionId() { return enclosingScope.getConnectionId(); } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/RequestSenderOrchestratorTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/RequestSenderOrchestratorTest.java index 5fda5f7cf..a8701076d 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/RequestSenderOrchestratorTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/RequestSenderOrchestratorTest.java @@ -52,7 +52,7 @@ public void testThatSchedulingWorks() throws Exception { } var connectionCtx = TestRequestKey.getTestConnectionRequestContext(NUM_REQUESTS_TO_SCHEDULE); var closeFuture = senderOrchestrator.scheduleClose( - connectionCtx.getChannelKey(), NUM_REQUESTS_TO_SCHEDULE, connectionCtx.getChannelKeyContext(), + connectionCtx.getEnclosingScope(), NUM_REQUESTS_TO_SCHEDULE, lastEndTime.plus(Duration.ofMillis(100))); Assertions.assertEquals(NUM_REQUESTS_TO_SCHEDULE, scheduledItems.size()); From 628884407afe2e93b1f0c1c1e80b8bad9590b79e Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Mon, 11 Dec 2023 12:54:23 -0500 Subject: [PATCH 18/20] Double down on using Context objects in lieu of String labels and fix a test bug. Signed-off-by: Greg Schohn --- .../migrations/replay/AddCompressionEncodingTest.java | 2 +- .../replay/PacketToTransformingHttpHandlerFactory.java | 2 +- .../datahandlers/http/HttpJsonTransformingConsumer.java | 3 +-- ...NettyDecodedHttpRequestPreliminaryConvertHandler.java | 3 +-- .../http/NettySendByteBufsToPacketHandlerHandler.java | 9 +++++---- .../datahandlers/http/RequestPipelineOrchestrator.java | 9 +++------ .../migrations/replay/tracing/ChannelKeyContext.java | 7 +++++++ .../migrations/replay/tracing/RequestContext.java | 5 +++++ .../migrations/replay/HeaderTransformerTest.java | 6 +++--- .../datahandlers/NettyPacketToHttpConsumerTest.java | 7 ++----- .../http/HttpJsonTransformingConsumerTest.java | 9 ++++----- .../org/opensearch/migrations/replay/TestRequestKey.java | 9 ++++++++- .../java/org/opensearch/migrations/replay/TestUtils.java | 2 +- 13 files changed, 42 insertions(+), 31 deletions(-) diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java index 33817c884..3bb3c936b 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java @@ -33,7 +33,7 @@ public void addingCompressionRequestHeaderCompressesPayload() throws ExecutionEx var compressingTransformer = new HttpJsonTransformingConsumer( JsonJoltTransformer.newBuilder() .addCannedOperation(JsonJoltTransformBuilder.CANNED_OPERATION.ADD_GZIP) - .build(), null, testPacketCapture, "TEST", + .build(), null, testPacketCapture, TestRequestKey.getTestConnectionRequestContext(0)); final var payloadPartSize = 511; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java index d827f28bf..005c7896e 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PacketToTransformingHttpHandlerFactory.java @@ -30,6 +30,6 @@ public PacketToTransformingHttpHandlerFactory(IJsonTransformer jsonTransformer, create(UniqueReplayerRequestKey requestKey, RequestContext requestContext) { log.trace("creating HttpJsonTransformingConsumer"); return new HttpJsonTransformingConsumer<>(jsonTransformer, authTransformerFactory, - new TransformedPacketReceiver(), requestKey.toString(), requestContext); + new TransformedPacketReceiver(), requestContext); } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java index dae5e2d05..ee25911e3 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java @@ -75,7 +75,6 @@ public class HttpJsonTransformingConsumer implements IPacketFinalizingConsume public HttpJsonTransformingConsumer(IJsonTransformer transformer, IAuthTransformerFactory authTransformerFactory, IPacketFinalizingConsumer transformedPacketReceiver, - String diagnosticLabel, RequestContext requestContext) { this.transformationContext = new IWithStartTimeAndAttributes<>() { @Getter Span currentSpan = METERING_CLOSURE.makeSpanContinuation("httpRequestTransformation") @@ -88,7 +87,7 @@ public HttpJsonTransformingConsumer(IJsonTransformer transformer, chunks = new ArrayList<>(HTTP_MESSAGE_NUM_SEGMENTS + EXPECTED_PACKET_COUNT_GUESS_FOR_HEADERS); channel = new EmbeddedChannel(); pipelineOrchestrator = new RequestPipelineOrchestrator<>(chunkSizes, transformedPacketReceiver, - authTransformerFactory, diagnosticLabel, requestContext); + authTransformerFactory, requestContext); pipelineOrchestrator.addInitialHandlers(channel.pipeline(), transformer); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java index 5db40bb9a..2957a7a70 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java @@ -33,12 +33,11 @@ public class NettyDecodedHttpRequestPreliminaryConvertHandler extends Channel public NettyDecodedHttpRequestPreliminaryConvertHandler(IJsonTransformer transformer, List> chunkSizes, RequestPipelineOrchestrator requestPipelineOrchestrator, - String diagnosticLabel, RequestContext requestContext) { this.transformer = transformer; this.chunkSizes = chunkSizes; this.requestPipelineOrchestrator = requestPipelineOrchestrator; - this.diagnosticLabel = "[" + diagnosticLabel + "] "; + this.diagnosticLabel = "[" + requestContext + "] "; this.requestContext = requestContext; } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettySendByteBufsToPacketHandlerHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettySendByteBufsToPacketHandlerHandler.java index a0f405c8e..e46763082 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettySendByteBufsToPacketHandlerHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettySendByteBufsToPacketHandlerHandler.java @@ -8,6 +8,7 @@ import org.opensearch.migrations.replay.datatypes.HttpRequestTransformationStatus; import org.opensearch.migrations.replay.datatypes.TransformedOutputAndResult; import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer; +import org.opensearch.migrations.replay.tracing.RequestContext; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture; @@ -28,15 +29,15 @@ public class NettySendByteBufsToPacketHandlerHandler extends ChannelInboundHa DiagnosticTrackableCompletableFuture currentFuture; private AtomicReference>> packetReceiverCompletionFutureRef; - String diagnosticLabel; + RequestContext requestContext; public NettySendByteBufsToPacketHandlerHandler(IPacketFinalizingConsumer packetReceiver, - String diagnosticLabel) { + RequestContext requestContext) { this.packetReceiver = packetReceiver; this.packetReceiverCompletionFutureRef = new AtomicReference<>(); - this.diagnosticLabel = diagnosticLabel; + this.requestContext = requestContext; currentFuture = DiagnosticTrackableCompletableFuture.Factory.completedFuture(null, - ()->"currentFuture for NettySendByteBufsToPacketHandlerHandler initialized to the base case for " + diagnosticLabel); + ()->"currentFuture for NettySendByteBufsToPacketHandlerHandler initialized to the base case for " + requestContext); } @Override diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/RequestPipelineOrchestrator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/RequestPipelineOrchestrator.java index 650c5e003..36c72e28b 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/RequestPipelineOrchestrator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/RequestPipelineOrchestrator.java @@ -42,7 +42,6 @@ public class RequestPipelineOrchestrator { public static final String HTTP_REQUEST_DECODER_NAME = "HTTP_REQUEST_DECODER"; private final List> chunkSizes; final IPacketFinalizingConsumer packetReceiver; - final String diagnosticLabel; private RequestContext requestContext; @Getter final IAuthTransformerFactory authTransfomerFactory; @@ -50,13 +49,11 @@ public class RequestPipelineOrchestrator { public RequestPipelineOrchestrator(List> chunkSizes, IPacketFinalizingConsumer packetReceiver, IAuthTransformerFactory incomingAuthTransformerFactory, - String diagnosticLabel, RequestContext requestContext) { this.chunkSizes = chunkSizes; this.packetReceiver = packetReceiver; this.authTransfomerFactory = incomingAuthTransformerFactory != null ? incomingAuthTransformerFactory : IAuthTransformerFactory.NullAuthTransformerFactory.instance; - this.diagnosticLabel = diagnosticLabel; this.requestContext = requestContext; } @@ -99,8 +96,8 @@ void addInitialHandlers(ChannelPipeline pipeline, IJsonTransformer transformer) // Note3: ByteBufs will be sent through when there were pending bytes left to be parsed by the // HttpRequestDecoder when the HttpRequestDecoder is removed from the pipeline BEFORE the // NettyDecodedHttpRequestHandler is removed. - pipeline.addLast(new NettyDecodedHttpRequestPreliminaryConvertHandler(transformer, chunkSizes, this, - diagnosticLabel, requestContext)); + pipeline.addLast(new NettyDecodedHttpRequestPreliminaryConvertHandler(transformer, chunkSizes, + this, requestContext)); addLoggingHandler(pipeline, "B"); } @@ -150,7 +147,7 @@ void addBaselineHandlers(ChannelPipeline pipeline) { // OUT: nothing - terminal! ByteBufs are routed to the packet handler! addLoggingHandler(pipeline, "K"); pipeline.addLast(OFFLOADING_HANDLER_NAME, - new NettySendByteBufsToPacketHandlerHandler(packetReceiver, diagnosticLabel)); + new NettySendByteBufsToPacketHandlerHandler(packetReceiver, requestContext)); } private void addLoggingHandler(ChannelPipeline pipeline, String name) { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelKeyContext.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelKeyContext.java index 3a8780fbe..419793eca 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelKeyContext.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelKeyContext.java @@ -6,6 +6,8 @@ import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.tracing.commoncontexts.IConnectionContext; +import java.util.StringJoiner; + public class ChannelKeyContext implements IConnectionContext { @Getter final ISourceTrafficChannelKey channelKey; @@ -26,4 +28,9 @@ public String getConnectionId() { public String getNodeId() { return channelKey.getNodeId(); } + + @Override + public String toString() { + return channelKey.toString(); + } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java index 29953654c..e90bfdf14 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/RequestContext.java @@ -47,4 +47,9 @@ public long replayerRequestIndex() { public ISourceTrafficChannelKey getChannelKey() { return replayerRequestKey.trafficStreamKey; } + + @Override + public String toString() { + return replayerRequestKey.toString(); + } } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java index ea1bb6eb9..69e8ad8b0 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java @@ -34,7 +34,7 @@ public void testTransformer() throws Exception { var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse); var transformer = new TransformationLoader().getTransformerFactoryLoader(SILLY_TARGET_CLUSTER_NAME); var transformingHandler = new HttpJsonTransformingConsumer(transformer, null, testPacketCapture, - "TEST", TestRequestKey.getTestConnectionRequestContext(0)); + TestRequestKey.getTestConnectionRequestContext(0)); runRandomPayloadWithTransformer(transformingHandler, dummyAggregatedResponse, testPacketCapture, contentLength -> "GET / HTTP/1.1\r\n" + "HoSt: " + SOURCE_CLUSTER_NAME + "\r\n" + @@ -86,7 +86,7 @@ public void testMalformedPayloadIsPassedThrough() throws Exception { var httpBasicAuthTransformer = new StaticAuthTransformerFactory("Basic YWRtaW46YWRtaW4="); var transformingHandler = new HttpJsonTransformingConsumer( new TransformationLoader().getTransformerFactoryLoader(SILLY_TARGET_CLUSTER_NAME), - httpBasicAuthTransformer, testPacketCapture, "TEST", + httpBasicAuthTransformer, testPacketCapture, TestRequestKey.getTestConnectionRequestContext(0)); runRandomPayloadWithTransformer(transformingHandler, dummyAggregatedResponse, testPacketCapture, @@ -113,7 +113,7 @@ public void testMalformedPayload_andTypeMappingUri_IsPassedThrough() throws Exce var transformingHandler = new HttpJsonTransformingConsumer( new TransformationLoader().getTransformerFactoryLoader(SILLY_TARGET_CLUSTER_NAME, null, "[{\"JsonTransformerForOpenSearch23PlusTargetTransformerProvider\":\"\"}]"), - null, testPacketCapture, "TEST", TestRequestKey.getTestConnectionRequestContext(0)); + null, testPacketCapture, TestRequestKey.getTestConnectionRequestContext(0)); Random r = new Random(2); var stringParts = IntStream.range(0, 1).mapToObj(i-> TestUtils.makeRandomString(r, 10)).map(o->(String)o) diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java index e6f833c7b..05d27763c 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java @@ -164,12 +164,9 @@ public void testThatConnectionsAreKeptAliveAndShared(boolean useTls) new TestFlowController(), timeShifter); for (int j=0; j<2; ++j) { for (int i = 0; i < 2; ++i) { - String connId = "TEST_" + j; - var trafficStreamKey = new PojoTrafficStreamKey("testNodeId", connId, 0); - var requestKey = new UniqueReplayerRequestKey(trafficStreamKey, 0, i); - var ctx = TestRequestKey.getTestConnectionRequestContext(0); + var ctx = TestRequestKey.getTestConnectionRequestContext("TEST_"+i, j); var requestFinishFuture = TrafficReplayer.transformAndSendRequest(transformingHttpHandlerFactory, - sendingFactory, ctx, Instant.now(), Instant.now(), requestKey, + sendingFactory, ctx, Instant.now(), Instant.now(), ctx.getReplayerRequestKey(), ()->Stream.of(EXPECTED_REQUEST_STRING.getBytes(StandardCharsets.UTF_8))); log.info("requestFinishFuture="+requestFinishFuture); var aggregatedResponse = requestFinishFuture.get(); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java index f4c7031ff..50c2aaa44 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java @@ -27,7 +27,7 @@ public void testPassThroughSinglePacketPost() throws Exception { var transformingHandler = new HttpJsonTransformingConsumer(new TransformationLoader() .getTransformerFactoryLoader(null), - null, testPacketCapture, "TEST", + null, testPacketCapture, TestRequestKey.getTestConnectionRequestContext(0)); byte[] testBytes; try (var sampleStream = HttpJsonTransformingConsumer.class.getResourceAsStream( @@ -48,7 +48,7 @@ public void testPassThroughSinglePacketWithoutBodyTransformationPost() throws Ex var transformingHandler = new HttpJsonTransformingConsumer( new TransformationLoader().getTransformerFactoryLoader("test.domain"), - null, testPacketCapture, "TEST", + null, testPacketCapture, TestRequestKey.getTestConnectionRequestContext(0)); byte[] testBytes; try (var sampleStream = HttpJsonTransformingConsumer.class.getResourceAsStream( @@ -73,8 +73,7 @@ public void testRemoveAuthHeadersWorks() throws Exception { var transformingHandler = new HttpJsonTransformingConsumer( new TransformationLoader().getTransformerFactoryLoader("test.domain"), - RemovingAuthTransformerFactory.instance, - testPacketCapture, "TEST", + RemovingAuthTransformerFactory.instance, testPacketCapture, TestRequestKey.getTestConnectionRequestContext(0)); byte[] testBytes; try (var sampleStream = HttpJsonTransformingConsumer.class.getResourceAsStream( @@ -114,7 +113,7 @@ private void walkMaps(Object o) { }); var transformingHandler = new HttpJsonTransformingConsumer(complexTransformer, null, - testPacketCapture, "TEST", TestRequestKey.getTestConnectionRequestContext(0)); + testPacketCapture, TestRequestKey.getTestConnectionRequestContext(0)); byte[] testBytes; try (var sampleStream = HttpJsonTransformingConsumer.class.getResourceAsStream( "/requests/raw/post_formUrlEncoded_withFixedLength.txt")) { diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java index 5089e6d7f..a9b232ae4 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java @@ -8,11 +8,18 @@ public class TestRequestKey { + public static final String TEST_NODE_ID = "testNodeId"; + public static final String DEFAULT_TEST_CONNECTION = "testConnection"; + private TestRequestKey() {} public static final RequestContext getTestConnectionRequestContext(int replayerIdx) { + return getTestConnectionRequestContext(DEFAULT_TEST_CONNECTION, replayerIdx); + } + + public static final RequestContext getTestConnectionRequestContext(String connectionId, int replayerIdx) { var rk = new UniqueReplayerRequestKey( - new PojoTrafficStreamKey("testNodeId", "testConnectionId", 0), + new PojoTrafficStreamKey(TEST_NODE_ID, connectionId, 0), 0, replayerIdx); var smc = new SimpleMeteringClosure("test"); var channelKeyContext = new ChannelKeyContext(rk.trafficStreamKey, smc.makeSpanContinuation("test", null)); diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java index 59d4576f2..3439bbf8f 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java @@ -141,7 +141,7 @@ static void runPipelineAndValidate(IJsonTransformer transformer, var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), new AggregatedRawResponse(-1, Duration.ZERO, new ArrayList<>(), null)); var transformingHandler = new HttpJsonTransformingConsumer<>(transformer, authTransformer, testPacketCapture, - "TEST", TestRequestKey.getTestConnectionRequestContext(0)); + TestRequestKey.getTestConnectionRequestContext("TEST_CONNECTION", 0)); var contentLength = stringParts.stream().mapToInt(String::length).sum(); var headerString = "GET / HTTP/1.1\r\n" + From c14da6a4f170a3a0ef70fb84853de9e776053dd4 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Tue, 12 Dec 2023 10:08:58 -0500 Subject: [PATCH 19/20] Update the Http Logging Handler to suppress response packet captures when the request was ignored and remove the now-unused file for responses. I'd like to revisit this eventually to make sure that it's as efficient as possible and to organize it better. However, it does get the job done for now and tests were updated to confirm. Signed-off-by: Greg Schohn --- .../netty/LoggingHttpRequestHandler.java | 92 +++++++++++++------ .../netty/LoggingHttpResponseHandler.java | 76 --------------- ...ReliableLoggingHttpRequestHandlerTest.java | 64 +++++++------ .../netty/ProxyChannelInitializer.java | 3 - 4 files changed, 102 insertions(+), 133 deletions(-) delete mode 100644 TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java index f941acf3c..5c1193720 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java @@ -41,11 +41,40 @@ public class LoggingHttpRequestHandler extends ChannelDuplexHandler { public static final String GATHERING_RESPONSE = "gatheringResponse"; public static final String BLOCKED = "blocked"; + static class CaptureIgnoreState { + static final byte CAPTURE = 0; + static final byte IGNORE_REQUEST = 1; + static final byte IGNORE_RESPONSE = 2; + private CaptureIgnoreState() {} + } + + static class CaptureState { + byte captureIgnoreState = CaptureIgnoreState.CAPTURE; + boolean liveReadObservationsInOffloader = false; + + boolean shouldCapture() { + return captureIgnoreState == CaptureIgnoreState.CAPTURE; + } + + public void setShouldCaptureForRequest(boolean b) { + captureIgnoreState = b ? CaptureIgnoreState.CAPTURE : CaptureIgnoreState.IGNORE_REQUEST; + } + + public void advanceStateModelIntoResponseGather() { + if (CaptureIgnoreState.CAPTURE != captureIgnoreState) { + captureIgnoreState = CaptureIgnoreState.IGNORE_RESPONSE; + } + } + } + static class SimpleHttpRequestDecoder extends HttpRequestDecoder { private final PassThruHttpHeaders.HttpHeadersToPreserve headersToPreserve; + private final CaptureState captureState; - public SimpleHttpRequestDecoder(@NonNull PassThruHttpHeaders.HttpHeadersToPreserve headersToPreserve) { + public SimpleHttpRequestDecoder(@NonNull PassThruHttpHeaders.HttpHeadersToPreserve headersToPreserve, + CaptureState captureState) { this.headersToPreserve = headersToPreserve; + this.captureState = captureState; } /** @@ -60,34 +89,40 @@ public HttpMessage createMessage(String[] initialLine) throws Exception { , new PassThruHttpHeaders(headersToPreserve) ); } - } + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + if (captureState.captureIgnoreState == CaptureIgnoreState.IGNORE_RESPONSE) { + captureState.captureIgnoreState = CaptureIgnoreState.CAPTURE; + } + super.channelRead(ctx, msg); + } + } + static class SimpleDecodedHttpRequestHandler extends ChannelInboundHandlerAdapter { @Getter private HttpRequest currentRequest; final RequestCapturePredicate requestCapturePredicate; - boolean isDone; - boolean shouldCapture; - boolean liveReadObservationsInOffloader; - - SimpleDecodedHttpRequestHandler(RequestCapturePredicate requestCapturePredicate) { + boolean haveParsedFullRequest; + final CaptureState captureState; + + SimpleDecodedHttpRequestHandler(RequestCapturePredicate requestCapturePredicate, CaptureState captureState) { this.requestCapturePredicate = requestCapturePredicate; this.currentRequest = null; - this.isDone = false; - this.shouldCapture = true; - liveReadObservationsInOffloader = false; + this.haveParsedFullRequest = false; + this.captureState = captureState; } @Override public void channelRead(@NonNull ChannelHandlerContext ctx, @NonNull Object msg) throws Exception { if (msg instanceof HttpRequest) { currentRequest = (HttpRequest) msg; - shouldCapture = RequestCapturePredicate.CaptureDirective.CAPTURE == - requestCapturePredicate.apply((HttpRequest) msg); + captureState.setShouldCaptureForRequest(RequestCapturePredicate.CaptureDirective.CAPTURE == + requestCapturePredicate.apply((HttpRequest) msg)); } else if (msg instanceof HttpContent) { ((HttpContent)msg).release(); if (msg instanceof LastHttpContent) { - isDone = true; + haveParsedFullRequest = true; } } else { super.channelRead(ctx, msg); @@ -95,11 +130,9 @@ public void channelRead(@NonNull ChannelHandlerContext ctx, @NonNull Object msg) } public HttpRequest resetCurrentRequest() { - this.shouldCapture = true; - this.isDone = false; + this.haveParsedFullRequest = false; var old = currentRequest; this.currentRequest = null; - this.liveReadObservationsInOffloader = false; return old; } } @@ -122,13 +155,13 @@ public LoggingHttpRequestHandler(String nodeId, String channelKey, METERING_CLOSURE.meterIncrementEvent(messageContext, "requestStarted"); this.trafficOffloader = trafficOffloaderFactory.createOffloader(parentContext, channelKey); + var captureState = new CaptureState(); httpDecoderChannel = new EmbeddedChannel( - new SimpleHttpRequestDecoder(httpHeadersCapturePredicate.getHeadersRequiredForMatcher()), - new SimpleDecodedHttpRequestHandler(httpHeadersCapturePredicate) + new SimpleHttpRequestDecoder(httpHeadersCapturePredicate.getHeadersRequiredForMatcher(), captureState), + new SimpleDecodedHttpRequestHandler(httpHeadersCapturePredicate, captureState) ); } - static String getSpanLabelForState(HttpMessageContext.HttpTransactionState state) { switch (state) { case REQUEST: @@ -217,16 +250,16 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception httpDecoderChannel.writeInbound(bb.retainedDuplicate()); // the ByteBuf is consumed/release by this method METERING_CLOSURE.meterIncrementEvent(messageContext, - getHandlerThatHoldsParsedHttpRequest().isDone ? "requestFullyParsed" : "requestPartiallyParsed"); + getHandlerThatHoldsParsedHttpRequest().haveParsedFullRequest ? "requestFullyParsed" : "requestPartiallyParsed"); - var shouldCapture = requestParsingHandler.shouldCapture; + var captureState = requestParsingHandler.captureState; + var shouldCapture = captureState.shouldCapture(); if (shouldCapture) { - requestParsingHandler.liveReadObservationsInOffloader = true; + captureState.liveReadObservationsInOffloader = true; trafficOffloader.addReadEvent(timestamp, bb); - - } else if (requestParsingHandler.liveReadObservationsInOffloader) { + } else if (captureState.liveReadObservationsInOffloader) { trafficOffloader.cancelCaptureForCurrentRequest(timestamp); - requestParsingHandler.liveReadObservationsInOffloader = false; + captureState.liveReadObservationsInOffloader = false; } metricsLogger.atSuccess(MetricsEvent.RECEIVED_REQUEST_COMPONENT) @@ -235,9 +268,12 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception METERING_CLOSURE.meterIncrementEvent(messageContext, "readBytes", bb.readableBytes()); - if (requestParsingHandler.isDone) { + if (requestParsingHandler.haveParsedFullRequest) { messageContext.getCurrentSpan().end(); var httpRequest = requestParsingHandler.resetCurrentRequest(); + captureState.liveReadObservationsInOffloader = false; + captureState.advanceStateModelIntoResponseGather(); + if (shouldCapture) { var decoderResultLoose = httpRequest.decoderResult(); if (decoderResultLoose instanceof HttpMessageDecoderResult) { @@ -260,7 +296,9 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) rotateNextMessageContext(HttpMessageContext.HttpTransactionState.RESPONSE); } var bb = (ByteBuf) msg; - trafficOffloader.addWriteEvent(Instant.now(), bb); + if (getHandlerThatHoldsParsedHttpRequest().captureState.shouldCapture()) { + trafficOffloader.addWriteEvent(Instant.now(), bb); + } metricsLogger.atSuccess(MetricsEvent.RECEIVED_RESPONSE_COMPONENT) .setAttribute(MetricsAttributeKey.CHANNEL_ID, ctx.channel().id().asLongText()).emit(); METERING_CLOSURE.meterIncrementEvent(messageContext, "write"); diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java deleted file mode 100644 index 50d263550..000000000 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java +++ /dev/null @@ -1,76 +0,0 @@ -package org.opensearch.migrations.trafficcapture.netty; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelOutboundHandlerAdapter; -import io.netty.channel.ChannelPromise; -import lombok.extern.slf4j.Slf4j; -import org.opensearch.migrations.coreutils.MetricsAttributeKey; -import org.opensearch.migrations.coreutils.MetricsEvent; -import org.opensearch.migrations.coreutils.MetricsLogger; -import org.opensearch.migrations.tracing.SimpleMeteringClosure; -import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; -import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; - -import java.time.Instant; - -@Slf4j -public class LoggingHttpResponseHandler extends ChannelOutboundHandlerAdapter { - public static final String TELEMETRY_SCOPE_NAME = "LoggingHttpOutboundHandler"; - public static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure(TELEMETRY_SCOPE_NAME); - private static final MetricsLogger metricsLogger = new MetricsLogger("LoggingHttpResponseHandler"); - - private final IChannelConnectionCaptureSerializer trafficOffloader; - private ConnectionContext telemetryContext; - - public LoggingHttpResponseHandler(ConnectionContext incomingContext, - IChannelConnectionCaptureSerializer trafficOffloader) { - this.trafficOffloader = trafficOffloader; - this.telemetryContext = incomingContext; - } -// -// @Override -// public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, SocketAddress localAddress, ChannelPromise promise) throws Exception { -// trafficOffloader.addConnectEvent(Instant.now(), remoteAddress, localAddress); -// -// telemetryContext = new ConnectionContext(telemetryContext, -// METERING_CLOSURE.makeSpanContinuation("backendConnection")); -// METERING_CLOSURE.meterIncrementEvent(telemetryContext, "connect"); -// METERING_CLOSURE.meterDeltaEvent(telemetryContext, "connections", 1); -// -// super.connect(ctx, remoteAddress, localAddress, promise); -// } - -// @Override -// public void disconnect(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { -// trafficOffloader.addDisconnectEvent(Instant.now()); -// METERING_CLOSURE.meterIncrementEvent(telemetryContext, "disconnect"); -// super.disconnect(ctx, promise); -// } - -// @Override -// public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { -// trafficOffloader.addCloseEvent(Instant.now()); -// -// METERING_CLOSURE.meterIncrementEvent(telemetryContext, "close"); -// METERING_CLOSURE.meterDeltaEvent(telemetryContext, "connections", -1); -// METERING_CLOSURE.meterHistogramMillis(telemetryContext, "connectionDuration"); -// telemetryContext.currentSpan.end(); -// } - -// @Override -// public void deregister(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { -// trafficOffloader.addDeregisterEvent(Instant.now()); -// METERING_CLOSURE.meterIncrementEvent(telemetryContext, "deregister"); -// super.deregister(ctx, promise); -// } - -// -// @Override -// public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { -// trafficOffloader.addExceptionCaughtEvent(Instant.now(), cause); -// METERING_CLOSURE.meterIncrementEvent(telemetryContext, "exception"); -// super.exceptionCaught(ctx, cause); -// } - -} diff --git a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java index b5a8936d3..6fe532efe 100644 --- a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java +++ b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java @@ -4,11 +4,9 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.embedded.EmbeddedChannel; -import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; -import lombok.AllArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; @@ -23,17 +21,17 @@ import org.opensearch.migrations.trafficcapture.CodedOutputStreamHolder; import org.opensearch.migrations.trafficcapture.OrderedStreamLifecyleManager; import org.opensearch.migrations.trafficcapture.StreamChannelConnectionCaptureSerializer; -import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; +import org.opensearch.migrations.trafficcapture.protos.TrafficObservation; import org.opensearch.migrations.trafficcapture.protos.TrafficStream; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.SequenceInputStream; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; @@ -41,6 +39,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.stream.Stream; @Slf4j public class ConditionallyReliableLoggingHttpRequestHandlerTest { @@ -185,18 +184,22 @@ public void testThatHealthCheckCaptureCanBeSuppressed(boolean singleBytes) throw var headerCapturePredicate = new HeaderValueFilteringCapturePredicate(Map.of("user-Agent", ".*uploader.*")); EmbeddedChannel channel = new EmbeddedChannel( new ConditionallyReliableLoggingHttpRequestHandler("n", "c", - (ctx, connectionId) -> offloader, headerCapturePredicate, x->true)); + (ctx, connectionId) -> offloader, headerCapturePredicate, x->false)); getWriter(singleBytes, true, SimpleRequests.HEALTH_CHECK.getBytes(StandardCharsets.UTF_8)).accept(channel); + channel.writeOutbound(Unpooled.wrappedBuffer("response1".getBytes(StandardCharsets.UTF_8))); getWriter(singleBytes, true, SimpleRequests.SMALL_POST.getBytes(StandardCharsets.UTF_8)).accept(channel); + var bytesForResponsePreserved = "response2".getBytes(StandardCharsets.UTF_8); + channel.writeOutbound(Unpooled.wrappedBuffer(bytesForResponsePreserved)); + channel.close(); var requestBytes = (SimpleRequests.HEALTH_CHECK + SimpleRequests.SMALL_POST).getBytes(StandardCharsets.UTF_8); // we wrote the correct data to the downstream handler/channel - var outputData = new SequenceInputStream(Collections.enumeration(channel.inboundMessages().stream() + var consumedData = new SequenceInputStream(Collections.enumeration(channel.inboundMessages().stream() .map(m->new ByteArrayInputStream(consumeIntoArray((ByteBuf)m))) .collect(Collectors.toList()))) .readAllBytes(); - log.info("outputdata = " + new String(outputData, StandardCharsets.UTF_8)); - Assertions.assertArrayEquals(requestBytes, outputData); + log.info("captureddata = " + new String(consumedData, StandardCharsets.UTF_8)); + Assertions.assertArrayEquals(requestBytes, consumedData); Assertions.assertNotNull(streamMgr.byteBufferAtomicReference, "This would be null if the handler didn't block until the output was written"); @@ -206,34 +209,41 @@ public void testThatHealthCheckCaptureCanBeSuppressed(boolean singleBytes) throw trafficStream.getSubStream(0).hasRead()); Assertions.assertEquals(1, streamMgr.flushCount.get()); var observations = trafficStream.getSubStreamList(); - if (singleBytes) { - var sawRequestDropped = new AtomicBoolean(false); - var observationsAfterDrop = observations.stream().dropWhile(o->{ - var wasDrop = o.hasRequestDropped(); - sawRequestDropped.compareAndSet(false, wasDrop); - return !sawRequestDropped.get() || wasDrop; - }).collect(Collectors.toList()); + { + var readObservationStreamToUse = singleBytes ? skipReadsBeforeDrop(observations) : observations.stream(); var combinedTrafficPacketsSteam = - new SequenceInputStream(Collections.enumeration(observationsAfterDrop.stream() - .filter(to->to.hasRead()) - .map(to->new ByteArrayInputStream(to.getRead().getData().toByteArray())) + new SequenceInputStream(Collections.enumeration(readObservationStreamToUse + .filter(to -> to.hasRead()) + .map(to -> new ByteArrayInputStream(to.getRead().getData().toByteArray())) .collect(Collectors.toList()))); + var reconstitutedTrafficStreamReads = combinedTrafficPacketsSteam.readAllBytes(); Assertions.assertArrayEquals(SimpleRequests.SMALL_POST.getBytes(StandardCharsets.UTF_8), - combinedTrafficPacketsSteam.readAllBytes()); - } else { + reconstitutedTrafficStreamReads); + } + + // check that we only got one response + { var combinedTrafficPacketsSteam = new SequenceInputStream(Collections.enumeration(observations.stream() - .filter(to->to.hasRead()) - .map(to->new ByteArrayInputStream(to.getRead().getData().toByteArray())) + .filter(to->to.hasWrite()) + .map(to->new ByteArrayInputStream(to.getWrite().getData().toByteArray())) .collect(Collectors.toList()))); - var reconstitutedTrafficStreamReads = combinedTrafficPacketsSteam.readAllBytes(); - log.info("reconstitutedTrafficStreamReads="+ - new String(reconstitutedTrafficStreamReads, StandardCharsets.UTF_8)); - Assertions.assertArrayEquals(SimpleRequests.SMALL_POST.getBytes(StandardCharsets.UTF_8), - reconstitutedTrafficStreamReads); + var reconstitutedTrafficStreamWrites = combinedTrafficPacketsSteam.readAllBytes(); + log.info("reconstitutedTrafficStreamWrites="+ + new String(reconstitutedTrafficStreamWrites, StandardCharsets.UTF_8)); + Assertions.assertArrayEquals(bytesForResponsePreserved, reconstitutedTrafficStreamWrites); } } + private static Stream skipReadsBeforeDrop(List observations) { + var sawRequestDropped = new AtomicBoolean(false); + return observations.stream().dropWhile(o->{ + var wasDrop = o.hasRequestDropped(); + sawRequestDropped.compareAndSet(false, wasDrop); + return !sawRequestDropped.get() || wasDrop; + }); + } + private Consumer getWriter(boolean singleBytes, boolean usePool, byte[] bytes) { if (singleBytes) { return getSingleByteAtATimeWriter(usePool, bytes); diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java index d9dc29a63..449abcc5e 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java @@ -5,14 +5,11 @@ import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.ssl.SslHandler; -import org.opensearch.migrations.tracing.IWithAttributes; import org.opensearch.migrations.tracing.SimpleMeteringClosure; import lombok.NonNull; import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; import org.opensearch.migrations.trafficcapture.netty.ConditionallyReliableLoggingHttpRequestHandler; import org.opensearch.migrations.trafficcapture.netty.RequestCapturePredicate; -import org.opensearch.migrations.trafficcapture.netty.LoggingHttpResponseHandler; -import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; import javax.net.ssl.SSLEngine; import java.io.IOException; From 7de4009181f92f713c54627cc71718131347ca18 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Tue, 12 Dec 2023 10:10:01 -0500 Subject: [PATCH 20/20] File rename since the LoggingHttpRequest handler now handles both requests and responses. Signed-off-by: Greg Schohn --- ...nditionallyReliableLoggingHttpHandler.java} | 18 +++++------------- ...estHandler.java => LoggingHttpHandler.java} | 8 ++++---- ...ionallyReliableLoggingHttpHandlerTest.java} | 8 ++++---- .../netty/ProxyChannelInitializer.java | 4 ++-- 4 files changed, 15 insertions(+), 23 deletions(-) rename TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/{ConditionallyReliableLoggingHttpRequestHandler.java => ConditionallyReliableLoggingHttpHandler.java} (74%) rename TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/{LoggingHttpRequestHandler.java => LoggingHttpHandler.java} (97%) rename TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/{ConditionallyReliableLoggingHttpRequestHandlerTest.java => ConditionallyReliableLoggingHttpHandlerTest.java} (97%) diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpHandler.java similarity index 74% rename from TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java rename to TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpHandler.java index a641b4bfe..05a31e8cd 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpHandler.java @@ -3,31 +3,23 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.HttpRequest; import io.netty.util.ReferenceCountUtil; -import io.opentelemetry.api.trace.Span; -import lombok.Getter; import lombok.Lombok; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.opensearch.migrations.tracing.IWithStartTimeAndAttributes; -import org.opensearch.migrations.tracing.commoncontexts.IConnectionContext; -import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; import org.opensearch.migrations.trafficcapture.netty.tracing.HttpMessageContext; -import org.opensearch.migrations.trafficcapture.tracing.ConnectionContext; import java.io.IOException; -import java.time.Instant; -import java.util.function.Function; import java.util.function.Predicate; @Slf4j -public class ConditionallyReliableLoggingHttpRequestHandler extends LoggingHttpRequestHandler { +public class ConditionallyReliableLoggingHttpHandler extends LoggingHttpHandler { private final Predicate shouldBlockPredicate; - public ConditionallyReliableLoggingHttpRequestHandler(@NonNull String nodeId, String connectionId, - @NonNull IConnectionCaptureFactory trafficOffloaderFactory, - @NonNull RequestCapturePredicate requestCapturePredicate, - @NonNull Predicate headerPredicateForWhenToBlock) + public ConditionallyReliableLoggingHttpHandler(@NonNull String nodeId, String connectionId, + @NonNull IConnectionCaptureFactory trafficOffloaderFactory, + @NonNull RequestCapturePredicate requestCapturePredicate, + @NonNull Predicate headerPredicateForWhenToBlock) throws IOException { super(nodeId, connectionId, trafficOffloaderFactory, requestCapturePredicate); this.shouldBlockPredicate = headerPredicateForWhenToBlock; diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpHandler.java similarity index 97% rename from TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java rename to TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpHandler.java index 5c1193720..05061cdcc 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpHandler.java @@ -32,7 +32,7 @@ import java.time.Instant; @Slf4j -public class LoggingHttpRequestHandler extends ChannelDuplexHandler { +public class LoggingHttpHandler extends ChannelDuplexHandler { public static final String TELEMETRY_SCOPE_NAME = "CapturingHttpHandler"; public static final SimpleMeteringClosure METERING_CLOSURE = new SimpleMeteringClosure(TELEMETRY_SCOPE_NAME); private static final MetricsLogger metricsLogger = new MetricsLogger("LoggingHttpRequestHandler"); @@ -143,9 +143,9 @@ public HttpRequest resetCurrentRequest() { protected HttpMessageContext messageContext; - public LoggingHttpRequestHandler(String nodeId, String channelKey, - @NonNull IConnectionCaptureFactory trafficOffloaderFactory, - @NonNull RequestCapturePredicate httpHeadersCapturePredicate) + public LoggingHttpHandler(String nodeId, String channelKey, + @NonNull IConnectionCaptureFactory trafficOffloaderFactory, + @NonNull RequestCapturePredicate httpHeadersCapturePredicate) throws IOException { var parentContext = new ConnectionContext(channelKey, nodeId, METERING_CLOSURE.makeSpanContinuation("connectionLifetime", null)); diff --git a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpHandlerTest.java similarity index 97% rename from TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java rename to TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpHandlerTest.java index 6fe532efe..da9c1ea8d 100644 --- a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java +++ b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpHandlerTest.java @@ -42,7 +42,7 @@ import java.util.stream.Stream; @Slf4j -public class ConditionallyReliableLoggingHttpRequestHandlerTest { +public class ConditionallyReliableLoggingHttpHandlerTest { @RegisterExtension static final OpenTelemetryExtension otelTesting = OpenTelemetryExtension.create(); private final Tracer tracer = otelTesting.getOpenTelemetry().getTracer("test"); @@ -86,7 +86,7 @@ private static void writeMessageAndVerify(byte[] fullTrafficBytes, Consumer offloader, + new ConditionallyReliableLoggingHttpHandler("n", "c", (ctx, connectionId) -> offloader, new RequestCapturePredicate(), x->true)); // true: block every request channelWriter.accept(channel); @@ -159,7 +159,7 @@ public void testThatSuppressedCaptureWorks() throws Exception { var headerCapturePredicate = new HeaderValueFilteringCapturePredicate(Map.of("user-Agent", "uploader")); EmbeddedChannel channel = new EmbeddedChannel( - new ConditionallyReliableLoggingHttpRequestHandler("n", "c", + new ConditionallyReliableLoggingHttpHandler("n", "c", (ctx, connectionId) -> offloader, headerCapturePredicate, x->true)); getWriter(false, true, SimpleRequests.HEALTH_CHECK.getBytes(StandardCharsets.UTF_8)).accept(channel); channel.close(); @@ -183,7 +183,7 @@ public void testThatHealthCheckCaptureCanBeSuppressed(boolean singleBytes) throw var headerCapturePredicate = new HeaderValueFilteringCapturePredicate(Map.of("user-Agent", ".*uploader.*")); EmbeddedChannel channel = new EmbeddedChannel( - new ConditionallyReliableLoggingHttpRequestHandler("n", "c", + new ConditionallyReliableLoggingHttpHandler("n", "c", (ctx, connectionId) -> offloader, headerCapturePredicate, x->false)); getWriter(singleBytes, true, SimpleRequests.HEALTH_CHECK.getBytes(StandardCharsets.UTF_8)).accept(channel); channel.writeOutbound(Unpooled.wrappedBuffer("response1".getBytes(StandardCharsets.UTF_8))); diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java index 449abcc5e..6ff60ee50 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java @@ -8,7 +8,7 @@ import org.opensearch.migrations.tracing.SimpleMeteringClosure; import lombok.NonNull; import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; -import org.opensearch.migrations.trafficcapture.netty.ConditionallyReliableLoggingHttpRequestHandler; +import org.opensearch.migrations.trafficcapture.netty.ConditionallyReliableLoggingHttpHandler; import org.opensearch.migrations.trafficcapture.netty.RequestCapturePredicate; import javax.net.ssl.SSLEngine; @@ -48,7 +48,7 @@ protected void initChannel(SocketChannel ch) throws IOException { } var connectionId = ch.id().asLongText(); - ch.pipeline().addLast(new ConditionallyReliableLoggingHttpRequestHandler("n", "c", + ch.pipeline().addLast(new ConditionallyReliableLoggingHttpHandler("n", "c", connectionCaptureFactory, requestCapturePredicate, this::shouldGuaranteeMessageOffloading)); ch.pipeline().addLast(new FrontsideHandler(backsideConnectionPool)); }