From 4f63298823b36f137106561659ce46a560702e12 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Fri, 5 Apr 2024 09:47:53 -0500 Subject: [PATCH 01/10] Fix Chunked header processing in NettyJsonToByteBufHandler Signed-off-by: Andre Kurait --- .../http/NettyJsonToByteBufHandler.java | 47 ++++--- .../HttpJsonTransformingConsumerTest.java | 119 ++++++++++++------ .../resources/requests/raw/.gitattributes | 2 + ...st_formUrlEncoded_withDuplicateHeaders.txt | 50 ++++++++ .../post_formUrlEncoded_withLargeHeader.txt | 71 +++++++++++ .../TestCapturePacketToHttpHandler.java | 5 + 6 files changed, 230 insertions(+), 64 deletions(-) create mode 100644 TrafficCapture/trafficReplayer/src/test/resources/requests/raw/.gitattributes create mode 100644 TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withDuplicateHeaders.txt create mode 100644 TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withLargeHeader.txt diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonToByteBufHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonToByteBufHandler.java index 0195b810b..54bf569fa 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonToByteBufHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonToByteBufHandler.java @@ -1,16 +1,13 @@ package org.opensearch.migrations.replay.datahandlers.http; import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; import io.netty.buffer.ByteBufOutputStream; -import io.netty.buffer.CompositeByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.handler.codec.http.HttpContent; import io.netty.handler.codec.http.LastHttpContent; -import io.netty.util.ResourceLeakDetector; -import io.netty.util.ResourceLeakDetectorFactory; +import lombok.Lombok; import lombok.extern.slf4j.Slf4j; import java.io.ByteArrayOutputStream; @@ -19,7 +16,6 @@ import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; /** * This class does the remaining serialization of the contents coming into it into ByteBuf @@ -37,8 +33,6 @@ */ @Slf4j public class NettyJsonToByteBufHandler extends ChannelInboundHandlerAdapter { - // TODO: Eventually, we can count up the size of all of the entries in the headers - but for now, I'm being lazy - public static final int MAX_HEADERS_BYTE_SIZE = 64 * 1024; List> sharedInProgressChunkSizes; ByteBuf inProgressByteBuf; int payloadBufferIndex; @@ -133,7 +127,7 @@ private void writeHeadersIntoByteBufs(ChannelHandlerContext ctx, var headerChunkSizes = sharedInProgressChunkSizes.get(0); try { if (headerChunkSizes.size() > 1) { - writeHeadersAsChunks(ctx, httpJson, headerChunkSizes, MAX_HEADERS_BYTE_SIZE); + writeHeadersAsChunks(ctx, httpJson, headerChunkSizes); return; } } catch (Exception e) { @@ -149,28 +143,31 @@ private void writeHeadersIntoByteBufs(ChannelHandlerContext ctx, private static void writeHeadersAsChunks(ChannelHandlerContext ctx, HttpJsonMessageWithFaultingPayload httpJson, - List headerChunkSizes, - int maxLastBufferSize) - throws IOException - { - AtomicInteger chunkIdx = new AtomicInteger(headerChunkSizes.size()); - var bufs = headerChunkSizes.stream() - .map(i -> ctx.alloc().buffer(chunkIdx.decrementAndGet()==0?maxLastBufferSize:i).retain()) - .toArray(ByteBuf[]::new); - CompositeByteBuf cbb = null; + List headerChunkSizes) throws IOException { + var initialSize = headerChunkSizes.stream().mapToInt(Integer::intValue).sum(); + + ByteBuf buf = null; try { - cbb = ctx.alloc().compositeBuffer(bufs.length); - cbb.addComponents(true, bufs); - log.debug("cbb.refcnt=" + cbb.refCnt()); - try (var bbos = new ByteBufOutputStream(cbb)) { + buf = ctx.alloc().buffer(initialSize); + try (var bbos = new ByteBufOutputStream(buf)) { writeHeadersIntoStream(httpJson, bbos); } - for (var bb : bufs) { - ctx.fireChannelRead(bb); + + int index = 0; + var chunkSizeIterator = headerChunkSizes.iterator(); + while (index < buf.writerIndex()) { + if (!chunkSizeIterator.hasNext()) { + throw Lombok.sneakyThrow(new RuntimeException("Ran out of input chunks for mapping")); + } + var inputChunkSize = chunkSizeIterator.next(); + var scaledChunkSize = (int) (((long) buf.writerIndex() * inputChunkSize) + (initialSize - 1)) / initialSize; + int actualChunkSize = Math.min(buf.writerIndex() - index, scaledChunkSize); + ctx.fireChannelRead(buf.retainedSlice(index, actualChunkSize)); + index += actualChunkSize; } } finally { - if (cbb != null) { - cbb.release(); + if (buf != null) { + buf.release(); } } } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java index f3ee45482..91163d03a 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java @@ -2,6 +2,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.opensearch.migrations.replay.AggregatedRawResponse; import org.opensearch.migrations.replay.TestCapturePacketToHttpHandler; import org.opensearch.migrations.replay.TransformationLoader; @@ -14,56 +17,67 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.Random; +import java.util.stream.Stream; @WrapWithNettyLeakDetection class HttpJsonTransformingConsumerTest extends InstrumentationTest { - @Test - public void testPassThroughSinglePacketPost() throws Exception { - final var dummyAggregatedResponse = - new AggregatedRawResponse(17, null, null, null); - var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse); - var transformingHandler = - new HttpJsonTransformingConsumer(new TransformationLoader() - .getTransformerFactoryLoader(null), - null, testPacketCapture, - rootContext.getTestConnectionRequestContext(0)); - byte[] testBytes; - try (var sampleStream = HttpJsonTransformingConsumer.class.getResourceAsStream( - "/requests/raw/post_formUrlEncoded_withFixedLength.txt")) { - testBytes = sampleStream.readAllBytes(); - } - transformingHandler.consumeBytes(testBytes); - var returnedResponse = transformingHandler.finalizeRequest().get(); - Assertions.assertEquals(new String(testBytes, StandardCharsets.UTF_8), testPacketCapture.getCapturedAsString()); - Assertions.assertArrayEquals(testBytes, testPacketCapture.getBytesCaptured()); - Assertions.assertEquals(HttpRequestTransformationStatus.SKIPPED, returnedResponse.transformationStatus); + + private static Stream provideTestParameters() { + Integer[] attemptedChunks = {1, 2, 4, 8, 100, 1000, Integer.MAX_VALUE}; + Boolean[] transformationOptions = {true, false}; + String[] requestFiles = { + "/requests/raw/post_formUrlEncoded_withFixedLength.txt", + "/requests/raw/post_formUrlEncoded_withLargeHeader.txt", + "/requests/raw/post_formUrlEncoded_withDuplicateHeaders.txt", + "/requests/raw/get_withAuthHeader.txt" + }; + + return Stream.of(attemptedChunks) + .flatMap(chunks -> Stream.of(transformationOptions) + .flatMap(transformation -> Stream.of(requestFiles) + .map(file -> Arguments.of(chunks, transformation, file)))); } - @Test - public void testPassThroughSinglePacketWithoutBodyTransformationPost() throws Exception { + @ParameterizedTest + @MethodSource("provideTestParameters") + public void testRequestProcessing(Integer attemptedChunks, Boolean hostTransformation, String requestFile) throws Exception { final var dummyAggregatedResponse = new AggregatedRawResponse(17, null, null, null); - var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse); + var testPacketCapture = new TestCapturePacketToHttpHandler( + Duration.ofMillis(Math.min(100 / attemptedChunks, 1)), dummyAggregatedResponse); var transformingHandler = - new HttpJsonTransformingConsumer( - new TransformationLoader().getTransformerFactoryLoader("test.domain"), - null, testPacketCapture, - rootContext.getTestConnectionRequestContext(0)); + new HttpJsonTransformingConsumer(new TransformationLoader() + .getTransformerFactoryLoader(hostTransformation ? "bar.example" : null), + null, testPacketCapture, + rootContext.getTestConnectionRequestContext(0)); byte[] testBytes; - try (var sampleStream = HttpJsonTransformingConsumer.class.getResourceAsStream( - "/requests/raw/post_formUrlEncoded_withFixedLength.txt")) { + try (var sampleStream = HttpJsonTransformingConsumer.class.getResourceAsStream(requestFile)) { testBytes = sampleStream.readAllBytes(); - testBytes = new String(testBytes, StandardCharsets.UTF_8) - .replace("foo.example", "test.domain") - .getBytes(StandardCharsets.UTF_8); } - transformingHandler.consumeBytes(testBytes); + + var chunks = Math.min(attemptedChunks, testBytes.length); + sliceRandomChunks(testBytes, chunks).forEach(chunk -> transformingHandler.consumeBytes(chunk)); + var returnedResponse = transformingHandler.finalizeRequest().get(); - Assertions.assertEquals(new String(testBytes, StandardCharsets.UTF_8), - testPacketCapture.getCapturedAsString()); - Assertions.assertArrayEquals(testBytes, testPacketCapture.getBytesCaptured()); - Assertions.assertEquals(HttpRequestTransformationStatus.SKIPPED, returnedResponse.transformationStatus); + + var expectedBytes = (hostTransformation) ? new String(testBytes, StandardCharsets.UTF_8) + .replace("foo.example", "bar.example").getBytes(StandardCharsets.UTF_8) : testBytes; + + var expectedTransformationStatus = (hostTransformation) ? HttpRequestTransformationStatus.COMPLETED : HttpRequestTransformationStatus.SKIPPED; + + + Assertions.assertEquals(new String(expectedBytes, StandardCharsets.UTF_8), testPacketCapture.getCapturedAsString()); + Assertions.assertArrayEquals(expectedBytes, testPacketCapture.getBytesCaptured()); + Assertions.assertEquals(expectedTransformationStatus, returnedResponse.transformationStatus); + + var numConsumes = testPacketCapture.getNumConsumes().get(); + Assertions.assertTrue(chunks + 1 == numConsumes || chunks == numConsumes, + "Expected output consumes to equal input consumes or input consumes + 1, but was " + numConsumes); } @Test @@ -81,7 +95,7 @@ public void testRemoveAuthHeadersWorks() throws Exception { testBytes = sampleStream.readAllBytes(); testBytes = new String(testBytes, StandardCharsets.UTF_8) .replace("foo.example", "test.domain") - .replace("auTHorization: Basic YWRtaW46YWRtaW4=\n", "") + .replace("auTHorization: Basic YWRtaW46YWRtaW4=\r\n", "") .getBytes(StandardCharsets.UTF_8); } transformingHandler.consumeBytes(testBytes); @@ -118,7 +132,7 @@ private void walkMaps(Object o) { try (var sampleStream = HttpJsonTransformingConsumer.class.getResourceAsStream( "/requests/raw/post_formUrlEncoded_withFixedLength.txt")) { var allBytes = sampleStream.readAllBytes(); - Arrays.fill(allBytes, allBytes.length-10, allBytes.length, (byte)' '); + Arrays.fill(allBytes, allBytes.length-30, allBytes.length, (byte)' '); testBytes = Arrays.copyOfRange(allBytes, 0, allBytes.length); } transformingHandler.consumeBytes(testBytes); @@ -130,4 +144,31 @@ private void walkMaps(Object o) { Assertions.assertInstanceOf(NettyJsonBodyAccumulateHandler.IncompleteJsonBodyException.class, returnedResponse.error); } + + public static List sliceRandomChunks(byte[] bytes, int numChunks) { + Random random = new Random(); + List chunkSizes = new ArrayList<>(numChunks); + int totalSizeRemaining = bytes.length; + for (int i = 0; i < numChunks - 1; i++) { + // Ensure at least one byte per remaining chunk + int remainingChunks = numChunks - i; + int sizeForThisChunk = 1 + random.nextInt(totalSizeRemaining - remainingChunks + 1); + chunkSizes.add(sizeForThisChunk); + totalSizeRemaining -= sizeForThisChunk; + } + // Last chunk gets the remaining bytes + chunkSizes.add(totalSizeRemaining); + + // Shuffle the array to distribute large chunks randomly + Collections.shuffle(chunkSizes, random); + + var byteList = new ArrayList(numChunks); + int start = 0; + for (int size : chunkSizes) { + byte[] chunk = Arrays.copyOfRange(bytes, start, start + size); + byteList.add(chunk); + start += size; + } + return byteList; + } } \ No newline at end of file diff --git a/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/.gitattributes b/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/.gitattributes new file mode 100644 index 000000000..d479abe31 --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/.gitattributes @@ -0,0 +1,2 @@ +; forcing CRLF for correct HTTP spec adherence +* text=auto eol=crlf \ No newline at end of file diff --git a/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withDuplicateHeaders.txt b/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withDuplicateHeaders.txt new file mode 100644 index 000000000..5257fb68e --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withDuplicateHeaders.txt @@ -0,0 +1,50 @@ +POST /test HTTP/1.1 +Host: foo.example +Content-Type: application/json +Header1: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header1: abc +Header1: bar +Header1: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header1: abc +Header1: bar +Header1: bar +Header1: abc +Header1: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Content-Length: 616 + +{ + "settings": { + "index": { + "number_of_shards": 7, + "number_of_replicas": 3 + }, + "analysis": { + "analyzer": { + "nameAnalyzer": { + "type": "custom", + "tokenizer": "keyword", + "filter": "uppercase" + } + } + } + }, + "mappings": { + "employee": { + "properties": { + "age": { + "type": "long" + }, + "level": { + "type": "long" + }, + "title": { + "type": "text" + }, + "name": { + "type": "text", + "analyzer": "nameAnalyzer" + } + } + } + } +} diff --git a/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withLargeHeader.txt b/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withLargeHeader.txt new file mode 100644 index 000000000..7399b5db7 --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withLargeHeader.txt @@ -0,0 +1,71 @@ +POST /test HTTP/1.1 +Host: foo.example +Content-Type: application/json +Header1: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header2: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header3: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header4: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header5: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header6: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header7: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header8: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header9: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header10: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header11: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header12: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header13: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header14: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header15: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header16: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header17: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header18: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header19: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header20: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header21: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header22: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header23: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header24: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header25: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header26: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header27: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header28: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header29: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Header30: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl +Content-Length: 616 + +{ + "settings": { + "index": { + "number_of_shards": 7, + "number_of_replicas": 3 + }, + "analysis": { + "analyzer": { + "nameAnalyzer": { + "type": "custom", + "tokenizer": "keyword", + "filter": "uppercase" + } + } + } + }, + "mappings": { + "employee": { + "properties": { + "age": { + "type": "long" + }, + "level": { + "type": "long" + }, + "title": { + "type": "text" + }, + "name": { + "type": "text", + "analyzer": "nameAnalyzer" + } + } + } + } +} diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java index a0ff5a8ae..c7b6c81c5 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java @@ -23,6 +23,9 @@ public class TestCapturePacketToHttpHandler implements IPacketFinalizingConsumer private final AtomicInteger numFinalizations; @Getter private byte[] bytesCaptured; + + @Getter + private final AtomicInteger numConsumes; private final AggregatedRawResponse dummyAggregatedResponse; ByteArrayOutputStream byteArrayOutputStream; @@ -30,12 +33,14 @@ public TestCapturePacketToHttpHandler(Duration consumeDuration, @NonNull AggregatedRawResponse dummyAggregatedResponse) { this.consumeDuration = consumeDuration; this.numFinalizations = new AtomicInteger(); + this.numConsumes = new AtomicInteger(); this.dummyAggregatedResponse = dummyAggregatedResponse; byteArrayOutputStream = new ByteArrayOutputStream(); } @Override public DiagnosticTrackableCompletableFuture consumeBytes(ByteBuf nextRequestPacket) { + numConsumes.incrementAndGet(); log.info("incoming buffer refcnt="+nextRequestPacket.refCnt()); var duplicatedPacket = nextRequestPacket.duplicate().retain(); return new DiagnosticTrackableCompletableFuture<>(CompletableFuture.runAsync(() -> { From 199b3a0a955dfb6403da6ce4b825ee9a650043f2 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Fri, 5 Apr 2024 09:52:44 -0500 Subject: [PATCH 02/10] Fix duplicate header behavior when evaluating equality Signed-off-by: Andre Kurait --- .../http/NettyDecodedHttpRequestPreliminaryConvertHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java index 5ca61964b..6c19bab0a 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java @@ -136,9 +136,10 @@ private void handlePayloadNeutralTransformationOrThrow(ChannelHandlerContext ctx private boolean headerFieldsAreIdentical(HttpRequest request, HttpJsonMessageWithFaultingPayload httpJsonMessage) { if (!request.uri().equals(httpJsonMessage.path()) || !request.method().toString().equals(httpJsonMessage.method()) || - request.headers().size() != httpJsonMessage.headers().strictHeadersMap.size()) { + request.headers().names().size() != httpJsonMessage.headers().strictHeadersMap.size()) { return false; } + // Depends on header size check above for correctness for (var headerName : httpJsonMessage.headers().keySet()) { if (!headerFieldIsIdentical(headerName, request, httpJsonMessage)) { return false; From 296710ad92255d9ce15e8bbb80ab96e7e3682c8f Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Fri, 5 Apr 2024 09:53:12 -0500 Subject: [PATCH 03/10] Update gitignore with subproject build output dir Signed-off-by: Andre Kurait --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2389bcea0..58319b627 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ logs # Build files plugins/opensearch/loggable-transport-netty4/.gradle/ +TrafficCapture/**/out/ RFS/.gradle/ RFS/bin/ From c37cad218fd100792c4ed46a993bcf0f5833dff5 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Fri, 5 Apr 2024 10:52:52 -0500 Subject: [PATCH 04/10] Correct ByteBuf Formatter with CRLF line endings Signed-off-by: Andre Kurait --- .../replay/HttpByteBufFormatter.java | 5 +- .../replay/HttpByteBufFormatterTest.java | 61 +++++++++++-------- ...st_formUrlEncoded_withDuplicateHeaders.txt | 2 +- .../post_formUrlEncoded_withFixedLength.txt | 2 +- .../post_formUrlEncoded_withLargeHeader.txt | 2 +- 5 files changed, 41 insertions(+), 31 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpByteBufFormatter.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpByteBufFormatter.java index 9e3f1491c..8ff279f75 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpByteBufFormatter.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpByteBufFormatter.java @@ -29,6 +29,7 @@ @Slf4j public class HttpByteBufFormatter { + private static final String LINE_DELIMITER = "\r\n"; private static final ThreadLocal> printStyle = ThreadLocal.withInitial(Optional::empty); @@ -105,13 +106,13 @@ public static String httpPacketsToPrettyPrintedString(HttpMessageType msgType, S } public static String prettyPrintNettyRequest(FullHttpRequest msg, boolean sortHeaders) { - var sj = new StringJoiner("\n"); + var sj = new StringJoiner(LINE_DELIMITER); sj.add(msg.method() + " " + msg.uri() + " " + msg.protocolVersion().text()); return prettyPrintNettyMessage(sj, sortHeaders, msg, msg.content()); } static String prettyPrintNettyResponse(FullHttpResponse msg, boolean sortHeaders) { - var sj = new StringJoiner("\n"); + var sj = new StringJoiner(LINE_DELIMITER); sj.add(msg.protocolVersion().text() + " " + msg.status().code() + " " + msg.status().reasonPhrase()); return prettyPrintNettyMessage(sj, sortHeaders, msg, msg.content()); } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HttpByteBufFormatterTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HttpByteBufFormatterTest.java index 4447adeb9..c895bc367 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HttpByteBufFormatterTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HttpByteBufFormatterTest.java @@ -32,31 +32,25 @@ public static void setup() { "User-Agent: UnitTest\r\n" + "\r\n"; - final static String SAMPLE_REQUEST_AS_BLOCKS = "[G],[E],[T],[ ],[/],[ ],[H],[T],[T],[P],[/],[1],[.],[1],[\r" + - "],[\n" + - "],[H],[o],[s],[t],[:],[ ],[l],[o],[c],[a],[l],[h],[o],[s],[t],[\r" + - "],[\n" + - "],[C],[o],[n],[n],[e],[c],[t],[i],[o],[n],[:],[ ],[K],[e],[e],[p],[-],[A],[l],[i],[v],[e],[\r" + - "],[\n" + - "],[U],[s],[e],[r],[-],[A],[g],[e],[n],[t],[:],[ ],[U],[n],[i],[t],[T],[e],[s],[t],[\r" + - "],[\n" + - "],[\r" + - "],[\n" + - "]"; - - final static String SAMPLE_REQUEST_AS_PARSED_HTTP = "GET / HTTP/1.1\n" + - "Host: localhost\n" + - "Connection: Keep-Alive\n" + - "User-Agent: UnitTest\n" + - "content-length: 0\n" + - "\n"; - - final static String SAMPLE_REQUEST_AS_PARSED_HTTP_SORTED = "GET / HTTP/1.1\n" + - "Connection: Keep-Alive\n" + - "Host: localhost\n" + - "User-Agent: UnitTest\n" + - "content-length: 0\n" + - "\n"; + final static String SAMPLE_REQUEST_AS_BLOCKS = "[G],[E],[T],[ ],[/],[ ],[H],[T],[T],[P],[/],[1],[.],[1]," + "[\r],[\n]," + + "[H],[o],[s],[t],[:],[ ],[l],[o],[c],[a],[l],[h],[o],[s],[t]," + "[\r],[\n]," + + "[C],[o],[n],[n],[e],[c],[t],[i],[o],[n],[:],[ ],[K],[e],[e],[p],[-],[A],[l],[i],[v],[e]," + "[\r],[\n]," + + "[U],[s],[e],[r],[-],[A],[g],[e],[n],[t],[:],[ ],[U],[n],[i],[t],[T],[e],[s],[t]," + "[\r],[\n]," + + "[\r],[\n]"; + + final static String SAMPLE_REQUEST_AS_PARSED_HTTP = "GET / HTTP/1.1\r\n" + + "Host: localhost\r\n" + + "Connection: Keep-Alive\r\n" + + "User-Agent: UnitTest\r\n" + + "content-length: 0\r\n" + + "\r\n"; + + final static String SAMPLE_REQUEST_AS_PARSED_HTTP_SORTED = "GET / HTTP/1.1\r\n" + + "Connection: Keep-Alive\r\n" + + "Host: localhost\r\n" + + "User-Agent: UnitTest\r\n" + + "content-length: 0\r\n" + + "\r\n"; enum BufferType { BYTE_ARRAY, UNPOOLED_BYTEBUF, POOLED_BYTEBUF @@ -98,7 +92,22 @@ public void httpPacketBufsToString(HttpByteBufFormatter.PacketPrintFormat format String outputString = HttpByteBufFormatter.setPrintStyleFor(format, ()-> prettyPrint(byteArrays, HttpByteBufFormatter.HttpMessageType.REQUEST, bufferType)); - Assertions.assertEquals(getExpectedResult(format, contentDirective), outputString); + Assertions.assertEquals(getExpectedResult(format, contentDirective), outputString, + "Strings did not match, after escaping, showing expected and actual on different lines: \n" + + escapeSpecialCharacters(getExpectedResult(format, contentDirective)) + "\n" + + escapeSpecialCharacters(outputString)); + } + + public static String escapeSpecialCharacters(String input) { + return input + .replace("\\", "\\\\") + .replace("\b", "\\b") + .replace("\n", "\\n") + .replace("\t", "\\t") + .replace("\f", "\\f") + .replace("\r", "\\r") + .replace("\"", "\\\"") + .replace("'", "\\'"); } @Test diff --git a/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withDuplicateHeaders.txt b/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withDuplicateHeaders.txt index 5257fb68e..fa56b3a53 100644 --- a/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withDuplicateHeaders.txt +++ b/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withDuplicateHeaders.txt @@ -10,7 +10,7 @@ Header1: bar Header1: bar Header1: abc Header1: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl -Content-Length: 616 +Content-Length: 652 { "settings": { diff --git a/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withFixedLength.txt b/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withFixedLength.txt index ad4e964e7..fbd89a948 100644 --- a/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withFixedLength.txt +++ b/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withFixedLength.txt @@ -1,7 +1,7 @@ POST /test HTTP/1.1 Host: foo.example Content-Type: application/json -Content-Length: 616 +Content-Length: 652 { "settings": { diff --git a/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withLargeHeader.txt b/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withLargeHeader.txt index 7399b5db7..9a1d4aa87 100644 --- a/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withLargeHeader.txt +++ b/TrafficCapture/trafficReplayer/src/test/resources/requests/raw/post_formUrlEncoded_withLargeHeader.txt @@ -31,7 +31,7 @@ Header27: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGH Header28: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl Header29: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl Header30: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijkl -Content-Length: 616 +Content-Length: 652 { "settings": { From c6713179d4322e6422db597f92afe72947fa77de Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Fri, 5 Apr 2024 12:45:07 -0500 Subject: [PATCH 05/10] Fix parsing body in AddCompressionEncodingTest Signed-off-by: Andre Kurait --- .../replay/AddCompressionEncodingTest.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java index 4eb408162..3eac3e974 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java @@ -1,5 +1,10 @@ package org.opensearch.migrations.replay; +import io.netty.buffer.Unpooled; +import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpServerCodec; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -59,7 +64,17 @@ public void addingCompressionRequestHeaderCompressesPayload() throws ExecutionEx () -> "AddCompressionEncodingTest.fullyProcessedResponse"); fullyProcessedResponse.get(); - try (var bais = new ByteArrayInputStream(testPacketCapture.getBytesCaptured()); + + EmbeddedChannel channel = new EmbeddedChannel( + new HttpServerCodec(), + new HttpObjectAggregator(Utils.MAX_PAYLOAD_SIZE_TO_PRINT) // Set max content length if needed + ); + + channel.writeInbound(Unpooled.wrappedBuffer(testPacketCapture.getBytesCaptured())); + var compressedRequest = ((FullHttpRequest) channel.readInbound()); + var compressedByteArr = new byte[compressedRequest.content().readableBytes()]; + compressedRequest.content().getBytes(0, compressedByteArr); + try (var bais = new ByteArrayInputStream(compressedByteArr); var unzipStream = new GZIPInputStream(bais); var isr = new InputStreamReader(unzipStream, StandardCharsets.UTF_8); var br = new BufferedReader(isr)) { @@ -76,5 +91,6 @@ public void addingCompressionRequestHeaderCompressesPayload() throws ExecutionEx } while (true); Assertions.assertEquals(numParts*payloadPartSize, counter); } + compressedRequest.release(); } } From 677b6b45fccb297488536f336a6c199b89cb73b4 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Fri, 5 Apr 2024 15:06:11 -0500 Subject: [PATCH 06/10] Fix CRLF in ResultsToLogsConsumerTest Signed-off-by: Andre Kurait --- .../replay/ResultsToLogsConsumerTest.java | 198 +++++++++--------- 1 file changed, 99 insertions(+), 99 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ResultsToLogsConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ResultsToLogsConsumerTest.java index 45b16bdf6..72e1f63cc 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ResultsToLogsConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ResultsToLogsConsumerTest.java @@ -45,7 +45,7 @@ class ResultsToLogsConsumerTest extends InstrumentationTest { "Funtime: checkIt!\r\n" + "\r\n" + "1e\r\n" + - "I should be decrypted tester!\n" + + "I should be decrypted tester!\r\n" + "\r\n" + "0\r\n" + "\r\n"; @@ -148,56 +148,56 @@ private static byte[] loadResourceAsBytes(String path) throws IOException { public void testOutputterForGet() throws IOException { final String EXPECTED_LOGGED_OUTPUT = "" + - "{\n" + - " \"sourceRequest\": {\n" + - " \"Request-URI\": \"/test\",\n" + - " \"Method\": \"GET\",\n" + - " \"HTTP-Version\": \"HTTP/1.1\",\n" + - " \"body\": \"\",\n" + - " \"Host\": \"foo.example\",\n" + - " \"auTHorization\": \"Basic YWRtaW46YWRtaW4=\",\n" + - " \"Content-Type\": \"application/json\",\n" + - " \"content-length\": \"0\"\n" + - " },\n" + - " \"sourceResponse\": {\n" + - " \"HTTP-Version\": {\n" + - " \"keepAliveDefault\": true\n" + - " },\n" + - " \"Status-Code\": 200,\n" + - " \"Reason-Phrase\": \"OK\",\n" + - " \"response_time_ms\": 0,\n" + - " \"body\": \"SSBzaG91bGQgYmUgZGVjcnlwdGVkIHRlc3RlciEK\",\n" + - " \"Content-transfer-encoding\": \"chunked\",\n" + - " \"Date\": \"Thu, 08 Jun 2023 23:06:23 GMT\",\n" + - " \"Content-type\": \"text/plain\",\n" + - " \"Funtime\": \"checkIt!\",\n" + - " \"content-length\": \"30\"\n" + - " },\n" + - " \"targetRequest\": {\n" + - " \"Request-URI\": \"/test\",\n" + - " \"Method\": \"GET\",\n" + - " \"HTTP-Version\": \"HTTP/1.1\",\n" + - " \"body\": \"\",\n" + - " \"Host\": \"foo.example\",\n" + - " \"auTHorization\": \"Basic YWRtaW46YWRtaW4=\",\n" + - " \"Content-Type\": \"application/json\",\n" + - " \"content-length\": \"0\"\n" + - " },\n" + - " \"targetResponse\": {\n" + - " \"HTTP-Version\": {\n" + - " \"keepAliveDefault\": true\n" + - " },\n" + - " \"Status-Code\": 200,\n" + - " \"Reason-Phrase\": \"OK\",\n" + - " \"response_time_ms\": 267,\n" + - " \"body\": \"SSBzaG91bGQgYmUgZGVjcnlwdGVkIHRlc3RlciEK\",\n" + - " \"Content-transfer-encoding\": \"chunked\",\n" + - " \"Date\": \"Thu, 08 Jun 2023 23:06:23 GMT\",\n" + - " \"Content-type\": \"text/plain\",\n" + - " \"Funtime\": \"checkIt!\",\n" + - " \"content-length\": \"30\"\n" + - " },\n" + - " \"connectionId\": \"testConnection.1\"\n" + + "{\r\n" + + " \"sourceRequest\": {\r\n" + + " \"Request-URI\": \"/test\",\r\n" + + " \"Method\": \"GET\",\r\n" + + " \"HTTP-Version\": \"HTTP/1.1\",\r\n" + + " \"body\": \"\",\r\n" + + " \"Host\": \"foo.example\",\r\n" + + " \"auTHorization\": \"Basic YWRtaW46YWRtaW4=\",\r\n" + + " \"Content-Type\": \"application/json\",\r\n" + + " \"content-length\": \"0\"\r\n" + + " },\r\n" + + " \"sourceResponse\": {\r\n" + + " \"HTTP-Version\": {\r\n" + + " \"keepAliveDefault\": true\r\n" + + " },\r\n" + + " \"Status-Code\": 200,\r\n" + + " \"Reason-Phrase\": \"OK\",\r\n" + + " \"response_time_ms\": 0,\r\n" + + " \"body\": \"SSBzaG91bGQgYmUgZGVjcnlwdGVkIHRlc3RlciEN\",\r\n" + + " \"Content-transfer-encoding\": \"chunked\",\r\n" + + " \"Date\": \"Thu, 08 Jun 2023 23:06:23 GMT\",\r\n" + + " \"Content-type\": \"text/plain\",\r\n" + + " \"Funtime\": \"checkIt!\",\r\n" + + " \"content-length\": \"30\"\r\n" + + " },\r\n" + + " \"targetRequest\": {\r\n" + + " \"Request-URI\": \"/test\",\r\n" + + " \"Method\": \"GET\",\r\n" + + " \"HTTP-Version\": \"HTTP/1.1\",\r\n" + + " \"body\": \"\",\r\n" + + " \"Host\": \"foo.example\",\r\n" + + " \"auTHorization\": \"Basic YWRtaW46YWRtaW4=\",\r\n" + + " \"Content-Type\": \"application/json\",\r\n" + + " \"content-length\": \"0\"\r\n" + + " },\r\n" + + " \"targetResponse\": {\r\n" + + " \"HTTP-Version\": {\r\n" + + " \"keepAliveDefault\": true\r\n" + + " },\r\n" + + " \"Status-Code\": 200,\r\n" + + " \"Reason-Phrase\": \"OK\",\r\n" + + " \"response_time_ms\": 267,\r\n" + + " \"body\": \"SSBzaG91bGQgYmUgZGVjcnlwdGVkIHRlc3RlciEN\",\r\n" + + " \"Content-transfer-encoding\": \"chunked\",\r\n" + + " \"Date\": \"Thu, 08 Jun 2023 23:06:23 GMT\",\r\n" + + " \"Content-type\": \"text/plain\",\r\n" + + " \"Funtime\": \"checkIt!\",\r\n" + + " \"content-length\": \"30\"\r\n" + + " },\r\n" + + " \"connectionId\": \"testConnection.1\"\r\n" + "}"; testOutputterForRequest("get_withAuthHeader.txt", EXPECTED_LOGGED_OUTPUT); } @@ -206,54 +206,54 @@ public void testOutputterForGet() throws IOException { @ResourceLock("TestContext") public void testOutputterForPost() throws IOException { final String EXPECTED_LOGGED_OUTPUT = "" + - "{\n" + - " \"sourceRequest\": {\n" + - " \"Request-URI\": \"/test\",\n" + - " \"Method\": \"POST\",\n" + - " \"HTTP-Version\": \"HTTP/1.1\",\n" + - " \"body\": \"ewogICJzZXR0aW5ncyI6IHsKICAgICJpbmRleCI6IHsKICAgICAgIm51bWJlcl9vZl9zaGFyZHMiOiA3LAogICAgICAibnVtYmVyX29mX3JlcGxpY2FzIjogMwogICAgfSwKICAgICJhbmFseXNpcyI6IHsKICAgICAgImFuYWx5emVyIjogewogICAgICAgICJuYW1lQW5hbHl6ZXIiOiB7CiAgICAgICAgICAidHlwZSI6ICJjdXN0b20iLAogICAgICAgICAgInRva2VuaXplciI6ICJrZXl3b3JkIiwKICAgICAgICAgICJmaWx0ZXIiOiAidXBwZXJjYXNlIgogICAgICAgIH0KICAgICAgfQogICAgfQogIH0sCiAgIm1hcHBpbmdzIjogewogICAgImVtcGxveWVlIjogewogICAgICAicHJvcGVydGllcyI6IHsKICAgICAgICAiYWdlIjogewogICAgICAgICAgInR5cGUiOiAibG9uZyIKICAgICAgICB9LAogICAgICAgICJsZXZlbCI6IHsKICAgICAgICAgICJ0eXBlIjogImxvbmciCiAgICAgICAgfSwKICAgICAgICAidGl0bGUiOiB7CiAgICAgICAgICAidHlwZSI6ICJ0ZXh0IgogICAgICAgIH0sCiAgICAgICAgIm5hbWUiOiB7CiAgICAgICAgICAidHlwZSI6ICJ0ZXh0IiwKICAgICAgICAgICJhbmFseXplciI6ICJuYW1lQW5hbHl6ZXIiCiAgICAgICAgfQogICAgICB9CiAgICB9CiAgfQp9Cg==\",\n" + - " \"Host\": \"foo.example\",\n" + - " \"Content-Type\": \"application/json\",\n" + - " \"Content-Length\": \"616\"\n" + - " },\n" + - " \"sourceResponse\": {\n" + - " \"HTTP-Version\": {\n" + - " \"keepAliveDefault\": true\n" + - " },\n" + - " \"Status-Code\": 200,\n" + - " \"Reason-Phrase\": \"OK\",\n" + - " \"response_time_ms\": 0,\n" + - " \"body\": \"SSBzaG91bGQgYmUgZGVjcnlwdGVkIHRlc3RlciEK\",\n" + - " \"Content-transfer-encoding\": \"chunked\",\n" + - " \"Date\": \"Thu, 08 Jun 2023 23:06:23 GMT\",\n" + - " \"Content-type\": \"text/plain\",\n" + - " \"Funtime\": \"checkIt!\",\n" + - " \"content-length\": \"30\"\n" + - " },\n" + - " \"targetRequest\": {\n" + - " \"Request-URI\": \"/test\",\n" + - " \"Method\": \"POST\",\n" + - " \"HTTP-Version\": \"HTTP/1.1\",\n" + - " \"body\": \"ewogICJzZXR0aW5ncyI6IHsKICAgICJpbmRleCI6IHsKICAgICAgIm51bWJlcl9vZl9zaGFyZHMiOiA3LAogICAgICAibnVtYmVyX29mX3JlcGxpY2FzIjogMwogICAgfSwKICAgICJhbmFseXNpcyI6IHsKICAgICAgImFuYWx5emVyIjogewogICAgICAgICJuYW1lQW5hbHl6ZXIiOiB7CiAgICAgICAgICAidHlwZSI6ICJjdXN0b20iLAogICAgICAgICAgInRva2VuaXplciI6ICJrZXl3b3JkIiwKICAgICAgICAgICJmaWx0ZXIiOiAidXBwZXJjYXNlIgogICAgICAgIH0KICAgICAgfQogICAgfQogIH0sCiAgIm1hcHBpbmdzIjogewogICAgImVtcGxveWVlIjogewogICAgICAicHJvcGVydGllcyI6IHsKICAgICAgICAiYWdlIjogewogICAgICAgICAgInR5cGUiOiAibG9uZyIKICAgICAgICB9LAogICAgICAgICJsZXZlbCI6IHsKICAgICAgICAgICJ0eXBlIjogImxvbmciCiAgICAgICAgfSwKICAgICAgICAidGl0bGUiOiB7CiAgICAgICAgICAidHlwZSI6ICJ0ZXh0IgogICAgICAgIH0sCiAgICAgICAgIm5hbWUiOiB7CiAgICAgICAgICAidHlwZSI6ICJ0ZXh0IiwKICAgICAgICAgICJhbmFseXplciI6ICJuYW1lQW5hbHl6ZXIiCiAgICAgICAgfQogICAgICB9CiAgICB9CiAgfQp9Cg==\",\n" + - " \"Host\": \"foo.example\",\n" + - " \"Content-Type\": \"application/json\",\n" + - " \"Content-Length\": \"616\"\n" + - " },\n" + - " \"targetResponse\": {\n" + - " \"HTTP-Version\": {\n" + - " \"keepAliveDefault\": true\n" + - " },\n" + - " \"Status-Code\": 200,\n" + - " \"Reason-Phrase\": \"OK\",\n" + - " \"response_time_ms\": 267,\n" + - " \"body\": \"SSBzaG91bGQgYmUgZGVjcnlwdGVkIHRlc3RlciEK\",\n" + - " \"Content-transfer-encoding\": \"chunked\",\n" + - " \"Date\": \"Thu, 08 Jun 2023 23:06:23 GMT\",\n" + - " \"Content-type\": \"text/plain\",\n" + - " \"Funtime\": \"checkIt!\",\n" + - " \"content-length\": \"30\"\n" + - " },\n" + - " \"connectionId\": \"testConnection.1\"\n" + + "{\r\n" + + " \"sourceRequest\": {\r\n" + + " \"Request-URI\": \"/test\",\r\n" + + " \"Method\": \"POST\",\r\n" + + " \"HTTP-Version\": \"HTTP/1.1\",\r\n" + + " \"body\": \"ew0KICAic2V0dGluZ3MiOiB7DQogICAgImluZGV4Ijogew0KICAgICAgIm51bWJlcl9vZl9zaGFyZHMiOiA3LA0KICAgICAgIm51bWJlcl9vZl9yZXBsaWNhcyI6IDMNCiAgICB9LA0KICAgICJhbmFseXNpcyI6IHsNCiAgICAgICJhbmFseXplciI6IHsNCiAgICAgICAgIm5hbWVBbmFseXplciI6IHsNCiAgICAgICAgICAidHlwZSI6ICJjdXN0b20iLA0KICAgICAgICAgICJ0b2tlbml6ZXIiOiAia2V5d29yZCIsDQogICAgICAgICAgImZpbHRlciI6ICJ1cHBlcmNhc2UiDQogICAgICAgIH0NCiAgICAgIH0NCiAgICB9DQogIH0sDQogICJtYXBwaW5ncyI6IHsNCiAgICAiZW1wbG95ZWUiOiB7DQogICAgICAicHJvcGVydGllcyI6IHsNCiAgICAgICAgImFnZSI6IHsNCiAgICAgICAgICAidHlwZSI6ICJsb25nIg0KICAgICAgICB9LA0KICAgICAgICAibGV2ZWwiOiB7DQogICAgICAgICAgInR5cGUiOiAibG9uZyINCiAgICAgICAgfSwNCiAgICAgICAgInRpdGxlIjogew0KICAgICAgICAgICJ0eXBlIjogInRleHQiDQogICAgICAgIH0sDQogICAgICAgICJuYW1lIjogew0KICAgICAgICAgICJ0eXBlIjogInRleHQiLA0KICAgICAgICAgICJhbmFseXplciI6ICJuYW1lQW5hbHl6ZXIiDQogICAgICAgIH0NCiAgICAgIH0NCiAgICB9DQogIH0NCn0NCg==\",\r\n" + + " \"Host\": \"foo.example\",\r\n" + + " \"Content-Type\": \"application/json\",\r\n" + + " \"Content-Length\": \"652\"\r\n" + + " },\r\n" + + " \"sourceResponse\": {\r\n" + + " \"HTTP-Version\": {\r\n" + + " \"keepAliveDefault\": true\r\n" + + " },\r\n" + + " \"Status-Code\": 200,\r\n" + + " \"Reason-Phrase\": \"OK\",\r\n" + + " \"response_time_ms\": 0,\r\n" + + " \"body\": \"SSBzaG91bGQgYmUgZGVjcnlwdGVkIHRlc3RlciEN\",\r\n" + + " \"Content-transfer-encoding\": \"chunked\",\r\n" + + " \"Date\": \"Thu, 08 Jun 2023 23:06:23 GMT\",\r\n" + + " \"Content-type\": \"text/plain\",\r\n" + + " \"Funtime\": \"checkIt!\",\r\n" + + " \"content-length\": \"30\"\r\n" + + " },\r\n" + + " \"targetRequest\": {\r\n" + + " \"Request-URI\": \"/test\",\r\n" + + " \"Method\": \"POST\",\r\n" + + " \"HTTP-Version\": \"HTTP/1.1\",\r\n" + + " \"body\": \"ew0KICAic2V0dGluZ3MiOiB7DQogICAgImluZGV4Ijogew0KICAgICAgIm51bWJlcl9vZl9zaGFyZHMiOiA3LA0KICAgICAgIm51bWJlcl9vZl9yZXBsaWNhcyI6IDMNCiAgICB9LA0KICAgICJhbmFseXNpcyI6IHsNCiAgICAgICJhbmFseXplciI6IHsNCiAgICAgICAgIm5hbWVBbmFseXplciI6IHsNCiAgICAgICAgICAidHlwZSI6ICJjdXN0b20iLA0KICAgICAgICAgICJ0b2tlbml6ZXIiOiAia2V5d29yZCIsDQogICAgICAgICAgImZpbHRlciI6ICJ1cHBlcmNhc2UiDQogICAgICAgIH0NCiAgICAgIH0NCiAgICB9DQogIH0sDQogICJtYXBwaW5ncyI6IHsNCiAgICAiZW1wbG95ZWUiOiB7DQogICAgICAicHJvcGVydGllcyI6IHsNCiAgICAgICAgImFnZSI6IHsNCiAgICAgICAgICAidHlwZSI6ICJsb25nIg0KICAgICAgICB9LA0KICAgICAgICAibGV2ZWwiOiB7DQogICAgICAgICAgInR5cGUiOiAibG9uZyINCiAgICAgICAgfSwNCiAgICAgICAgInRpdGxlIjogew0KICAgICAgICAgICJ0eXBlIjogInRleHQiDQogICAgICAgIH0sDQogICAgICAgICJuYW1lIjogew0KICAgICAgICAgICJ0eXBlIjogInRleHQiLA0KICAgICAgICAgICJhbmFseXplciI6ICJuYW1lQW5hbHl6ZXIiDQogICAgICAgIH0NCiAgICAgIH0NCiAgICB9DQogIH0NCn0NCg==\",\r\n" + + " \"Host\": \"foo.example\",\r\n" + + " \"Content-Type\": \"application/json\",\r\n" + + " \"Content-Length\": \"652\"\r\n" + + " },\r\n" + + " \"targetResponse\": {\r\n" + + " \"HTTP-Version\": {\r\n" + + " \"keepAliveDefault\": true\r\n" + + " },\r\n" + + " \"Status-Code\": 200,\r\n" + + " \"Reason-Phrase\": \"OK\",\r\n" + + " \"response_time_ms\": 267,\r\n" + + " \"body\": \"SSBzaG91bGQgYmUgZGVjcnlwdGVkIHRlc3RlciEN\",\r\n" + + " \"Content-transfer-encoding\": \"chunked\",\r\n" + + " \"Date\": \"Thu, 08 Jun 2023 23:06:23 GMT\",\r\n" + + " \"Content-type\": \"text/plain\",\r\n" + + " \"Funtime\": \"checkIt!\",\r\n" + + " \"content-length\": \"30\"\r\n" + + " },\r\n" + + " \"connectionId\": \"testConnection.1\"\r\n" + "}"; testOutputterForRequest("post_formUrlEncoded_withFixedLength.txt", EXPECTED_LOGGED_OUTPUT); } From 1feffd254d5890cce50cb59917b705f101985f55 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Fri, 5 Apr 2024 16:50:39 -0500 Subject: [PATCH 07/10] Fix memory leak in NettyDecodedHttpRequestPreliminaryConvertHandler Signed-off-by: Andre Kurait --- ...tyDecodedHttpRequestPreliminaryConvertHandler.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java index 6c19bab0a..5c89047d5 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java @@ -1,5 +1,6 @@ package org.opensearch.migrations.replay.datahandlers.http; +import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.handler.codec.http.HttpContent; @@ -67,8 +68,14 @@ public void channelRead(@NonNull ChannelHandlerContext ctx, @NonNull Object msg) } else { // ByteBufs shouldn't come through, but in case there's a regression in // RequestPipelineOrchestrator.removeThisAndPreviousHandlers to remove the handlers - // in order rather in reverse order - super.channelRead(ctx, msg); + // in order rather in reverse order this will release the byte buffs and complete the channel + if (msg instanceof ByteBuf) { + ((ByteBuf) msg).release(); + super.channelReadComplete(ctx); + } + else { + super.channelRead(ctx, msg); + } } } From 15ad146b9c38c6ff4a5d0166109ed08be5b66baa Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Tue, 9 Apr 2024 12:43:43 -0500 Subject: [PATCH 08/10] Address PR Comments Signed-off-by: Andre Kurait --- .../replay/HttpByteBufFormatter.java | 47 +++++++++++++------ .../replay/HttpMessageAndTimestamp.java | 3 +- .../replay/SourceTargetCaptureTuple.java | 6 ++- .../http/HttpJsonTransformingConsumer.java | 1 + ...dHttpRequestPreliminaryConvertHandler.java | 22 ++++----- .../http/NettyJsonToByteBufHandler.java | 2 +- .../http/RequestPipelineOrchestrator.java | 6 +++ .../HttpJsonTransformingConsumerTest.java | 2 +- 8 files changed, 55 insertions(+), 34 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpByteBufFormatter.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpByteBufFormatter.java index 8ff279f75..a7d8065c2 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpByteBufFormatter.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpByteBufFormatter.java @@ -15,7 +15,6 @@ import lombok.extern.slf4j.Slf4j; import java.nio.charset.StandardCharsets; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -29,7 +28,10 @@ @Slf4j public class HttpByteBufFormatter { - private static final String LINE_DELIMITER = "\r\n"; + public static final String CRLF_LINE_DELIMITER = "\r\n"; + public static final String LF_LINE_DELIMITER = "\n"; + private static final String DEFAULT_LINE_DELIMITER = CRLF_LINE_DELIMITER; + private static final ThreadLocal> printStyle = ThreadLocal.withInitial(Optional::empty); @@ -59,41 +61,56 @@ public static T setPrintStyleFor(PacketPrintFormat packetPrintFormat, Suppli public enum HttpMessageType { REQUEST, RESPONSE } public static String httpPacketBytesToString(HttpMessageType msgType, List byteArrStream) { - return httpPacketBytesToString(msgType, - Optional.ofNullable(byteArrStream).map(p -> p.stream()).orElse(Stream.of())); + return httpPacketBytesToString(msgType, byteArrStream, DEFAULT_LINE_DELIMITER); } public static String httpPacketBytesToString(HttpMessageType msgType, Stream byteArrStream) { + return httpPacketBytesToString(msgType, byteArrStream, DEFAULT_LINE_DELIMITER); + } + + public static String httpPacketBufsToString(HttpMessageType msgType, Stream byteBufStream, + boolean releaseByteBufs) { + return httpPacketBufsToString(msgType, byteBufStream, releaseByteBufs, DEFAULT_LINE_DELIMITER); + } + + public static String httpPacketBytesToString(HttpMessageType msgType, List byteArrStream, String lineDelimiter) { + return httpPacketBytesToString(msgType, + Optional.ofNullable(byteArrStream).map(p -> p.stream()).orElse(Stream.of()), lineDelimiter); + } + + public static String httpPacketBytesToString(HttpMessageType msgType, Stream byteArrStream, String lineDelimiter) { // This isn't memory efficient, // but stringifying byte bufs through a full parse and reserializing them was already really slow! - return httpPacketBufsToString(msgType, byteArrStream.map(Unpooled::wrappedBuffer), true); + return httpPacketBufsToString(msgType, byteArrStream.map(Unpooled::wrappedBuffer), true, lineDelimiter); } public static String httpPacketBufsToString(HttpMessageType msgType, Stream byteBufStream, - boolean releaseByteBufs) { + boolean releaseByteBufs, String lineDelimiter) { switch (printStyle.get().orElse(PacketPrintFormat.TRUNCATED)) { case TRUNCATED: return httpPacketBufsToString(byteBufStream, Utils.MAX_BYTES_SHOWN_FOR_TO_STRING, releaseByteBufs); case FULL_BYTES: return httpPacketBufsToString(byteBufStream, Long.MAX_VALUE, releaseByteBufs); case PARSED_HTTP: - return httpPacketsToPrettyPrintedString(msgType, byteBufStream, false, releaseByteBufs); + return httpPacketsToPrettyPrintedString(msgType, byteBufStream, false, releaseByteBufs, + lineDelimiter); case PARSED_HTTP_SORTED_HEADERS: - return httpPacketsToPrettyPrintedString(msgType, byteBufStream, true, releaseByteBufs); + return httpPacketsToPrettyPrintedString(msgType, byteBufStream, true, releaseByteBufs, + lineDelimiter); default: throw new IllegalStateException("Unknown PacketPrintFormat: " + printStyle.get()); } } public static String httpPacketsToPrettyPrintedString(HttpMessageType msgType, Stream byteBufStream, - boolean sortHeaders, boolean releaseByteBufs) { + boolean sortHeaders, boolean releaseByteBufs, String lineDelimiter) { HttpMessage httpMessage = parseHttpMessageFromBufs(msgType, byteBufStream, releaseByteBufs); var holderOp = Optional.ofNullable((httpMessage instanceof ByteBufHolder) ? (ByteBufHolder) httpMessage : null); try { if (httpMessage instanceof FullHttpRequest) { - return prettyPrintNettyRequest((FullHttpRequest) httpMessage, sortHeaders); + return prettyPrintNettyRequest((FullHttpRequest) httpMessage, sortHeaders, lineDelimiter); } else if (httpMessage instanceof FullHttpResponse) { - return prettyPrintNettyResponse((FullHttpResponse) httpMessage, sortHeaders); + return prettyPrintNettyResponse((FullHttpResponse) httpMessage, sortHeaders, lineDelimiter); } else if (httpMessage == null) { return "[NULL]"; } else { @@ -105,14 +122,14 @@ public static String httpPacketsToPrettyPrintedString(HttpMessageType msgType, S } } - public static String prettyPrintNettyRequest(FullHttpRequest msg, boolean sortHeaders) { - var sj = new StringJoiner(LINE_DELIMITER); + public static String prettyPrintNettyRequest(FullHttpRequest msg, boolean sortHeaders, String lineDelimiter) { + var sj = new StringJoiner(lineDelimiter); sj.add(msg.method() + " " + msg.uri() + " " + msg.protocolVersion().text()); return prettyPrintNettyMessage(sj, sortHeaders, msg, msg.content()); } - static String prettyPrintNettyResponse(FullHttpResponse msg, boolean sortHeaders) { - var sj = new StringJoiner(LINE_DELIMITER); + public static String prettyPrintNettyResponse(FullHttpResponse msg, boolean sortHeaders, String lineDelimiter) { + var sj = new StringJoiner(lineDelimiter); sj.add(msg.protocolVersion().text() + " " + msg.status().code() + " " + msg.status().reasonPhrase()); return prettyPrintNettyMessage(sj, sortHeaders, msg, msg.content()); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpMessageAndTimestamp.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpMessageAndTimestamp.java index 709b6faea..cb9089c3d 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpMessageAndTimestamp.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpMessageAndTimestamp.java @@ -65,7 +65,8 @@ public Stream stream() { } public String format(Optional messageTypeOp) { - var packetBytesAsStr = messageTypeOp.map(mt-> HttpByteBufFormatter.httpPacketBytesToString(mt, packetBytes)) + var packetBytesAsStr = messageTypeOp.map(mt-> HttpByteBufFormatter.httpPacketBytesToString(mt, packetBytes, + HttpByteBufFormatter.LF_LINE_DELIMITER)) .orElseGet(()-> HttpByteBufFormatter.httpPacketBufsToString( packetBytes.stream().map(Unpooled::wrappedBuffer), Utils.MAX_PAYLOAD_SIZE_TO_PRINT, true)); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java index f957b6cc7..b63f1aa15 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java @@ -1,5 +1,7 @@ package org.opensearch.migrations.replay; +import static org.opensearch.migrations.replay.HttpByteBufFormatter.LF_LINE_DELIMITER; + import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.replay.datatypes.HttpRequestTransformationStatus; @@ -52,9 +54,9 @@ public String toString() { if (targetResponseDuration != null) { sj.add("targetResponseDuration=").add(targetResponseDuration+""); } Optional.ofNullable(targetRequestData).ifPresent(d-> sj.add("targetRequestData=") .add(d.isClosed() ? "CLOSED" : HttpByteBufFormatter.httpPacketBufsToString( - HttpByteBufFormatter.HttpMessageType.REQUEST, d.streamUnretained(), false))); + HttpByteBufFormatter.HttpMessageType.REQUEST, d.streamUnretained(), false, LF_LINE_DELIMITER))); Optional.ofNullable(targetResponseData).filter(d->!d.isEmpty()).ifPresent(d -> sj.add("targetResponseData=") - .add(HttpByteBufFormatter.httpPacketBytesToString(HttpByteBufFormatter.HttpMessageType.RESPONSE, d))); + .add(HttpByteBufFormatter.httpPacketBytesToString(HttpByteBufFormatter.HttpMessageType.RESPONSE, d, LF_LINE_DELIMITER))); sj.add("transformStatus=").add(transformationStatus+""); sj.add("errorCause=").add(errorCause == null ? "none" : errorCause.toString()); return sj.toString(); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java index b0d8cc0db..5b767c995 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java @@ -119,6 +119,7 @@ public DiagnosticTrackableCompletableFuture sliceRandomChunks(byte[] bytes, int numChunks) { - Random random = new Random(); + Random random = new Random(0); List chunkSizes = new ArrayList<>(numChunks); int totalSizeRemaining = bytes.length; for (int i = 0; i < numChunks - 1; i++) { From 4a0a4e5a590d9f190cbbc581586e334335dd8abe Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Tue, 9 Apr 2024 18:05:21 -0500 Subject: [PATCH 09/10] Add Artifacts in format for maven central Signed-off-by: Andre Kurait --- TrafficCapture/build.gradle | 17 +++++++++++++++++ .../NettyJsonContentStreamToByteBufHandler.java | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/TrafficCapture/build.gradle b/TrafficCapture/build.gradle index bc38b456d..506d9a33d 100644 --- a/TrafficCapture/build.gradle +++ b/TrafficCapture/build.gradle @@ -12,6 +12,15 @@ allprojects { subprojects { apply plugin: 'java' apply plugin: 'maven-publish' + task javadocJar(type: Jar, dependsOn: javadoc) { + archiveClassifier.set('javadoc') + from javadoc.destinationDir + } + task sourcesJar(type: Jar) { + archiveClassifier.set('sources') + from sourceSets.main.allSource + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } def excludedProjects = [ 'buildSrc', 'dockerSolution', @@ -21,6 +30,9 @@ subprojects { publications { mavenJava(MavenPublication) { from components.java + artifact javadocJar + artifact sourcesJar + group = 'org.opensearch.migrations.trafficcapture' version = '0.1.0' @@ -48,6 +60,11 @@ subprojects { url = "https://github.com/opensearch-project/opensearch-migrations" } } + scm { + connection = "scm:git@github.com:opensearch-project/opensearch-migrations.git" + developerConnection = "scm:git@github.com:opensearch-project/opensearch-migrations.git" + url = "git@github.com:opensearch-project/opensearch-migrations.git" + } } // Suppress POM metadata warnings for test fixtures diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentStreamToByteBufHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentStreamToByteBufHandler.java index b81453deb..185a91f74 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentStreamToByteBufHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentStreamToByteBufHandler.java @@ -25,7 +25,7 @@ * responsibility to serialize that for the final HTTP Request. * * Notice that this class will emit ByteBufs and the next handler in the pipeline, - * @NettyJsonToByteBufHandler will simply pass those ByteBufs through, while repackaging HttpContent + * NettyJsonToByteBufHandler will simply pass those ByteBufs through, while repackaging HttpContent * messages, seemingly similar to what this class does! However, these two handlers have slightly * calling contexts. This handler will only be utilized when there needed to be a material change * on the incoming HttpContent objects from the original request. The next handler will be called From c81c7dcff7873e344644734b5cdceb1a175a2095 Mon Sep 17 00:00:00 2001 From: Rishabh Singh Date: Wed, 10 Apr 2024 17:54:41 +0000 Subject: [PATCH 10/10] Remove unused 'arguments' parameter Signed-off-by: Rishabh Singh --- jenkins/release.jenkinsFile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/release.jenkinsFile b/jenkins/release.jenkinsFile index 1dea5fc4e..f876c9d72 100644 --- a/jenkins/release.jenkinsFile +++ b/jenkins/release.jenkinsFile @@ -89,7 +89,7 @@ pipeline { post { success { script { - if (arguments.publishRelease && release_url != null) { + if (release_url != null) { withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) { sh "curl -X PATCH -H 'Accept: application/vnd.github+json' -H 'Authorization: Bearer ${GITHUB_TOKEN}' ${release_url} -d '{\"tag_name\":\"${tag}\",\"draft\":false,\"prerelease\":false}'" }