From 361f169a99b63d82cc7413375b98cedbbfc0005e Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Tue, 24 Oct 2023 10:52:30 -0500 Subject: [PATCH 01/55] Add test to run opensearch benchmark and compare doc counts Signed-off-by: Omar Khasawneh --- test/tests.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/test/tests.py b/test/tests.py index ae77127899..4db760c20f 100644 --- a/test/tests.py +++ b/test/tests.py @@ -1,3 +1,6 @@ +import json +import subprocess + from operations import create_index, check_index, create_document, \ delete_document, delete_index, get_document from http import HTTPStatus @@ -15,6 +18,23 @@ logger = logging.getLogger(__name__) +def get_indices(endpoint, auth): + response = requests.get(f'{endpoint}/_cat/indices', auth=auth, verify=False) + indices = [] + response_lines = response.text.strip().split('\n') + for line in response_lines: + parts = line.split() + index_name = parts[2] + indices.append(index_name) + return indices + + +def get_doc_count(endpoint, index, auth): + response = requests.get(f'{endpoint}/{index}/_count', auth=auth, verify=False) + count = json.loads(response.text)['count'] + return count + + # The following "retry_request" function's purpose is to retry a certain request for "max_attempts" # times every "delay" seconds IF the requests returned a status code other than what's expected. # So this "retry_request" function's arguments are a request function's name and whatever arguments that function @@ -200,3 +220,29 @@ def test_0006_invalidIncorrectUri(self): incorrectUri = "/_cluster/incorrectUri" response = requests.get(f'{self.proxy_endpoint}{incorrectUri}', auth=self.auth, verify=False) self.assertEqual(response.status_code, HTTPStatus.METHOD_NOT_ALLOWED) + + def test_0007_OSB(self): + cmd = "docker ps | grep 'migrations/migration_console:latest' | awk '{print $NF}'" + container_name = subprocess.getoutput(cmd).strip() + + if container_name: + cmd_exec = f"docker exec {container_name} ./runTestBenchmarks.sh" + logger.warning(f"Running command: {cmd_exec}") + subprocess.run(cmd_exec, shell=True) + else: + logger.error("Migration-console container was not found, please double check that deployment was a success") + self.assert_(False) + + source_indices = get_indices(self.source_endpoint, self.auth) + target_indices = get_indices(self.target_endpoint, self.auth) + + for index in set(source_indices) & set(target_indices): + source_count = get_doc_count(self.source_endpoint, index, self.auth) + target_count = get_doc_count(self.target_endpoint, index, self.auth) + + if source_count != source_count: + logger.error(f'{index}: doc counts do not match - Source = {source_count}, Target = {target_count}') + self.assertEqual(source_count, target_count) + else: + self.assertEqual(source_count, target_count) + logger.info(f'{index}: doc counts match on both source and target endpoints - {source_count}') From 26fc191ed67e55d20d9c1e9573eab2b92c1a491f Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Tue, 24 Oct 2023 11:14:26 -0500 Subject: [PATCH 02/55] Add ignore list logic to test Signed-off-by: Omar Khasawneh --- test/tests.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/test/tests.py b/test/tests.py index 4db760c20f..8c000a4e43 100644 --- a/test/tests.py +++ b/test/tests.py @@ -79,6 +79,7 @@ def set_common_values(self): self.auth = (self.username, self.password) self.index = f"my_index_{uuid.uuid4()}" self.doc_id = '7' + self.ignore_list = [] def setUp(self): self.set_common_values() @@ -237,12 +238,13 @@ def test_0007_OSB(self): target_indices = get_indices(self.target_endpoint, self.auth) for index in set(source_indices) & set(target_indices): - source_count = get_doc_count(self.source_endpoint, index, self.auth) - target_count = get_doc_count(self.target_endpoint, index, self.auth) - - if source_count != source_count: - logger.error(f'{index}: doc counts do not match - Source = {source_count}, Target = {target_count}') - self.assertEqual(source_count, target_count) - else: - self.assertEqual(source_count, target_count) - logger.info(f'{index}: doc counts match on both source and target endpoints - {source_count}') + if index not in self.ignore_list: + source_count = get_doc_count(self.source_endpoint, index, self.auth) + target_count = get_doc_count(self.target_endpoint, index, self.auth) + + if source_count != source_count: + logger.error(f'{index}: doc counts do not match - Source = {source_count}, Target = {target_count}') + self.assertEqual(source_count, target_count) + else: + self.assertEqual(source_count, target_count) + logger.info(f'{index}: doc counts match on both source and target endpoints - {source_count}') From 3b92bd45a4b316d731f169f8847b5c5fdc9362a8 Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Wed, 25 Oct 2023 14:22:21 -0500 Subject: [PATCH 03/55] Add logic to check if OpenSearch Benchmark ran correctly Signed-off-by: Omar Khasawneh --- test/tests.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/tests.py b/test/tests.py index 8c000a4e43..2973d20b7e 100644 --- a/test/tests.py +++ b/test/tests.py @@ -236,11 +236,18 @@ def test_0007_OSB(self): source_indices = get_indices(self.source_endpoint, self.auth) target_indices = get_indices(self.target_endpoint, self.auth) + processed_indices = [] for index in set(source_indices) & set(target_indices): if index not in self.ignore_list: - source_count = get_doc_count(self.source_endpoint, index, self.auth) - target_count = get_doc_count(self.target_endpoint, index, self.auth) + if index != "searchguard" + source_count = get_doc_count(self.source_endpoint, index, self.auth) + target_count = get_doc_count(self.target_endpoint, index, self.auth) + processed_indices.append(index) + + if not processed_indices: + logger.error("There were no indices to compare, check that OpenSearch Benchmark ran successfully") + self.assert_(False) if source_count != source_count: logger.error(f'{index}: doc counts do not match - Source = {source_count}, Target = {target_count}') From 4a68ddd452d89f4ebbe247e3552c2b6c2bca598e Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Wed, 25 Oct 2023 14:30:44 -0500 Subject: [PATCH 04/55] Fix logic to check if OpenSearch Benchmark ran correctly Signed-off-by: Omar Khasawneh --- test/tests.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/tests.py b/test/tests.py index 2973d20b7e..f560040e31 100644 --- a/test/tests.py +++ b/test/tests.py @@ -227,9 +227,10 @@ def test_0007_OSB(self): container_name = subprocess.getoutput(cmd).strip() if container_name: - cmd_exec = f"docker exec {container_name} ./runTestBenchmarks.sh" - logger.warning(f"Running command: {cmd_exec}") - subprocess.run(cmd_exec, shell=True) + cmd = [] + #cmd_exec = f"docker exec {container_name} ./runTestBenchmarks.sh" + #logger.warning(f"Running command: {cmd_exec}") + #subprocess.run(cmd_exec, shell=True) else: logger.error("Migration-console container was not found, please double check that deployment was a success") self.assert_(False) @@ -240,18 +241,18 @@ def test_0007_OSB(self): for index in set(source_indices) & set(target_indices): if index not in self.ignore_list: - if index != "searchguard" + if index != "searchguard": source_count = get_doc_count(self.source_endpoint, index, self.auth) target_count = get_doc_count(self.target_endpoint, index, self.auth) processed_indices.append(index) - if not processed_indices: - logger.error("There were no indices to compare, check that OpenSearch Benchmark ran successfully") - self.assert_(False) - if source_count != source_count: logger.error(f'{index}: doc counts do not match - Source = {source_count}, Target = {target_count}') self.assertEqual(source_count, target_count) else: self.assertEqual(source_count, target_count) logger.info(f'{index}: doc counts match on both source and target endpoints - {source_count}') + + if not processed_indices: + logger.error("There were no indices to compare, check that OpenSearch Benchmark ran successfully") + self.assert_(False) From e1c060bdc64e2862051647da85b691c395fd608f Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Wed, 25 Oct 2023 14:58:13 -0500 Subject: [PATCH 05/55] uncomment mistakenly commented out code that runs OSB Signed-off-by: Omar Khasawneh --- test/tests.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/tests.py b/test/tests.py index f560040e31..aae053b03d 100644 --- a/test/tests.py +++ b/test/tests.py @@ -227,10 +227,9 @@ def test_0007_OSB(self): container_name = subprocess.getoutput(cmd).strip() if container_name: - cmd = [] - #cmd_exec = f"docker exec {container_name} ./runTestBenchmarks.sh" - #logger.warning(f"Running command: {cmd_exec}") - #subprocess.run(cmd_exec, shell=True) + cmd_exec = f"docker exec {container_name} ./runTestBenchmarks.sh" + logger.warning(f"Running command: {cmd_exec}") + subprocess.run(cmd_exec, shell=True) else: logger.error("Migration-console container was not found, please double check that deployment was a success") self.assert_(False) From 498e6bea1a5084483f88c2f0be398b5367ffd7fa Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Thu, 26 Oct 2023 13:31:38 -0500 Subject: [PATCH 06/55] Use container ID instead of container name Signed-off-by: Omar Khasawneh --- test/tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/tests.py b/test/tests.py index aae053b03d..4af404d0cf 100644 --- a/test/tests.py +++ b/test/tests.py @@ -223,11 +223,11 @@ def test_0006_invalidIncorrectUri(self): self.assertEqual(response.status_code, HTTPStatus.METHOD_NOT_ALLOWED) def test_0007_OSB(self): - cmd = "docker ps | grep 'migrations/migration_console:latest' | awk '{print $NF}'" - container_name = subprocess.getoutput(cmd).strip() + cmd = ['docker', 'ps', '--format="{{.ID}}"', '--filter', 'name=migration'] + container_id = subprocess.run(cmd, stdout=subprocess.PIPE, text=True).stdout.strip().replace('"', '') - if container_name: - cmd_exec = f"docker exec {container_name} ./runTestBenchmarks.sh" + if container_id: + cmd_exec = f"docker exec {container_id} ./runTestBenchmarks.sh" logger.warning(f"Running command: {cmd_exec}") subprocess.run(cmd_exec, shell=True) else: From b049a60e65a7c0e2a23d250bd0c6cb36894ec472 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Sun, 5 Nov 2023 12:08:03 -0500 Subject: [PATCH 07/55] Part I of a large refactoring to pull IJsonTransformer into a separate package and to package a number of implementations into their own packages. I'm breaking this up to make rename mapping easier for git. As it is, this code won't compile or run. Signed-off-by: Greg Schohn --- TrafficCapture/replayerPlugins/build.gradle | 0 .../jsonMessageTransformers/build.gradle | 0 .../jsonJoltMessageTransformer/build.gradle | 0 .../transform/JsonJoltTransformBuilder.java | 0 .../transform/JsonJoltTransformer.java | 0 .../main/resources}/operations/addGzip.jolt | 0 .../operations/hostSwitch.jolt.template | 0 .../resources}/operations/makeChunked.jolt | 0 .../main/resources}/operations/passThru.jolt | 0 .../build.gradle | 8 +++++ ...rations.transform.IJsonTransformerProvider | 1 + .../JsonJoltTransformerProvider.java | 2 ++ .../replay/AddCompressionEncodingTest.java | 0 .../replay}/JsonTransformerTest.java | 0 .../replay/PayloadRepackingTest.java | 0 .../build.gradle | 34 +++++++++++++++++++ .../migrations/transform}/IHttpMessage.java | 0 .../transform/IJsonTransformer.java | 0 .../transform/IJsonTransformerFactory.java | 2 ++ .../transform/IJsonTransformerProvider.java | 2 ++ .../transform/JsonCompositeTransformer.java | 0 .../transform/JsonKeysForHttpMessage.java | 11 ++++++ ...Search23PlusTargetTransformerProvider.java | 8 +++++ .../transform/JsonTypeMappingTransformer.java | 16 ++++----- .../replay/TransformationLoader.java | 2 ++ .../migrations/PruferTreeGenerator.java | 0 .../GenerateRandomNestedJsonObject.java | 0 .../TestCapturePacketToHttpHandler.java | 0 .../replay/TestHttpServerContext.java | 0 .../migrations/replay/TestRequestKey.java | 0 .../migrations/replay/TestUtils.java | 0 31 files changed, 77 insertions(+), 9 deletions(-) create mode 100644 TrafficCapture/replayerPlugins/build.gradle create mode 100644 TrafficCapture/replayerPlugins/jsonMessageTransformers/build.gradle create mode 100644 TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/build.gradle rename TrafficCapture/{trafficReplayer => replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer}/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformBuilder.java (100%) rename TrafficCapture/{trafficReplayer => replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer}/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformer.java (100%) rename TrafficCapture/{trafficReplayer/src/main/resources/jolt => replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources}/operations/addGzip.jolt (100%) rename TrafficCapture/{trafficReplayer/src/main/resources/jolt => replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources}/operations/hostSwitch.jolt.template (100%) rename TrafficCapture/{trafficReplayer/src/main/resources/jolt => replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources}/operations/makeChunked.jolt (100%) rename TrafficCapture/{trafficReplayer/src/main/resources/jolt => replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources}/operations/passThru.jolt (100%) create mode 100644 TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/build.gradle create mode 100644 TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider create mode 100644 TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java rename TrafficCapture/{trafficReplayer => replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider}/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java (100%) rename TrafficCapture/{trafficReplayer/src/test/java/org/opensearch/migrations/transform => replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay}/JsonTransformerTest.java (100%) rename TrafficCapture/{trafficReplayer => replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider}/src/test/java/org/opensearch/migrations/replay/PayloadRepackingTest.java (100%) create mode 100644 TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/build.gradle rename TrafficCapture/{trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http => replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform}/IHttpMessage.java (100%) rename TrafficCapture/{trafficReplayer => replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface}/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java (100%) create mode 100644 TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerFactory.java create mode 100644 TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java rename TrafficCapture/{trafficReplayer => replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface}/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java (100%) create mode 100644 TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java create mode 100644 TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java rename TrafficCapture/{trafficReplayer => replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider}/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java (79%) create mode 100644 TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java rename TrafficCapture/trafficReplayer/src/{test => testFixtures}/java/org/opensearch/migrations/PruferTreeGenerator.java (100%) rename TrafficCapture/trafficReplayer/src/{test => testFixtures}/java/org/opensearch/migrations/replay/GenerateRandomNestedJsonObject.java (100%) rename TrafficCapture/trafficReplayer/src/{test => testFixtures}/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java (100%) rename TrafficCapture/trafficReplayer/src/{test => testFixtures}/java/org/opensearch/migrations/replay/TestHttpServerContext.java (100%) rename TrafficCapture/trafficReplayer/src/{test => testFixtures}/java/org/opensearch/migrations/replay/TestRequestKey.java (100%) rename TrafficCapture/trafficReplayer/src/{test => testFixtures}/java/org/opensearch/migrations/replay/TestUtils.java (100%) diff --git a/TrafficCapture/replayerPlugins/build.gradle b/TrafficCapture/replayerPlugins/build.gradle new file mode 100644 index 0000000000..e69de29bb2 diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/build.gradle b/TrafficCapture/replayerPlugins/jsonMessageTransformers/build.gradle new file mode 100644 index 0000000000..e69de29bb2 diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/build.gradle b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/build.gradle new file mode 100644 index 0000000000..e69de29bb2 diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformBuilder.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformBuilder.java similarity index 100% rename from TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformBuilder.java rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformBuilder.java diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformer.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformer.java similarity index 100% rename from TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformer.java rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformer.java diff --git a/TrafficCapture/trafficReplayer/src/main/resources/jolt/operations/addGzip.jolt b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/operations/addGzip.jolt similarity index 100% rename from TrafficCapture/trafficReplayer/src/main/resources/jolt/operations/addGzip.jolt rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/operations/addGzip.jolt diff --git a/TrafficCapture/trafficReplayer/src/main/resources/jolt/operations/hostSwitch.jolt.template b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/operations/hostSwitch.jolt.template similarity index 100% rename from TrafficCapture/trafficReplayer/src/main/resources/jolt/operations/hostSwitch.jolt.template rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/operations/hostSwitch.jolt.template diff --git a/TrafficCapture/trafficReplayer/src/main/resources/jolt/operations/makeChunked.jolt b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/operations/makeChunked.jolt similarity index 100% rename from TrafficCapture/trafficReplayer/src/main/resources/jolt/operations/makeChunked.jolt rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/operations/makeChunked.jolt diff --git a/TrafficCapture/trafficReplayer/src/main/resources/jolt/operations/passThru.jolt b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/operations/passThru.jolt similarity index 100% rename from TrafficCapture/trafficReplayer/src/main/resources/jolt/operations/passThru.jolt rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/operations/passThru.jolt diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/build.gradle b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/build.gradle new file mode 100644 index 0000000000..bcd13841d4 --- /dev/null +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/build.gradle @@ -0,0 +1,8 @@ +plugins { + id "io.freefair.lombok" version "8.0.1" +} + +dependencies { + implementation project(':replayerPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') + implementation project(':replayerPlugins:jsonMessageTransformers:jsonJoltMessageTransformer') +} diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider new file mode 100644 index 0000000000..34d314de61 --- /dev/null +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider @@ -0,0 +1 @@ +org.opensearch.migrations.transform.JsonTransformerForOpenSearch23PlusTargetTransformerProvider \ No newline at end of file diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java new file mode 100644 index 0000000000..35b0ac8fbc --- /dev/null +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java @@ -0,0 +1,2 @@ +package org.opensearch.migrations.transform;public class JsonJoltTransformerProvider { +} diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java similarity index 100% rename from TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/transform/JsonTransformerTest.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonTransformerTest.java similarity index 100% rename from TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/transform/JsonTransformerTest.java rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonTransformerTest.java diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/PayloadRepackingTest.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/PayloadRepackingTest.java similarity index 100% rename from TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/PayloadRepackingTest.java rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/PayloadRepackingTest.java diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/build.gradle b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/build.gradle new file mode 100644 index 0000000000..016a4af282 --- /dev/null +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/build.gradle @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + + +buildscript { + dependencies { + classpath 'org.junit.platform:junit-platform-gradle-plugin:1.0.1' + } +} + +plugins { + id 'org.opensearch.migrations.java-library-conventions' + id 'org.owasp.dependencycheck' version '8.2.1' + id "io.freefair.lombok" version "8.0.1" +} + +repositories { + mavenCentral() +} + +dependencies { +} + +tasks.named('test') { + useJUnitPlatform() +} diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/IHttpMessage.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IHttpMessage.java similarity index 100% rename from TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/IHttpMessage.java rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IHttpMessage.java diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java similarity index 100% rename from TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerFactory.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerFactory.java new file mode 100644 index 0000000000..b6dd5fca18 --- /dev/null +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerFactory.java @@ -0,0 +1,2 @@ +package org.opensearch.migrations.transform;public interface IJsonTransformerFactory { +} diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java new file mode 100644 index 0000000000..0ed379a339 --- /dev/null +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java @@ -0,0 +1,2 @@ +package org.opensearch.migrations.transform;public class IJsonTransformerProvider { +} diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java similarity index 100% rename from TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java new file mode 100644 index 0000000000..8d61e25764 --- /dev/null +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java @@ -0,0 +1,11 @@ +package org.opensearch.migrations.transform; + +public class JsonKeys { + public static final String INLINED_JSON_BODY_DOCUMENT_KEY = "inlinedJsonBody"; + + public static final String METHOD_KEY = "method"; + public static final String URI_KEY = "URI"; + public static final String PROTOCOL_KEY = "protocol"; + public static final String HEADERS_KEY = "headers"; + public static final String PAYLOAD_KEY = "payload"; +} diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java new file mode 100644 index 0000000000..21a0d915ba --- /dev/null +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java @@ -0,0 +1,8 @@ +package org.opensearch.migrations.transform; + +public class JsonTransformerForOpenSearch23PlusTargetTransformerProvider implements IJsonTransformerProvider { + @Override + public IJsonTransformerFactory createTransformerFactory(String[] args) { + return () -> new JsonTypeMappingTransformer(); + } +} diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java similarity index 79% rename from TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java index 05e99be886..3c83776460 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java @@ -1,7 +1,5 @@ package org.opensearch.migrations.transform; -import org.opensearch.migrations.replay.datahandlers.PayloadAccessFaultingMap; -import org.opensearch.migrations.replay.datahandlers.http.HttpJsonMessageWithFaultingPayload; import java.util.Map; import java.util.regex.Pattern; @@ -37,7 +35,7 @@ public Object transformJson(Object incomingJson) { } private Object transformHttpMessage(Map httpMsg) { - var incomingMethod = httpMsg.get(HttpJsonMessageWithFaultingPayload.METHOD_KEY); + var incomingMethod = httpMsg.get(JsonKeysForHttpMessage.METHOD_KEY); if ("GET".equals(incomingMethod)) { processGet(httpMsg); } else if ("PUT".equals(incomingMethod)) { @@ -47,31 +45,31 @@ private Object transformHttpMessage(Map httpMsg) { } private void processGet(Map httpMsg) { - var incomingUri = (String) httpMsg.get(HttpJsonMessageWithFaultingPayload.URI_KEY); + var incomingUri = (String) httpMsg.get(JsonKeysForHttpMessage.URI_KEY); var matchedUri = TYPED_OPERATION_URI_PATTERN_WITH_SIDE_CAPTURES.matcher(incomingUri); if (matchedUri.matches()) { var operationStr = matchedUri.group(2); if (operationStr.equals(SEARCH_URI_COMPONENT)) { - httpMsg.put(HttpJsonMessageWithFaultingPayload.URI_KEY, matchedUri.group(1) + operationStr); + httpMsg.put(JsonKeysForHttpMessage.URI_KEY, matchedUri.group(1) + operationStr); } } } private void processPut(Map httpMsg) { - final var uriStr = (String) httpMsg.get(HttpJsonMessageWithFaultingPayload.URI_KEY); + final var uriStr = (String) httpMsg.get(JsonKeysForHttpMessage.URI_KEY); var matchedTriple = TYPED_OPERATION_URI_PATTERN_WITH_SIDE_CAPTURES.matcher(uriStr); if (matchedTriple.matches()) { // TODO: Add support for multiple type mappings per index (something possible with // versions before ES7) - httpMsg.put(HttpJsonMessageWithFaultingPayload.URI_KEY, + httpMsg.put(JsonKeysForHttpMessage.URI_KEY, matchedTriple.group(1) + DOC_URI_COMPONENT + matchedTriple.group(2)); return; } var matchedSingle = SINGLE_LEVEL_OPERATION_PATTERN_WITH_CAPTURE.matcher(uriStr); if (matchedSingle.matches()) { var topPayloadElement = - (Map) ((Map) httpMsg.get(HttpJsonMessageWithFaultingPayload.PAYLOAD_KEY)) - .get(PayloadAccessFaultingMap.INLINED_JSON_BODY_DOCUMENT_KEY); + (Map) ((Map) httpMsg.get(JsonKeysForHttpMessage.PAYLOAD_KEY)) + .get(JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY); var mappingsValue = (Map) topPayloadElement.get(MAPPINGS_KEYNAME); if (mappingsValue != null) { exciseMappingsType(topPayloadElement, mappingsValue); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java new file mode 100644 index 0000000000..8f59651fbc --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java @@ -0,0 +1,2 @@ +package org.opensearch.migrations.replay;public class TransformationLoader { +} diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/PruferTreeGenerator.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/PruferTreeGenerator.java similarity index 100% rename from TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/PruferTreeGenerator.java rename to TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/PruferTreeGenerator.java diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/GenerateRandomNestedJsonObject.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/GenerateRandomNestedJsonObject.java similarity index 100% rename from TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/GenerateRandomNestedJsonObject.java rename to TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/GenerateRandomNestedJsonObject.java diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java similarity index 100% rename from TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java rename to TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TestHttpServerContext.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestHttpServerContext.java similarity index 100% rename from TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TestHttpServerContext.java rename to TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestHttpServerContext.java diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TestRequestKey.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java similarity index 100% rename from TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TestRequestKey.java rename to TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TestUtils.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java similarity index 100% rename from TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TestUtils.java rename to TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java From 7d95e3a422d3e8298467da6dc7b9f45dbec3b554 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Mon, 6 Nov 2023 06:46:49 -0500 Subject: [PATCH 08/55] WIP to finish setting up various replayer transformation plugin projects. I've added a TransformationLoader helper object to replace buildlDefaultJsonTransformer() for the TrafficReplayer. This loads transformation plugins via ServiceLoader and configures them. The configuration part still has work to be done on it. TrafficReplayer has a new parameter to pass a configuration string, which is expected to be JSON. When there's only one transformerProvider, the string is optional. When there's more than one that's been loaded, the string is mandatory so that the loader knows what order to run the transformations in. It turns out, that we can probably build very complex transformations dynamically just with configurations (e.g., here's 3 transformations that use a JSON manipulation script like jolt or JMESPath and each can be configured with their own script). I've moved test support code for the trafficReplayer from test to testFixtures. That gives better clarity and also allows those to be reused from other projects. I'm not sure whether it's better to run full transformer tests from the plugin projects or the replayer. The replayer won't be aware of every future, hypothetical transformer plugin. Providing a more testable interface utilizing replayer components will give those components the ability to do more complex tests which are completely at the discretion of the plugin author. What I don't like about this approach right now is the dependency list for the testFixture. It includes netty, jackson, and junit5. Considering how terse and lightweight the contract is for a transformer, those could be heavy and potentially conflicting assumptions. There is still one failing test: org.opensearch.migrations.replay.HeaderTransformerTest#testMalformedPayload_andTypeMappingUri_IsPassedThrough Signed-off-by: Greg Schohn --- TrafficCapture/replayerPlugins/build.gradle | 5 + .../jsonJoltMessageTransformer/build.gradle | 13 +++ .../build.gradle | 26 +++++ .../JsonJoltTransformerProvider.java | 11 +- .../replay/AddCompressionEncodingTest.java | 6 - .../replay/JsonTransformerTest.java | 12 +- .../replay/PayloadRepackingTest.java | 3 - .../migrations/transform/IHttpMessage.java | 2 +- .../transform/IJsonTransformerFactory.java | 2 - .../transform/IJsonTransformerProvider.java | 7 +- .../transform/JsonKeysForHttpMessage.java | 2 +- .../build.gradle | 8 ++ ...rations.transform.IJsonTransformerProvider | 1 + ...Search23PlusTargetTransformerProvider.java | 6 +- TrafficCapture/settings.gradle | 21 ++++ TrafficCapture/trafficReplayer/build.gradle | 12 +- .../migrations/replay/SigV4Signer.java | 3 +- .../migrations/replay/TrafficReplayer.java | 29 +++-- .../replay/TransformationLoader.java | 105 +++++++++++++++++- .../PayloadAccessFaultingMap.java | 13 +-- .../HttpJsonMessageWithFaultingPayload.java | 27 ++--- .../http/NettyJsonBodyAccumulateHandler.java | 4 +- .../http/NettyJsonBodySerializeHandler.java | 5 +- .../transform/IAuthTransformerFactory.java | 2 - .../RemovingAuthTransformerFactory.java | 1 - .../StaticAuthTransformerFactory.java | 1 - .../replay/FullTrafficReplayerTest.java | 2 +- .../replay/HeaderTransformerTest.java | 11 +- .../SigV4SigningTransformationTest.java | 3 +- .../replay/TrafficReplayerTest.java | 4 +- .../NettyPacketToHttpConsumerTest.java | 4 +- .../HttpJsonTransformingConsumerTest.java | 14 +-- .../NettyJsonBodySerializeHandlerTest.java | 7 +- .../migrations/replay/TestUtils.java | 8 ++ 34 files changed, 283 insertions(+), 97 deletions(-) delete mode 100644 TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerFactory.java create mode 100644 TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/build.gradle create mode 100644 TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider diff --git a/TrafficCapture/replayerPlugins/build.gradle b/TrafficCapture/replayerPlugins/build.gradle index e69de29bb2..fb3c3f88a2 100644 --- a/TrafficCapture/replayerPlugins/build.gradle +++ b/TrafficCapture/replayerPlugins/build.gradle @@ -0,0 +1,5 @@ +subprojects { + repositories { + mavenCentral() + } +} \ No newline at end of file diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/build.gradle b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/build.gradle index e69de29bb2..0ccc6dd66e 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/build.gradle +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/build.gradle @@ -0,0 +1,13 @@ +plugins { + id("io.freefair.lombok") version "8.0.1" +} + +dependencies { + implementation project(':replayerPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') + + implementation group: 'com.bazaarvoice.jolt', name: 'jolt-core', version: '0.1.7' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.0' + implementation group: 'org.slf4j', name:"slf4j-api", version:"2.0.7" + + testImplementation project(':trafficReplayer') +} \ No newline at end of file diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/build.gradle b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/build.gradle index bcd13841d4..d218c9ca7b 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/build.gradle +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/build.gradle @@ -1,8 +1,34 @@ +buildscript { + dependencies { + classpath 'org.junit.platform:junit-platform-gradle-plugin:1.0.1' + } +} + plugins { id "io.freefair.lombok" version "8.0.1" + id 'java-test-fixtures' +} + +repositories { + mavenCentral() } dependencies { implementation project(':replayerPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') implementation project(':replayerPlugins:jsonMessageTransformers:jsonJoltMessageTransformer') + + testImplementation project(':replayerPlugins:jsonMessageTransformers:jsonJoltMessageTransformer') + testImplementation project(':trafficReplayer') + testImplementation testFixtures(project(path: ':testUtilities')) + testImplementation testFixtures(project(path: ':trafficReplayer')) + + testImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.0' + testImplementation group: 'io.netty', name: 'netty-all', version: '4.1.94.Final' + testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-api', version:'5.9.3' + testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-params', version:'5.9.3' + testImplementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7' +} + +tasks.named('test') { + useJUnitPlatform() } diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java index 35b0ac8fbc..805f8c01e6 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java @@ -1,2 +1,11 @@ -package org.opensearch.migrations.transform;public class JsonJoltTransformerProvider { +package org.opensearch.migrations.transform; + +import java.util.Optional; + +public class JsonJoltTransformerProvider implements IJsonTransformerProvider { + @Override + public IJsonTransformer createTransformer(Optional config) { + var builder = JsonJoltTransformer.newBuilder(); + //builder.addOperationObject() + } } diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java index 2bd6979169..44da58363e 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/AddCompressionEncodingTest.java @@ -1,13 +1,11 @@ package org.opensearch.migrations.replay; -import io.netty.util.ResourceLeakDetector; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.opensearch.migrations.replay.datahandlers.http.HttpJsonTransformingConsumer; import org.opensearch.migrations.replay.datatypes.HttpRequestTransformationStatus; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; -import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; import org.opensearch.migrations.transform.JsonJoltTransformBuilder; import org.opensearch.migrations.transform.JsonJoltTransformer; @@ -23,16 +21,12 @@ import java.util.zip.GZIPInputStream; @Slf4j -@WrapWithNettyLeakDetection public class AddCompressionEncodingTest { public static final byte BYTE_FILL_VALUE = (byte) '7'; @Test - @WrapWithNettyLeakDetection(repetitions = 2) public void addingCompressionRequestHeaderCompressesPayload() throws ExecutionException, InterruptedException, IOException { - ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); - final var dummyAggregatedResponse = new TransformedTargetRequestAndResponse(null, 17, null, null, HttpRequestTransformationStatus.COMPLETED, null); var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse); diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonTransformerTest.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonTransformerTest.java index 6d9d51f2e0..99165a0189 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonTransformerTest.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonTransformerTest.java @@ -1,15 +1,19 @@ -package org.opensearch.migrations.transform; +package org.opensearch.migrations.replay; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; +import org.opensearch.migrations.transform.JsonJoltTransformBuilder; +import org.opensearch.migrations.transform.JsonJoltTransformer; import java.io.IOException; import java.io.InputStream; +import java.util.LinkedHashMap; import java.util.Map; @Slf4j @@ -17,19 +21,20 @@ class JsonTransformerTest { public static final String DUMMY_HOSTNAME_TEST_STRING = "THIS_IS_A_TEST_STRING_THAT_ONLY_EXISTS_IN_ONE_PLACE"; ObjectMapper mapper = new ObjectMapper(); + static final TypeReference> TYPE_REFERENCE_FOR_MAP_TYPE = new TypeReference<>(){}; public JsonTransformerTest() { mapper.configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_COMMENTS, true); } private Map parseStringAsJson(String jsonStr) throws JsonProcessingException { - return mapper.readValue(jsonStr, JsonJoltTransformBuilder.TYPE_REFERENCE_FOR_MAP_TYPE); + return mapper.readValue(jsonStr, TYPE_REFERENCE_FOR_MAP_TYPE); } @SneakyThrows private Map parseSampleRequestFromResource(String path) { try (InputStream inputStream = JsonJoltTransformBuilder.class.getResourceAsStream("/requests/"+path)) { - return mapper.readValue(inputStream, JsonJoltTransformBuilder.TYPE_REFERENCE_FOR_MAP_TYPE); + return mapper.readValue(inputStream, TYPE_REFERENCE_FOR_MAP_TYPE); } } @@ -64,5 +69,4 @@ public void testHttpTransform() throws IOException { log.error("transformed json document: "+transformedJsonOutputStr); Assertions.assertTrue(transformedJsonOutputStr.contains(DUMMY_HOSTNAME_TEST_STRING)); } - } \ No newline at end of file diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/PayloadRepackingTest.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/PayloadRepackingTest.java index 6aebd72203..8ba273ce16 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/PayloadRepackingTest.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/PayloadRepackingTest.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import io.netty.handler.codec.http.DefaultHttpHeaders; -import io.netty.util.ResourceLeakDetector; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -44,8 +43,6 @@ public static Arguments[] makeCombinations() { @ParameterizedTest @MethodSource("makeCombinations") public void testSimplePayloadTransform(boolean doGzip, boolean doChunked) throws Exception { - ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); - var transformerBuilder = JsonJoltTransformer.newBuilder(); if (doGzip) { transformerBuilder.addCannedOperation(JsonJoltTransformBuilder.CANNED_OPERATION.ADD_GZIP); } diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IHttpMessage.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IHttpMessage.java index d1fadfa77b..ed97a4bb85 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IHttpMessage.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IHttpMessage.java @@ -1,4 +1,4 @@ -package org.opensearch.migrations.replay.datahandlers.http; +package org.opensearch.migrations.transform; import java.util.List; import java.util.Map; diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerFactory.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerFactory.java deleted file mode 100644 index b6dd5fca18..0000000000 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerFactory.java +++ /dev/null @@ -1,2 +0,0 @@ -package org.opensearch.migrations.transform;public interface IJsonTransformerFactory { -} diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java index 0ed379a339..b379acfa49 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java @@ -1,2 +1,7 @@ -package org.opensearch.migrations.transform;public class IJsonTransformerProvider { +package org.opensearch.migrations.transform; + +import java.util.Optional; + +public interface IJsonTransformerProvider { + IJsonTransformer createTransformer(Optional config); } diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java index 8d61e25764..3c7115722b 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java @@ -1,6 +1,6 @@ package org.opensearch.migrations.transform; -public class JsonKeys { +public class JsonKeysForHttpMessage { public static final String INLINED_JSON_BODY_DOCUMENT_KEY = "inlinedJsonBody"; public static final String METHOD_KEY = "method"; diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/build.gradle b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/build.gradle new file mode 100644 index 0000000000..bcd13841d4 --- /dev/null +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/build.gradle @@ -0,0 +1,8 @@ +plugins { + id "io.freefair.lombok" version "8.0.1" +} + +dependencies { + implementation project(':replayerPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') + implementation project(':replayerPlugins:jsonMessageTransformers:jsonJoltMessageTransformer') +} diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider new file mode 100644 index 0000000000..34d314de61 --- /dev/null +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider @@ -0,0 +1 @@ +org.opensearch.migrations.transform.JsonTransformerForOpenSearch23PlusTargetTransformerProvider \ No newline at end of file diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java index 21a0d915ba..523ffee93d 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java @@ -1,8 +1,10 @@ package org.opensearch.migrations.transform; +import java.util.Optional; + public class JsonTransformerForOpenSearch23PlusTargetTransformerProvider implements IJsonTransformerProvider { @Override - public IJsonTransformerFactory createTransformerFactory(String[] args) { - return () -> new JsonTypeMappingTransformer(); + public IJsonTransformer createTransformer(Optional args) { + return new JsonTypeMappingTransformer(); } } diff --git a/TrafficCapture/settings.gradle b/TrafficCapture/settings.gradle index 74681f6f68..05b6ca6194 100644 --- a/TrafficCapture/settings.gradle +++ b/TrafficCapture/settings.gradle @@ -7,6 +7,22 @@ * in the user manual at https://docs.gradle.org/8.0.2/userguide/multi_project_builds.html */ +void addSubProjects(String path, File dir) { + if (dir.isDirectory() == false) return; + if (dir.name == 'buildSrc') return; + if (new File(dir, 'build.gradle').exists() == false) return; + if (findProject(dir) != null) return; + + final String projectName = "${path}:${dir.name}" + include projectName + if (path.isEmpty() || path.startsWith(':example-plugins')) { + project(projectName).projectDir = dir + } + for (File subdir : dir.listFiles()) { + addSubProjects(projectName, subdir) + } +} + rootProject.name = 'TrafficCapture' include('captureKafkaOffloader', 'captureOffloader', @@ -19,3 +35,8 @@ include('captureKafkaOffloader', 'dockerSolution', 'trafficCaptureProxyServerTest' ) + +addSubProjects('', new File(rootProject.projectDir,'replayerPlugins')) +include 'replayerPlugins:jsonMessageTransformers:jsonJoltMessageTransformerProvider:untitled' +findProject(':replayerPlugins:jsonMessageTransformers:jsonJoltMessageTransformerProvider:untitled')?.name = 'untitled' + diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle index 89388ded7e..aed114879d 100644 --- a/TrafficCapture/trafficReplayer/build.gradle +++ b/TrafficCapture/trafficReplayer/build.gradle @@ -15,6 +15,7 @@ plugins { // id 'checkstyle' id 'org.owasp.dependencycheck' version '8.2.1' id "io.freefair.lombok" version "8.0.1" + id 'java-test-fixtures' } //spotbugs { @@ -38,9 +39,10 @@ dependencies { implementation project(':captureProtobufs') implementation project(':coreUtilities') + implementation project(':replayerPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') + implementation project(':replayerPlugins:jsonMessageTransformers:openSearch23PlusTargetTransformerProvider') implementation group: 'com.beust', name: 'jcommander', version: '1.82' - implementation group: 'com.bazaarvoice.jolt', name: 'jolt-core', version: '0.1.7' implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.0' implementation group: 'com.google.guava', name: 'guava', version: '32.0.1-jre' implementation group: 'com.google.protobuf', name: 'protobuf-java', version: '3.22.2' @@ -59,6 +61,14 @@ dependencies { implementation group: 'software.amazon.awssdk', name: 'secretsmanager', version: '2.20.127' implementation group: 'software.amazon.msk', name: 'aws-msk-iam-auth', version: '1.1.9' + testFixturesImplementation project(':replayerPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') + testFixturesImplementation testFixtures(project(path: ':testUtilities')) + + testFixturesImplementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7' + testFixturesImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.0' + testFixturesImplementation group: 'io.netty', name: 'netty-all', version: '4.1.94.Final' + testFixturesImplementation group: 'org.junit.jupiter', name:'junit-jupiter-api', version:'5.9.3' + testImplementation project(':captureOffloader') testImplementation testFixtures(project(path: ':captureOffloader')) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java index 166f773712..b42b004609 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java @@ -15,10 +15,9 @@ import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.replay.datahandlers.http.HttpJsonMessageWithFaultingPayload; -import org.opensearch.migrations.replay.datahandlers.http.IHttpMessage; +import org.opensearch.migrations.transform.IHttpMessage; import org.opensearch.migrations.transform.IAuthTransformer; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.signer.Aws4Signer; import software.amazon.awssdk.auth.signer.internal.BaseAws4Signer; import software.amazon.awssdk.auth.signer.params.Aws4SignerParams; import software.amazon.awssdk.core.checksums.SdkChecksum; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index cd9cb6565a..79c115ea1a 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -3,6 +3,7 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; +import com.fasterxml.jackson.core.JsonProcessingException; import io.netty.buffer.Unpooled; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; @@ -11,7 +12,7 @@ import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer; -import org.opensearch.migrations.replay.datahandlers.http.IHttpMessage; +import org.opensearch.migrations.transform.IHttpMessage; import org.opensearch.migrations.replay.datatypes.HttpRequestTransformationStatus; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.datatypes.TransformedPackets; @@ -25,10 +26,7 @@ import org.opensearch.migrations.trafficcapture.protos.TrafficStreamUtils; import org.opensearch.migrations.transform.IAuthTransformer; import org.opensearch.migrations.transform.IAuthTransformerFactory; -import org.opensearch.migrations.transform.JsonCompositeTransformer; -import org.opensearch.migrations.transform.JsonJoltTransformer; import org.opensearch.migrations.transform.IJsonTransformer; -import org.opensearch.migrations.transform.JsonTypeMappingTransformer; import org.opensearch.migrations.transform.RemovingAuthTransformerFactory; import org.opensearch.migrations.transform.StaticAuthTransformerFactory; import org.slf4j.event.Level; @@ -52,7 +50,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; @@ -104,29 +101,24 @@ public TerminationException(Throwable shutdownCause, Throwable immediateCause) { } } - public static IJsonTransformer buildDefaultJsonTransformer(String newHostName) { - var joltJsonTransformerBuilder = JsonJoltTransformer.newBuilder() - .addHostSwitchOperation(newHostName); - var joltJsonTransformer = joltJsonTransformerBuilder.build(); - return new JsonCompositeTransformer(joltJsonTransformer, new JsonTypeMappingTransformer()); - } - public TrafficReplayer(URI serverUri, + String fullTransformerConfig, IAuthTransformerFactory authTransformerFactory, boolean allowInsecureConnections) throws SSLException { - this(serverUri, authTransformerFactory, allowInsecureConnections, 0, 1024); + this(serverUri, fullTransformerConfig, authTransformerFactory, allowInsecureConnections, 0, 1024); } public TrafficReplayer(URI serverUri, + String fullTransformerConfig, IAuthTransformerFactory authTransformerFactory, boolean allowInsecureConnections, int numSendingThreads, int maxConcurrentOutstandingRequests) throws SSLException { this(serverUri, authTransformerFactory, allowInsecureConnections, numSendingThreads, maxConcurrentOutstandingRequests, - buildDefaultJsonTransformer(serverUri.getHost()) + new TransformationLoader().getTransformerFactoryLoader(serverUri.getHost(), fullTransformerConfig) ); } @@ -292,6 +284,13 @@ static class Parameters { arity=1, description = "File path for Kafka properties file to use for additional or overriden Kafka properties") String kafkaTrafficPropertyFile; + + @Parameter(required = false, + names = "--transformer-config", + arity = 1, + description = "Json configuration of message transformers. Keys are the names of the loaded transformers " + + "(shortname or longname) and values are the configuration passed to each of the transformers.") + String transformerConfig; } public static Parameters parseArgs(String[] args) { @@ -333,7 +332,7 @@ public static void main(String[] args) Duration.ofSeconds(params.lookaheadTimeSeconds)); var authTransformer = buildAuthTransformerFactory(params)) { - var tr = new TrafficReplayer(uri, authTransformer, + var tr = new TrafficReplayer(uri, params.transformerConfig, authTransformer, params.allowInsecureConnections, params.numClientThreads, params.maxConcurrentRequests); setupShutdownHookForReplayer(tr); var tupleWriter = new SourceTargetCaptureTuple.TupleToFileWriter(bufferedOutputStream); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java index 8f59651fbc..0d359ce1e0 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java @@ -1,2 +1,105 @@ -package org.opensearch.migrations.replay;public class TransformationLoader { +package org.opensearch.migrations.replay; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; +import org.opensearch.migrations.transform.IJsonTransformer; +import org.opensearch.migrations.transform.IJsonTransformerProvider; +import org.opensearch.migrations.transform.JsonCompositeTransformer; +import org.opensearch.migrations.transform.JsonKeysForHttpMessage; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.ServiceLoader; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class TransformationLoader { + private final List providers; + ObjectMapper objMapper = new ObjectMapper(); + + public TransformationLoader() { + ServiceLoader transformerProviders = + ServiceLoader.load(IJsonTransformerProvider.class); + var inProgressProviders = new ArrayList(); + for (var provider : transformerProviders) { + inProgressProviders.add(provider); + } + providers = Collections.unmodifiableList(inProgressProviders); + } + + List> parseFullConfig(String fullConfig) throws JsonProcessingException { + return objMapper.readValue(fullConfig, new TypeReference<>() { + }); + } + + protected Stream getTransformerFactoryFromServiceLoader(String fullConfig) + throws JsonProcessingException { + var configList = fullConfig == null ? List.of() : parseFullConfig(fullConfig); + if (providers.size() > 1 && configList.isEmpty()) { + throw new IllegalArgumentException("Must provide a configuration when multiple IJsonTransformerProvider " + + "are loaded (" + providers.stream().map(p -> p.getClass().toString()) + .collect(Collectors.joining(",")) + ")"); + } else if (providers.isEmpty()) { + return Stream.of(); + } else if (!configList.isEmpty()) { + return configList.stream().map(c -> configureTransformerFromConfig((Map) c)); + } else { + // send in Optional.empty because we would have hit the other case in the previous branch + return Stream.of(providers.get(0).createTransformer(Optional.empty())); + } + } + + @SneakyThrows // JsonProcessingException should be impossible since the contents are those that were just parsed + private IJsonTransformer configureTransformerFromConfig(Map c) { + var keys = c.keySet(); + if (keys.size() != 1) { + throw new IllegalArgumentException("Must specify the configuration list with a sequence of maps " + + "with only one key each, where the key is the name of the transformer."); + } + var key = keys.stream().findFirst().get(); + var transformerConfigStr = objMapper.writeValueAsString(c.get(key)); + for (var p : providers) { + if (p.getClass().getSimpleName().equals(key)) { + return p.createTransformer(Optional.of(transformerConfigStr)); + } + } + return null; + } + public IJsonTransformer getTransformerFactoryLoader(String newHostName) { + return getTransformerFactoryLoader(newHostName, null); + } + + public IJsonTransformer getTransformerFactoryLoader(String newHostName, String fullConfig) { + try { + var loadedTransformers = getTransformerFactoryFromServiceLoader(fullConfig); + return new JsonCompositeTransformer(Stream.concat( + loadedTransformers, + Optional.ofNullable(newHostName).map(h->Stream.of(new HostTransformer(h))).orElse(Stream.of()) + ).toArray(IJsonTransformer[]::new)); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException("Could not parse the transformer configuration as a json list", e); + } + } + + private class HostTransformer implements IJsonTransformer { + private final String newHostName; + + @Override + public Object transformJson(Object incomingJson) { + var asMap = (Map) incomingJson; + var headers = (Map) asMap.get(JsonKeysForHttpMessage.HEADERS_KEY); + headers.replace("host", newHostName); + return asMap; + } + + public HostTransformer(String newHostName) { + this.newHostName = newHostName; + } + + } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java index f0c61a2a1c..1a729cf9f5 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java @@ -2,8 +2,9 @@ import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; -import org.opensearch.migrations.replay.datahandlers.http.IHttpMessage; +import org.opensearch.migrations.transform.IHttpMessage; import org.opensearch.migrations.replay.datahandlers.http.StrictCaseInsensitiveHttpHeadersMap; +import org.opensearch.migrations.transform.JsonKeysForHttpMessage; import java.util.AbstractMap; import java.util.AbstractSet; @@ -23,8 +24,6 @@ @Slf4j public class PayloadAccessFaultingMap extends AbstractMap { - public static final String INLINED_JSON_BODY_DOCUMENT_KEY = "inlinedJsonBody"; - private final boolean isJson; private Object onlyValue; @@ -36,7 +35,7 @@ public PayloadAccessFaultingMap(StrictCaseInsensitiveHttpHeadersMap headers) { @Override public Object get(Object key) { - if (!INLINED_JSON_BODY_DOCUMENT_KEY.equals(key) || !isJson) { return null; } + if (!JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY.equals(key) || !isJson) { return null; } if (onlyValue == null) { throw PayloadNotLoadedException.getInstance(); } else { @@ -47,7 +46,7 @@ public Object get(Object key) { @Override public Set> entrySet() { if (onlyValue != null) { - return Set.of(new SimpleEntry<>(INLINED_JSON_BODY_DOCUMENT_KEY, onlyValue)); + return Set.of(new SimpleEntry<>(JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY, onlyValue)); } else { return new AbstractSet>() { @Override @@ -64,7 +63,7 @@ public Entry next() { if (isJson && count == 0) { ++count; if (onlyValue != null) { - return new SimpleEntry<>(INLINED_JSON_BODY_DOCUMENT_KEY, onlyValue); + return new SimpleEntry<>(JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY, onlyValue); } else { throw PayloadNotLoadedException.getInstance(); } @@ -85,7 +84,7 @@ public int size() { @Override public Object put(String key, Object value) { - if (!INLINED_JSON_BODY_DOCUMENT_KEY.equals(key)) { return null; } + if (!JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY.equals(key)) { return null; } Object old = onlyValue; onlyValue = value; return old; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonMessageWithFaultingPayload.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonMessageWithFaultingPayload.java index 257f58a4db..eeba756113 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonMessageWithFaultingPayload.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonMessageWithFaultingPayload.java @@ -1,17 +1,14 @@ package org.opensearch.migrations.replay.datahandlers.http; import org.opensearch.migrations.replay.datahandlers.PayloadAccessFaultingMap; +import org.opensearch.migrations.transform.IHttpMessage; +import org.opensearch.migrations.transform.JsonKeysForHttpMessage; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; public class HttpJsonMessageWithFaultingPayload extends LinkedHashMap implements IHttpMessage { - public static final String METHOD_KEY = "method"; - public static final String URI_KEY = "URI"; - public static final String PROTOCOL_KEY = "protocol"; - public static final String HEADERS_KEY = "headers"; - public static final String PAYLOAD_KEY = "payload"; public HttpJsonMessageWithFaultingPayload() { } @@ -22,25 +19,25 @@ public HttpJsonMessageWithFaultingPayload(Map m) { @Override public String method() { - return (String) this.get(METHOD_KEY); + return (String) this.get(JsonKeysForHttpMessage.METHOD_KEY); } public void setMethod(String value) { - this.put(METHOD_KEY, value); + this.put(JsonKeysForHttpMessage.METHOD_KEY, value); } @Override public String path() { - return (String) this.get(URI_KEY); + return (String) this.get(JsonKeysForHttpMessage.URI_KEY); } public void setPath(String value) { - this.put(URI_KEY, value); + this.put(JsonKeysForHttpMessage.URI_KEY, value); } @Override public String protocol() { - return (String) this.get(PROTOCOL_KEY); + return (String) this.get(JsonKeysForHttpMessage.PROTOCOL_KEY); } public void setProtocol(String value) { - this.put(PROTOCOL_KEY, value); + this.put(JsonKeysForHttpMessage.PROTOCOL_KEY, value); } @@ -50,15 +47,15 @@ public Map headersMap() { } public ListKeyAdaptingCaseInsensitiveHeadersMap headers() { - return (ListKeyAdaptingCaseInsensitiveHeadersMap) this.get(HEADERS_KEY); + return (ListKeyAdaptingCaseInsensitiveHeadersMap) this.get(JsonKeysForHttpMessage.HEADERS_KEY); } public void setHeaders(ListKeyAdaptingCaseInsensitiveHeadersMap value) { - this.put(HEADERS_KEY, value); + this.put(JsonKeysForHttpMessage.HEADERS_KEY, value); } public Map payload() { - return (Map) this.get(PAYLOAD_KEY); + return (Map) this.get(JsonKeysForHttpMessage.PAYLOAD_KEY); } public void setPayloadFaultMap(PayloadAccessFaultingMap value) { - this.put(PAYLOAD_KEY, value); + this.put(JsonKeysForHttpMessage.PAYLOAD_KEY, value); } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyAccumulateHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyAccumulateHandler.java index 2b48950b59..be256a0d64 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyAccumulateHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyAccumulateHandler.java @@ -6,7 +6,7 @@ import io.netty.handler.codec.http.LastHttpContent; import lombok.SneakyThrows; import org.opensearch.migrations.replay.datahandlers.JsonAccumulator; -import org.opensearch.migrations.replay.datahandlers.PayloadAccessFaultingMap; +import org.opensearch.migrations.transform.JsonKeysForHttpMessage; /** * This accumulates HttpContent messages through a JsonAccumulator and eventually fires off a @@ -35,7 +35,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception } else if (msg instanceof HttpContent) { var jsonObject = jsonAccumulator.consumeByteBuffer(((HttpContent)msg).content().nioBuffer()); if (jsonObject != null) { - capturedHttpJsonMessage.payload().put(PayloadAccessFaultingMap.INLINED_JSON_BODY_DOCUMENT_KEY, jsonObject); + capturedHttpJsonMessage.payload().put(JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY, jsonObject); ctx.fireChannelRead(capturedHttpJsonMessage); } else if (msg instanceof LastHttpContent) { throw new IncompleteJsonBodyException(); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodySerializeHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodySerializeHandler.java index f72f6550b2..39c4722d69 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodySerializeHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodySerializeHandler.java @@ -6,7 +6,7 @@ import io.netty.handler.codec.http.LastHttpContent; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.replay.datahandlers.JsonEmitter; -import org.opensearch.migrations.replay.datahandlers.PayloadAccessFaultingMap; +import org.opensearch.migrations.transform.JsonKeysForHttpMessage; import java.io.IOException; import java.util.Map; @@ -21,7 +21,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception var jsonMessage = (HttpJsonMessageWithFaultingPayload) msg; var payload = jsonMessage.payload(); jsonMessage.setPayloadFaultMap(null); - var payloadContents = (Map) payload.get(PayloadAccessFaultingMap.INLINED_JSON_BODY_DOCUMENT_KEY); + var payloadContents = + (Map) payload.get(JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY); ctx.fireChannelRead(msg); if (payloadContents != null) { serializePayload(ctx, payloadContents); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/IAuthTransformerFactory.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/IAuthTransformerFactory.java index 0dfb472d93..3401d0347e 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/IAuthTransformerFactory.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/IAuthTransformerFactory.java @@ -1,7 +1,5 @@ package org.opensearch.migrations.transform; -import org.opensearch.migrations.replay.datahandlers.http.IHttpMessage; - import java.io.IOException; public interface IAuthTransformerFactory extends AutoCloseable { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/RemovingAuthTransformerFactory.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/RemovingAuthTransformerFactory.java index 0ef986147d..5b5b6f3a54 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/RemovingAuthTransformerFactory.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/RemovingAuthTransformerFactory.java @@ -1,7 +1,6 @@ package org.opensearch.migrations.transform; import org.opensearch.migrations.replay.datahandlers.http.HttpJsonMessageWithFaultingPayload; -import org.opensearch.migrations.replay.datahandlers.http.IHttpMessage; public class RemovingAuthTransformerFactory implements IAuthTransformerFactory { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/StaticAuthTransformerFactory.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/StaticAuthTransformerFactory.java index 8b47a777e0..618bec7a8a 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/StaticAuthTransformerFactory.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/StaticAuthTransformerFactory.java @@ -1,7 +1,6 @@ package org.opensearch.migrations.transform; import org.opensearch.migrations.replay.datahandlers.http.HttpJsonMessageWithFaultingPayload; -import org.opensearch.migrations.replay.datahandlers.http.IHttpMessage; public class StaticAuthTransformerFactory implements IAuthTransformerFactory { private final String authHeaderValue; diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java index 5d61e10819..357372c9d7 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java @@ -338,7 +338,7 @@ private static void runTrafficReplayer(Supplier cap var tr = new TrafficReplayer(httpServer.localhostEndpoint(), new StaticAuthTransformerFactory("TEST"), true, 10, 10*1024, - TrafficReplayer.buildDefaultJsonTransformer(httpServer.localhostEndpoint().getHost())); + new TransformationLoader().getTransformerFactoryLoader(httpServer.localhostEndpoint().getHost())); try (var os = new NullOutputStream(); var trafficSource = captureSourceSupplier.get(); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java index f7095e689f..98d5d93a1c 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java @@ -7,7 +7,6 @@ import org.opensearch.migrations.replay.datatypes.HttpRequestTransformationStatus; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; -import org.opensearch.migrations.transform.JsonJoltTransformer; import org.opensearch.migrations.transform.StaticAuthTransformerFactory; import java.time.Duration; @@ -33,10 +32,8 @@ public void testTransformer() throws Exception { final var dummyAggregatedResponse = new TransformedTargetRequestAndResponse(null, 17, null, null, HttpRequestTransformationStatus.COMPLETED, null); var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse); - var jsonHandler = JsonJoltTransformer.newBuilder() - .addHostSwitchOperation(SILLY_TARGET_CLUSTER_NAME) - .build(); - var transformingHandler = new HttpJsonTransformingConsumer(jsonHandler, null, testPacketCapture, + var transformer = new TransformationLoader().getTransformerFactoryLoader(SILLY_TARGET_CLUSTER_NAME, null); + var transformingHandler = new HttpJsonTransformingConsumer(transformer, null, testPacketCapture, "TEST", TestRequestKey.getTestConnectionRequestId(0)); runRandomPayloadWithTransformer(transformingHandler, dummyAggregatedResponse, testPacketCapture, contentLength -> "GET / HTTP/1.1\r\n" + @@ -88,7 +85,7 @@ public void testMalformedPayloadIsPassedThrough() throws Exception { var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse); var httpBasicAuthTransformer = new StaticAuthTransformerFactory("Basic YWRtaW46YWRtaW4="); var transformingHandler = new HttpJsonTransformingConsumer( - TrafficReplayer.buildDefaultJsonTransformer(SILLY_TARGET_CLUSTER_NAME), + new TransformationLoader().getTransformerFactoryLoader(SILLY_TARGET_CLUSTER_NAME, null), httpBasicAuthTransformer, testPacketCapture, "TEST", TestRequestKey.getTestConnectionRequestId(0)); @@ -113,7 +110,7 @@ public void testMalformedPayload_andTypeMappingUri_IsPassedThrough() throws Exce null, HttpRequestTransformationStatus.COMPLETED, null); var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse); var transformingHandler = new HttpJsonTransformingConsumer( - TrafficReplayer.buildDefaultJsonTransformer(SILLY_TARGET_CLUSTER_NAME), + new TransformationLoader().getTransformerFactoryLoader(SILLY_TARGET_CLUSTER_NAME), null, testPacketCapture, "TEST", TestRequestKey.getTestConnectionRequestId(0)); Random r = new Random(2); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SigV4SigningTransformationTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SigV4SigningTransformationTest.java index cdd7909ad9..8d7ad72067 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SigV4SigningTransformationTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SigV4SigningTransformationTest.java @@ -6,7 +6,6 @@ import io.netty.util.ResourceLeakDetector; import org.junit.jupiter.api.Test; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; -import org.opensearch.migrations.transform.JsonJoltTransformer; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; @@ -56,7 +55,7 @@ public void testSignatureProperlyApplied() throws Exception { "fc0e8e9a1f7697f510bfdd4d55b8612df8a0140b4210967efd87ee9cb7104362"); expectedRequestHeaders.add("X-Amz-Date", "19700101T000000Z"); - TestUtils.runPipelineAndValidate(JsonJoltTransformer.newBuilder().build(), + TestUtils.runPipelineAndValidate( msg -> new SigV4Signer(mockCredentialsProvider, "es", "us-east-1", "https", () -> Clock.fixed(Instant.EPOCH, ZoneOffset.UTC)), null, stringParts, expectedRequestHeaders, diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java index 64c835ac38..e289bfea4e 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java @@ -147,7 +147,7 @@ static byte[] synthesizeTrafficStreamsIntoByteArray(Instant timestamp, int numSt @Test public void testReader() throws Exception { - var tr = new TrafficReplayer(new URI("http://localhost:9200"), null,false); + var tr = new TrafficReplayer(new URI("http://localhost:9200"), null, null, false); List> byteArrays = new ArrayList<>(); CapturedTrafficToHttpTransactionAccumulator trafficAccumulator = new CapturedTrafficToHttpTransactionAccumulator(Duration.ofSeconds(30), null, @@ -185,7 +185,7 @@ public void onConnectionClose(UniqueReplayerRequestKey key, Instant when) {} @Test public void testCapturedReadsAfterCloseAreHandledAsNew() throws Exception { - var tr = new TrafficReplayer(new URI("http://localhost:9200"), null,false); + var tr = new TrafficReplayer(new URI("http://localhost:9200"), null, null, false); List> byteArrays = new ArrayList<>(); var remainingAccumulations = new AtomicInteger(); CapturedTrafficToHttpTransactionAccumulator trafficAccumulator = diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java index d855f28a47..e4de2b02f2 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java @@ -18,6 +18,7 @@ import org.opensearch.migrations.replay.TestRequestKey; import org.opensearch.migrations.replay.TimeShifter; import org.opensearch.migrations.replay.TrafficReplayer; +import org.opensearch.migrations.replay.TransformationLoader; import org.opensearch.migrations.replay.datatypes.PojoTrafficStreamKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; import org.opensearch.migrations.replay.traffic.source.BufferedFlowController; @@ -27,7 +28,6 @@ import org.opensearch.migrations.testutils.SimpleHttpClientForTesting; import org.opensearch.migrations.testutils.SimpleHttpServer; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; -import org.opensearch.migrations.transform.JsonJoltTransformBuilder; import javax.net.ssl.SSLException; import java.io.IOException; @@ -150,7 +150,7 @@ public void testThatConnectionsAreKeptAliveAndShared(boolean useTls) var sslContext = !testServer.localhostEndpoint().getScheme().toLowerCase().equals("https") ? null : SslContextBuilder.forClient().trustManager(InsecureTrustManagerFactory.INSTANCE).build(); var transformingHttpHandlerFactory = new PacketToTransformingHttpHandlerFactory( - new JsonJoltTransformBuilder().build(), null); + new TransformationLoader().getTransformerFactoryLoader(null), null); var timeShifter = new TimeShifter(); timeShifter.setFirstTimestamp(Instant.now()); var sendingFactory = new ReplayEngine( diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java index 1735304b80..5afc70e277 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java @@ -5,11 +5,11 @@ import org.opensearch.migrations.replay.AggregatedRawResponse; import org.opensearch.migrations.replay.TestCapturePacketToHttpHandler; import org.opensearch.migrations.replay.TestRequestKey; +import org.opensearch.migrations.replay.TransformationLoader; import org.opensearch.migrations.replay.datatypes.HttpRequestTransformationStatus; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; -import org.opensearch.migrations.transform.JsonCompositeTransformer; -import org.opensearch.migrations.transform.JsonJoltTransformer; import org.opensearch.migrations.transform.IJsonTransformer; +import org.opensearch.migrations.transform.JsonCompositeTransformer; import org.opensearch.migrations.transform.RemovingAuthTransformerFactory; import java.nio.charset.StandardCharsets; @@ -25,7 +25,7 @@ public void testPassThroughSinglePacketPost() throws Exception { new AggregatedRawResponse(17, null, null, null); var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse); var transformingHandler = - new HttpJsonTransformingConsumer(JsonJoltTransformer.newBuilder().build(), + new HttpJsonTransformingConsumer(new TransformationLoader().getTransformerFactoryLoader(null, null), null, testPacketCapture, "TEST", TestRequestKey.getTestConnectionRequestId(0)); byte[] testBytes; @@ -46,9 +46,7 @@ public void testPassThroughSinglePacketWithoutBodyTransformationPost() throws Ex var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse); var transformingHandler = new HttpJsonTransformingConsumer( - JsonJoltTransformer.newBuilder() - .addHostSwitchOperation("test.domain") - .build(), + new TransformationLoader().getTransformerFactoryLoader("test.domain"), null, testPacketCapture, "TEST", TestRequestKey.getTestConnectionRequestId(0)); byte[] testBytes; @@ -73,9 +71,7 @@ public void testRemoveAuthHeadersWorks() throws Exception { var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse); var transformingHandler = new HttpJsonTransformingConsumer( - JsonJoltTransformer.newBuilder() - .addHostSwitchOperation("test.domain") - .build(), + new TransformationLoader().getTransformerFactoryLoader("test.domain"), RemovingAuthTransformerFactory.instance, testPacketCapture, "TEST", TestRequestKey.getTestConnectionRequestId(0)); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodySerializeHandlerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodySerializeHandlerTest.java index c835e03e4d..9d3be6d262 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodySerializeHandlerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodySerializeHandlerTest.java @@ -11,15 +11,14 @@ import org.opensearch.migrations.replay.ReplayUtils; import org.opensearch.migrations.replay.datahandlers.PayloadAccessFaultingMap; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; +import org.opensearch.migrations.transform.IHttpMessage; +import org.opensearch.migrations.transform.JsonKeysForHttpMessage; -import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Random; import java.util.stream.Stream; -import static org.junit.jupiter.api.Assertions.*; - @Slf4j @WrapWithNettyLeakDetection public class NettyJsonBodySerializeHandlerTest { @@ -31,7 +30,7 @@ public void testJsonSerializerHandler() throws Exception { headers.put(IHttpMessage.CONTENT_TYPE, List.of(IHttpMessage.APPLICATION_JSON)); var fullHttpMessageWithJsonBody = new HttpJsonMessageWithFaultingPayload(headers); fullHttpMessageWithJsonBody.setPayloadFaultMap(new PayloadAccessFaultingMap(headers)); - fullHttpMessageWithJsonBody.payload().put(PayloadAccessFaultingMap.INLINED_JSON_BODY_DOCUMENT_KEY, randomJson); + fullHttpMessageWithJsonBody.payload().put(JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY, randomJson); var channel = new EmbeddedChannel(new NettyJsonBodySerializeHandler()); channel.writeInbound(fullHttpMessageWithJsonBody); diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java index 116afb25f9..c41d1e06b3 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java @@ -116,6 +116,14 @@ private static String getStringFromContent(FullHttpRequest fullRequest) throws I return new String(baos.toByteArray(), StandardCharsets.UTF_8); } } + static void runPipelineAndValidate(IAuthTransformerFactory authTransformer, + String extraHeaders, + List stringParts, + DefaultHttpHeaders expectedRequestHeaders, + Function expectedOutputGenerator) throws Exception { + runPipelineAndValidate(x -> x, + authTransformer, extraHeaders, stringParts, expectedRequestHeaders, expectedOutputGenerator); + } static void runPipelineAndValidate(IJsonTransformer transformer, IAuthTransformerFactory authTransformer, From 08f5987984aed3df6f911d2eef2262319947c013 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Mon, 6 Nov 2023 11:18:53 -0500 Subject: [PATCH 09/55] Fix the the build by correcting some build.gradle files, rearranging resource directories to advertise implementations and to fix tests, and properly loading of IJsonTransfomerProviders from jarfiles. I've also * overhauled the README section of the TrafficReplayer on transformations * changed the configuration value from a string to an (json-friendly) object. * added a version key to the HTTP messages to future-proof replayer plugins at least a bit. Signed-off-by: Greg Schohn --- .../jsonJoltMessageTransformer/build.gradle | 14 ++++- .../transform/JsonJoltTransformer.java | 5 +- .../{ => jolt}/operations/addGzip.jolt | 0 .../operations/hostSwitch.jolt.template | 0 .../{ => jolt}/operations/makeChunked.jolt | 0 .../{ => jolt}/operations/passThru.jolt | 0 .../post_formUrlEncoded_withFixedLength.json | 0 .../build.gradle | 6 +- .../JsonJoltTransformerProvider.java | 3 +- ...rations.transform.IJsonTransformerProvider | 1 + .../transform/IJsonTransformer.java | 4 +- .../transform/IJsonTransformerProvider.java | 10 +++- .../transform/JsonCompositeTransformer.java | 4 +- .../transform/JsonKeysForHttpMessage.java | 9 ++- ...rations.transform.IJsonTransformerProvider | 1 - ...Search23PlusTargetTransformerProvider.java | 2 +- .../transform/JsonTypeMappingTransformer.java | 10 +--- ...rations.transform.IJsonTransformerProvider | 0 TrafficCapture/trafficReplayer/README.md | 57 +++++++++++++++++-- .../replay/TransformationLoader.java | 24 +++++--- .../HttpJsonMessageWithFaultingPayload.java | 4 ++ .../http/NettyJsonBodyConvertHandler.java | 2 +- .../replay/HeaderTransformerTest.java | 4 +- .../HttpJsonTransformingConsumerTest.java | 2 +- .../transform/TypeMappingsExcisionTest.java | 7 ++- 25 files changed, 125 insertions(+), 44 deletions(-) rename TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/{ => jolt}/operations/addGzip.jolt (100%) rename TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/{ => jolt}/operations/hostSwitch.jolt.template (100%) rename TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/{ => jolt}/operations/makeChunked.jolt (100%) rename TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/{ => jolt}/operations/passThru.jolt (100%) rename TrafficCapture/{trafficReplayer/src/test => replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main}/resources/requests/parsed/post_formUrlEncoded_withFixedLength.json (100%) create mode 100644 TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/resources/META-INFO/services/org.opensearch.migrations.transform.IJsonTransformerProvider delete mode 100644 TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider rename TrafficCapture/replayerPlugins/jsonMessageTransformers/{jsonJoltMessageTransformerProvider/src/main/java => openSearch23PlusTargetTransformerProvider/src/main/resources}/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider (100%) diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/build.gradle b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/build.gradle index 0ccc6dd66e..e90cb412dc 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/build.gradle +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/build.gradle @@ -1,3 +1,9 @@ +buildscript { + dependencies { + classpath 'org.junit.platform:junit-platform-gradle-plugin:1.0.1' + } +} + plugins { id("io.freefair.lombok") version "8.0.1" } @@ -10,4 +16,10 @@ dependencies { implementation group: 'org.slf4j', name:"slf4j-api", version:"2.0.7" testImplementation project(':trafficReplayer') -} \ No newline at end of file + testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-api', version:'5.9.3' + testRuntimeOnly group:'org.junit.jupiter', name:'junit-jupiter-engine', version:'5.x.x' +} + +tasks.named('test') { + useJUnitPlatform() +} diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformer.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformer.java index a645b48042..a76cd3ef3f 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformer.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformer.java @@ -3,6 +3,7 @@ import com.bazaarvoice.jolt.Chainr; import java.util.List; +import java.util.Map; public class JsonJoltTransformer implements IJsonTransformer { @@ -18,7 +19,7 @@ public static JsonJoltTransformBuilder newBuilder() { } @Override - public Object transformJson(Object incomingJson) { - return this.spec.transform(incomingJson); + public Map transformJson(Map incomingJson) { + return (Map) this.spec.transform(incomingJson); } } diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/operations/addGzip.jolt b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/jolt/operations/addGzip.jolt similarity index 100% rename from TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/operations/addGzip.jolt rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/jolt/operations/addGzip.jolt diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/operations/hostSwitch.jolt.template b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/jolt/operations/hostSwitch.jolt.template similarity index 100% rename from TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/operations/hostSwitch.jolt.template rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/jolt/operations/hostSwitch.jolt.template diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/operations/makeChunked.jolt b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/jolt/operations/makeChunked.jolt similarity index 100% rename from TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/operations/makeChunked.jolt rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/jolt/operations/makeChunked.jolt diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/operations/passThru.jolt b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/jolt/operations/passThru.jolt similarity index 100% rename from TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/operations/passThru.jolt rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/jolt/operations/passThru.jolt diff --git a/TrafficCapture/trafficReplayer/src/test/resources/requests/parsed/post_formUrlEncoded_withFixedLength.json b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/requests/parsed/post_formUrlEncoded_withFixedLength.json similarity index 100% rename from TrafficCapture/trafficReplayer/src/test/resources/requests/parsed/post_formUrlEncoded_withFixedLength.json rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/resources/requests/parsed/post_formUrlEncoded_withFixedLength.json diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/build.gradle b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/build.gradle index d218c9ca7b..73bc1b03d3 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/build.gradle +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/build.gradle @@ -6,11 +6,6 @@ buildscript { plugins { id "io.freefair.lombok" version "8.0.1" - id 'java-test-fixtures' -} - -repositories { - mavenCentral() } dependencies { @@ -27,6 +22,7 @@ dependencies { testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-api', version:'5.9.3' testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-params', version:'5.9.3' testImplementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7' + testRuntimeOnly group:'org.junit.jupiter', name:'junit-jupiter-engine', version:'5.x.x' } tasks.named('test') { diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java index 805f8c01e6..f843f94fdd 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java @@ -4,8 +4,9 @@ public class JsonJoltTransformerProvider implements IJsonTransformerProvider { @Override - public IJsonTransformer createTransformer(Optional config) { + public IJsonTransformer createTransformer(Object jsonConfig) { var builder = JsonJoltTransformer.newBuilder(); //builder.addOperationObject() + return builder.build(); } } diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/resources/META-INFO/services/org.opensearch.migrations.transform.IJsonTransformerProvider b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/resources/META-INFO/services/org.opensearch.migrations.transform.IJsonTransformerProvider new file mode 100644 index 0000000000..de15cf3863 --- /dev/null +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/resources/META-INFO/services/org.opensearch.migrations.transform.IJsonTransformerProvider @@ -0,0 +1 @@ +org.opensearch.migrations.transform.JsonJoltTransformerProvider \ No newline at end of file diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java index fa35a862ad..c3bc6b3432 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java @@ -1,9 +1,11 @@ package org.opensearch.migrations.transform; +import java.util.Map; + /** * This is a simple interface to convert a JSON object (String, Map, or Array) into another * JSON object. Any changes to datastructures, nesting, order, etc should be intentional. */ public interface IJsonTransformer { - Object transformJson(Object incomingJson); + Map transformJson(Map incomingJson); } diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java index b379acfa49..0dd9cfb9bc 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java @@ -3,5 +3,13 @@ import java.util.Optional; public interface IJsonTransformerProvider { - IJsonTransformer createTransformer(Optional config); + /** + * Create a new transformer from the given configuration. This transformer + * will be used repeatedly and concurrently from different threads to modify + * messages. + * @param jsonConfig is a List, Map, String, or null that should be used to configure the + * IJsonTransformer that is being created + * @return + */ + IJsonTransformer createTransformer(Object jsonConfig); } diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java index 43edaada8a..00044d5686 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java @@ -1,6 +1,7 @@ package org.opensearch.migrations.transform; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; public class JsonCompositeTransformer implements IJsonTransformer { @@ -10,7 +11,8 @@ public JsonCompositeTransformer(IJsonTransformer... jsonTransformers) { this.jsonTransformerList = List.of(jsonTransformers); } - public Object transformJson(Object incomingJson) { + @Override + public Map transformJson(Map incomingJson) { var lastOutput = new AtomicReference<>(incomingJson); jsonTransformerList.forEach(t->lastOutput.set(t.transformJson(lastOutput.get()))); return lastOutput.get(); diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java index 3c7115722b..054e4875a5 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java @@ -1,11 +1,16 @@ package org.opensearch.migrations.transform; public class JsonKeysForHttpMessage { - public static final String INLINED_JSON_BODY_DOCUMENT_KEY = "inlinedJsonBody"; - + public static final String HTTP_MESSAGE_SCHEMA_VERSION_KEY = "transformerMessageVersion"; public static final String METHOD_KEY = "method"; public static final String URI_KEY = "URI"; public static final String PROTOCOL_KEY = "protocol"; public static final String HEADERS_KEY = "headers"; public static final String PAYLOAD_KEY = "payload"; + /** + * This is the key under the 'payload' object whose value is the parsed json from the HTTP message payload. + * Notice that there aren't yet other ways to access the payload contents. If the content-type was not json, + * the payload object will be an empty map. + */ + public static final String INLINED_JSON_BODY_DOCUMENT_KEY = "inlinedJsonBody"; } diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider deleted file mode 100644 index 34d314de61..0000000000 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider +++ /dev/null @@ -1 +0,0 @@ -org.opensearch.migrations.transform.JsonTransformerForOpenSearch23PlusTargetTransformerProvider \ No newline at end of file diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java index 523ffee93d..aed93bff4f 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java @@ -4,7 +4,7 @@ public class JsonTransformerForOpenSearch23PlusTargetTransformerProvider implements IJsonTransformerProvider { @Override - public IJsonTransformer createTransformer(Optional args) { + public IJsonTransformer createTransformer(Object jsonConfig) { return new JsonTypeMappingTransformer(); } } diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java index 3c83776460..1ed89b481e 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java @@ -26,15 +26,11 @@ public class JsonTypeMappingTransformer implements IJsonTransformer { public static final String MAPPINGS_KEYNAME = "mappings"; @Override - public Object transformJson(Object incomingJson) { - if (!(incomingJson instanceof Map)) { - return incomingJson; - } else { - return transformHttpMessage((Map) incomingJson); - } + public Map transformJson(Map incomingJson) { + return transformHttpMessage(incomingJson); } - private Object transformHttpMessage(Map httpMsg) { + private Map transformHttpMessage(Map httpMsg) { var incomingMethod = httpMsg.get(JsonKeysForHttpMessage.METHOD_KEY); if ("GET".equals(incomingMethod)) { processGet(httpMsg); diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider similarity index 100% rename from TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider diff --git a/TrafficCapture/trafficReplayer/README.md b/TrafficCapture/trafficReplayer/README.md index a6222bb91e..dd8984d038 100644 --- a/TrafficCapture/trafficReplayer/README.md +++ b/TrafficCapture/trafficReplayer/README.md @@ -76,12 +76,57 @@ target URI. ## Transformations -Transformations are performed via a simple class defined by -[IJsonTransformer](../trafficReplayer/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java). Currently, -this class uses [JOLT](https://github.com/bazaarvoice/jolt) to perform transforms that are composed of modular -operations that are defined in the [resources](../trafficReplayer/src/main/resources/jolt/operations) associated with -the package. Future work will include adding more JSON transformations and other potential JSON transformation tools -(like [JMESPath](https://jmespath.org/)). +Transformations are performed via a simple interface defined by +[IJsonTransformer](../TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java) ('transformer'). They are loaded dynamically and are designed to allow for easy extension +of the TrafficReplayer to support a diverse set of needs. + +The input to the transformer will be an HTTP message represented as a json-like Map with +top-level key-value pairs defined in +[JsonKeysForHttpMessage.java](../replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java) +Only bodies that are json-formatted will be accessible, and they will be accessible as a fully-parsed Map (at +the keypath `'payload'->'inlinedJsonBody'`). Transformers have the option to rewrite none, or any of the keys and +values within the original message. The transformer can return either the original message or a completely new message. +Transformers may be used simultaneously from concurrent threads over the lifetime of the replayer. However, +a message will only be processed by one transformer at a time. + +Transformer implementations are loaded via [Java's ServiceLoader](https://docs.oracle.com/javase/8/docs/api/java/util/ServiceLoader.html) +by loading a jarfile that implements the [IJsonTransformerProvider] +(../TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java). +That jarfile will be loaded by specifying the provider jarfile (and any of its dependencies) in the classpath. +For the ServiceLoader to load the IJsonTransformerProvider, the provided jarfile needs +to supply a _provider-configuration_ file (`META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider`) +with the fully qualified class name of the IJsonTransformerProvider implementation that should be loaded. The contents +of that file must be in plain text with a single class on a line. + +If multiple IJsonTransformerProviders are loaded, or if the provider/transformer require additional configuration +parameters, the user must specify additional configuration to the TrafficReplayer via the "--transformer-config" +argument. This argument must be a json formatted list of maps. The order of the list will define the order that +the transformations are run. Within each map is a single key-value pair whose name is the fully-qualified classname +of the IJsonTransformerProvider. The value of that key is then passed to the provider to fully instantiate an +IJsonTransformer object. If only one IJsonTransformerProvider was loaded, configuration may be excluded. Note that +when the transformer-config is specified, it will control which of the loaded IJsonTransformerProviders will be +created, how they'll be created, and what order they will run in. + +The base [jsonJoltMessageTransformerProvider](../replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider) +package includes [JsonCompositeTransformer.java] +(../replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java), +which run transformers in serial. That composite transformer is also utilized by the TrafficReplayer to combine the +list of loaded transformations with a transformer to rewrite the 'Host' header. That host transformation changes the +host header of every HTTP message to use the target domain-name rather than the source's. That will be run after +all loaded/specified transformations. + +Currently, there are multiple, nascent implementations included in the repository. The +[jsonJoltMessageTransformerProvider](../replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider) +package uses [JOLT](https://github.com/bazaarvoice/jolt) to perform transforms. Some simple transformations are +included to change headers to add compression or to force an HTTP message payload to be chunked. Another transformer, +[JsonTypeMappingTransformer.java](../replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java), +is a work-in-progress to excise type mapping references from URIs and message payloads since versions of OpenSource +greater than 2.3 do not support them. + +When these transformations are enabled, they will be run on each message. If a transformation should only be run +for certain request types, that is the responsibility of the transformer implementation. While a transformer doesn't +have the ability to return control information to a next transformer, it does send back and entire HTTP message that +can be augmented however it may choose. ## Authorization Header for Replayed Requests diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java index 0d359ce1e0..f3fbeb6f2a 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.transform.IJsonTransformer; import org.opensearch.migrations.transform.IJsonTransformerProvider; import org.opensearch.migrations.transform.JsonCompositeTransformer; @@ -18,6 +19,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +@Slf4j public class TransformationLoader { private final List providers; ObjectMapper objMapper = new ObjectMapper(); @@ -27,14 +29,16 @@ public TransformationLoader() { ServiceLoader.load(IJsonTransformerProvider.class); var inProgressProviders = new ArrayList(); for (var provider : transformerProviders) { + log.info("Adding IJsonTransfomerProvider: " + provider); inProgressProviders.add(provider); } providers = Collections.unmodifiableList(inProgressProviders); + log.atInfo().setMessage(()->"IJsonTransformerProviders loaded: " + + providers.stream().map(p->p.getClass().toString()).collect(Collectors.joining())).log(); } List> parseFullConfig(String fullConfig) throws JsonProcessingException { - return objMapper.readValue(fullConfig, new TypeReference<>() { - }); + return objMapper.readValue(fullConfig, new TypeReference<>() {}); } protected Stream getTransformerFactoryFromServiceLoader(String fullConfig) @@ -58,17 +62,19 @@ protected Stream getTransformerFactoryFromServiceLoader(String private IJsonTransformer configureTransformerFromConfig(Map c) { var keys = c.keySet(); if (keys.size() != 1) { - throw new IllegalArgumentException("Must specify the configuration list with a sequence of maps " + - "with only one key each, where the key is the name of the transformer."); + throw new IllegalArgumentException("Must specify the top-level configuration list with a sequence of " + + "maps that have only one key each, where the key is the name of the transformer to be configured."); } var key = keys.stream().findFirst().get(); - var transformerConfigStr = objMapper.writeValueAsString(c.get(key)); for (var p : providers) { - if (p.getClass().getSimpleName().equals(key)) { - return p.createTransformer(Optional.of(transformerConfigStr)); + var className = p.getClass().getName(); + if (className.equals(key)) { + var configuration = c.get(key); + log.atInfo().setMessage(()->"Creating a transformer with configuration="+configuration).log(); + return p.createTransformer(Optional.of(configuration)); } } - return null; + throw new IllegalArgumentException("Could not find a provider named: " + key); } public IJsonTransformer getTransformerFactoryLoader(String newHostName) { return getTransformerFactoryLoader(newHostName, null); @@ -90,7 +96,7 @@ private class HostTransformer implements IJsonTransformer { private final String newHostName; @Override - public Object transformJson(Object incomingJson) { + public Map transformJson(Map incomingJson) { var asMap = (Map) incomingJson; var headers = (Map) asMap.get(JsonKeysForHttpMessage.HEADERS_KEY); headers.replace("host", newHostName); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonMessageWithFaultingPayload.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonMessageWithFaultingPayload.java index eeba756113..35ea9652c1 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonMessageWithFaultingPayload.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonMessageWithFaultingPayload.java @@ -10,11 +10,15 @@ public class HttpJsonMessageWithFaultingPayload extends LinkedHashMap implements IHttpMessage { + public static final String MESSAGE_SCHEMA_VERSION = "1.0"; + public HttpJsonMessageWithFaultingPayload() { } public HttpJsonMessageWithFaultingPayload(Map m) { + super(m); + put(JsonKeysForHttpMessage.HTTP_MESSAGE_SCHEMA_VERSION_KEY, MESSAGE_SCHEMA_VERSION); } @Override diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyConvertHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyConvertHandler.java index d022185e8c..fb3ddd58c6 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyConvertHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyConvertHandler.java @@ -16,7 +16,7 @@ public NettyJsonBodyConvertHandler(IJsonTransformer transformer) { @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { if (msg instanceof HttpJsonMessageWithFaultingPayload) { - var output = transformer.transformJson(msg); + var output = transformer.transformJson((HttpJsonMessageWithFaultingPayload)msg); var newHttpJson = new HttpJsonMessageWithFaultingPayload(((Map)output)); ctx.fireChannelRead(newHttpJson); } else { diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java index 98d5d93a1c..b0e28042d2 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java @@ -109,8 +109,10 @@ public void testMalformedPayload_andTypeMappingUri_IsPassedThrough() throws Exce final var dummyAggregatedResponse = new TransformedTargetRequestAndResponse(null, 12, null, null, HttpRequestTransformationStatus.COMPLETED, null); var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse); + var transformingHandler = new HttpJsonTransformingConsumer( - new TransformationLoader().getTransformerFactoryLoader(SILLY_TARGET_CLUSTER_NAME), + new TransformationLoader().getTransformerFactoryLoader(SILLY_TARGET_CLUSTER_NAME, + "[{\"org.opensearch.migrations.transform.JsonTransformerForOpenSearch23PlusTargetTransformerProvider\":\"\"}]"), null, testPacketCapture, "TEST", TestRequestKey.getTestConnectionRequestId(0)); Random r = new Random(2); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java index 5afc70e277..fafed72456 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java @@ -98,7 +98,7 @@ public void testPartialBodyThrowsAndIsRedriven() throws Exception { var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse); var complexTransformer = new JsonCompositeTransformer(new IJsonTransformer() { @Override - public Object transformJson(Object incomingJson) { + public Map transformJson(Map incomingJson) { // just walk everything - that's enough to touch the payload and throw walkMaps(incomingJson); return incomingJson; diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/transform/TypeMappingsExcisionTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/transform/TypeMappingsExcisionTest.java index bbd3e5171c..eac2347128 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/transform/TypeMappingsExcisionTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/transform/TypeMappingsExcisionTest.java @@ -13,6 +13,7 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.LinkedHashMap; +import java.util.Map; @WrapWithNettyLeakDetection(disableLeakChecks = true) public class TypeMappingsExcisionTest { @@ -45,16 +46,16 @@ public void removesTypeMappingsFrom_queryGet() throws Exception { transformAndVerifyResult(json, "get_query_output.txt"); } - private static Object parseJsonFromResourceName(String resourceName) throws Exception { + private static Map parseJsonFromResourceName(String resourceName) throws Exception { var jsonAccumulator = new JsonAccumulator(); try (var resourceStream = getInputStreamForTypeMappingResource(resourceName); var isr = new InputStreamReader(resourceStream, StandardCharsets.UTF_8)) { var expectedBytes = CharStreams.toString(isr).getBytes(StandardCharsets.UTF_8); - return jsonAccumulator.consumeByteBuffer(ByteBuffer.wrap(expectedBytes)); + return (Map) jsonAccumulator.consumeByteBuffer(ByteBuffer.wrap(expectedBytes)); } } - private static void transformAndVerifyResult(Object json, String expectedValueSource) throws Exception { + private static void transformAndVerifyResult(Map json, String expectedValueSource) throws Exception { var jsonTransformer = getJsonTransformer(); json = jsonTransformer.transformJson(json); var jsonAsStr = objectMapper.writeValueAsString(json); From b2184595a94dc9a5d4cd742f232eb0e707f260a4 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Mon, 6 Nov 2023 14:18:50 -0500 Subject: [PATCH 10/55] Setup Jolt transformations to be loaded via configuration. Add some tests that should serve as examples for how to configure complex Jolt scripts as well as complex non-Jolt scripts. Some bugs were fixed along the way. Signed-off-by: Greg Schohn --- .../JsonJoltTransformerProvider.java | 60 ++++++++++++++++++- ...rations.transform.IJsonTransformerProvider | 0 .../replay/MultipleJoltScriptsTest.java | 58 ++++++++++++++++++ .../replay/TransformationLoader.java | 4 +- .../replay/TransformationLoaderTest.java | 31 ++++++++++ .../migrations/replay/SampleContents.java | 15 +++++ .../resources/sampleRequestMessage.json | 47 +++++++++++++++ 7 files changed, 211 insertions(+), 4 deletions(-) rename TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/resources/{META-INFO => META-INF}/services/org.opensearch.migrations.transform.IJsonTransformerProvider (100%) create mode 100644 TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/MultipleJoltScriptsTest.java create mode 100644 TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TransformationLoaderTest.java create mode 100644 TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/SampleContents.java create mode 100644 TrafficCapture/trafficReplayer/src/testFixtures/resources/sampleRequestMessage.json diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java index f843f94fdd..e8d93384a6 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java @@ -1,12 +1,68 @@ package org.opensearch.migrations.transform; -import java.util.Optional; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; public class JsonJoltTransformerProvider implements IJsonTransformerProvider { + + public static final String SCRIPT_KEY = "script"; + public static final String CANNED_KEY = "canned"; + @Override public IJsonTransformer createTransformer(Object jsonConfig) { var builder = JsonJoltTransformer.newBuilder(); - //builder.addOperationObject() + var configs = new ArrayList>(); + try { + if (jsonConfig instanceof Map) { + configs.add((Map) jsonConfig); + } else if (jsonConfig instanceof List) { + for (var c : (List) jsonConfig) { + configs.add((Map) c); + } + } else { + throw new IllegalArgumentException(getConfigUsageStr()); + } + for (var c : configs) { + if (c.size() != 1) { + throw new IllegalArgumentException(getConfigUsageStr()); + } + var cannedValue = c.get(CANNED_KEY); + var scriptValue = c.get(SCRIPT_KEY); + if (cannedValue != null) { + var cannedValueStr = (String) cannedValue; + var cannedOperation = getCannedOperationOrThrow(cannedValueStr); + builder.addCannedOperation(cannedOperation); + } else { + builder.addOperationObject((Map)scriptValue); + } + } + } catch (ClassCastException e) { + throw new IllegalArgumentException(getConfigUsageStr(), e); + } return builder.build(); } + + private JsonJoltTransformBuilder.CANNED_OPERATION getCannedOperationOrThrow(String cannedValueStr) { + try { + return JsonJoltTransformBuilder.CANNED_OPERATION.valueOf(cannedValueStr); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(getConfigUsageStr(), e); + } + } + + private String getConfigUsageStr() { + return this.getClass().getName()+" expects the incoming configuration " + + "to be a Map or a List>. " + + "Each of the Maps should have one key-value, either \"canned\" or \"script\". " + + "Canned values should be a string that specifies the name of the pre-built transformation to use " + + Arrays.stream(JsonJoltTransformBuilder.CANNED_OPERATION.values()) + .map(e->e.toString()).collect(Collectors.joining(",")) + + ". " + + "Script values should be a fully-formed inlined Jolt transformation in json form. " + + "All of the values (canned or inlined) within a configuration will be concatenated " + + "into one chained Jolt transformation"; + } } diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/resources/META-INFO/services/org.opensearch.migrations.transform.IJsonTransformerProvider b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider similarity index 100% rename from TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/resources/META-INFO/services/org.opensearch.migrations.transform.IJsonTransformerProvider rename to TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/MultipleJoltScriptsTest.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/MultipleJoltScriptsTest.java new file mode 100644 index 0000000000..b83c6f8923 --- /dev/null +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/MultipleJoltScriptsTest.java @@ -0,0 +1,58 @@ +package org.opensearch.migrations.replay; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.opensearch.migrations.transform.JsonJoltTransformBuilder; +import org.opensearch.migrations.transform.JsonKeysForHttpMessage; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +public class MultipleJoltScriptsTest { + + private static final ObjectMapper mapper = new ObjectMapper(); + + private static Map parseAsMap(String contents) throws Exception { + return mapper.readValue(contents.getBytes(), new TypeReference<>() {}); + } + + @Test + public void testAddGzip() throws Exception { + final var addGzip = + "[{\"org.opensearch.migrations.transform.JsonJoltTransformerProvider\": { \"canned\": \"ADD_GZIP\" }}]"; + var toNewHostTransformer = new TransformationLoader().getTransformerFactoryLoader("testhostname", + addGzip); + var origDocStr = SampleContents.loadSampleJsonRequestAsString(); + var origDoc = parseAsMap(origDocStr); + var newDoc = toNewHostTransformer.transformJson(origDoc); + Assertions.assertEquals("testhostname", ((Map) newDoc.get(JsonKeysForHttpMessage.HEADERS_KEY)) + .get("host")); + Assertions.assertEquals("gzip", ((Map) newDoc.get(JsonKeysForHttpMessage.HEADERS_KEY)) + .get("content-encoding")); + } + + @Test + public void testAddGzipAndCustom() throws Exception { + final var addGzip = "[" + + "{\"org.opensearch.migrations.transform.JsonJoltTransformerProvider\": { \"canned\": \"ADD_GZIP\" }}," + + "{ \"org.opensearch.migrations.transform.JsonJoltTransformerProvider\":" + + " {\"script\": \n" + + " { \"operation\": \"modify-overwrite-beta\", \"spec\": " + + " { \"headers\": {\"newHeader\": \"newValue\"}}}}}" + + "]"; + var toNewHostTransformer = new TransformationLoader().getTransformerFactoryLoader("testhostname", + addGzip); + var origDocStr = SampleContents.loadSampleJsonRequestAsString(); + var origDoc = parseAsMap(origDocStr); + var newDoc = toNewHostTransformer.transformJson(origDoc); + Assertions.assertEquals("testhostname", ((Map) newDoc.get(JsonKeysForHttpMessage.HEADERS_KEY)) + .get("host")); + var headers = (Map) newDoc.get(JsonKeysForHttpMessage.HEADERS_KEY); + Assertions.assertEquals("gzip", headers.get("content-encoding")); + Assertions.assertEquals("newValue", headers.get("newHeader")); + } + +} diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java index f3fbeb6f2a..6b0ffc1653 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java @@ -34,7 +34,7 @@ public TransformationLoader() { } providers = Collections.unmodifiableList(inProgressProviders); log.atInfo().setMessage(()->"IJsonTransformerProviders loaded: " + - providers.stream().map(p->p.getClass().toString()).collect(Collectors.joining())).log(); + providers.stream().map(p->p.getClass().toString()).collect(Collectors.joining("; "))).log(); } List> parseFullConfig(String fullConfig) throws JsonProcessingException { @@ -71,7 +71,7 @@ private IJsonTransformer configureTransformerFromConfig(Map c) { if (className.equals(key)) { var configuration = c.get(key); log.atInfo().setMessage(()->"Creating a transformer with configuration="+configuration).log(); - return p.createTransformer(Optional.of(configuration)); + return p.createTransformer(configuration); } } throw new IllegalArgumentException("Could not find a provider named: " + key); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TransformationLoaderTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TransformationLoaderTest.java new file mode 100644 index 0000000000..5f1e36c453 --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TransformationLoaderTest.java @@ -0,0 +1,31 @@ +package org.opensearch.migrations.replay; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +public class TransformationLoaderTest { + private static final ObjectMapper mapper = new ObjectMapper(); + + private static Map parseAsMap(String contents) throws Exception { + return mapper.readValue(contents.getBytes(), new TypeReference<>() {}); + } + + @Test + public void testTransformationLoader() throws Exception { + var toNewHostTransformer = new TransformationLoader().getTransformerFactoryLoader("testhostname"); + var toOldHostTransformer = new TransformationLoader().getTransformerFactoryLoader("localhost"); + var origDoc = parseAsMap(SampleContents.loadSampleJsonRequestAsString()); + var origDocStr = mapper.writeValueAsString(origDoc); + var outputWithNewHostname = toNewHostTransformer.transformJson(origDoc); + var docWithNewHostnameStr = mapper.writeValueAsString(outputWithNewHostname); + var outputWithOldHostname = toOldHostTransformer.transformJson(outputWithNewHostname); + var docWithOldHostnameStr = mapper.writeValueAsString(outputWithOldHostname); + Assertions.assertEquals(origDocStr, docWithOldHostnameStr); + Assertions.assertNotEquals(origDocStr, docWithNewHostnameStr); + } + +} diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/SampleContents.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/SampleContents.java new file mode 100644 index 0000000000..87e53caef1 --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/SampleContents.java @@ -0,0 +1,15 @@ +package org.opensearch.migrations.replay; + +import java.nio.charset.StandardCharsets; + +public class SampleContents { + + public static final String SAMPLE_REQUEST_MESSAGE_JSON_RESOURCE_NAME = "/sampleRequestMessage.json"; + + public static String loadSampleJsonRequestAsString() throws Exception { + try (var inputStream = + SampleContents.class.getResourceAsStream(SAMPLE_REQUEST_MESSAGE_JSON_RESOURCE_NAME)) { + return new String(inputStream.readAllBytes(), StandardCharsets.UTF_8); + } + } +} diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/resources/sampleRequestMessage.json b/TrafficCapture/trafficReplayer/src/testFixtures/resources/sampleRequestMessage.json new file mode 100644 index 0000000000..89a8a2e032 --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/testFixtures/resources/sampleRequestMessage.json @@ -0,0 +1,47 @@ +{ + "transformerMessageVersion": "1.0", + "method": "GET", + "protocol": "HTTP/1.1", + "URI": "/testindex/_search", + "headers": { + "host": "localhost" + }, + "payload": { + "inlinedJsonBody" : { + "query": { + "intervals": { + "title": { + "all_of": { + "ordered": true, + "intervals": [ + { + "match": { + "query": "key-value pairs", + "max_gaps": 0, + "ordered": true + } + }, + { + "any_of": { + "intervals": [ + { + "match": { + "query": "hash table" + } + }, + { + "match": { + "query": "hash map" + } + } + ] + } + } + ] + } + } + } + } + } + } +} \ No newline at end of file From c45a4188a1d157140b48eabe2d576b46ffc171bf Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Mon, 6 Nov 2023 22:17:32 -0500 Subject: [PATCH 11/55] fix link to a source file in the README Signed-off-by: Greg Schohn --- TrafficCapture/trafficReplayer/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TrafficCapture/trafficReplayer/README.md b/TrafficCapture/trafficReplayer/README.md index dd8984d038..8921dfed03 100644 --- a/TrafficCapture/trafficReplayer/README.md +++ b/TrafficCapture/trafficReplayer/README.md @@ -77,7 +77,7 @@ target URI. ## Transformations Transformations are performed via a simple interface defined by -[IJsonTransformer](../TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java) ('transformer'). They are loaded dynamically and are designed to allow for easy extension +[IJsonTransformer](../replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java) ('transformer'). They are loaded dynamically and are designed to allow for easy extension of the TrafficReplayer to support a diverse set of needs. The input to the transformer will be an HTTP message represented as a json-like Map with @@ -91,7 +91,7 @@ a message will only be processed by one transformer at a time. Transformer implementations are loaded via [Java's ServiceLoader](https://docs.oracle.com/javase/8/docs/api/java/util/ServiceLoader.html) by loading a jarfile that implements the [IJsonTransformerProvider] -(../TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java). +(../replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java). That jarfile will be loaded by specifying the provider jarfile (and any of its dependencies) in the classpath. For the ServiceLoader to load the IJsonTransformerProvider, the provided jarfile needs to supply a _provider-configuration_ file (`META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider`) From 11aef4d97faef86d3f3a6f081c49fda37ffb3066 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Tue, 7 Nov 2023 13:23:02 -0500 Subject: [PATCH 12/55] Cleaning up some code and fixed some NPEs in outputting tuples. I'm still preparing to commit messages. There's some minor refactoring to make that easier. The FullTrafficReplayerTest continues to undergo changes. In making those changes, I also realized that during shutdown, some rare exceptions fire & tickle code that's otherwise tough to reach. One spot caught an exception before almost anything other than a UniqueResponseRequestKey was parsed. That left most of the tuple fields as null, except for the error. Unfortunately, we weren't outputting the error at all and would throw an exception on the null fields. Now we output the error and don't emit each specific field when the data to serialize was null. Unit tests are included. The contract stipulates that ONLY the request key must not be null. Signed-off-by: Greg Schohn --- .../replay/AccumulationCallbacks.java | 3 +- .../replay/AggregatedRawResponse.java | 4 +- .../replay/SourceTargetCaptureTuple.java | 68 +++++---- .../migrations/replay/TrafficReplayer.java | 138 ++++++++++-------- .../datatypes/PojoTrafficStreamKey.java | 4 + .../replay/datatypes/TransformedPackets.java | 1 + .../datatypes/UniqueSourceRequestKey.java | 7 + .../replay/FullTrafficReplayerTest.java | 108 +++++++++----- .../replay/SourceTargetCaptureTupleTest.java | 59 ++++++++ .../replay/TrafficStreamGenerator.java | 6 +- 10 files changed, 262 insertions(+), 136 deletions(-) create mode 100644 TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SourceTargetCaptureTupleTest.java diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java index b729f42626..8afff5760b 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java @@ -9,6 +9,7 @@ public interface AccumulationCallbacks { void onRequestReceived(UniqueReplayerRequestKey key, HttpMessageAndTimestamp request); void onFullDataReceived(UniqueReplayerRequestKey key, RequestResponsePacketPair rrpp); - void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, List trafficStreamKeysBeingHeld); + void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, + List trafficStreamKeysBeingHeld); void onConnectionClose(UniqueReplayerRequestKey key, Instant when); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AggregatedRawResponse.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AggregatedRawResponse.java index 3a5af05eec..92ad6eaf19 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AggregatedRawResponse.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AggregatedRawResponse.java @@ -83,8 +83,8 @@ public String toString() { final StringBuilder sb = new StringBuilder("IResponseSummary{"); sb.append("responseSizeInBytes=").append(responseSizeInBytes); sb.append(", responseDuration=").append(responseDuration); - sb.append(", # of responsePackets=").append(""+ - (this.responsePackets==null ? "-1" : "" + this.responsePackets.size())); + sb.append(", # of responsePackets=").append("" + + (this.responsePackets == null ? "-1" : "" + this.responsePackets.size())); addSubclassInfoForToString(sb); sb.append('}'); return sb.toString(); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java index 3625617997..d1282c3729 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java @@ -19,7 +19,9 @@ import java.util.Base64; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Scanner; +import java.util.StringJoiner; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -33,7 +35,7 @@ public class SourceTargetCaptureTuple implements AutoCloseable { final Throwable errorCause; Duration targetResponseDuration; - public SourceTargetCaptureTuple(UniqueSourceRequestKey uniqueRequestKey, + public SourceTargetCaptureTuple(@NonNull UniqueSourceRequestKey uniqueRequestKey, RequestResponsePacketPair sourcePair, TransformedPackets targetRequestData, List targetResponseData, @@ -51,14 +53,14 @@ public SourceTargetCaptureTuple(UniqueSourceRequestKey uniqueRequestKey, @Override public void close() { - targetRequestData.close(); + Optional.ofNullable(targetRequestData).ifPresent(d->d.close()); } - public static class TupleToFileWriter implements Consumer { + public static class TupleToStreamConsumer implements Consumer { OutputStream outputStream; Logger tupleLogger = LogManager.getLogger("OutputTupleJsonLogger"); - public TupleToFileWriter(OutputStream outputStream){ + public TupleToStreamConsumer(OutputStream outputStream){ this.outputStream = outputStream; } @@ -100,22 +102,28 @@ private JSONObject jsonFromHttpData(@NonNull List data, Duration latency return message; } - private JSONObject toJSONObject(SourceTargetCaptureTuple triple) { + private JSONObject toJSONObject(SourceTargetCaptureTuple tuple) { // TODO: Use Netty to parse the packets as HTTP rather than json.org (we can also remove it as a dependency) JSONObject meta = new JSONObject(); - meta.put("sourceRequest", jsonFromHttpData(triple.sourcePair.requestData.packetBytes)); - meta.put("targetRequest", jsonFromHttpData(triple.targetRequestData.asByteArrayStream() - .collect(Collectors.toList()))); - //log.warn("TODO: These durations are not measuring the same values!"); - if (triple.sourcePair.responseData != null) { - meta.put("sourceResponse", jsonFromHttpData(triple.sourcePair.responseData.packetBytes, - Duration.between(triple.sourcePair.requestData.getLastPacketTimestamp(), - triple.sourcePair.responseData.getLastPacketTimestamp()))); - } - if (triple.targetResponseData != null && !triple.targetResponseData.isEmpty()) { - meta.put("targetResponse", jsonFromHttpData(triple.targetResponseData, triple.targetResponseDuration)); - } - meta.put("connectionId", triple.uniqueRequestKey); + Optional.ofNullable(tuple.sourcePair).ifPresent(p-> { + Optional.ofNullable(p.requestData).flatMap(d -> Optional.ofNullable(d.packetBytes)) + .ifPresent(d -> meta.put("sourceRequest", jsonFromHttpData(d))); + Optional.ofNullable(p.responseData).flatMap(d -> Optional.ofNullable(d.packetBytes)) + .ifPresent(d -> meta.put("sourceResponse", jsonFromHttpData(d, + //log.warn("TODO: These durations are not measuring the same values!"); + Duration.between(tuple.sourcePair.requestData.getLastPacketTimestamp(), + tuple.sourcePair.responseData.getLastPacketTimestamp())))); + }); + + Optional.ofNullable(tuple.targetRequestData) + .map(d->d.asByteArrayStream()) + .ifPresent(d->meta.put("targetRequest", jsonFromHttpData(d.collect(Collectors.toList())))); + + Optional.ofNullable(tuple.targetResponseData) + .filter(r->!r.isEmpty()) + .ifPresent(d-> meta.put("targetResponse", jsonFromHttpData(d, tuple.targetResponseDuration))); + meta.put("connectionId", tuple.uniqueRequestKey); + Optional.ofNullable(tuple.errorCause).ifPresent(e->meta.put("error", e)); return meta; } @@ -177,18 +185,18 @@ public void accept(SourceTargetCaptureTuple triple) { @Override public String toString() { return PrettyPrinter.setPrintStyleFor(PrettyPrinter.PacketPrintFormat.TRUNCATED, () -> { - final StringBuilder sb = new StringBuilder("SourceTargetCaptureTuple{"); - sb.append("\n diagnosticLabel=").append(uniqueRequestKey); - sb.append("\n sourcePair=").append(sourcePair); - sb.append("\n targetResponseDuration=").append(targetResponseDuration); - sb.append("\n targetRequestData=") - .append(PrettyPrinter.httpPacketBufsToString(PrettyPrinter.HttpMessageType.REQUEST, targetRequestData.streamUnretained())); - sb.append("\n targetResponseData=") - .append(PrettyPrinter.httpPacketBytesToString(PrettyPrinter.HttpMessageType.RESPONSE, targetResponseData)); - sb.append("\n transformStatus=").append(transformationStatus); - sb.append("\n errorCause=").append(errorCause == null ? "null" : errorCause.toString()); - sb.append('}'); - return sb.toString(); + final StringJoiner sj = new StringJoiner("\n ", "SourceTargetCaptureTuple{","}"); + sj.add("diagnosticLabel=").add(uniqueRequestKey.toString()); + if (sourcePair != null) { sj.add("sourcePair=").add(sourcePair.toString()); } + if (targetResponseDuration != null) { sj.add("targetResponseDuration=").add(targetResponseDuration+""); } + Optional.ofNullable(targetRequestData).filter(d->!d.isEmpty()).ifPresent(d-> sj.add("targetRequestData=") + .add(PrettyPrinter.httpPacketBufsToString(PrettyPrinter.HttpMessageType.REQUEST, + d.streamUnretained()))); + Optional.ofNullable(targetResponseData).filter(d->!d.isEmpty()).ifPresent(d -> sj.add("targetResponseData=") + .add(PrettyPrinter.httpPacketBytesToString(PrettyPrinter.HttpMessageType.RESPONSE, d))); + sj.add("transformStatus=").add(transformationStatus+""); + sj.add("errorCause=").add(errorCause == null ? "none" : errorCause.toString()); + return sj.toString(); }); } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index cd9cb6565a..cb944edb37 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -52,7 +52,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; @@ -83,7 +82,7 @@ public class TrafficReplayer { private ConcurrentHashMap> requestFutureMap; private ConcurrentHashMap> requestToFinalWorkFuturesMap; + DiagnosticTrackableCompletableFuture> requestToFinalWorkFuturesMap; private AtomicBoolean stopReadingRef; private AtomicReference> allRemainingWorkFutureOrShutdownSignalRef; @@ -92,18 +91,27 @@ public class TrafficReplayer { private AtomicReference>> nextChunkFutureRef; private ConcurrentHashMap liveRequests = new ConcurrentHashMap<>(); - public class TerminationException extends Exception { - public final Throwable shutdownCause; + public class DualException extends Exception { + public final Throwable originalCause; public final Throwable immediateCause; - public TerminationException(Throwable shutdownCause, Throwable immediateCause) { - // use one of these two so that anybody handling this as any other exception can get - // at least one of the root errors - super(Optional.ofNullable(shutdownCause).orElse(immediateCause)); - this.shutdownCause = shutdownCause; + public DualException(Throwable originalCause, Throwable immediateCause) { + this(null, originalCause, immediateCause); + } + // use one of these two so that anybody handling this as any other exception can get + // at least one of the root errors + public DualException(String message, Throwable originalCause, Throwable immediateCause) { + super(message, Optional.ofNullable(originalCause).orElse(immediateCause)); + this.originalCause = originalCause; this.immediateCause = immediateCause; } } + public class TerminationException extends DualException { + public TerminationException(Throwable originalCause, Throwable immediateCause) { + super(originalCause, immediateCause); + } + } + public static IJsonTransformer buildDefaultJsonTransformer(String newHostName) { var joltJsonTransformerBuilder = JsonJoltTransformer.newBuilder() .addHostSwitchOperation(newHostName); @@ -329,17 +337,17 @@ public static void main(String[] args) try (OutputStream outputStream = params.outputFilename == null ? System.out : new FileOutputStream(params.outputFilename, true); var bufferedOutputStream = new BufferedOutputStream(outputStream); - var blockingTrafficStream = TrafficCaptureSourceFactory.createTrafficCaptureSource(params, + var blockingTrafficSource = TrafficCaptureSourceFactory.createTrafficCaptureSource(params, Duration.ofSeconds(params.lookaheadTimeSeconds)); var authTransformer = buildAuthTransformerFactory(params)) { var tr = new TrafficReplayer(uri, authTransformer, params.allowInsecureConnections, params.numClientThreads, params.maxConcurrentRequests); setupShutdownHookForReplayer(tr); - var tupleWriter = new SourceTargetCaptureTuple.TupleToFileWriter(bufferedOutputStream); + var tupleWriter = new SourceTargetCaptureTuple.TupleToStreamConsumer(bufferedOutputStream); var timeShifter = new TimeShifter(params.speedupFactor); tr.setupRunAndWaitForReplayWithShutdownChecks(Duration.ofSeconds(params.observedPacketConnectionTimeout), - blockingTrafficStream, timeShifter, tupleWriter); + blockingTrafficSource, timeShifter, tupleWriter); log.info("Done processing TrafficStreams"); } } @@ -438,20 +446,20 @@ public void close() { } void setupRunAndWaitForReplay(Duration observedPacketConnectionTimeout, - BlockingTrafficSource trafficChunkStream, + BlockingTrafficSource trafficSource, TimeShifter timeShifter, Consumer resultTupleConsumer) throws InterruptedException, ExecutionException { var senderOrchestrator = new RequestSenderOrchestrator(clientConnectionPool); - var replayEngine = new ReplayEngine(senderOrchestrator, trafficChunkStream, timeShifter); + var replayEngine = new ReplayEngine(senderOrchestrator, trafficSource, timeShifter); CapturedTrafficToHttpTransactionAccumulator trafficToHttpTransactionAccumulator = new CapturedTrafficToHttpTransactionAccumulator(observedPacketConnectionTimeout, "(see " + PACKET_TIMEOUT_SECONDS_PARAMETER_NAME + ")", - new TrafficReplayerAccumulationCallbacks(replayEngine, resultTupleConsumer)); + new TrafficReplayerAccumulationCallbacks(replayEngine, resultTupleConsumer, trafficSource)); try { - pullCaptureFromSourceToAccumulator(trafficChunkStream, trafficToHttpTransactionAccumulator); + pullCaptureFromSourceToAccumulator(trafficSource, trafficToHttpTransactionAccumulator); } catch (InterruptedException ex) { throw ex; } catch (Exception e) { @@ -509,12 +517,12 @@ private void wrapUpWorkAndEmitSummary(ReplayEngine replayEngine, CapturedTraffic } void setupRunAndWaitForReplayWithShutdownChecks(Duration observedPacketConnectionTimeout, - BlockingTrafficSource trafficChunkStream, + BlockingTrafficSource trafficSource, TimeShifter timeShifter, Consumer resultTupleConsumer) throws TerminationException, ExecutionException, InterruptedException { try { - setupRunAndWaitForReplay(observedPacketConnectionTimeout, trafficChunkStream, + setupRunAndWaitForReplay(observedPacketConnectionTimeout, trafficSource, timeShifter, resultTupleConsumer); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -533,6 +541,7 @@ void setupRunAndWaitForReplayWithShutdownChecks(Duration observedPacketConnectio class TrafficReplayerAccumulationCallbacks implements AccumulationCallbacks { private final ReplayEngine replayEngine; private Consumer resultTupleConsumer; + private ITrafficCaptureSource trafficCaptureSource; @Override public void onRequestReceived(UniqueReplayerRequestKey requestKey, HttpMessageAndTimestamp request) { @@ -552,46 +561,11 @@ public void onRequestReceived(UniqueReplayerRequestKey requestKey, HttpMessageAn } @Override - public void onFullDataReceived(UniqueReplayerRequestKey requestKey, RequestResponsePacketPair rrPair) { + public void onFullDataReceived(@NonNull UniqueReplayerRequestKey requestKey, RequestResponsePacketPair rrPair) { log.atTrace().setMessage(()->"Done receiving captured stream for this " + rrPair.requestData).log(); var resultantCf = requestFutureMap.remove(requestKey) - .map(f -> - f.handle((summary, t) -> { - try { - // if this comes in with a serious Throwable (not an Exception), don't bother - // packaging it up and calling the callback. - // Escalate it up out handling stack and shutdown. - if (t == null || t instanceof Exception) { - return packageAndWriteResponse(resultTupleConsumer, requestKey, rrPair, summary, - (Exception) t); - } else if (t instanceof Error) { - throw (Error) t; - } else { - throw new Error("Unknown throwable type passed to handle().", t) { }; - } - } catch (Error error) { - log.atError() - .setCause(error) - .setMessage(()->"Caught error and initiating TrafficReplayer shutdown") - .log(); - shutdown(error); - throw error; - } catch (Exception e) { - log.atError() - .setMessage("Unexpected exception while sending the " + - "aggregated response and context for {} to the callback. " + - "Proceeding, but the tuple receiver context may be compromised.") - .addArgument(requestKey) - .setCause(e) - .log(); - throw e; - } finally { - requestToFinalWorkFuturesMap.remove(requestKey); - log.trace("removed rrPair.requestData to " + - "targetTransactionInProgressMap for " + - requestKey); - } - }), () -> "TrafficReplayer.runReplayWithIOStreams.progressTracker"); + .map(f -> f.handle((summary,t)->handleCompletedTransaction(requestKey, rrPair, summary, t)), + () -> "TrafficReplayer.runReplayWithIOStreams.progressTracker"); if (!resultantCf.future.isDone()) { log.trace("Adding " + requestKey + " to targetTransactionInProgressMap"); requestToFinalWorkFuturesMap.put(requestKey, resultantCf); @@ -601,6 +575,46 @@ public void onFullDataReceived(UniqueReplayerRequestKey requestKey, RequestRespo } } + Void handleCompletedTransaction(@NonNull UniqueReplayerRequestKey requestKey, RequestResponsePacketPair rrPair, + TransformedTargetRequestAndResponse summary, Throwable t) { + try { + // if this comes in with a serious Throwable (not an Exception), don't bother + // packaging it up and calling the callback. + // Escalate it up out handling stack and shutdown. + if (t == null || t instanceof Exception) { + packageAndWriteResponse(resultTupleConsumer, requestKey, rrPair, summary, + (Exception) t); + return null; + } else if (t instanceof Error) { + throw (Error) t; + } else { + throw new Error("Unknown throwable type passed to handle().", t) { + }; + } + } catch (Error error) { + log.atError() + .setCause(error) + .setMessage(() -> "Caught error and initiating TrafficReplayer shutdown") + .log(); + shutdown(error); + throw error; + } catch (Exception e) { + log.atError() + .setMessage("Unexpected exception while sending the " + + "aggregated response and context for {} to the callback. " + + "Proceeding, but the tuple receiver context may be compromised.") + .addArgument(requestKey) + .setCause(e) + .log(); + throw e; + } finally { + requestToFinalWorkFuturesMap.remove(requestKey); + log.trace("removed rrPair.requestData to " + + "targetTransactionInProgressMap for " + + requestKey); + } + } + @Override public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, List trafficStreamKeysBeingHeld) { @@ -617,7 +631,7 @@ public void onConnectionClose(UniqueReplayerRequestKey requestKey, Instant times private TransformedTargetRequestAndResponse packageAndWriteResponse(Consumer tupleWriter, - UniqueReplayerRequestKey requestKey, + @NonNull UniqueReplayerRequestKey requestKey, RequestResponsePacketPair rrPair, TransformedTargetRequestAndResponse summary, Exception t) { @@ -738,10 +752,12 @@ private static String formatWorkItem(DiagnosticTrackableCompletableFuture streamUnretained() { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueSourceRequestKey.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueSourceRequestKey.java index 6430252518..5439f687b6 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueSourceRequestKey.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueSourceRequestKey.java @@ -2,6 +2,8 @@ import com.google.common.base.Objects; +import java.util.StringJoiner; + public abstract class UniqueSourceRequestKey { public abstract ITrafficStreamKey getTrafficStreamKey(); @@ -16,6 +18,11 @@ public boolean equals(Object o) { Objects.equal(getTrafficStreamKey(), that.getTrafficStreamKey()); } + @Override + public String toString() { + return getTrafficStreamKey() + "." + getSourceRequestIndex(); + } + @Override public int hashCode() { return Objects.hashCode(getTrafficStreamKey(), getSourceRequestIndex()); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java index 5d61e10819..6b7e353872 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java @@ -5,6 +5,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.SneakyThrows; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.producer.KafkaProducer; @@ -15,6 +16,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; +import org.opensearch.migrations.replay.datatypes.PojoTrafficStreamKey; import org.opensearch.migrations.replay.kafka.KafkaProtobufConsumer; import org.opensearch.migrations.replay.traffic.source.BlockingTrafficSource; import org.opensearch.migrations.replay.traffic.source.ISimpleTrafficCaptureSource; @@ -45,8 +47,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.Supplier; -import java.util.function.ToIntFunction; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -85,66 +87,91 @@ private static class FabricatedErrorToKillTheReplayer extends Error { @Test @Tag("longTest") public void fullTest() throws Exception { - Random r = new Random(1); - var nextStopPointRef = new AtomicInteger(INITIAL_STOP_REPLAYER_REQUEST_COUNT); - var httpServer = SimpleNettyHttpServer.makeServer(false, Duration.ofMillis(2), TestHttpServerContext::makeResponse); var streamAndConsumer = generateStreamAndTupleConsumerWithSomeChecks(); - var onNewTupleReceived = streamAndConsumer._2; + var numExpectedRequests = streamAndConsumer._2; var trafficSourceSupplier = loadStreamsToCursorArraySource(streamAndConsumer._1.collect(Collectors.toList())); - - Consumer tupleReceiver = t -> { - var stopPoint = nextStopPointRef.get(); - if (onNewTupleReceived.applyAsInt(t) > stopPoint) { - var roughlyDoubled = stopPoint + new Random(stopPoint).nextInt(stopPoint+1); - if (nextStopPointRef.compareAndSet(stopPoint, roughlyDoubled)) { - throw new FabricatedErrorToKillTheReplayer(false); - } else { - // somebody else got to this, don't worry about it - } - } - }; - - runReplayerUntilSourceWasExhausted(httpServer, trafficSourceSupplier, tupleReceiver); - - //Assertions.assertEquals(); + runReplayerUntilSourceWasExhausted(numExpectedRequests, httpServer, trafficSourceSupplier); log.error("done"); } - private static void runReplayerUntilSourceWasExhausted(SimpleNettyHttpServer httpServer, - Supplier trafficSourceSupplier, - Consumer tupleReceiver) + private static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, + SimpleNettyHttpServer httpServer, + Supplier trafficSourceSupplier) throws Exception { - for (AtomicInteger runNumberRef = new AtomicInteger(); true; runNumberRef.incrementAndGet()) { + AtomicInteger runNumberRef = new AtomicInteger(); + var totalUniqueEverReceived = new AtomicInteger(); + var nextStopPointRef = new AtomicInteger(INITIAL_STOP_REPLAYER_REQUEST_COUNT); + + var receivedPerRun = new ArrayList(); + var totalUniqueEverReceivedSizeAfterEachRun = new ArrayList(); + var previouslyCompletelyHandledItems = new ConcurrentHashMap(); + + for (; true; runNumberRef.incrementAndGet()) { + var stopPoint = nextStopPointRef.get(); int runNumber = runNumberRef.get(); + var counter = new AtomicInteger(); try { runTrafficReplayer(trafficSourceSupplier, httpServer, (t) -> { Assertions.assertEquals(runNumber, runNumberRef.get()); - tupleReceiver.accept(t); + synchronized (nextStopPointRef) { + var key = t.uniqueRequestKey; + if (((TrafficStreamCursorKey)(key.getTrafficStreamKey())).sourceListIndex > stopPoint) { + log.error("Stopping"); + var roughlyDoubled = stopPoint + new Random(stopPoint).nextInt(stopPoint + 1); + if (nextStopPointRef.compareAndSet(stopPoint, roughlyDoubled)) { + throw new FabricatedErrorToKillTheReplayer(false); + } else { + // somebody else already threw to stop the loop. + return; + } + } + + var keyString = new PojoTrafficStreamKey(key.getTrafficStreamKey()) + "_" + key.getSourceRequestIndex(); + var totalUnique = null != previouslyCompletelyHandledItems.put(keyString, t) ? + totalUniqueEverReceived.get() : + totalUniqueEverReceived.incrementAndGet(); + + var c = counter.incrementAndGet(); + log.info("counter="+c+" totalUnique="+totalUnique+" runNum="+runNumber+" key="+key); + } }); // if this finished running without an exception, we need to stop the loop break; } catch (TrafficReplayer.TerminationException e) { - log.atLevel(e.shutdownCause instanceof FabricatedErrorToKillTheReplayer ? Level.INFO : Level.ERROR) - .setCause(e.shutdownCause) + log.atLevel(e.originalCause instanceof FabricatedErrorToKillTheReplayer ? Level.INFO : Level.ERROR) + .setCause(e.originalCause) .setMessage(()->"broke out of the replayer, with this shutdown reason") .log(); log.atLevel(e.immediateCause == null ? Level.INFO : Level.ERROR) .setCause(e.immediateCause) - .setMessage(()->"broke out of the replayer, with the shutdown cause=" + e.shutdownCause + + .setMessage(()->"broke out of the replayer, with the shutdown cause=" + e.originalCause + " and this immediate reason") .log(); + } finally { + log.info("Upon appending.... counter="+counter.get()+" totalUnique="+totalUniqueEverReceived.get()); + receivedPerRun.add(counter.get()); + totalUniqueEverReceivedSizeAfterEachRun.add(totalUniqueEverReceived.get()); } } + var skippedPerRun = IntStream.range(0, receivedPerRun.size()) + .map(i->totalUniqueEverReceivedSizeAfterEachRun.get(i)-receivedPerRun.get(i)).toArray(); + var skippedPerRunDiffs = IntStream.range(0, receivedPerRun.size()-1) + .map(i->(skippedPerRun[i]<=skippedPerRun[i+1]) ? 1 : 0) + .toArray(); + var expectedSkipArray = new int[skippedPerRunDiffs.length]; + Arrays.fill(expectedSkipArray, 1); + //Assertions.assertArrayEquals(expectedSkipArray, skippedPerRunDiffs); + Assertions.assertEquals(numExpectedRequests, totalUniqueEverReceived.get()); } - private Tuple2, ToIntFunction> + private Tuple2, Integer> generateStreamAndTupleConsumerWithSomeChecks() { return generateStreamAndTupleConsumerWithSomeChecks(-1); } - private Tuple2, ToIntFunction> + private Tuple2, Integer> generateStreamAndTupleConsumerWithSomeChecks(int count) { Random r = new Random(1); var generatedCases = count > 0 ? @@ -154,13 +181,8 @@ private static void runReplayerUntilSourceWasExhausted(SimpleNettyHttpServer htt var shuffledStreams = randomlyInterleaveStreams(r, Arrays.stream(testCaseArr).map(c->Arrays.stream(c.trafficStreams))); - var previouslyCompletelyHandledItems = new ConcurrentHashMap(); - return new Tuple2<>(shuffledStreams, t -> { - var key = t.uniqueRequestKey; - var keyString = key.getTrafficStreamKey() + "_" + key.getSourceRequestIndex(); - previouslyCompletelyHandledItems.put(keyString, t); - return previouslyCompletelyHandledItems.size(); - }); + var numExpectedRequests = Arrays.stream(testCaseArr).mapToInt(c->c.requestByteSizes.length).sum(); + return new Tuple2<>(shuffledStreams, numExpectedRequests); } private void @@ -223,6 +245,7 @@ Producer buildKafkaProducer() { } @Getter + @ToString private static class TrafficStreamCursorKey implements ITrafficStreamKey, Comparable { public final String connectionId; public final String nodeId; @@ -234,13 +257,14 @@ public TrafficStreamCursorKey(TrafficStream stream, int sourceListIndex) { connectionId = stream.getConnectionId(); nodeId = stream.getNodeId(); trafficStreamIndex = TrafficStreamUtils.getTrafficStreamIndex(stream); - this.sourceListIndex = this.getSourceListIndex(); + this.sourceListIndex = sourceListIndex; } @Override public int compareTo(TrafficStreamCursorKey other) { return Integer.compare(sourceListIndex, other.sourceListIndex); } + } @AllArgsConstructor @@ -260,6 +284,7 @@ private Supplier loadStreamsToCursorArraySource(Lis @Override public CompletableFuture> readNextTrafficStreamChunk() { var idx = readCursor.getAndIncrement(); + log.info("reading chunk from index="+idx); if (streams.size() <= idx) { return CompletableFuture.failedFuture(new EOFException()); } @@ -271,14 +296,17 @@ public CompletableFuture> readNextTrafficStreamChunk @Override public void commitTrafficStream(ITrafficStreamKey trafficStreamKey) { - synchronized (readCursor) { // figure out if I need to do something faster later + synchronized (readCursor) { // figure out if I need to do something more efficient later int topCursor = pQueue.peek().sourceListIndex; var didRemove = pQueue.remove(trafficStreamKey); assert didRemove; var incomingCursor = trafficStreamKey.getTrafficStreamIndex(); if (topCursor == incomingCursor) { topCursor = Optional.ofNullable(pQueue.peek()).map(k->k.getSourceListIndex()).orElse(topCursor); + log.info("Commit called for "+trafficStreamKey+", but and new topCursor="+topCursor); commitCursor.set(topCursor); + } else { + log.info("Commit called for "+trafficStreamKey+", but topCursor="+topCursor); } } } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SourceTargetCaptureTupleTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SourceTargetCaptureTupleTest.java new file mode 100644 index 0000000000..355a12d5ad --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SourceTargetCaptureTupleTest.java @@ -0,0 +1,59 @@ +package org.opensearch.migrations.replay; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.opensearch.migrations.replay.datatypes.PojoTrafficStreamKey; +import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; +import org.opensearch.migrations.replay.datatypes.UniqueSourceRequestKey; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.*; + +@Slf4j +class SourceTargetCaptureTupleTest { + private static final String NODE_ID = "n"; + public static final String TEST_EXCEPTION_MESSAGE = "TEST_EXCEPTION"; + + @Test + public void testTupleNewWithNullKeyThrows() { + Assertions.assertThrows(Exception.class, + ()->new SourceTargetCaptureTuple(null, null, null, + null, null, null, null)); + } + + @Test + public void testOutputterWithNulls() throws IOException { + var emptyTuple = new SourceTargetCaptureTuple( + new UniqueReplayerRequestKey(new PojoTrafficStreamKey(NODE_ID,"c",0), 0, 0), + null, null, null, null, null, null); + String contents; + try (var byteArrayOutputStream = new ByteArrayOutputStream()) { + var consumer = new SourceTargetCaptureTuple.TupleToStreamConsumer(byteArrayOutputStream); + consumer.accept(emptyTuple); + contents = new String(byteArrayOutputStream.toByteArray(), StandardCharsets.UTF_8); + } + log.info("Output="+contents); + Assertions.assertTrue(contents.contains(NODE_ID)); + } + + @Test + public void testOutputterWithNullsShowsException() throws IOException { + var exception = new Exception(TEST_EXCEPTION_MESSAGE); + var emptyTuple = new SourceTargetCaptureTuple( + new UniqueReplayerRequestKey(new PojoTrafficStreamKey(NODE_ID,"c",0), 0, 0), + null, null, null, null, exception, null); + String contents; + try (var byteArrayOutputStream = new ByteArrayOutputStream()) { + var consumer = new SourceTargetCaptureTuple.TupleToStreamConsumer(byteArrayOutputStream); + consumer.accept(emptyTuple); + contents = new String(byteArrayOutputStream.toByteArray(), StandardCharsets.UTF_8); + } + log.info("Output="+contents); + Assertions.assertTrue(contents.contains(NODE_ID)); + Assertions.assertTrue(contents.contains(TEST_EXCEPTION_MESSAGE)); + } +} \ No newline at end of file diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficStreamGenerator.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficStreamGenerator.java index 9851db955e..07689f8c85 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficStreamGenerator.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficStreamGenerator.java @@ -145,7 +145,8 @@ static int classifyTrafficStream(HashSet possibilitiesLeftToTest, Traff } private static void addCommands(Random r, double flushLikelihood, int numPacketCommands, - List outgoingCommands, List outgoingSizes, + List outgoingCommands, + List outgoingSizes, Supplier directiveSupplier) { int aggregateBufferSize = 0; for (var cmdCount = new AtomicInteger(numPacketCommands); cmdCount.get()>0;) { @@ -172,7 +173,8 @@ private static T supplyRandomly(Random r, double p1, Supplier supplier1, } private static void fillCommandsAndSizes(int bufferSize, Random r, double flushLikelihood, int bufferBound, - List commands, List sizes) { + List commands, + List sizes) { var numTransactions = r.nextInt(MAX_COMMANDS_IN_CONNECTION); for (int i=numTransactions; i>0; --i) { addCommands(r, flushLikelihood, r.nextInt(MAX_READS_IN_REQUEST)+1, commands, sizes, From be1a339bc0dd1ac4538dabf1745df7971a46edf9 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Tue, 7 Nov 2023 17:05:52 -0500 Subject: [PATCH 13/55] Setup a simpler test and make changes to get it to work. Create a traffic stream source with one stream which has ONE CLOSE observation. Make sure that that causes the commit cursor to move. That required more connecting dots and making additional refinements on the FullTrafficReplayerTest. Signed-off-by: Greg Schohn --- .../replay/AccumulationCallbacks.java | 3 +- ...edTrafficToHttpTransactionAccumulator.java | 11 +- .../migrations/replay/TrafficReplayer.java | 20 ++- .../traffic/source/BlockingTrafficSource.java | 6 + .../replay/FullTrafficReplayerTest.java | 120 ++++++++++++------ ...afficToHttpTransactionAccumulatorTest.java | 3 +- .../replay/TrafficReplayerTest.java | 6 +- 7 files changed, 115 insertions(+), 54 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java index 8afff5760b..5b81050312 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java @@ -11,5 +11,6 @@ public interface AccumulationCallbacks { void onFullDataReceived(UniqueReplayerRequestKey key, RequestResponsePacketPair rrpp); void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, List trafficStreamKeysBeingHeld); - void onConnectionClose(UniqueReplayerRequestKey key, Instant when); + void onConnectionClose(UniqueReplayerRequestKey key, Instant when, + List trafficStreamKeysBeingHeld); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java index 9e6e70a247..1311b0be2b 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java @@ -14,6 +14,7 @@ import java.time.Duration; import java.time.Instant; import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.StringJoiner; import java.util.concurrent.atomic.AtomicInteger; @@ -140,6 +141,7 @@ public void accept(ITrafficStreamWithKey trafficStreamAndKey) { accum.getRrPair().holdTrafficStream(tsk); } else { assert accum.state == Accumulation.State.WAITING_FOR_NEXT_READ_CHUNK || + accum.state == Accumulation.State.IGNORING_LAST_REQUEST || trafficStream.getSubStreamCount() == 0 || trafficStream.getSubStream(trafficStream.getSubStreamCount()-1).hasClose(); } @@ -177,8 +179,8 @@ public CONNECTION_STATUS addObservationToAccumulation(@NonNull Accumulation accu Optional.of(observation.getTs()).map(TrafficStreamUtils::instantFromProtoTimestamp).get(); liveStreams.expireOldEntries(trafficStreamKey, accum, timestamp); - return handleObservationForSkipState(accum, observation) - .or(() -> handleCloseObservationThatAffectEveryState(accum, observation, trafficStreamKey, timestamp)) + return handleCloseObservationThatAffectEveryState(accum, observation, trafficStreamKey, timestamp) + .or(() -> handleObservationForSkipState(accum, observation)) .or(() -> handleObservationForReadState(accum, observation, trafficStreamKey, timestamp)) .or(() -> handleObservationForWriteState(accum, observation, trafficStreamKey, timestamp)) .orElseGet(() -> { @@ -214,7 +216,7 @@ private static Optional handleObservationForSkipState(Accumul accum.getOrCreateTransactionPair().holdTrafficStream(trafficStreamKey); rotateAccumulationIfNecessary(trafficStreamKey.getConnectionId(), accum); closedConnectionCounter.incrementAndGet(); - listener.onConnectionClose(accum.getRequestKey(), timestamp); + listener.onConnectionClose(accum.getRequestKey(), timestamp, accum.getRrPair().trafficStreamKeysBeingHeld); return Optional.of(CONNECTION_STATUS.CLOSED); } else if (observation.hasConnectionException()) { accum.getOrCreateTransactionPair().holdTrafficStream(trafficStreamKey); @@ -392,7 +394,8 @@ private void fireAccumulationsCallbacksAndClose(Accumulation accumulation, } } finally { if (accumulation.hasSignaledRequests()) { - listener.onConnectionClose(accumulation.getRequestKey(), accumulation.getLastTimestamp()); + listener.onConnectionClose(accumulation.getRequestKey(), accumulation.getLastTimestamp(), + accumulation.hasRrPair() ? accumulation.getRrPair().trafficStreamKeysBeingHeld : List.of()); } } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index cb944edb37..8d0b4a2c5a 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -9,6 +9,7 @@ import io.netty.handler.ssl.util.InsecureTrustManagerFactory; import lombok.AllArgsConstructor; import lombok.NonNull; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer; import org.opensearch.migrations.replay.datahandlers.http.IHttpMessage; @@ -582,14 +583,13 @@ Void handleCompletedTransaction(@NonNull UniqueReplayerRequestKey requestKey, Re // packaging it up and calling the callback. // Escalate it up out handling stack and shutdown. if (t == null || t instanceof Exception) { - packageAndWriteResponse(resultTupleConsumer, requestKey, rrPair, summary, - (Exception) t); + packageAndWriteResponse(resultTupleConsumer, requestKey, rrPair, summary, (Exception) t); + commitTrafficStreams(rrPair.trafficStreamKeysBeingHeld); return null; } else if (t instanceof Error) { throw (Error) t; } else { - throw new Error("Unknown throwable type passed to handle().", t) { - }; + throw new Error("Unknown throwable type passed to handle().", t); } } catch (Error error) { log.atError() @@ -620,13 +620,23 @@ public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStat List trafficStreamKeysBeingHeld) { if (status == RequestResponsePacketPair.ReconstructionStatus.EXPIRED_PREMATURELY) { // eventually fill this in to commit the message + commitTrafficStreams(trafficStreamKeysBeingHeld); + } + } + + @SneakyThrows + private void commitTrafficStreams(List trafficStreamKeysBeingHeld) { + for (var tsk : trafficStreamKeysBeingHeld) { + trafficCaptureSource.commitTrafficStream(tsk); } } @Override - public void onConnectionClose(UniqueReplayerRequestKey requestKey, Instant timestamp) { + public void onConnectionClose(UniqueReplayerRequestKey requestKey, Instant timestamp, + List trafficStreamKeysBeingHeld) { replayEngine.setFirstTimestamp(timestamp); replayEngine.closeConnection(requestKey, timestamp); + commitTrafficStreams(trafficStreamKeysBeingHeld); } private TransformedTargetRequestAndResponse diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/BlockingTrafficSource.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/BlockingTrafficSource.java index 3cace9b767..a22ad25490 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/BlockingTrafficSource.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/BlockingTrafficSource.java @@ -3,6 +3,7 @@ import com.google.protobuf.Timestamp; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.replay.Utils; +import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.trafficcapture.protos.TrafficStreamUtils; import org.slf4j.event.Level; @@ -125,6 +126,11 @@ public Duration getBufferTimeWindow() { }); } + @Override + public void commitTrafficStream(ITrafficStreamKey trafficStreamKey) throws IOException { + underlyingSource.commitTrafficStream(trafficStreamKey); + } + @Override public void close() throws IOException { underlyingSource.close(); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java index 6b7e353872..6d88a3748e 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java @@ -3,6 +3,7 @@ import com.google.common.collect.Streams; import io.vavr.Tuple2; import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.SneakyThrows; import lombok.ToString; @@ -24,6 +25,8 @@ import org.opensearch.migrations.replay.traffic.source.TrafficStreamWithEmbeddedKey; import org.opensearch.migrations.testutils.SimpleNettyHttpServer; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; +import org.opensearch.migrations.trafficcapture.protos.CloseObservation; +import org.opensearch.migrations.trafficcapture.protos.TrafficObservation; import org.opensearch.migrations.trafficcapture.protos.TrafficStream; import org.opensearch.migrations.trafficcapture.protos.TrafficStreamUtils; import org.opensearch.migrations.transform.StaticAuthTransformerFactory; @@ -47,7 +50,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; -import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -73,6 +75,7 @@ public class FullTrafficReplayerTest { public static final int PRODUCER_SLEEP_INTERVAL_MS = 100; public static final Duration MAX_WAIT_TIME_FOR_TOPIC = Duration.ofMillis(PRODUCER_SLEEP_INTERVAL_MS*2); public static final int INITIAL_STOP_REPLAYER_REQUEST_COUNT = 1; + public static final String TEST_CONNECTION_ID = "testConnectionId"; @AllArgsConstructor private static class FabricatedErrorToKillTheReplayer extends Error { @@ -86,20 +89,38 @@ private static class FabricatedErrorToKillTheReplayer extends Error { @Test @Tag("longTest") - public void fullTest() throws Exception { + public void testSingleStreamWithCloseIsCommitted() throws Throwable { var httpServer = SimpleNettyHttpServer.makeServer(false, Duration.ofMillis(2), TestHttpServerContext::makeResponse); - var streamAndConsumer = generateStreamAndTupleConsumerWithSomeChecks(); + var trafficStreamWithJustClose = TrafficStream.newBuilder() + .setNodeId(TEST_NODE_ID) + .setConnectionId(TEST_CONNECTION_ID) + .addSubStream(TrafficObservation.newBuilder() + .setClose(CloseObservation.newBuilder().build()).build()) + .build(); + var trafficSourceSupplier = new ArrayCursorTrafficSourceFactory(List.of(trafficStreamWithJustClose)); + runReplayerUntilSourceWasExhausted(0, httpServer, trafficSourceSupplier); + Assertions.assertEquals(0, trafficSourceSupplier.commitCursor.get()); + log.error("done"); + } + + @Test + @Tag("longTest") + public void fullTest() throws Throwable { + var httpServer = SimpleNettyHttpServer.makeServer(false, Duration.ofMillis(2), + TestHttpServerContext::makeResponse); + var streamAndConsumer = generateStreamAndTupleConsumerWithSomeChecks(2); var numExpectedRequests = streamAndConsumer._2; - var trafficSourceSupplier = loadStreamsToCursorArraySource(streamAndConsumer._1.collect(Collectors.toList())); + var trafficSourceSupplier = new ArrayCursorTrafficSourceFactory(streamAndConsumer._1.collect(Collectors.toList())); runReplayerUntilSourceWasExhausted(numExpectedRequests, httpServer, trafficSourceSupplier); + Assertions.assertEquals(numExpectedRequests, trafficSourceSupplier.commitCursor.get()); log.error("done"); } private static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, SimpleNettyHttpServer httpServer, Supplier trafficSourceSupplier) - throws Exception { + throws Throwable { AtomicInteger runNumberRef = new AtomicInteger(); var totalUniqueEverReceived = new AtomicInteger(); var nextStopPointRef = new AtomicInteger(INITIAL_STOP_REPLAYER_REQUEST_COUNT); @@ -149,6 +170,9 @@ private static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, .setMessage(()->"broke out of the replayer, with the shutdown cause=" + e.originalCause + " and this immediate reason") .log(); + if (!(e.originalCause instanceof FabricatedErrorToKillTheReplayer)) { + throw e.immediateCause; + } } finally { log.info("Upon appending.... counter="+counter.get()+" totalUnique="+totalUniqueEverReceived.get()); receivedPerRun.add(counter.get()); @@ -178,11 +202,12 @@ private static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, TrafficStreamGenerator.generateRandomTrafficStreamsAndSizes(IntStream.range(0,count)) : TrafficStreamGenerator.generateAllIndicativeRandomTrafficStreamsAndSizes(); var testCaseArr = generatedCases.toArray(TrafficStreamGenerator.RandomTrafficStreamAndTransactionSizes[]::new); - var shuffledStreams = - randomlyInterleaveStreams(r, Arrays.stream(testCaseArr).map(c->Arrays.stream(c.trafficStreams))); + var aggregatedStreams = + Arrays.stream(testCaseArr).flatMap(c->Arrays.stream(c.trafficStreams)); + //randomlyInterleaveStreams(r, Arrays.stream(testCaseArr).map(c->Arrays.stream(c.trafficStreams))); var numExpectedRequests = Arrays.stream(testCaseArr).mapToInt(c->c.requestByteSizes.length).sum(); - return new Tuple2<>(shuffledStreams, numExpectedRequests); + return new Tuple2<>(aggregatedStreams, numExpectedRequests); } private void @@ -246,12 +271,14 @@ Producer buildKafkaProducer() { @Getter @ToString + @EqualsAndHashCode private static class TrafficStreamCursorKey implements ITrafficStreamKey, Comparable { + public final int sourceListIndex; + public final String connectionId; public final String nodeId; public final int trafficStreamIndex; - public final int sourceListIndex; public TrafficStreamCursorKey(TrafficStream stream, int sourceListIndex) { connectionId = stream.getConnectionId(); @@ -264,7 +291,6 @@ public TrafficStreamCursorKey(TrafficStream stream, int sourceListIndex) { public int compareTo(TrafficStreamCursorKey other) { return Integer.compare(sourceListIndex, other.sourceListIndex); } - } @AllArgsConstructor @@ -274,43 +300,55 @@ private static class PojoTrafficStreamWithKey implements ITrafficStreamWithKey { ITrafficStreamKey key; } - private Supplier loadStreamsToCursorArraySource(List streams) { - var commitCursor = new AtomicInteger(-1); + private static class ArrayCursorTrafficSourceFactory implements Supplier { + List streams; + AtomicInteger commitCursor = new AtomicInteger(-1); - return () -> new ISimpleTrafficCaptureSource() { - AtomicInteger readCursor = new AtomicInteger(commitCursor.get()+1); - PriorityQueue pQueue = new PriorityQueue<>(); + public ArrayCursorTrafficSourceFactory(List streams) { + this.streams = streams; + } - @Override - public CompletableFuture> readNextTrafficStreamChunk() { - var idx = readCursor.getAndIncrement(); - log.info("reading chunk from index="+idx); - if (streams.size() <= idx) { - return CompletableFuture.failedFuture(new EOFException()); + public ISimpleTrafficCaptureSource get() { + return new ISimpleTrafficCaptureSource() { + AtomicInteger readCursor = new AtomicInteger(commitCursor.get()+1); + PriorityQueue pQueue = new PriorityQueue<>(); + + @Override + public CompletableFuture> readNextTrafficStreamChunk() { + var idx = readCursor.getAndIncrement(); + log.info("reading chunk from index="+idx); + if (streams.size() <= idx) { + return CompletableFuture.failedFuture(new EOFException()); + } + var stream = streams.get(idx); + var key = new TrafficStreamCursorKey(stream, idx); + synchronized (pQueue) { + if (pQueue.isEmpty()) { + commitCursor.set(key.sourceListIndex); + } + pQueue.add(key); + } + return CompletableFuture.supplyAsync(()->List.of(new PojoTrafficStreamWithKey(stream, key))); } - var stream = streams.get(idx); - var key = new TrafficStreamCursorKey(stream, idx); - pQueue.add(key); - return CompletableFuture.supplyAsync(()->List.of(new PojoTrafficStreamWithKey(stream, key))); - } - @Override - public void commitTrafficStream(ITrafficStreamKey trafficStreamKey) { - synchronized (readCursor) { // figure out if I need to do something more efficient later - int topCursor = pQueue.peek().sourceListIndex; - var didRemove = pQueue.remove(trafficStreamKey); - assert didRemove; - var incomingCursor = trafficStreamKey.getTrafficStreamIndex(); - if (topCursor == incomingCursor) { - topCursor = Optional.ofNullable(pQueue.peek()).map(k->k.getSourceListIndex()).orElse(topCursor); - log.info("Commit called for "+trafficStreamKey+", but and new topCursor="+topCursor); - commitCursor.set(topCursor); - } else { - log.info("Commit called for "+trafficStreamKey+", but topCursor="+topCursor); + @Override + public void commitTrafficStream(ITrafficStreamKey trafficStreamKey) { + synchronized (pQueue) { // figure out if I need to do something more efficient later + int topCursor = pQueue.peek().sourceListIndex; + var didRemove = pQueue.remove(trafficStreamKey); + assert didRemove; + var incomingCursor = ((TrafficStreamCursorKey)trafficStreamKey).sourceListIndex; + if (topCursor == incomingCursor) { + topCursor = Optional.ofNullable(pQueue.peek()).map(k->k.getSourceListIndex()).orElse(topCursor); + log.info("Commit called for "+trafficStreamKey+", but and new topCursor="+topCursor); + commitCursor.set(topCursor); + } else { + log.info("Commit called for "+trafficStreamKey+", but topCursor="+topCursor); + } } } - } - }; + }; + } } private Supplier loadStreamsToKafka(Stream streams) throws Exception { diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java index 142f45a409..6f8973fb9f 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java @@ -216,7 +216,8 @@ public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStat List trafficStreamKeysBeingHeld) {} @Override - public void onConnectionClose(UniqueReplayerRequestKey key, Instant when) {} + public void onConnectionClose(UniqueReplayerRequestKey key, Instant when, + List keysHeld) {} }); var tsList = trafficStreams.collect(Collectors.toList()); trafficStreams = tsList.stream(); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java index 64c835ac38..e1b156d2cd 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java @@ -170,7 +170,8 @@ public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStat List trafficStreamKeysBeingHeld) {} @Override - public void onConnectionClose(UniqueReplayerRequestKey key, Instant when) {} + public void onConnectionClose(UniqueReplayerRequestKey key, Instant when, + List keysHeld) {} }); var bytes = synthesizeTrafficStreamsIntoByteArray(Instant.now(), 1); @@ -211,7 +212,8 @@ public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStat List trafficStreamKeysBeingHeld) {} @Override - public void onConnectionClose(UniqueReplayerRequestKey key, Instant when) {} + public void onConnectionClose(UniqueReplayerRequestKey key, Instant when, + List keysHeld) {} } ); byte[] serializedChunks; From c4c19fd16118d7166f0736c9bd8dd4b27bf6b136 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Wed, 8 Nov 2023 10:01:37 -0500 Subject: [PATCH 14/55] Sigv4 IAM Documentation and Sonar Vulnerability Fixes (#388) * Allow MSK public endpoints to be restricted, documentation for sigv4, and sonar fixes Signed-off-by: Tanner Lewis --- .../opensearch-service-migration/README.md | 16 +++++- .../lib/lambda/msk-public-endpoint-handler.ts | 2 - .../lib/migration-assistance-stack.ts | 35 ++++++++++--- .../lib/msk-utility-stack.ts | 6 +-- .../lib/network-stack.ts | 6 +-- .../lib/opensearch-domain-stack.ts | 40 ++++++++------- .../lib/service-stacks/elasticsearch-stack.ts | 2 +- .../service-stacks/migration-console-stack.ts | 1 - .../service-stacks/migration-service-core.ts | 49 ++++++++++--------- .../service-stacks/traffic-replayer-stack.ts | 2 +- .../lib/stack-composer.ts | 8 ++- .../opensearch-service-migration/options.md | 12 +++-- .../opensearch-service-migration/package.json | 6 +++ 13 files changed, 117 insertions(+), 68 deletions(-) diff --git a/deployment/cdk/opensearch-service-migration/README.md b/deployment/cdk/opensearch-service-migration/README.md index d69d1a6061..4a6a67b538 100644 --- a/deployment/cdk/opensearch-service-migration/README.md +++ b/deployment/cdk/opensearch-service-migration/README.md @@ -29,7 +29,7 @@ More details can be found [here](../../../TrafficCapture/dockerSolution/README.m 5- There is a known issue where service linked roles fail to get applied when deploying certain AWS services for the first time in an account. This can be resolved by simply deploying again (for each failing role) or avoided entirely by creating the service linked role initially like seen below: ```shell -aws iam create-service-linked-role --aws-service-name opensearchservice.amazonaws.com && aws iam create-service-linked-role --aws-service-name ecs.amazonaws.com +aws iam create-service-linked-role --aws-service-name opensearchservice.amazonaws.com; aws iam create-service-linked-role --aws-service-name ecs.amazonaws.com ``` ### First time using CDK in this region? @@ -41,7 +41,7 @@ If this is your first experience with CDK, follow the steps below to get started npm install -g aws-cdk ``` -2- **Bootstrap CDK**: if you have not run CDK previously in the configured region of you account, it is necessary to run the following command from the `opensearch-service-migration` directory to set up a small CloudFormation stack of resources that CDK needs to function within your account +2- **Bootstrap CDK**: if you have not run CDK previously in the configured region of you account, it is necessary to run the following command from the same directory as this README to set up a small CloudFormation stack of resources that CDK needs to function within your account ```shell # Execute from the deployment/cdk/opensearch-service-migration directory @@ -126,6 +126,18 @@ After the benchmark has been run, the indices and documents of the source and ta ./catIndices.sh ``` +## Configuring SigV4 access +When this solution is deployed, it will attempt to configure needed IAM permissions on its applicable Migration services (Traffic Replayer and Migration Console) to allow SigV4 communication between the service and the AWS OpenSearch Service or OpenSearch Serverless target cluster. In addition to this, the target cluster must be configured to allow these resources as well, which the following subsections will discuss. + +There is also an assumption here that security groups are in place that allow communication between these Migration services and the target cluster. If the OpenSearch Service is created with this CDK, the relevant security group will be created and configured automatically. Otherwise, imported target clusters will need to add the `osClusterAccessSG` security group (created after deployment) to their target cluster or modify their existing security groups to allow communication. + +#### OpenSearch Service +If an open access policy is in place for the Domain, either configured manually or with the `openAccessPolicyEnabled` [option](./options.md) for Domains created with this CDK, this should be sufficient and no changes should be needed on this front. If a custom access policy is in place it must allow the IAM task roles for the applicable Migration services (Traffic Replayer and Migration Console) to access. + +#### OpenSearch Serverless +When using an existing OpenSearch Serverless Collection, some additional configuration is needed for the data access policies to allow the Traffic Replayer and Migration Console services to communicate with the Serverless cluster. Mainly, the IAM task roles for these Migration services should be added to a data access policy that allows them permission to perform all [index operations](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-data-access.html#serverless-data-supported-permissions) (`aoss:*`) for all indexes in the given collection. This will allow the Replayer to create, modify, and delete indexes as it mirrors incoming traffic. This can also be focused down to specific indexes or operations for sensitive cases, but should be monitored as this could prevent some requests from being replayed. + + ## Kicking off Fetch Migration * First, access the Migration Console container diff --git a/deployment/cdk/opensearch-service-migration/lib/lambda/msk-public-endpoint-handler.ts b/deployment/cdk/opensearch-service-migration/lib/lambda/msk-public-endpoint-handler.ts index ca27008020..448fb726ef 100644 --- a/deployment/cdk/opensearch-service-migration/lib/lambda/msk-public-endpoint-handler.ts +++ b/deployment/cdk/opensearch-service-migration/lib/lambda/msk-public-endpoint-handler.ts @@ -105,8 +105,6 @@ async function invokeNextLambda(payload: any, functionName: string) { await lambdaClient.send(command).then( (data) => { console.log("Invoke lambda response: " + JSON.stringify(data)) - data.$metadata.httpStatusCode - }, (error) => { console.error("Invoke lambda error: " + JSON.stringify(error)) diff --git a/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts b/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts index bf8a1963d7..7b8cb924b6 100644 --- a/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts @@ -1,5 +1,5 @@ import {Stack} from "aws-cdk-lib"; -import {IVpc, Peer, Port, SecurityGroup, SubnetType, Vpc} from "aws-cdk-lib/aws-ec2"; +import {IPeer, IVpc, Peer, Port, SecurityGroup, SubnetType} from "aws-cdk-lib/aws-ec2"; import {FileSystem} from 'aws-cdk-lib/aws-efs'; import {Construct} from "constructs"; import {CfnCluster, CfnConfiguration} from "aws-cdk-lib/aws-msk"; @@ -9,21 +9,42 @@ import {LogGroup, RetentionDays} from "aws-cdk-lib/aws-logs"; import {NamespaceType} from "aws-cdk-lib/aws-servicediscovery"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; -export interface migrationStackProps extends StackPropsExt { +export interface MigrationStackProps extends StackPropsExt { readonly vpc: IVpc, readonly trafficComparatorEnabled: boolean, // Future support needed to allow importing an existing MSK cluster readonly mskImportARN?: string, - readonly mskEnablePublicEndpoints?: boolean + readonly mskEnablePublicEndpoints?: boolean, + readonly mskRestrictPublicAccessTo?: string, + readonly mskRestrictPublicAccessType?: string, readonly mskBrokerNodeCount?: number } export class MigrationAssistanceStack extends Stack { - constructor(scope: Construct, id: string, props: migrationStackProps) { + getPublicEndpointAccess(restrictPublicAccessTo: string, restrictPublicAccessType: string): IPeer { + switch (restrictPublicAccessType) { + case 'ipv4': + return Peer.ipv4(restrictPublicAccessTo); + case 'ipv6': + return Peer.ipv6(restrictPublicAccessTo); + case 'prefixList': + return Peer.prefixList(restrictPublicAccessTo); + case 'securityGroupId': + return Peer.securityGroupId(restrictPublicAccessTo); + default: + throw new Error('mskRestrictPublicAccessType should be one of the below values: ipv4, ipv6, prefixList or securityGroupId'); + } + } + + constructor(scope: Construct, id: string, props: MigrationStackProps) { super(scope, id, props); + if (props.mskEnablePublicEndpoints && (!props.mskRestrictPublicAccessTo || !props.mskRestrictPublicAccessType)) { + throw new Error("The 'mskEnablePublicEndpoints' option requires both 'mskRestrictPublicAccessTo' and 'mskRestrictPublicAccessType' options to be provided") + } + // Create MSK cluster config const mskClusterConfig = new CfnConfiguration(this, "migrationMSKClusterConfig", { name: `migration-msk-config-${props.stage}`, @@ -34,10 +55,8 @@ export class MigrationAssistanceStack extends Stack { vpc: props.vpc, allowAllOutbound: false }); - // This will allow all ip access for all TCP ports by default when public access is enabled, - // it should be updated if further restriction is desired - if (props.mskEnablePublicEndpoints) { - mskSecurityGroup.addIngressRule(Peer.anyIpv4(), Port.allTcp()) + if (props.mskEnablePublicEndpoints && props.mskRestrictPublicAccessTo && props.mskRestrictPublicAccessType) { + mskSecurityGroup.addIngressRule(this.getPublicEndpointAccess(props.mskRestrictPublicAccessTo, props.mskRestrictPublicAccessType), Port.allTcp()) } mskSecurityGroup.addIngressRule(mskSecurityGroup, Port.allTraffic()) new StringParameter(this, 'SSMParameterMSKAccessGroupId', { diff --git a/deployment/cdk/opensearch-service-migration/lib/msk-utility-stack.ts b/deployment/cdk/opensearch-service-migration/lib/msk-utility-stack.ts index 77ab9ada0c..6d5d6cf428 100644 --- a/deployment/cdk/opensearch-service-migration/lib/msk-utility-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/msk-utility-stack.ts @@ -9,7 +9,7 @@ import {Provider} from "aws-cdk-lib/custom-resources"; import * as path from "path"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; -export interface mskUtilityStackProps extends StackPropsExt { +export interface MskUtilityStackProps extends StackPropsExt { readonly vpc: IVpc, readonly mskEnablePublicEndpoints?: boolean } @@ -21,7 +21,7 @@ export interface mskUtilityStackProps extends StackPropsExt { */ export class MSKUtilityStack extends Stack { - constructor(scope: Construct, id: string, props: mskUtilityStackProps) { + constructor(scope: Construct, id: string, props: MskUtilityStackProps) { super(scope, id, props); const mskARN = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/mskClusterARN`) @@ -32,7 +32,7 @@ export class MSKUtilityStack extends Stack { const lambdaInvokeStatement = new PolicyStatement({ effect: Effect.ALLOW, actions: ["lambda:InvokeFunction"], - resources: ["*"] + resources: [`arn:aws:lambda:${props.env?.region}:${props.env?.account}:function:OSMigrations*`] }) // Updating connectivity for an MSK cluster requires some VPC permissions // (https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazonmanagedstreamingforapachekafka.html#amazonmanagedstreamingforapachekafka-cluster) diff --git a/deployment/cdk/opensearch-service-migration/lib/network-stack.ts b/deployment/cdk/opensearch-service-migration/lib/network-stack.ts index 0f2f2c02e5..3854945da4 100644 --- a/deployment/cdk/opensearch-service-migration/lib/network-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/network-stack.ts @@ -8,7 +8,7 @@ import {Construct} from "constructs"; import {StackPropsExt} from "./stack-composer"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; -export interface networkStackProps extends StackPropsExt { +export interface NetworkStackProps extends StackPropsExt { readonly vpcId?: string readonly availabilityZoneCount?: number } @@ -16,7 +16,7 @@ export interface networkStackProps extends StackPropsExt { export class NetworkStack extends Stack { public readonly vpc: IVpc; - constructor(scope: Construct, id: string, props: networkStackProps) { + constructor(scope: Construct, id: string, props: NetworkStackProps) { super(scope, id, props); // Retrieve original deployment VPC for addon deployments @@ -65,7 +65,7 @@ export class NetworkStack extends Stack { }); // Create a default SG which only allows members of this SG to access the Domain endpoints - const defaultSecurityGroup = new SecurityGroup(this, 'domainMigrationAccessSG', { + const defaultSecurityGroup = new SecurityGroup(this, 'osClusterAccessSG', { vpc: this.vpc, allowAllOutbound: false, }); diff --git a/deployment/cdk/opensearch-service-migration/lib/opensearch-domain-stack.ts b/deployment/cdk/opensearch-service-migration/lib/opensearch-domain-stack.ts index 86992ed1b0..8bbe3bc126 100644 --- a/deployment/cdk/opensearch-service-migration/lib/opensearch-domain-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/opensearch-domain-stack.ts @@ -16,7 +16,7 @@ import {StackPropsExt} from "./stack-composer"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; -export interface opensearchDomainStackProps extends StackPropsExt { +export interface OpensearchDomainStackProps extends StackPropsExt { readonly version: EngineVersion, readonly domainName: string, readonly dataNodeInstanceType?: string, @@ -52,7 +52,26 @@ export interface opensearchDomainStackProps extends StackPropsExt { export class OpenSearchDomainStack extends Stack { - constructor(scope: Construct, id: string, props: opensearchDomainStackProps) { + createSSMParameters(domain: Domain, adminUserName: string|undefined, adminUserSecret: ISecret|undefined, stage: string, deployId: string) { + new StringParameter(this, 'SSMParameterOpenSearchEndpoint', { + description: 'OpenSearch migration parameter for OpenSearch endpoint', + parameterName: `/migration/${stage}/${deployId}/osClusterEndpoint`, + stringValue: domain.domainEndpoint + }); + + if (domain.masterUserPassword && !adminUserSecret) { + console.log(`An OpenSearch domain fine-grained access control user was configured without an existing Secrets Manager secret, will not create SSM Parameter: /migration/${stage}/${deployId}/osUserAndSecret`) + } + else if (domain.masterUserPassword && adminUserSecret) { + new StringParameter(this, 'SSMParameterOpenSearchFGACUserAndSecretArn', { + description: 'OpenSearch migration parameter for OpenSearch configured fine-grained access control user and associated Secrets Manager secret ARN ', + parameterName: `/migration/${stage}/${deployId}/osUserAndSecretArn`, + stringValue: `${adminUserName} ${adminUserSecret.secretArn}` + }); + } + } + + constructor(scope: Construct, id: string, props: OpensearchDomainStackProps) { super(scope, id, props); const deployId = props.addOnMigrationDeployId ? props.addOnMigrationDeployId : props.defaultDeployId @@ -104,7 +123,6 @@ export class OpenSearchDomainStack extends Stack { } } - const domain = new Domain(this, 'Domain', { version: props.version, domainName: props.domainName, @@ -147,21 +165,7 @@ export class OpenSearchDomainStack extends Stack { removalPolicy: props.domainRemovalPolicy }); - new StringParameter(this, 'SSMParameterOpenSearchEndpoint', { - description: 'OpenSearch migration parameter for OpenSearch endpoint', - parameterName: `/migration/${props.stage}/${deployId}/osClusterEndpoint`, - stringValue: domain.domainEndpoint - }); + this.createSSMParameters(domain, adminUserName, adminUserSecret, props.stage, deployId) - if (domain.masterUserPassword && !adminUserSecret) { - console.log(`An OpenSearch domain fine-grained access control user was configured without an existing Secrets Manager secret, will not create SSM Parameter: /migration/${props.stage}/${deployId}/osUserAndSecret`) - } - else if (domain.masterUserPassword && adminUserSecret) { - new StringParameter(this, 'SSMParameterOpenSearchFGACUserAndSecretArn', { - description: 'OpenSearch migration parameter for OpenSearch configured fine-grained access control user and associated Secrets Manager secret ARN ', - parameterName: `/migration/${props.stage}/${deployId}/osUserAndSecretArn`, - stringValue: `${adminUserName} ${adminUserSecret.secretArn}` - }); - } } } diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts index e8318eb26d..5492761a21 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts @@ -4,7 +4,7 @@ import {PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; -import {CloudMapOptions, ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service"; +import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts index e03f6337bf..2b91aa289d 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts @@ -18,7 +18,6 @@ export class MigrationConsoleStack extends MigrationServiceCore { constructor(scope: Construct, id: string, props: MigrationConsoleProps) { super(scope, id, props) let securityGroups = [ - // TODO see about egress rule change here SecurityGroup.fromSecurityGroupId(this, "serviceConnectSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceConnectSecurityGroupId`)), SecurityGroup.fromSecurityGroupId(this, "mskAccessSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/mskAccessSecurityGroupId`)), SecurityGroup.fromSecurityGroupId(this, "defaultDomainAccessSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/osAccessSecurityGroupId`)), diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts index 07b960471f..b2566dd09f 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts @@ -44,6 +44,31 @@ export interface MigrationServiceCoreProps extends StackPropsExt { export class MigrationServiceCore extends Stack { + // Use CDK escape hatch to modify the underlying CFN for the generated AWS::ServiceDiscovery::Service to allow + // multiple DnsRecords. GitHub issue can be found here: https://github.com/aws/aws-cdk/issues/18894 + addServiceDiscoveryRecords(fargateService: FargateService, serviceDiscoveryPort: number|undefined) { + const multipleDnsRecords = { + DnsRecords: [ + { + TTL: 10, + Type: "A" + }, + { + TTL: 10, + Type: "SRV" + } + ] + } + const cloudMapCfn = fargateService.node.findChild("CloudmapService") + const cloudMapServiceCfn = cloudMapCfn.node.defaultChild as DiscoveryCfnService + cloudMapServiceCfn.addPropertyOverride("DnsConfig", multipleDnsRecords) + + if (serviceDiscoveryPort) { + const fargateCfn = fargateService.node.defaultChild as FargateCfnService + fargateCfn.addPropertyOverride("ServiceRegistries.0.Port", serviceDiscoveryPort) + } + } + createService(props: MigrationServiceCoreProps) { if ((!props.dockerFilePath && !props.dockerImageRegistryName) || (props.dockerFilePath && props.dockerImageRegistryName)) { throw new Error(`Exactly one option [dockerFilePath, dockerImageRegistryName] is required to create the "${props.serviceName}" service`) @@ -151,29 +176,9 @@ export class MigrationServiceCore extends Stack { }, cloudMapOptions: cloudMapOptions }); - // Use CDK escape hatch to modify the underlying CFN for the generated AWS::ServiceDiscovery::Service to allow - // multiple DnsRecords. GitHub issue can be found here: https://github.com/aws/aws-cdk/issues/18894 - if (props.serviceDiscoveryEnabled) { - const multipleDnsRecords = { - DnsRecords: [ - { - TTL: 10, - Type: "A" - }, - { - TTL: 10, - Type: "SRV" - } - ] - } - const cloudMapCfn = fargateService.node.findChild("CloudmapService") - const cloudMapServiceCfn = cloudMapCfn.node.defaultChild as DiscoveryCfnService - cloudMapServiceCfn.addPropertyOverride("DnsConfig", multipleDnsRecords) - if (props.serviceDiscoveryPort) { - const fargateCfn = fargateService.node.defaultChild as FargateCfnService - fargateCfn.addPropertyOverride("ServiceRegistries.0.Port", props.serviceDiscoveryPort) - } + if (props.serviceDiscoveryEnabled) { + this.addServiceDiscoveryRecords(fargateService, props.serviceDiscoveryPort) } } diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts index 6760687f78..54b3732a5c 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts @@ -1,6 +1,6 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {MountPoint, PortMapping, Protocol, Volume} from "aws-cdk-lib/aws-ecs"; +import {MountPoint, Volume} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts index bf676368ac..cf1f81f384 100644 --- a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts +++ b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts @@ -77,6 +77,8 @@ export class StackComposer { const migrationAssistanceEnabled = getContextForType('migrationAssistanceEnabled', 'boolean') const mskARN = getContextForType('mskARN', 'string') const mskEnablePublicEndpoints = getContextForType('mskEnablePublicEndpoints', 'boolean') + const mskRestrictPublicAccessTo = getContextForType('mskRestrictPublicAccessTo', 'string') + const mskRestrictPublicAccessType = getContextForType('mskRestrictPublicAccessType', 'string') const mskBrokerNodeCount = getContextForType('mskBrokerNodeCount', 'number') const addOnMigrationDeployId = getContextForType('addOnMigrationDeployId', 'string') const captureProxyESServiceEnabled = getContextForType('captureProxyESServiceEnabled', 'boolean') @@ -219,6 +221,8 @@ export class StackComposer { trafficComparatorEnabled: trafficComparatorServiceEnabled, mskImportARN: mskARN, mskEnablePublicEndpoints: mskEnablePublicEndpoints, + mskRestrictPublicAccessTo: mskRestrictPublicAccessTo, + mskRestrictPublicAccessType: mskRestrictPublicAccessType, mskBrokerNodeCount: mskBrokerNodeCount, stackName: `OSMigrations-${stage}-${region}-MigrationInfra`, description: "This stack contains resources to assist migrating an OpenSearch Service domain", @@ -456,8 +460,8 @@ export class StackComposer { } // Access policies can provide a single Statement block or an array of Statement blocks if (Array.isArray(statements)) { - for (let i = 0; i < statements.length; i++) { - const statement = PolicyStatement.fromJson(statements[i]) + for (let statementBlock of statements) { + const statement = PolicyStatement.fromJson(statementBlock) accessPolicies.push(statement) } } diff --git a/deployment/cdk/opensearch-service-migration/options.md b/deployment/cdk/opensearch-service-migration/options.md index 7bde6bf141..5aec5eb5f6 100644 --- a/deployment/cdk/opensearch-service-migration/options.md +++ b/deployment/cdk/opensearch-service-migration/options.md @@ -71,8 +71,10 @@ These tables list all CDK context configuration values a user can specify for th | availabilityZoneCount (shared) | number | 1 | The number of Availability Zones for the Domain to use. If not specified a single AZ is used. If specified the Domain CDK construct requires at least 2 AZs | ### MSK(Kafka) Options -| Name | Type | Example | Description | -|----------------------------------|---------|---------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------| -| mskEnablePublicEndpoints | boolean | true | Specify if public endpoints should be enabled on the MSK cluster | -| mskBrokerNodeCount | number | 2 | The number of broker nodes in the MSK cluster | -| mskARN (Not currently available) | string | `"arn:aws:kafka:us-east-2:12345678912:cluster/msk-cluster-test/81fbae45-5d25-44bb-aff0-108e71cc079b-7"` | Supply an existing MSK cluster ARN to use. **NOTE** As MSK is using an L1 construct this is not currently available for use | +| Name | Type | Example | Description | +|----------------------------------|---------|---------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| mskEnablePublicEndpoints | boolean | true | Specify if public endpoints should be enabled on the MSK cluster | +| mskRestrictPublicAccessTo | string | "10.0.0.0/32", "2002::1234:abcd:ffff:c0a8:101/64", "pl-123456abcde123456", "sg-123456789abcdefgh" | Restrict access for the public endpoints to a specific audience. For ipv4 and ipv6 this will be a CIDR range of IP addresses and for prefixList and securityGroupId this will be a particular id. Open access to anyone can be achieved by setting a value of (0.0.0.0/0) for ipv4 or (::/0) for ipv6. **Required for mskEnablePublicEndpoints**. | +| mskRestrictPublicAccessType | string | "ipv4", "ipv6", "prefixList", "securityGroupId" | Specify the type used for mskRestrictPublicAccessTo from the provided examples. **Required for mskEnablePublicEndpoints**. | +| mskBrokerNodeCount | number | 2 | The number of broker nodes in the MSK cluster | +| mskARN (Not currently available) | string | `"arn:aws:kafka:us-east-2:12345678912:cluster/msk-cluster-test/81fbae45-5d25-44bb-aff0-108e71cc079b-7"` | Supply an existing MSK cluster ARN to use. **NOTE** As MSK is using an L1 construct this is not currently available for use | diff --git a/deployment/cdk/opensearch-service-migration/package.json b/deployment/cdk/opensearch-service-migration/package.json index 3967657298..7a3e1b0feb 100644 --- a/deployment/cdk/opensearch-service-migration/package.json +++ b/deployment/cdk/opensearch-service-migration/package.json @@ -1,6 +1,12 @@ { "name": "opensearch-service-domain-cdk", "version": "0.1.0", + "description": "CDK infrastructure for migrating to AWS OpenSearch", + "license": "Apache-2.0", + "author": { + "name": "Amazon Web Services", + "url": "https://github.com/opensearch-project" + }, "scripts": { "build": "tsc", "watch": "tsc -w", From e65124b57a56238a4da1cb0e5ea2531a625ed6b4 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Wed, 8 Nov 2023 10:04:10 -0500 Subject: [PATCH 15/55] Allow Existing App Registry for CDK (#391) * MIGRATIONS-1411: Allow adding CDK stacks to an existing AppRegistry application Signed-off-by: Tanner Lewis --- .../opensearch-service-migration/bin/app.ts | 7 +++++++ .../buildDockerImages.sh | 2 +- .../lib/stack-composer.ts | 18 +++++++++++++++++- .../package-lock.json | 13 +++++++++++++ .../opensearch-service-migration/package.json | 1 + 5 files changed, 39 insertions(+), 2 deletions(-) diff --git a/deployment/cdk/opensearch-service-migration/bin/app.ts b/deployment/cdk/opensearch-service-migration/bin/app.ts index e188f1ca26..e5597a2d80 100644 --- a/deployment/cdk/opensearch-service-migration/bin/app.ts +++ b/deployment/cdk/opensearch-service-migration/bin/app.ts @@ -9,7 +9,14 @@ const version = readFileSync('../../../VERSION', 'utf-8') Tags.of(app).add("migration_deployment", version) const account = process.env.CDK_DEFAULT_ACCOUNT const region = process.env.CDK_DEFAULT_REGION +// Environment setting to allow providing an existing AWS AppRegistry application ARN which each created CDK stack +// from this CDK app will be added to. +const migrationsAppRegistryARN = process.env.MIGRATIONS_APP_REGISTRY_ARN +if (migrationsAppRegistryARN) { + console.info(`App Registry mode is enabled for CFN stack tracking. Will attempt to import the App Registry application from the MIGRATIONS_APP_REGISTRY_ARN env variable of ${migrationsAppRegistryARN} and looking in the configured region of ${region}`) +} new StackComposer(app, { + migrationsAppRegistryARN: migrationsAppRegistryARN, env: { account: account, region: region } }); \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/buildDockerImages.sh b/deployment/cdk/opensearch-service-migration/buildDockerImages.sh index 798f21f133..f9a4bee1dc 100755 --- a/deployment/cdk/opensearch-service-migration/buildDockerImages.sh +++ b/deployment/cdk/opensearch-service-migration/buildDockerImages.sh @@ -6,4 +6,4 @@ script_dir_abs_path=$(dirname "$script_abs_path") cd $script_dir_abs_path || exit cd ../../../TrafficCapture || exit -./gradlew :dockerSolution:buildDockerImages \ No newline at end of file +./gradlew :dockerSolution:buildDockerImages -x test \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts index cf1f81f384..7e117a19f4 100644 --- a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts +++ b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts @@ -18,6 +18,7 @@ import {CaptureProxyStack} from "./service-stacks/capture-proxy-stack"; import {ElasticsearchStack} from "./service-stacks/elasticsearch-stack"; import {KafkaBrokerStack} from "./service-stacks/kafka-broker-stack"; import {KafkaZookeeperStack} from "./service-stacks/kafka-zookeeper-stack"; +import {Application} from "@aws-cdk/aws-servicecatalogappregistry-alpha"; export interface StackPropsExt extends StackProps { readonly stage: string, @@ -25,10 +26,21 @@ export interface StackPropsExt extends StackProps { readonly addOnMigrationDeployId?: string } +export interface StackComposerProps extends StackProps { + readonly migrationsAppRegistryARN?: string +} + export class StackComposer { public stacks: Stack[] = []; - constructor(scope: Construct, props: StackProps) { + private addStacksToAppRegistry(scope: Construct, appRegistryAppARN: string, allStacks: Stack[]) { + for (let stack of allStacks) { + const appRegistryApp = Application.fromApplicationArn(stack, 'AppRegistryApplicationImport', appRegistryAppARN) + appRegistryApp.associateApplicationWithStack(stack) + } + } + + constructor(scope: Construct, props: StackComposerProps) { const defaultValues: { [x: string]: (any); } = defaultValuesJson const account = props.env?.account @@ -424,6 +436,10 @@ export class StackComposer { this.stacks.push(migrationConsoleStack) } + if (props.migrationsAppRegistryARN) { + this.addStacksToAppRegistry(scope, props.migrationsAppRegistryARN, this.stacks) + } + function getContextForType(optionName: string, expectedType: string): any { const option = contextJSON[optionName] diff --git a/deployment/cdk/opensearch-service-migration/package-lock.json b/deployment/cdk/opensearch-service-migration/package-lock.json index 1f5925cffd..ddaa7f49c8 100644 --- a/deployment/cdk/opensearch-service-migration/package-lock.json +++ b/deployment/cdk/opensearch-service-migration/package-lock.json @@ -8,6 +8,7 @@ "name": "opensearch-service-domain-cdk", "version": "0.1.0", "dependencies": { + "@aws-cdk/aws-servicecatalogappregistry-alpha": "2.100.0-alpha.0", "@aws-sdk/client-kafka": "^3.410.0", "@aws-sdk/client-lambda": "^3.359.0", "@types/aws-lambda": "^8.10.117", @@ -55,6 +56,18 @@ "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.0.1.tgz", "integrity": "sha512-DDt4SLdLOwWCjGtltH4VCST7hpOI5DzieuhGZsBpZ+AgJdSI2GCjklCXm0GCTwJG/SolkL5dtQXyUKgg9luBDg==" }, + "node_modules/@aws-cdk/aws-servicecatalogappregistry-alpha": { + "version": "2.100.0-alpha.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-servicecatalogappregistry-alpha/-/aws-servicecatalogappregistry-alpha-2.100.0-alpha.0.tgz", + "integrity": "sha512-+CVSZ9KVspdXENqD8TV3inD1TNybvgdBwq00t0b9xbFCZpsRHKeD3bkU9ZTd/0+zXhxDDkJyAr09TJ3iJUrKFw==", + "engines": { + "node": ">= 14.15.0" + }, + "peerDependencies": { + "aws-cdk-lib": "^2.100.0", + "constructs": "^10.0.0" + } + }, "node_modules/@aws-crypto/crc32": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-3.0.0.tgz", diff --git a/deployment/cdk/opensearch-service-migration/package.json b/deployment/cdk/opensearch-service-migration/package.json index 7a3e1b0feb..49cf53e3e4 100644 --- a/deployment/cdk/opensearch-service-migration/package.json +++ b/deployment/cdk/opensearch-service-migration/package.json @@ -23,6 +23,7 @@ "typescript": "~4.9.4" }, "dependencies": { + "@aws-cdk/aws-servicecatalogappregistry-alpha": "2.100.0-alpha.0", "@aws-sdk/client-kafka": "^3.410.0", "@aws-sdk/client-lambda": "^3.359.0", "@types/aws-lambda": "^8.10.117", From 7130e2740276c80b24559b62bec1cea102d16d10 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Wed, 8 Nov 2023 15:42:52 -0500 Subject: [PATCH 16/55] Bugfix, at least for when debugging. If the byte bufs for a tuple's request have already been output and closed, just print CLOSED in toString() rather than crashing. Signed-off-by: Greg Schohn --- .../replay/SourceTargetCaptureTuple.java | 6 +++--- .../replay/datatypes/TransformedPackets.java | 14 +++++++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java index d1282c3729..eff13575a2 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java @@ -189,9 +189,9 @@ public String toString() { sj.add("diagnosticLabel=").add(uniqueRequestKey.toString()); if (sourcePair != null) { sj.add("sourcePair=").add(sourcePair.toString()); } if (targetResponseDuration != null) { sj.add("targetResponseDuration=").add(targetResponseDuration+""); } - Optional.ofNullable(targetRequestData).filter(d->!d.isEmpty()).ifPresent(d-> sj.add("targetRequestData=") - .add(PrettyPrinter.httpPacketBufsToString(PrettyPrinter.HttpMessageType.REQUEST, - d.streamUnretained()))); + Optional.ofNullable(targetRequestData).ifPresent(d-> sj.add("targetRequestData=") + .add(d.isClosed() ? "CLOSED" : PrettyPrinter.httpPacketBufsToString( + PrettyPrinter.HttpMessageType.REQUEST, d.streamUnretained()))); Optional.ofNullable(targetResponseData).filter(d->!d.isEmpty()).ifPresent(d -> sj.add("targetResponseData=") .add(PrettyPrinter.httpPacketBytesToString(PrettyPrinter.HttpMessageType.RESPONSE, d))); sj.add("transformStatus=").add(transformationStatus+""); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/TransformedPackets.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/TransformedPackets.java index 7fafb801d8..ecda9bd3d9 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/TransformedPackets.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/TransformedPackets.java @@ -3,6 +3,7 @@ import io.netty.buffer.ByteBuf; import java.util.ArrayList; +import java.util.StringJoiner; import java.util.stream.Stream; public class TransformedPackets implements AutoCloseable { @@ -13,7 +14,8 @@ public boolean add(ByteBuf nextRequestPacket) { return data.add(nextRequestPacket.retainedDuplicate()); } - public boolean isEmpty() { return data.isEmpty(); } + public boolean isClosed() { return data == null; } + public boolean isEmpty() { return data != null && data.isEmpty(); } public int size() { return data.size(); } public Stream streamUnretained() { @@ -41,4 +43,14 @@ public void close() { data.clear(); data = null; } + + @Override + public String toString() { + if (data == null) { + return "CLOSED"; + } + return new StringJoiner(", ", TransformedPackets.class.getSimpleName() + "[", "]") + .add("data=" + data) + .toString(); + } } From 94fcf0155a34f8e088b3ebe92e517ffc3c514594 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Wed, 8 Nov 2023 15:55:18 -0500 Subject: [PATCH 17/55] Make the distinction between the ids of a TrafficStream for its requests and its channel by introducing a Hopefully, this clarifies some of the code (& hopefully logs). Signed-off-by: Greg Schohn --- .../migrations/replay/Accumulation.java | 13 ++-- .../replay/AccumulationCallbacks.java | 3 +- .../replay/ClientConnectionPool.java | 17 ++--- .../migrations/replay/ReplayEngine.java | 20 +++--- .../replay/RequestSenderOrchestrator.java | 63 +++++++++++-------- .../migrations/replay/TrafficReplayer.java | 5 +- .../datatypes/ConnectionReplaySession.java | 2 +- .../datatypes/ISourceTrafficChannelKey.java | 7 +++ .../replay/datatypes/ITrafficStreamKey.java | 6 +- .../datatypes/IndexedChannelInteraction.java | 11 ++++ .../datatypes/UniqueReplayerRequestKey.java | 6 +- .../datatypes/UniqueSourceRequestKey.java | 4 +- .../replay/RequestSenderOrchestratorTest.java | 4 +- ...afficToHttpTransactionAccumulatorTest.java | 7 ++- .../replay/TrafficReplayerTest.java | 13 ++-- 15 files changed, 108 insertions(+), 73 deletions(-) create mode 100644 TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/ISourceTrafficChannelKey.java create mode 100644 TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/IndexedChannelInteraction.java diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java index ce6e2c479c..a467e3802b 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java @@ -1,6 +1,7 @@ package org.opensearch.migrations.replay; import lombok.NonNull; +import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; @@ -20,20 +21,20 @@ enum State { ACCUMULATING_WRITES } - public final ITrafficStreamKey trafficStreamKey; + public final ISourceTrafficChannelKey trafficChannelKey; private RequestResponsePacketPair rrPair; AtomicLong newestPacketTimestampInMillis; State state; AtomicInteger numberOfResets; final int startingSourceRequestIndex; - public Accumulation(@NonNull ITrafficStreamKey trafficStreamKey, int startingSourceRequestIndex) { - this(trafficStreamKey, startingSourceRequestIndex, false); + public Accumulation(@NonNull ITrafficStreamKey trafficChannelKey, int startingSourceRequestIndex) { + this(trafficChannelKey, startingSourceRequestIndex, false); } - public Accumulation(@NonNull ITrafficStreamKey trafficStreamKey, + public Accumulation(@NonNull ITrafficStreamKey trafficChannelKey, int startingSourceRequestIndex, boolean dropObservationsLeftoverFromPrevious) { - this.trafficStreamKey = trafficStreamKey; + this.trafficChannelKey = trafficChannelKey; numberOfResets = new AtomicInteger(); this.newestPacketTimestampInMillis = new AtomicLong(0); this.startingSourceRequestIndex = startingSourceRequestIndex; @@ -50,7 +51,7 @@ public RequestResponsePacketPair getOrCreateTransactionPair() { } public UniqueReplayerRequestKey getRequestKey() { - return new UniqueReplayerRequestKey(trafficStreamKey, startingSourceRequestIndex, getIndexOfCurrentRequest()); + return new UniqueReplayerRequestKey(trafficChannelKey, startingSourceRequestIndex, getIndexOfCurrentRequest()); } public boolean hasSignaledRequests() { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java index 5b81050312..422f390f18 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java @@ -1,5 +1,6 @@ package org.opensearch.migrations.replay; +import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; @@ -11,6 +12,6 @@ public interface AccumulationCallbacks { void onFullDataReceived(UniqueReplayerRequestKey key, RequestResponsePacketPair rrpp); void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, List trafficStreamKeysBeingHeld); - void onConnectionClose(UniqueReplayerRequestKey key, Instant when, + void onConnectionClose(ISourceTrafficChannelKey key, int channelInteractionNumber, Instant when, List trafficStreamKeysBeingHeld); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java index 1c87cab07d..367ce5a4ed 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java @@ -15,6 +15,7 @@ import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.replay.datahandlers.NettyPacketToHttpConsumer; import org.opensearch.migrations.replay.datatypes.ConnectionReplaySession; +import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture; @@ -132,9 +133,9 @@ public void closeConnection(String connId) { } public Future - submitEventualChannelGet(UniqueReplayerRequestKey requestKey, boolean ignoreIfNotPresent) { + submitEventualChannelGet(ISourceTrafficChannelKey channelKey, boolean ignoreIfNotPresent) { ConnectionReplaySession channelFutureAndSchedule = - getCachedSession(requestKey, ignoreIfNotPresent); + getCachedSession(channelKey, ignoreIfNotPresent); if (channelFutureAndSchedule == null) { var rval = new DefaultPromise(eventLoopGroup.next()); rval.setSuccess(null); @@ -144,18 +145,18 @@ public void closeConnection(String connId) { if (channelFutureAndSchedule.getChannelFutureFuture() == null) { channelFutureAndSchedule.setChannelFutureFuture( getResilientClientChannelProducer(channelFutureAndSchedule.eventLoop, - requestKey.getTrafficStreamKey().getConnectionId())); + channelKey.getConnectionId())); } return channelFutureAndSchedule; }); } @SneakyThrows - public ConnectionReplaySession getCachedSession(UniqueReplayerRequestKey requestKey, boolean dontCreate) { - var crs = dontCreate ? connectionId2ChannelCache.getIfPresent(requestKey.getTrafficStreamKey().getConnectionId()) : - connectionId2ChannelCache.get(requestKey.getTrafficStreamKey().getConnectionId()); + public ConnectionReplaySession getCachedSession(ISourceTrafficChannelKey channelKey, boolean dontCreate) { + var crs = dontCreate ? connectionId2ChannelCache.getIfPresent(channelKey.getConnectionId()) : + connectionId2ChannelCache.get(channelKey.getConnectionId()); if (crs != null) { - crs.setCurrentConnectionId(requestKey); + crs.setChannelId(channelKey); } return crs; } @@ -180,7 +181,7 @@ public ConnectionReplaySession getCachedSession(UniqueReplayerRequestKey request if (channelAndFutureWork.hasWorkRemaining()) { log.atWarn().setMessage(()->"Work items are still remaining for this connection session" + "(last associated with connection=" + - channelAndFutureWork.getCurrentConnectionId() + + channelAndFutureWork.getChannelId() + "). " + channelAndFutureWork.calculateSizeSlowly() + " requests that were enqueued won't be run").log(); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java index 56780e657a..1ae430f143 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java @@ -5,6 +5,7 @@ import io.netty.util.concurrent.ScheduledFuture; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsLogger; +import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; import org.opensearch.migrations.replay.traffic.source.BufferedFlowController; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; @@ -97,13 +98,13 @@ public boolean isWorkOutstanding() { private DiagnosticTrackableCompletableFuture hookWorkFinishingUpdates(DiagnosticTrackableCompletableFuture future, Instant timestamp, - UniqueReplayerRequestKey requestKey, String taskDescription) { + Object stringableKey, String taskDescription) { return future.map(f->f .whenComplete((v,t)->Utils.setIfLater(lastCompletedSourceTimeEpochMs, timestamp.toEpochMilli())) .whenComplete((v,t)->{ var newCount = totalCountOfScheduledTasksOutstanding.decrementAndGet(); log.atInfo().setMessage(()->"Scheduled task '" + taskDescription + "' finished (" - + requestKey + ") decremented tasksOutstanding to "+newCount).log(); + + stringableKey + ") decremented tasksOutstanding to "+newCount).log(); }) .whenComplete((v,t)->contentTimeController.stopReadsPast(timestamp)) .whenComplete((v,t)->log.atDebug(). @@ -113,8 +114,8 @@ public boolean isWorkOutstanding() { ()->"Updating fields for callers to poll progress and updating backpressure"); } - private static void logStartOfWork(UniqueReplayerRequestKey requestKey, long newCount, Instant start, String label) { - log.atInfo().setMessage(()->"Scheduling '" + label + "' (" + requestKey + + private static void logStartOfWork(Object stringableKey, long newCount, Instant start, String label) { + log.atInfo().setMessage(()->"Scheduling '" + label + "' (" + stringableKey + ") to run at " + start + " incremented tasksOutstanding to "+ newCount).log(); } @@ -125,7 +126,8 @@ private static void logStartOfWork(UniqueReplayerRequestKey requestKey, long new final String label = "processing"; var start = timeShifter.transformSourceTimeToRealTime(originalStart); logStartOfWork(requestKey, newCount, start, label); - var result = networkSendOrchestrator.scheduleWork(requestKey, start.minus(EXPECTED_TRANSFORMATION_DURATION), task); + var result = networkSendOrchestrator.scheduleWork(requestKey.trafficStreamKey, + start.minus(EXPECTED_TRANSFORMATION_DURATION), task); return hookWorkFinishingUpdates(result, originalStart, requestKey, label); } @@ -148,13 +150,13 @@ private static void logStartOfWork(UniqueReplayerRequestKey requestKey, long new return hookWorkFinishingUpdates(sendResult, originalStart, requestKey, label); } - public void closeConnection(UniqueReplayerRequestKey requestKey, Instant timestamp) { + public void closeConnection(ISourceTrafficChannelKey channelKey, int channelInteractionNum, Instant timestamp) { var newCount = totalCountOfScheduledTasksOutstanding.incrementAndGet(); final String label = "close"; var atTime = timeShifter.transformSourceTimeToRealTime(timestamp); - logStartOfWork(requestKey, newCount, atTime, label); - var future = networkSendOrchestrator.scheduleClose(requestKey, atTime); - hookWorkFinishingUpdates(future, timestamp, requestKey, label); + logStartOfWork(channelKey, newCount, atTime, label); + var future = networkSendOrchestrator.scheduleClose(channelKey, channelInteractionNum, atTime); + hookWorkFinishingUpdates(future, timestamp, channelKey, label); } public DiagnosticTrackableCompletableFuture closeConnectionsAndShutdown() { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java index 40005df9b5..292b6ddb88 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java @@ -6,6 +6,8 @@ import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.replay.datahandlers.NettyPacketToHttpConsumer; import org.opensearch.migrations.replay.datatypes.ConnectionReplaySession; +import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; +import org.opensearch.migrations.replay.datatypes.IndexedChannelInteraction; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture; @@ -32,13 +34,13 @@ public RequestSenderOrchestrator(ClientConnectionPool clientConnectionPool) { } public DiagnosticTrackableCompletableFuture - scheduleWork(UniqueReplayerRequestKey requestKey, Instant timestamp, + scheduleWork(ISourceTrafficChannelKey channelKey, Instant timestamp, Supplier> task) { - var connectionSession = clientConnectionPool.getCachedSession(requestKey, false); + var connectionSession = clientConnectionPool.getCachedSession(channelKey, false); var finalTunneledResponse = new StringTrackableCompletableFuture(new CompletableFuture<>(), ()->"waiting for final signal to confirm close has finished"); - log.atDebug().setMessage(()->"Scheduling work for "+requestKey+" at time "+timestamp).log(); + log.atDebug().setMessage(()->"Scheduling work for "+channelKey+" at time "+timestamp).log(); connectionSession.eventLoop.schedule(()-> task.get().map(f->f.whenComplete((v,t) -> { if (t!=null) { @@ -58,22 +60,27 @@ public RequestSenderOrchestrator(ClientConnectionPool clientConnectionPool) { new StringTrackableCompletableFuture(new CompletableFuture<>(), ()->"waiting for final aggregated response"); log.atDebug().setMessage(()->"Scheduling request for "+requestKey+" at start time "+start).log(); - return asynchronouslyInvokeRunnableToSetupFuture(requestKey, false, finalTunneledResponse, - channelFutureAndRequestSchedule-> scheduleSendOnCffr(requestKey, channelFutureAndRequestSchedule, - finalTunneledResponse, start, interval, packets)); + return asynchronouslyInvokeRunnableToSetupFuture(requestKey.getTrafficStreamKey(), + requestKey.getSourceRequestIndex(), + false, finalTunneledResponse, + channelFutureAndRequestSchedule-> scheduleSendOnConnectionSession(requestKey, + channelFutureAndRequestSchedule, finalTunneledResponse, start, interval, packets)); } - public StringTrackableCompletableFuture scheduleClose(UniqueReplayerRequestKey requestKey, Instant timestamp) { + public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChannelKey channelKey, + int channelInteractionNum, Instant timestamp) { + var channelInteraction = new IndexedChannelInteraction(channelKey, channelInteractionNum); var finalTunneledResponse = new StringTrackableCompletableFuture(new CompletableFuture<>(), ()->"waiting for final signal to confirm close has finished"); - log.atDebug().setMessage(()->"Scheduling CLOSE for "+requestKey+" at time "+timestamp).log(); - asynchronouslyInvokeRunnableToSetupFuture(requestKey, true, finalTunneledResponse, + log.atDebug().setMessage(()->"Scheduling CLOSE for "+channelInteraction+" at time "+timestamp).log(); + asynchronouslyInvokeRunnableToSetupFuture(channelKey, channelInteractionNum, true, + finalTunneledResponse, channelFutureAndRequestSchedule-> - scheduleOnCffr(requestKey, channelFutureAndRequestSchedule, + scheduleOnConnectionSession(channelKey, channelInteractionNum, channelFutureAndRequestSchedule, finalTunneledResponse, timestamp, "close", () -> { - log.trace("Closing client connection " + requestKey); - clientConnectionPool.closeConnection(requestKey.getTrafficStreamKey().getConnectionId()); + log.trace("Closing client connection " + channelInteraction); + clientConnectionPool.closeConnection(channelKey.getConnectionId()); finalTunneledResponse.future.complete(null); }) ); @@ -81,7 +88,7 @@ public StringTrackableCompletableFuture scheduleClose(UniqueReplayerReques } private DiagnosticTrackableCompletableFuture - asynchronouslyInvokeRunnableToSetupFuture(UniqueReplayerRequestKey requestKey, + asynchronouslyInvokeRunnableToSetupFuture(ISourceTrafficChannelKey requestKey, int channelInteractionNumber, boolean ignoreIfChannelNotPresent, DiagnosticTrackableCompletableFuture finalTunneledResponse, Consumer successFn) { @@ -111,7 +118,7 @@ public StringTrackableCompletableFuture scheduleClose(UniqueReplayerReques runAfterChannelSetup(channelFutureAndRequestSchedule, finalTunneledResponse, cffr -> { - cffr.scheduleSequencer.add(requestKey.getReplayerRequestIndex(), + cffr.scheduleSequencer.add(channelInteractionNumber, () -> successFn.accept(channelFutureAndRequestSchedule), x -> x.run()); if (cffr.scheduleSequencer.hasPending()) { @@ -132,11 +139,12 @@ public StringTrackableCompletableFuture scheduleClose(UniqueReplayerReques return finalTunneledResponse; } - private void scheduleOnCffr(UniqueReplayerRequestKey requestKey, - ConnectionReplaySession channelFutureAndRequestSchedule, - StringTrackableCompletableFuture signalCleanupCompleteToFuture, - Instant atTime, String activityNameForLogging, Runnable task) { - log.atInfo().setMessage(()->requestKey + " scheduling " + activityNameForLogging + " at " + atTime).log(); + private void scheduleOnConnectionSession(ISourceTrafficChannelKey channelKey, int channelInteractionIdx, + ConnectionReplaySession channelFutureAndRequestSchedule, + StringTrackableCompletableFuture signalCleanupCompleteToFuture, + Instant atTime, String activityNameForLogging, Runnable task) { + var channelInteraction = new IndexedChannelInteraction(channelKey, channelInteractionIdx); + log.atInfo().setMessage(()->channelInteraction + " scheduling " + activityNameForLogging + " at " + atTime).log(); var schedule = channelFutureAndRequestSchedule.schedule; var eventLoop = channelFutureAndRequestSchedule.getInnerChannelFuture().channel().eventLoop(); @@ -145,7 +153,7 @@ private void scheduleOnCffr(UniqueReplayerRequestKey requestKey, var itemStartTimeOfPopped = schedule.removeFirstItem(); assert atTime.equals(itemStartTimeOfPopped): "Expected to have popped the item to match the start time for the responseFuture that finished"; - log.atDebug().setMessage(()->requestKey.toString() + " responseFuture completed - checking " + log.atDebug().setMessage(()->channelInteraction.toString() + " responseFuture completed - checking " + schedule + " for the next item to schedule").log(); Optional.ofNullable(schedule.peekFirstItem()).ifPresent(kvp-> { var sf = eventLoop.schedule(kvp.getValue(), getDelayFromNowMs(kvp.getKey()), TimeUnit.MILLISECONDS); @@ -164,7 +172,7 @@ private void scheduleOnCffr(UniqueReplayerRequestKey requestKey, if (!f.isSuccess()) { log.atError().setCause(f.cause()).setMessage(()->"Error scheduling task").log(); } else { - log.atInfo().setMessage(()->"scheduled future has finished for "+requestKey).log(); + log.atInfo().setMessage(()->"scheduled future has finished for "+channelInteraction).log(); } }); } else { @@ -173,21 +181,22 @@ private void scheduleOnCffr(UniqueReplayerRequestKey requestKey, } schedule.appendTask(atTime, task); - log.atTrace().setMessage(()->requestKey + " added a scheduled event at " + atTime + + log.atTrace().setMessage(()->channelInteraction + " added a scheduled event at " + atTime + "... " + schedule).log(); } - private void scheduleSendOnCffr(UniqueReplayerRequestKey requestKey, - ConnectionReplaySession channelFutureAndRequestSchedule, - StringTrackableCompletableFuture responseFuture, - Instant start, Duration interval, Stream packets) { + private void scheduleSendOnConnectionSession(UniqueReplayerRequestKey requestKey, + ConnectionReplaySession channelFutureAndRequestSchedule, + StringTrackableCompletableFuture responseFuture, + Instant start, Duration interval, Stream packets) { var eventLoop = channelFutureAndRequestSchedule.eventLoop; var packetReceiverRef = new AtomicReference(); Runnable packetSender = () -> sendNextPartAndContinue(() -> getPacketReceiver(requestKey, channelFutureAndRequestSchedule.getInnerChannelFuture(), packetReceiverRef), eventLoop, packets.iterator(), start, interval, new AtomicInteger(), responseFuture); - scheduleOnCffr(requestKey, channelFutureAndRequestSchedule, responseFuture, start, "send", packetSender); + scheduleOnConnectionSession(requestKey.trafficStreamKey, requestKey.getSourceRequestIndex(), + channelFutureAndRequestSchedule, responseFuture, start, "send", packetSender); } private void runAfterChannelSetup(ConnectionReplaySession channelFutureAndItsFutureRequests, diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index 8d0b4a2c5a..a35158bbb9 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -14,6 +14,7 @@ import org.opensearch.migrations.replay.datahandlers.IPacketFinalizingConsumer; import org.opensearch.migrations.replay.datahandlers.http.IHttpMessage; import org.opensearch.migrations.replay.datatypes.HttpRequestTransformationStatus; +import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.datatypes.TransformedPackets; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; @@ -632,10 +633,10 @@ private void commitTrafficStreams(List trafficStreamKeysBeing } @Override - public void onConnectionClose(UniqueReplayerRequestKey requestKey, Instant timestamp, + public void onConnectionClose(ISourceTrafficChannelKey channelKey, int channelInteractionNum, Instant timestamp, List trafficStreamKeysBeingHeld) { replayEngine.setFirstTimestamp(timestamp); - replayEngine.closeConnection(requestKey, timestamp); + replayEngine.closeConnection(channelKey, channelInteractionNum, timestamp); commitTrafficStreams(trafficStreamKeysBeingHeld); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/ConnectionReplaySession.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/ConnectionReplaySession.java index a56b83c557..ae6554c713 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/ConnectionReplaySession.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/ConnectionReplaySession.java @@ -32,7 +32,7 @@ public class ConnectionReplaySession { @Getter @Setter - private UniqueReplayerRequestKey currentConnectionId; + private ISourceTrafficChannelKey channelId; public ConnectionReplaySession(EventLoop eventLoop) { this.eventLoop = eventLoop; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/ISourceTrafficChannelKey.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/ISourceTrafficChannelKey.java new file mode 100644 index 0000000000..3a0eb4644c --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/ISourceTrafficChannelKey.java @@ -0,0 +1,7 @@ +package org.opensearch.migrations.replay.datatypes; + +public interface ISourceTrafficChannelKey { + String getNodeId(); + + String getConnectionId(); +} diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/ITrafficStreamKey.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/ITrafficStreamKey.java index 88b34303c7..ab9d6ced44 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/ITrafficStreamKey.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/ITrafficStreamKey.java @@ -1,9 +1,5 @@ package org.opensearch.migrations.replay.datatypes; -public interface ITrafficStreamKey { - String getNodeId(); - - String getConnectionId(); - +public interface ITrafficStreamKey extends ISourceTrafficChannelKey { int getTrafficStreamIndex(); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/IndexedChannelInteraction.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/IndexedChannelInteraction.java new file mode 100644 index 0000000000..e4643f0873 --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/IndexedChannelInteraction.java @@ -0,0 +1,11 @@ +package org.opensearch.migrations.replay.datatypes; + +import lombok.AllArgsConstructor; +import lombok.ToString; + +@AllArgsConstructor +@ToString +public class IndexedChannelInteraction { + public final ISourceTrafficChannelKey channelKey; + public final int index; +} diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueReplayerRequestKey.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueReplayerRequestKey.java index 556878d074..0e7d6bcafe 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueReplayerRequestKey.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueReplayerRequestKey.java @@ -4,18 +4,18 @@ @EqualsAndHashCode public class UniqueReplayerRequestKey extends UniqueSourceRequestKey { - public final ITrafficStreamKey trafficStreamKey; + public final ISourceTrafficChannelKey trafficStreamKey; public final int sourceRequestIndexOffsetAtFirstObservation; public final int replayerRequestIndex; - public UniqueReplayerRequestKey(ITrafficStreamKey streamKey, int sourceOffset, int replayerIndex) { + public UniqueReplayerRequestKey(ISourceTrafficChannelKey streamKey, int sourceOffset, int replayerIndex) { this.trafficStreamKey = streamKey; this.sourceRequestIndexOffsetAtFirstObservation = sourceOffset; this.replayerRequestIndex = replayerIndex; } @Override - public ITrafficStreamKey getTrafficStreamKey() { + public ISourceTrafficChannelKey getTrafficStreamKey() { return trafficStreamKey; } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueSourceRequestKey.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueSourceRequestKey.java index 5439f687b6..2951a1b4ba 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueSourceRequestKey.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueSourceRequestKey.java @@ -2,10 +2,8 @@ import com.google.common.base.Objects; -import java.util.StringJoiner; - public abstract class UniqueSourceRequestKey { - public abstract ITrafficStreamKey getTrafficStreamKey(); + public abstract ISourceTrafficChannelKey getTrafficStreamKey(); public abstract int getSourceRequestIndex(); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/RequestSenderOrchestratorTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/RequestSenderOrchestratorTest.java index f269fac0cf..83caf04167 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/RequestSenderOrchestratorTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/RequestSenderOrchestratorTest.java @@ -51,8 +51,8 @@ public void testThatSchedulingWorks() throws Exception { lastEndTime = startTimeForThisRequest.plus(perPacketShift.multipliedBy(requestPackets.size())); } var closeFuture = senderOrchestrator.scheduleClose( - TestRequestKey.getTestConnectionRequestId(NUM_REQUESTS_TO_SCHEDULE), - lastEndTime.plus(Duration.ofMillis(100))); + TestRequestKey.getTestConnectionRequestId(NUM_REQUESTS_TO_SCHEDULE).trafficStreamKey, + NUM_REQUESTS_TO_SCHEDULE, lastEndTime.plus(Duration.ofMillis(100))); Assertions.assertEquals(NUM_REQUESTS_TO_SCHEDULE, scheduledItems.size()); for (int i=0; i trafficStreamKeysBeingHeld) {} @Override - public void onConnectionClose(UniqueReplayerRequestKey key, Instant when, - List keysHeld) {} + public void onConnectionClose(ISourceTrafficChannelKey key, int channelInteractionNumber, + Instant when, + List trafficStreamKeysBeingHeld) { + } }); var tsList = trafficStreams.collect(Collectors.toList()); trafficStreams = tsList.stream(); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java index e1b156d2cd..637942bed6 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java @@ -5,6 +5,7 @@ import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; import org.opensearch.migrations.replay.traffic.source.InputStreamOfTraffic; @@ -170,8 +171,10 @@ public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStat List trafficStreamKeysBeingHeld) {} @Override - public void onConnectionClose(UniqueReplayerRequestKey key, Instant when, - List keysHeld) {} + public void onConnectionClose(ISourceTrafficChannelKey key, int channelInteractionNumber, + Instant when, + List trafficStreamKeysBeingHeld) { + } }); var bytes = synthesizeTrafficStreamsIntoByteArray(Instant.now(), 1); @@ -212,8 +215,10 @@ public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStat List trafficStreamKeysBeingHeld) {} @Override - public void onConnectionClose(UniqueReplayerRequestKey key, Instant when, - List keysHeld) {} + public void onConnectionClose(ISourceTrafficChannelKey key, int channelInteractionNumber, + Instant when, + List trafficStreamKeysBeingHeld) { + } } ); byte[] serializedChunks; From 4cd149478be5dae9c40e20968ed5635126da284b Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Wed, 8 Nov 2023 19:27:55 -0600 Subject: [PATCH 18/55] Bump Kafka version on Docker Signed-off-by: Brian Presley --- .../dockerSolution/src/main/docker/migrationConsole/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile index 368918ea58..d29bd9e4db 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile @@ -19,7 +19,7 @@ RUN chmod ug+x /root/humanReadableLogs.py RUN chmod ug+x /root/catIndices.sh WORKDIR /root/kafka-tools # Get kafka distribution and unpack to 'kafka' -RUN wget -qO- https://archive.apache.org/dist/kafka/3.5.0/kafka_2.13-3.5.0.tgz | tar --transform 's!^[^/]*!kafka!' -xvz +RUN wget -qO- https://archive.apache.org/dist/kafka/3.6.0/kafka_2.13-3.6.0.tgz | tar --transform 's!^[^/]*!kafka!' -xvz RUN wget -O kafka/libs/msk-iam-auth.jar https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.9/aws-msk-iam-auth-1.1.9-all.jar WORKDIR /root From 8999f4ce295cbd61527066a69ff292bc1c78f28f Mon Sep 17 00:00:00 2001 From: "mend-for-github.aaakk.us.kg[bot]" <50673670+mend-for-github.aaakk.us.kg[bot]@users.noreply.github.com> Date: Thu, 9 Nov 2023 03:56:38 +0000 Subject: [PATCH 19/55] Update dependency org.apache.kafka:kafka-clients to v3.6.0 --- TrafficCapture/captureKafkaOffloader/build.gradle | 2 +- TrafficCapture/trafficReplayer/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/TrafficCapture/captureKafkaOffloader/build.gradle b/TrafficCapture/captureKafkaOffloader/build.gradle index 013deb3db0..063cec0c81 100644 --- a/TrafficCapture/captureKafkaOffloader/build.gradle +++ b/TrafficCapture/captureKafkaOffloader/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation project(':coreUtilities') implementation 'org.projectlombok:lombok:1.18.26' implementation 'com.google.protobuf:protobuf-java:3.22.2' - implementation 'org.apache.kafka:kafka-clients:3.5.1' + implementation 'org.apache.kafka:kafka-clients:3.6.0' implementation 'software.amazon.msk:aws-msk-iam-auth:1.1.9' implementation 'org.slf4j:slf4j-api:2.0.7' testImplementation project(':captureProtobufs') diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle index 411a1d20ba..29d2e8cb31 100644 --- a/TrafficCapture/trafficReplayer/build.gradle +++ b/TrafficCapture/trafficReplayer/build.gradle @@ -47,7 +47,7 @@ dependencies { implementation group: 'io.github.resilience4j', name: 'resilience4j-ratelimiter', version:"${resilience4jVersion}" implementation group: 'io.github.resilience4j', name: 'resilience4j-retry', version:"${resilience4jVersion}" implementation group: 'io.netty', name: 'netty-all', version: '4.1.94.Final' - implementation group: 'org.apache.kafka', name: 'kafka-clients', version: '3.5.1' + implementation group: 'org.apache.kafka', name: 'kafka-clients', version: '3.6.0' implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.20.0' implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.20.0' implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl', version: '2.20.0' From 77010e0b64d82d5986f1aabb459db32830c0682b Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Thu, 9 Nov 2023 08:47:07 -0500 Subject: [PATCH 20/55] Refactoring and renaming. Make the constructor for RequestResponsePacketPair take a TrafficStreamKey so that it can immediately hold that. The first item in the held list is now used as the TrafficStreamKey component of the requestKey for the object being constructed. There was a critical bugfix within RequestSenderOrchestrator to cleanup an issue introduced in the last commit. When calling asynchronouslyInvokeRunnableToSetupFuture, the REPLAYER request index must be used as the sequence id. The source index was being passed and that was causing the sequencer to hang since the earlier items were never received. Lifecycle management of the threads in the ClientConnectionPool have had a couple tweaks to make shutting down smoother for the FullReplayerTest. The pool's guava cache will throw an exception when attempting to load a new session if the event loop group was shutting down. Within the test code, it now polls all of the threads within the process until the targetConnectionPool threads have been purged. WIthin the TrafficReplayer, Signed-off-by: Greg Schohn --- .../migrations/replay/Accumulation.java | 7 +- .../replay/AccumulationCallbacks.java | 11 +- ...edTrafficToHttpTransactionAccumulator.java | 19 +- .../replay/ClientConnectionPool.java | 9 +- .../migrations/replay/ReplayEngine.java | 7 +- .../replay/RequestResponsePacketPair.java | 16 +- .../replay/RequestSenderOrchestrator.java | 30 +-- .../datatypes/UniqueReplayerRequestKey.java | 13 +- ...xpiringTrafficStreamMapSequentialTest.java | 2 +- ...ExpiringTrafficStreamMapUnorderedTest.java | 2 +- .../replay/FullTrafficReplayerTest.java | 171 ++++++++++++------ 11 files changed, 184 insertions(+), 103 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java index a467e3802b..b3f65daf2a 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Accumulation.java @@ -42,16 +42,17 @@ public Accumulation(@NonNull ITrafficStreamKey trafficChannelKey, dropObservationsLeftoverFromPrevious ? State.IGNORING_LAST_REQUEST : State.WAITING_FOR_NEXT_READ_CHUNK; } - public RequestResponsePacketPair getOrCreateTransactionPair() { + public RequestResponsePacketPair getOrCreateTransactionPair(ITrafficStreamKey forTrafficStreamKey) { if (rrPair != null) { return rrPair; } - rrPair = new RequestResponsePacketPair(); + rrPair = new RequestResponsePacketPair(forTrafficStreamKey); return rrPair; } public UniqueReplayerRequestKey getRequestKey() { - return new UniqueReplayerRequestKey(trafficChannelKey, startingSourceRequestIndex, getIndexOfCurrentRequest()); + return new UniqueReplayerRequestKey(getRrPair().getBeginningTrafficStreamKey(), + startingSourceRequestIndex, getIndexOfCurrentRequest()); } public boolean hasSignaledRequests() { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java index 422f390f18..21c6e15f97 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java @@ -1,5 +1,6 @@ package org.opensearch.migrations.replay; +import lombok.NonNull; import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; @@ -8,10 +9,10 @@ import java.util.List; public interface AccumulationCallbacks { - void onRequestReceived(UniqueReplayerRequestKey key, HttpMessageAndTimestamp request); - void onFullDataReceived(UniqueReplayerRequestKey key, RequestResponsePacketPair rrpp); + void onRequestReceived(@NonNull UniqueReplayerRequestKey key, @NonNull HttpMessageAndTimestamp request); + void onFullDataReceived(@NonNull UniqueReplayerRequestKey key, @NonNull RequestResponsePacketPair rrpp); void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, - List trafficStreamKeysBeingHeld); - void onConnectionClose(ISourceTrafficChannelKey key, int channelInteractionNumber, Instant when, - List trafficStreamKeysBeingHeld); + @NonNull List trafficStreamKeysBeingHeld); + void onConnectionClose(@NonNull ISourceTrafficChannelKey key, int channelInteractionNumber, @NonNull Instant when, + @NonNull List trafficStreamKeysBeingHeld); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java index 1311b0be2b..817cc68810 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java @@ -207,19 +207,24 @@ private static Optional handleObservationForSkipState(Accumul return Optional.empty(); } + private static List getTrafficStreamsHeldByAccum(Accumulation accum) { + return accum.hasRrPair() ? accum.getRrPair().trafficStreamKeysBeingHeld : List.of(); + } + private Optional handleCloseObservationThatAffectEveryState(Accumulation accum, TrafficObservation observation, @NonNull ITrafficStreamKey trafficStreamKey, Instant timestamp) { if (observation.hasClose()) { - accum.getOrCreateTransactionPair().holdTrafficStream(trafficStreamKey); + accum.getOrCreateTransactionPair(trafficStreamKey).holdTrafficStream(trafficStreamKey); rotateAccumulationIfNecessary(trafficStreamKey.getConnectionId(), accum); closedConnectionCounter.incrementAndGet(); - listener.onConnectionClose(accum.getRequestKey(), timestamp, accum.getRrPair().trafficStreamKeysBeingHeld); + listener.onConnectionClose(accum.trafficChannelKey, accum.getIndexOfCurrentRequest(), + timestamp, getTrafficStreamsHeldByAccum(accum)); return Optional.of(CONNECTION_STATUS.CLOSED); } else if (observation.hasConnectionException()) { - accum.getOrCreateTransactionPair().holdTrafficStream(trafficStreamKey); + accum.getOrCreateTransactionPair(trafficStreamKey).holdTrafficStream(trafficStreamKey); rotateAccumulationIfNecessary(trafficStreamKey.getConnectionId(), accum); exceptionConnectionCounter.incrementAndGet(); accum.resetForNextRequest(); @@ -244,7 +249,7 @@ private Optional handleObservationForReadState(@NonNull Accum if (!accum.hasRrPair()) { requestCounter.incrementAndGet(); } - var rrPair = accum.getOrCreateTransactionPair(); + var rrPair = accum.getOrCreateTransactionPair(trafficStreamKey); log.atTrace().setMessage(() -> "Adding request data for accum[" + connectionId + "]=" + accum).log(); rrPair.addRequestData(timestamp, observation.getRead().getData().toByteArray()); log.atTrace().setMessage(() -> "Added request data for accum[" + connectionId + "]=" + accum).log(); @@ -253,7 +258,7 @@ private Optional handleObservationForReadState(@NonNull Accum handleEndOfRequest(accum); } else if (observation.hasReadSegment()) { log.atTrace().setMessage(()->"Adding request segment for accum[" + connectionId + "]=" + accum).log(); - var rrPair = accum.getOrCreateTransactionPair(); + var rrPair = accum.getOrCreateTransactionPair(trafficStreamKey); if (rrPair.requestData == null) { rrPair.requestData = new HttpMessageAndTimestamp.Request(timestamp); requestCounter.incrementAndGet(); @@ -394,8 +399,8 @@ private void fireAccumulationsCallbacksAndClose(Accumulation accumulation, } } finally { if (accumulation.hasSignaledRequests()) { - listener.onConnectionClose(accumulation.getRequestKey(), accumulation.getLastTimestamp(), - accumulation.hasRrPair() ? accumulation.getRrPair().trafficStreamKeysBeingHeld : List.of()); + listener.onConnectionClose(accumulation.trafficChannelKey, accumulation.getIndexOfCurrentRequest(), + accumulation.getLastTimestamp(), getTrafficStreamsHeldByAccum(accumulation)); } } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java index 367ce5a4ed..65b7649e0b 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java @@ -16,7 +16,6 @@ import org.opensearch.migrations.replay.datahandlers.NettyPacketToHttpConsumer; import org.opensearch.migrations.replay.datatypes.ConnectionReplaySession; import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; -import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; import org.opensearch.migrations.replay.util.StringTrackableCompletableFuture; @@ -28,6 +27,7 @@ @Slf4j public class ClientConnectionPool { + public static final String TARGET_CONNECTION_POOL_NAME = "targetConnectionPool"; private final URI serverUri; private final SslContext sslContext; public final NioEventLoopGroup eventLoopGroup; @@ -40,11 +40,14 @@ public ClientConnectionPool(URI serverUri, SslContext sslContext, int numThreads this.serverUri = serverUri; this.sslContext = sslContext; this.eventLoopGroup = - new NioEventLoopGroup(numThreads, new DefaultThreadFactory("targetConnectionPool")); + new NioEventLoopGroup(numThreads, new DefaultThreadFactory(TARGET_CONNECTION_POOL_NAME)); connectionId2ChannelCache = CacheBuilder.newBuilder().build(new CacheLoader<>() { @Override public ConnectionReplaySession load(final String s) { + if (eventLoopGroup.isShuttingDown()) { + throw new IllegalStateException("Event loop group is shutting down. Not creating a new session."); + } numConnectionsCreated.incrementAndGet(); log.trace("creating connection session"); // arguably the most only thing that matters here is associating this item with an @@ -133,7 +136,7 @@ public void closeConnection(String connId) { } public Future - submitEventualChannelGet(ISourceTrafficChannelKey channelKey, boolean ignoreIfNotPresent) { + submitEventualSessionGet(ISourceTrafficChannelKey channelKey, boolean ignoreIfNotPresent) { ConnectionReplaySession channelFutureAndSchedule = getCachedSession(channelKey, ignoreIfNotPresent); if (channelFutureAndSchedule == null) { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java index 1ae430f143..80e301feb4 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java @@ -6,6 +6,7 @@ import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.coreutils.MetricsLogger; import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; +import org.opensearch.migrations.replay.datatypes.IndexedChannelInteraction; import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; import org.opensearch.migrations.replay.traffic.source.BufferedFlowController; import org.opensearch.migrations.replay.util.DiagnosticTrackableCompletableFuture; @@ -154,7 +155,7 @@ public void closeConnection(ISourceTrafficChannelKey channelKey, int channelInte var newCount = totalCountOfScheduledTasksOutstanding.incrementAndGet(); final String label = "close"; var atTime = timeShifter.transformSourceTimeToRealTime(timestamp); - logStartOfWork(channelKey, newCount, atTime, label); + logStartOfWork(new IndexedChannelInteraction(channelKey, channelInteractionNum), newCount, atTime, label); var future = networkSendOrchestrator.scheduleClose(channelKey, channelInteractionNum, atTime); hookWorkFinishingUpdates(future, timestamp, channelKey, label); } @@ -163,10 +164,6 @@ public DiagnosticTrackableCompletableFuture closeConnectionsAndShu return networkSendOrchestrator.clientConnectionPool.closeConnectionsAndShutdown(); } - public Future shutdownNow() { - return networkSendOrchestrator.clientConnectionPool.shutdownNow(); - } - public int getNumConnectionsCreated() { return networkSendOrchestrator.clientConnectionPool.getNumConnectionsCreated(); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestResponsePacketPair.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestResponsePacketPair.java index 3f40983d58..21a3c3a65e 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestResponsePacketPair.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestResponsePacketPair.java @@ -1,6 +1,7 @@ package org.opensearch.migrations.replay; import com.google.common.base.Objects; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; @@ -25,6 +26,16 @@ public enum ReconstructionStatus { List trafficStreamKeysBeingHeld; ReconstructionStatus completionStatus; + public RequestResponsePacketPair(ITrafficStreamKey startingAtTrafficStreamKey) { + this.trafficStreamKeysBeingHeld = new ArrayList<>(); + this.trafficStreamKeysBeingHeld.add(startingAtTrafficStreamKey); + } + + @NonNull ITrafficStreamKey getBeginningTrafficStreamKey() { + assert trafficStreamKeysBeingHeld != null && !trafficStreamKeysBeingHeld.isEmpty(); + return trafficStreamKeysBeingHeld.get(0); + } + public void addRequestData(Instant packetTimeStamp, byte[] data) { if (log.isTraceEnabled()) { log.trace(this + " Adding request data: " + new String(data, StandardCharsets.UTF_8)); @@ -51,7 +62,10 @@ public void holdTrafficStream(ITrafficStreamKey trafficStreamKey) { if (trafficStreamKeysBeingHeld == null) { trafficStreamKeysBeingHeld = new ArrayList<>(); } - trafficStreamKeysBeingHeld.add(trafficStreamKey); + if (trafficStreamKeysBeingHeld.size() == 0 || + trafficStreamKey != trafficStreamKeysBeingHeld.get(trafficStreamKeysBeingHeld.size()-1)) { + trafficStreamKeysBeingHeld.add(trafficStreamKey); + } } private static final List emptyUnmodifiableList = List.of(); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java index 292b6ddb88..a0ce9c4378 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java @@ -61,9 +61,9 @@ public RequestSenderOrchestrator(ClientConnectionPool clientConnectionPool) { ()->"waiting for final aggregated response"); log.atDebug().setMessage(()->"Scheduling request for "+requestKey+" at start time "+start).log(); return asynchronouslyInvokeRunnableToSetupFuture(requestKey.getTrafficStreamKey(), - requestKey.getSourceRequestIndex(), + requestKey.getReplayerRequestIndex(), false, finalTunneledResponse, - channelFutureAndRequestSchedule-> scheduleSendOnConnectionSession(requestKey, + channelFutureAndRequestSchedule-> scheduleSendOnConnectionReplaySession(requestKey, channelFutureAndRequestSchedule, finalTunneledResponse, start, interval, packets)); } @@ -77,7 +77,7 @@ public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChanne asynchronouslyInvokeRunnableToSetupFuture(channelKey, channelInteractionNum, true, finalTunneledResponse, channelFutureAndRequestSchedule-> - scheduleOnConnectionSession(channelKey, channelInteractionNum, channelFutureAndRequestSchedule, + scheduleOnConnectionReplaySession(channelKey, channelInteractionNum, channelFutureAndRequestSchedule, finalTunneledResponse, timestamp, "close", () -> { log.trace("Closing client connection " + channelInteraction); clientConnectionPool.closeConnection(channelKey.getConnectionId()); @@ -93,7 +93,7 @@ public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChanne DiagnosticTrackableCompletableFuture finalTunneledResponse, Consumer successFn) { var channelFutureAndScheduleFuture = - clientConnectionPool.submitEventualChannelGet(requestKey, ignoreIfChannelNotPresent); + clientConnectionPool.submitEventualSessionGet(requestKey, ignoreIfChannelNotPresent); channelFutureAndScheduleFuture.addListener(submitFuture->{ if (!submitFuture.isSuccess()) { log.atError().setCause(submitFuture.cause()) @@ -101,7 +101,7 @@ public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChanne .log(); finalTunneledResponse.future.completeExceptionally(submitFuture.cause()); } else { - log.atTrace().setMessage(()->requestKey.toString() + " in submitFuture(success) callback").log(); + log.atInfo().setMessage(()->requestKey.toString() + " in submitFuture(success) callback").log(); var channelFutureAndRequestSchedule = ((ConnectionReplaySession) submitFuture.get()); if (channelFutureAndRequestSchedule == null) { finalTunneledResponse.future.complete(null); @@ -139,17 +139,17 @@ public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChanne return finalTunneledResponse; } - private void scheduleOnConnectionSession(ISourceTrafficChannelKey channelKey, int channelInteractionIdx, - ConnectionReplaySession channelFutureAndRequestSchedule, - StringTrackableCompletableFuture signalCleanupCompleteToFuture, - Instant atTime, String activityNameForLogging, Runnable task) { + private void scheduleOnConnectionReplaySession(ISourceTrafficChannelKey channelKey, int channelInteractionIdx, + ConnectionReplaySession channelFutureAndRequestSchedule, + StringTrackableCompletableFuture futureToBeCompletedByTask, + Instant atTime, String activityNameForLogging, Runnable task) { var channelInteraction = new IndexedChannelInteraction(channelKey, channelInteractionIdx); log.atInfo().setMessage(()->channelInteraction + " scheduling " + activityNameForLogging + " at " + atTime).log(); var schedule = channelFutureAndRequestSchedule.schedule; var eventLoop = channelFutureAndRequestSchedule.getInnerChannelFuture().channel().eventLoop(); - signalCleanupCompleteToFuture.map(f->f.whenComplete((v,t)-> { + futureToBeCompletedByTask.map(f->f.whenComplete((v,t)-> { var itemStartTimeOfPopped = schedule.removeFirstItem(); assert atTime.equals(itemStartTimeOfPopped): "Expected to have popped the item to match the start time for the responseFuture that finished"; @@ -185,17 +185,17 @@ private void scheduleOnConnectionSession(ISourceTrafficChannelKey channelKey "... " + schedule).log(); } - private void scheduleSendOnConnectionSession(UniqueReplayerRequestKey requestKey, - ConnectionReplaySession channelFutureAndRequestSchedule, - StringTrackableCompletableFuture responseFuture, - Instant start, Duration interval, Stream packets) { + private void scheduleSendOnConnectionReplaySession(UniqueReplayerRequestKey requestKey, + ConnectionReplaySession channelFutureAndRequestSchedule, + StringTrackableCompletableFuture responseFuture, + Instant start, Duration interval, Stream packets) { var eventLoop = channelFutureAndRequestSchedule.eventLoop; var packetReceiverRef = new AtomicReference(); Runnable packetSender = () -> sendNextPartAndContinue(() -> getPacketReceiver(requestKey, channelFutureAndRequestSchedule.getInnerChannelFuture(), packetReceiverRef), eventLoop, packets.iterator(), start, interval, new AtomicInteger(), responseFuture); - scheduleOnConnectionSession(requestKey.trafficStreamKey, requestKey.getSourceRequestIndex(), + scheduleOnConnectionReplaySession(requestKey.trafficStreamKey, requestKey.getSourceRequestIndex(), channelFutureAndRequestSchedule, responseFuture, start, "send", packetSender); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueReplayerRequestKey.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueReplayerRequestKey.java index 0e7d6bcafe..57f1cc55ba 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueReplayerRequestKey.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueReplayerRequestKey.java @@ -5,12 +5,13 @@ @EqualsAndHashCode public class UniqueReplayerRequestKey extends UniqueSourceRequestKey { public final ISourceTrafficChannelKey trafficStreamKey; - public final int sourceRequestIndexOffsetAtFirstObservation; + public final int sourceRequestIndexOffsetAtStartOfAccumulation; public final int replayerRequestIndex; - public UniqueReplayerRequestKey(ISourceTrafficChannelKey streamKey, int sourceOffset, int replayerIndex) { + public UniqueReplayerRequestKey(ISourceTrafficChannelKey streamKey, int sourceOffsetAtStartOfAccumulation, + int replayerIndex) { this.trafficStreamKey = streamKey; - this.sourceRequestIndexOffsetAtFirstObservation = sourceOffset; + this.sourceRequestIndexOffsetAtStartOfAccumulation = sourceOffsetAtStartOfAccumulation; this.replayerRequestIndex = replayerIndex; } @@ -21,7 +22,7 @@ public ISourceTrafficChannelKey getTrafficStreamKey() { @Override public int getSourceRequestIndex() { - return replayerRequestIndex + sourceRequestIndexOffsetAtFirstObservation; + return replayerRequestIndex + sourceRequestIndexOffsetAtStartOfAccumulation; } public int getReplayerRequestIndex() { @@ -44,7 +45,7 @@ public String toString() { // // That code currently resides in CapturedTrafficToHttpTransactionAccumulator. return trafficStreamKey + "." + getSourceRequestIndex() + - (sourceRequestIndexOffsetAtFirstObservation == 0 ? "" : - "(offset: "+sourceRequestIndexOffsetAtFirstObservation+")"); + (sourceRequestIndexOffsetAtStartOfAccumulation == 0 ? "" : + "(offset: "+ sourceRequestIndexOffsetAtStartOfAccumulation +")"); } } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ExpiringTrafficStreamMapSequentialTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ExpiringTrafficStreamMapSequentialTest.java index caa53aa52d..b74e23606b 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ExpiringTrafficStreamMapSequentialTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ExpiringTrafficStreamMapSequentialTest.java @@ -38,7 +38,7 @@ public void onExpireAccumulation(String partitionId, createdAccumulations.add(accumulation); expiringMap.expireOldEntries(new PojoTrafficStreamKey(TEST_NODE_ID_STRING, connectionGenerator.apply(i), 0), accumulation, ts); - var rrPair = createdAccumulations.get(i).getOrCreateTransactionPair(); + var rrPair = createdAccumulations.get(i).getOrCreateTransactionPair(new PojoTrafficStreamKey("n","c",1)); rrPair.addResponseData(ts, ("Add"+i).getBytes(StandardCharsets.UTF_8)); expiredCountsPerLoop.add(expiredAccumulations.size()); } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ExpiringTrafficStreamMapUnorderedTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ExpiringTrafficStreamMapUnorderedTest.java index 98467c912a..b214cdc5e8 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ExpiringTrafficStreamMapUnorderedTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ExpiringTrafficStreamMapUnorderedTest.java @@ -41,7 +41,7 @@ public void onExpireAccumulation(String partitionId, expiringMap.expireOldEntries(new PojoTrafficStreamKey(TEST_NODE_ID_STRING, connectionGenerator.apply(i), 0), accumulation, ts); createdAccumulations.add(accumulation); if (accumulation != null) { - var rrPair = accumulation.getOrCreateTransactionPair(); + var rrPair = accumulation.getOrCreateTransactionPair(new PojoTrafficStreamKey("n","c",1)); rrPair.addResponseData(ts, ("Add" + i).getBytes(StandardCharsets.UTF_8)); } expiredCountsPerLoop.add(expiredAccumulations.size()); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java index 6d88a3748e..b613867b68 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java @@ -1,5 +1,6 @@ package org.opensearch.migrations.replay; +import com.google.common.base.Strings; import com.google.common.collect.Streams; import io.vavr.Tuple2; import lombok.AllArgsConstructor; @@ -16,8 +17,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; -import org.opensearch.migrations.replay.datatypes.PojoTrafficStreamKey; import org.opensearch.migrations.replay.kafka.KafkaProtobufConsumer; import org.opensearch.migrations.replay.traffic.source.BlockingTrafficSource; import org.opensearch.migrations.replay.traffic.source.ISimpleTrafficCaptureSource; @@ -109,11 +110,14 @@ public void testSingleStreamWithCloseIsCommitted() throws Throwable { public void fullTest() throws Throwable { var httpServer = SimpleNettyHttpServer.makeServer(false, Duration.ofMillis(2), TestHttpServerContext::makeResponse); - var streamAndConsumer = generateStreamAndTupleConsumerWithSomeChecks(2); + var streamAndConsumer = generateStreamAndTupleConsumerWithSomeChecks(3); var numExpectedRequests = streamAndConsumer._2; - var trafficSourceSupplier = new ArrayCursorTrafficSourceFactory(streamAndConsumer._1.collect(Collectors.toList())); + var trafficStreams = streamAndConsumer._1.collect(Collectors.toList()); + log.atInfo().setMessage(()->trafficStreams.stream().map(ts->TrafficStreamUtils.summarizeTrafficStream(ts)) + .collect(Collectors.joining("\n"))).log(); + var trafficSourceSupplier = new ArrayCursorTrafficSourceFactory(trafficStreams); runReplayerUntilSourceWasExhausted(numExpectedRequests, httpServer, trafficSourceSupplier); - Assertions.assertEquals(numExpectedRequests, trafficSourceSupplier.commitCursor.get()); + Assertions.assertEquals(trafficSourceSupplier.streams.size()-1, trafficSourceSupplier.commitCursor.get()); log.error("done"); } @@ -135,21 +139,23 @@ private static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, var counter = new AtomicInteger(); try { runTrafficReplayer(trafficSourceSupplier, httpServer, (t) -> { + if (runNumber != runNumberRef.get()) { + // for an old replayer. I'm not sure why shutdown isn't blocking until all threads are dead, + // but that behavior only impacts this test as far as I can tell. + return; + } Assertions.assertEquals(runNumber, runNumberRef.get()); + var key = t.uniqueRequestKey; synchronized (nextStopPointRef) { - var key = t.uniqueRequestKey; - if (((TrafficStreamCursorKey)(key.getTrafficStreamKey())).sourceListIndex > stopPoint) { + ISourceTrafficChannelKey tsk = key.getTrafficStreamKey(); + var keyString = tsk.getConnectionId() + "_" + key.getSourceRequestIndex(); + if (((TrafficStreamCursorKey)(key.getTrafficStreamKey())).arrayIndex > stopPoint) { log.error("Stopping"); var roughlyDoubled = stopPoint + new Random(stopPoint).nextInt(stopPoint + 1); - if (nextStopPointRef.compareAndSet(stopPoint, roughlyDoubled)) { - throw new FabricatedErrorToKillTheReplayer(false); - } else { - // somebody else already threw to stop the loop. - return; - } + nextStopPointRef.compareAndSet(stopPoint, roughlyDoubled); + throw new FabricatedErrorToKillTheReplayer(false); } - var keyString = new PojoTrafficStreamKey(key.getTrafficStreamKey()) + "_" + key.getSourceRequestIndex(); var totalUnique = null != previouslyCompletelyHandledItems.put(keyString, t) ? totalUniqueEverReceived.get() : totalUniqueEverReceived.incrementAndGet(); @@ -174,7 +180,11 @@ private static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, throw e.immediateCause; } } finally { - log.info("Upon appending.... counter="+counter.get()+" totalUnique="+totalUniqueEverReceived.get()); + waitForWorkerThreadsToStop(); + log.info("Upon appending.... counter="+counter.get()+" totalUnique="+totalUniqueEverReceived.get()+ + " runNumber="+runNumber+" stopAt="+nextStopPointRef.get() + + " commitCursor="+((ArrayCursorTrafficSourceFactory)trafficSourceSupplier).commitCursor); + log.info(Strings.repeat("\n", 20)); receivedPerRun.add(counter.get()); totalUniqueEverReceivedSizeAfterEachRun.add(totalUniqueEverReceived.get()); } @@ -186,10 +196,52 @@ private static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, .toArray(); var expectedSkipArray = new int[skippedPerRunDiffs.length]; Arrays.fill(expectedSkipArray, 1); - //Assertions.assertArrayEquals(expectedSkipArray, skippedPerRunDiffs); + Assertions.assertArrayEquals(expectedSkipArray, skippedPerRunDiffs); Assertions.assertEquals(numExpectedRequests, totalUniqueEverReceived.get()); } + private static void waitForWorkerThreadsToStop() throws InterruptedException { + while (true) { + var rootThreadGroup = getRootThreadGroup(); + if (!foundClientPoolThread(rootThreadGroup)) { + log.info("No client connection pool threads, done polling."); + return; + } else { + log.trace("Found a client connection pool - waiting briefly and retrying."); + Thread.sleep(100); + } + } + } + + private static boolean foundClientPoolThread(ThreadGroup group) { + Thread[] threads = new Thread[group.activeCount()*2]; + var numThreads = group.enumerate(threads); + for (int i=0; i, Integer> generateStreamAndTupleConsumerWithSomeChecks() { return generateStreamAndTupleConsumerWithSomeChecks(-1); @@ -273,23 +325,23 @@ Producer buildKafkaProducer() { @ToString @EqualsAndHashCode private static class TrafficStreamCursorKey implements ITrafficStreamKey, Comparable { - public final int sourceListIndex; + public final int arrayIndex; public final String connectionId; public final String nodeId; public final int trafficStreamIndex; - public TrafficStreamCursorKey(TrafficStream stream, int sourceListIndex) { + public TrafficStreamCursorKey(TrafficStream stream, int arrayIndex) { connectionId = stream.getConnectionId(); nodeId = stream.getNodeId(); trafficStreamIndex = TrafficStreamUtils.getTrafficStreamIndex(stream); - this.sourceListIndex = sourceListIndex; + this.arrayIndex = arrayIndex; } @Override public int compareTo(TrafficStreamCursorKey other) { - return Integer.compare(sourceListIndex, other.sourceListIndex); + return Integer.compare(arrayIndex, other.arrayIndex); } } @@ -309,45 +361,52 @@ public ArrayCursorTrafficSourceFactory(List streams) { } public ISimpleTrafficCaptureSource get() { - return new ISimpleTrafficCaptureSource() { - AtomicInteger readCursor = new AtomicInteger(commitCursor.get()+1); - PriorityQueue pQueue = new PriorityQueue<>(); - - @Override - public CompletableFuture> readNextTrafficStreamChunk() { - var idx = readCursor.getAndIncrement(); - log.info("reading chunk from index="+idx); - if (streams.size() <= idx) { - return CompletableFuture.failedFuture(new EOFException()); - } - var stream = streams.get(idx); - var key = new TrafficStreamCursorKey(stream, idx); - synchronized (pQueue) { - if (pQueue.isEmpty()) { - commitCursor.set(key.sourceListIndex); - } - pQueue.add(key); - } - return CompletableFuture.supplyAsync(()->List.of(new PojoTrafficStreamWithKey(stream, key))); - } + var rval = new ArrayCursorTrafficCaptureSource(this); + log.error("trafficSource="+rval+" readCursor="+rval.readCursor.get()+" commitCursor="+commitCursor.get()); + return rval; + } + } - @Override - public void commitTrafficStream(ITrafficStreamKey trafficStreamKey) { - synchronized (pQueue) { // figure out if I need to do something more efficient later - int topCursor = pQueue.peek().sourceListIndex; - var didRemove = pQueue.remove(trafficStreamKey); - assert didRemove; - var incomingCursor = ((TrafficStreamCursorKey)trafficStreamKey).sourceListIndex; - if (topCursor == incomingCursor) { - topCursor = Optional.ofNullable(pQueue.peek()).map(k->k.getSourceListIndex()).orElse(topCursor); - log.info("Commit called for "+trafficStreamKey+", but and new topCursor="+topCursor); - commitCursor.set(topCursor); - } else { - log.info("Commit called for "+trafficStreamKey+", but topCursor="+topCursor); - } - } + private static class ArrayCursorTrafficCaptureSource implements ISimpleTrafficCaptureSource { + final AtomicInteger readCursor; + final PriorityQueue pQueue = new PriorityQueue<>(); + ArrayCursorTrafficSourceFactory arrayCursorTrafficSourceFactory; + + public ArrayCursorTrafficCaptureSource(ArrayCursorTrafficSourceFactory arrayCursorTrafficSourceFactory) { + this.readCursor = new AtomicInteger(arrayCursorTrafficSourceFactory.commitCursor.get()+1); + this.arrayCursorTrafficSourceFactory = arrayCursorTrafficSourceFactory; + } + + @Override + public CompletableFuture> readNextTrafficStreamChunk() { + var idx = readCursor.getAndIncrement(); + log.info("reading chunk from index="+idx); + if (arrayCursorTrafficSourceFactory.streams.size() <= idx) { + return CompletableFuture.failedFuture(new EOFException()); + } + var stream = arrayCursorTrafficSourceFactory.streams.get(idx); + var key = new TrafficStreamCursorKey(stream, idx); + synchronized (pQueue) { + pQueue.add(key); + } + return CompletableFuture.supplyAsync(()->List.of(new PojoTrafficStreamWithKey(stream, key))); + } + + @Override + public void commitTrafficStream(ITrafficStreamKey trafficStreamKey) { + synchronized (pQueue) { // figure out if I need to do something more efficient later + int topCursor = pQueue.peek().arrayIndex; + var didRemove = pQueue.remove(trafficStreamKey); + assert didRemove; + var incomingCursor = ((TrafficStreamCursorKey)trafficStreamKey).arrayIndex; + if (topCursor == incomingCursor) { + topCursor = Optional.ofNullable(pQueue.peek()).map(k->k.getArrayIndex()).orElse(topCursor); + log.info("Commit called for "+trafficStreamKey+", and new topCursor="+topCursor); + arrayCursorTrafficSourceFactory.commitCursor.set(topCursor); + } else { + log.info("Commit called for "+trafficStreamKey+", but topCursor="+topCursor); } - }; + } } } From 68579708ba3c8219b787763951e7c20466f195e4 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Thu, 9 Nov 2023 08:50:03 -0500 Subject: [PATCH 21/55] Minor TrafficReplayer shutdown changes. It now stops the traffic source pull loop just before calling readNextTrafficStreamChunk() instead of after. It also tracks the Future from netty that was returned by shutdown on the clientConnectionPool. Before leaving the top level loop, sync() is called on that. However, it seems to leave prematurely, just like how the netty signals the completable future to finish before all of the worker threads have died off. Signed-off-by: Greg Schohn --- .../migrations/replay/TrafficReplayer.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index a35158bbb9..f054d61496 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -7,6 +7,7 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import io.netty.util.concurrent.Future; import lombok.AllArgsConstructor; import lombok.NonNull; import lombok.SneakyThrows; @@ -92,6 +93,7 @@ public class TrafficReplayer { private AtomicReference> shutdownFutureRef; private AtomicReference>> nextChunkFutureRef; private ConcurrentHashMap liveRequests = new ConcurrentHashMap<>(); + private Future nettyShutdownFuture; public class DualException extends Exception { public final Throwable originalCause; @@ -537,6 +539,7 @@ void setupRunAndWaitForReplayWithShutdownChecks(Duration observedPacketConnectio } // if nobody has run shutdown yet, do so now so that we can tear down the netty resources shutdown(null).get(); // if somebody already HAD run shutdown, it will return the future already created + nettyShutdownFuture.sync(); } @AllArgsConstructor @@ -563,8 +566,10 @@ public void onRequestReceived(UniqueReplayerRequestKey requestKey, HttpMessageAn } @Override - public void onFullDataReceived(@NonNull UniqueReplayerRequestKey requestKey, RequestResponsePacketPair rrPair) { - log.atTrace().setMessage(()->"Done receiving captured stream for this " + rrPair.requestData).log(); + public void onFullDataReceived(@NonNull UniqueReplayerRequestKey requestKey, + @NonNull RequestResponsePacketPair rrPair) { + log.atInfo().setMessage(()->"Done receiving captured stream for " + requestKey + + ":" + rrPair.requestData).log(); var resultantCf = requestFutureMap.remove(requestKey) .map(f -> f.handle((summary,t)->handleCompletedTransaction(requestKey, rrPair, summary, t)), () -> "TrafficReplayer.runReplayWithIOStreams.progressTracker"); @@ -871,7 +876,7 @@ public void stopReadingAsync() { } stopReadingAsync(); shutdownReasonRef.compareAndSet(null, error); - clientConnectionPool.shutdownNow() + nettyShutdownFuture = clientConnectionPool.shutdownNow() .addListener(f->{ if (f.isSuccess()) { shutdownFutureRef.get().complete(null); @@ -890,7 +895,7 @@ public void stopReadingAsync() { } } var shutdownFuture = shutdownFutureRef.get(); - log.atWarn().setMessage(()->"Shutdown procedure has finished").log(); + log.atWarn().setMessage(()->"Shutdown setup has been initiated").log(); return shutdownFuture; } @@ -900,11 +905,11 @@ public void pullCaptureFromSourceToAccumulator( throws InterruptedException { while (true) { log.trace("Reading next chunk from TrafficStream supplier"); - this.nextChunkFutureRef.set(trafficChunkStream.readNextTrafficStreamChunk()); - List trafficStreams = null; if (stopReadingRef.get()) { break; } + this.nextChunkFutureRef.set(trafficChunkStream.readNextTrafficStreamChunk()); + List trafficStreams = null; try { trafficStreams = this.nextChunkFutureRef.get().get(); } catch (ExecutionException ex) { From 93dff281d1d46f4327df1e32c8940979d0892e74 Mon Sep 17 00:00:00 2001 From: "mend-for-github.aaakk.us.kg[bot]" <50673670+mend-for-github.aaakk.us.kg[bot]@users.noreply.github.com> Date: Thu, 9 Nov 2023 14:54:46 +0000 Subject: [PATCH 22/55] Update dependency io.netty:netty-all to v4.1.100.Final --- TrafficCapture/nettyWireLogging/build.gradle | 2 +- TrafficCapture/testUtilities/build.gradle | 2 +- TrafficCapture/trafficCaptureProxyServer/build.gradle | 2 +- TrafficCapture/trafficReplayer/build.gradle | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/TrafficCapture/nettyWireLogging/build.gradle b/TrafficCapture/nettyWireLogging/build.gradle index 3a9a1b973d..da49a0da6d 100644 --- a/TrafficCapture/nettyWireLogging/build.gradle +++ b/TrafficCapture/nettyWireLogging/build.gradle @@ -11,7 +11,7 @@ plugins { dependencies { implementation project(':captureOffloader') implementation project(':coreUtilities') - api group: 'io.netty', name: 'netty-all', version: '4.1.94.Final' + api group: 'io.netty', name: 'netty-all', version: '4.1.100.Final' implementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7' testImplementation project(':captureProtobufs') diff --git a/TrafficCapture/testUtilities/build.gradle b/TrafficCapture/testUtilities/build.gradle index 50d190ba0c..1554e7e9ae 100644 --- a/TrafficCapture/testUtilities/build.gradle +++ b/TrafficCapture/testUtilities/build.gradle @@ -47,7 +47,7 @@ dependencies { testFixturesImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.0' testFixturesImplementation group: 'com.google.guava', name: 'guava', version: '32.0.1-jre' - testFixturesImplementation group: 'io.netty', name: 'netty-all', version: '4.1.94.Final' + testFixturesImplementation group: 'io.netty', name: 'netty-all', version: '4.1.100.Final' testFixturesImplementation group: 'org.apache.httpcomponents.client5', name: 'httpclient5', version: '5.2.1' testFixturesImplementation group: 'org.bouncycastle', name: 'bcprov-jdk18on', version: '1.74' testFixturesImplementation group: 'org.bouncycastle', name: 'bcpkix-jdk18on', version: '1.74' diff --git a/TrafficCapture/trafficCaptureProxyServer/build.gradle b/TrafficCapture/trafficCaptureProxyServer/build.gradle index 4955e20fef..a5bdc5299b 100644 --- a/TrafficCapture/trafficCaptureProxyServer/build.gradle +++ b/TrafficCapture/trafficCaptureProxyServer/build.gradle @@ -26,7 +26,7 @@ dependencies { implementation project(':captureKafkaOffloader') implementation project(':coreUtilities') - implementation group: 'io.netty', name: 'netty-all', version: '4.1.94.Final' + implementation group: 'io.netty', name: 'netty-all', version: '4.1.100.Final' implementation 'org.opensearch:opensearch-common:2.6.0' implementation 'org.opensearch:opensearch-core:2.6.0' implementation 'org.opensearch:opensearch:2.8.0' diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle index 29d2e8cb31..bf1d4f0449 100644 --- a/TrafficCapture/trafficReplayer/build.gradle +++ b/TrafficCapture/trafficReplayer/build.gradle @@ -46,7 +46,7 @@ dependencies { implementation group: 'com.google.protobuf', name: 'protobuf-java', version: '3.22.2' implementation group: 'io.github.resilience4j', name: 'resilience4j-ratelimiter', version:"${resilience4jVersion}" implementation group: 'io.github.resilience4j', name: 'resilience4j-retry', version:"${resilience4jVersion}" - implementation group: 'io.netty', name: 'netty-all', version: '4.1.94.Final' + implementation group: 'io.netty', name: 'netty-all', version: '4.1.100.Final' implementation group: 'org.apache.kafka', name: 'kafka-clients', version: '3.6.0' implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.20.0' implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.20.0' From cfb50b66d356d6e6884a7733c1cf343994c77308 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Thu, 9 Nov 2023 11:17:47 -0500 Subject: [PATCH 23/55] Fixed an off-by one error in the FullTrafficReplayer test that was causing a moderately difficult test to break. Improved some log messages along the way to make it easier to understand what's really going on. Signed-off-by: Greg Schohn --- .../replay/ClientConnectionPool.java | 1 - .../replay/RequestSenderOrchestrator.java | 20 ++++++++-------- .../replay/FullTrafficReplayerTest.java | 23 +++++++++++-------- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java index 65b7649e0b..d647abb344 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ClientConnectionPool.java @@ -118,7 +118,6 @@ public DiagnosticTrackableCompletableFuture closeConnectionsAndShu (channelClosedFuturesArray.stream().filter(c -> !c.isDone()).count())); } catch (Exception e) { log.atError().setCause(e).setMessage("Caught error while closing cached connections").log(); - log.error("bad", e); eventLoopFuture.future.completeExceptionally(e); } }); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java index a0ce9c4378..c67444d687 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java @@ -39,7 +39,7 @@ public RequestSenderOrchestrator(ClientConnectionPool clientConnectionPool) { var connectionSession = clientConnectionPool.getCachedSession(channelKey, false); var finalTunneledResponse = new StringTrackableCompletableFuture(new CompletableFuture<>(), - ()->"waiting for final signal to confirm close has finished"); + ()->"waiting for final signal to confirm processing work has finished"); log.atDebug().setMessage(()->"Scheduling work for "+channelKey+" at time "+timestamp).log(); connectionSession.eventLoop.schedule(()-> task.get().map(f->f.whenComplete((v,t) -> { @@ -88,20 +88,21 @@ public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChanne } private DiagnosticTrackableCompletableFuture - asynchronouslyInvokeRunnableToSetupFuture(ISourceTrafficChannelKey requestKey, int channelInteractionNumber, + asynchronouslyInvokeRunnableToSetupFuture(ISourceTrafficChannelKey channelKey, int channelInteractionNumber, boolean ignoreIfChannelNotPresent, DiagnosticTrackableCompletableFuture finalTunneledResponse, Consumer successFn) { var channelFutureAndScheduleFuture = - clientConnectionPool.submitEventualSessionGet(requestKey, ignoreIfChannelNotPresent); + clientConnectionPool.submitEventualSessionGet(channelKey, ignoreIfChannelNotPresent); channelFutureAndScheduleFuture.addListener(submitFuture->{ if (!submitFuture.isSuccess()) { log.atError().setCause(submitFuture.cause()) - .setMessage(()->requestKey.toString() + " unexpected issue found from a scheduled task") + .setMessage(()->channelKey.toString() + " unexpected issue found from a scheduled task") .log(); finalTunneledResponse.future.completeExceptionally(submitFuture.cause()); } else { - log.atInfo().setMessage(()->requestKey.toString() + " in submitFuture(success) callback").log(); + log.atTrace().setMessage(()->channelKey.toString() + + " on the channel's thread... getting a ConnectionReplaySession for it").log(); var channelFutureAndRequestSchedule = ((ConnectionReplaySession) submitFuture.get()); if (channelFutureAndRequestSchedule == null) { finalTunneledResponse.future.complete(null); @@ -110,8 +111,8 @@ public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChanne channelFutureAndRequestSchedule.getChannelFutureFuture() .map(channelFutureGetAttemptFuture->channelFutureGetAttemptFuture .thenAccept(v->{ - log.atTrace().setMessage(()->requestKey.toString() + - " ChannelFuture was created with "+v).log(); + log.atTrace().setMessage(()->channelKey.toString() + " in submitFuture(success) and scheduling the task" + + " for " + finalTunneledResponse.toString()).log(); assert v.channel() == channelFutureAndRequestSchedule.getChannelFutureFuture().future .getNow(null).channel(); @@ -122,13 +123,13 @@ public StringTrackableCompletableFuture scheduleClose(ISourceTrafficChanne () -> successFn.accept(channelFutureAndRequestSchedule), x -> x.run()); if (cffr.scheduleSequencer.hasPending()) { - log.atDebug().setMessage(()->"Sequencer for "+requestKey+ + log.atDebug().setMessage(()->"Sequencer for "+channelKey+ " = "+cffr.scheduleSequencer).log(); } }); }) .exceptionally(t->{ - log.atTrace().setCause(t).setMessage(()->requestKey.toString() + + log.atTrace().setCause(t).setMessage(()->channelKey.toString() + " ChannelFuture creation threw an exception").log(); finalTunneledResponse.future.completeExceptionally(t); return null; @@ -203,7 +204,6 @@ private void runAfterChannelSetup(ConnectionReplaySession channelFutureAndIt DiagnosticTrackableCompletableFuture responseFuture, Consumer task) { var cf = channelFutureAndItsFutureRequests.getInnerChannelFuture(); - log.trace("cf="+cf); cf.addListener(f->{ log.atTrace().setMessage(()->"channel creation has finished initialized (success="+f.isSuccess()+")").log(); if (!f.isSuccess()) { diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java index b613867b68..ed39a3c49f 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java @@ -101,7 +101,7 @@ public void testSingleStreamWithCloseIsCommitted() throws Throwable { .build(); var trafficSourceSupplier = new ArrayCursorTrafficSourceFactory(List.of(trafficStreamWithJustClose)); runReplayerUntilSourceWasExhausted(0, httpServer, trafficSourceSupplier); - Assertions.assertEquals(0, trafficSourceSupplier.commitCursor.get()); + Assertions.assertEquals(1, trafficSourceSupplier.nextReadCursor.get()); log.error("done"); } @@ -117,7 +117,7 @@ public void fullTest() throws Throwable { .collect(Collectors.joining("\n"))).log(); var trafficSourceSupplier = new ArrayCursorTrafficSourceFactory(trafficStreams); runReplayerUntilSourceWasExhausted(numExpectedRequests, httpServer, trafficSourceSupplier); - Assertions.assertEquals(trafficSourceSupplier.streams.size()-1, trafficSourceSupplier.commitCursor.get()); + Assertions.assertEquals(trafficSourceSupplier.streams.size(), trafficSourceSupplier.nextReadCursor.get()); log.error("done"); } @@ -150,7 +150,7 @@ private static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, ISourceTrafficChannelKey tsk = key.getTrafficStreamKey(); var keyString = tsk.getConnectionId() + "_" + key.getSourceRequestIndex(); if (((TrafficStreamCursorKey)(key.getTrafficStreamKey())).arrayIndex > stopPoint) { - log.error("Stopping"); + log.error("Request received after our ingest threshold. Throwing. Discarding "+key); var roughlyDoubled = stopPoint + new Random(stopPoint).nextInt(stopPoint + 1); nextStopPointRef.compareAndSet(stopPoint, roughlyDoubled); throw new FabricatedErrorToKillTheReplayer(false); @@ -183,7 +183,7 @@ private static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, waitForWorkerThreadsToStop(); log.info("Upon appending.... counter="+counter.get()+" totalUnique="+totalUniqueEverReceived.get()+ " runNumber="+runNumber+" stopAt="+nextStopPointRef.get() + - " commitCursor="+((ArrayCursorTrafficSourceFactory)trafficSourceSupplier).commitCursor); + " nextReadCursor="+((ArrayCursorTrafficSourceFactory)trafficSourceSupplier).nextReadCursor); log.info(Strings.repeat("\n", 20)); receivedPerRun.add(counter.get()); totalUniqueEverReceivedSizeAfterEachRun.add(totalUniqueEverReceived.get()); @@ -201,6 +201,8 @@ private static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, } private static void waitForWorkerThreadsToStop() throws InterruptedException { + var sleepMs = 2; + final var MAX_SLEEP_MS = 100; while (true) { var rootThreadGroup = getRootThreadGroup(); if (!foundClientPoolThread(rootThreadGroup)) { @@ -208,7 +210,8 @@ private static void waitForWorkerThreadsToStop() throws InterruptedException { return; } else { log.trace("Found a client connection pool - waiting briefly and retrying."); - Thread.sleep(100); + Thread.sleep(sleepMs); + sleepMs = Math.max(MAX_SLEEP_MS, sleepMs*2); } } } @@ -354,7 +357,7 @@ private static class PojoTrafficStreamWithKey implements ITrafficStreamWithKey { private static class ArrayCursorTrafficSourceFactory implements Supplier { List streams; - AtomicInteger commitCursor = new AtomicInteger(-1); + AtomicInteger nextReadCursor = new AtomicInteger(); public ArrayCursorTrafficSourceFactory(List streams) { this.streams = streams; @@ -362,7 +365,7 @@ public ArrayCursorTrafficSourceFactory(List streams) { public ISimpleTrafficCaptureSource get() { var rval = new ArrayCursorTrafficCaptureSource(this); - log.error("trafficSource="+rval+" readCursor="+rval.readCursor.get()+" commitCursor="+commitCursor.get()); + log.error("trafficSource="+rval+" readCursor="+rval.readCursor.get()+" nextReadCursor="+ nextReadCursor.get()); return rval; } } @@ -373,7 +376,7 @@ private static class ArrayCursorTrafficCaptureSource implements ISimpleTrafficCa ArrayCursorTrafficSourceFactory arrayCursorTrafficSourceFactory; public ArrayCursorTrafficCaptureSource(ArrayCursorTrafficSourceFactory arrayCursorTrafficSourceFactory) { - this.readCursor = new AtomicInteger(arrayCursorTrafficSourceFactory.commitCursor.get()+1); + this.readCursor = new AtomicInteger(arrayCursorTrafficSourceFactory.nextReadCursor.get()); this.arrayCursorTrafficSourceFactory = arrayCursorTrafficSourceFactory; } @@ -400,9 +403,9 @@ public void commitTrafficStream(ITrafficStreamKey trafficStreamKey) { assert didRemove; var incomingCursor = ((TrafficStreamCursorKey)trafficStreamKey).arrayIndex; if (topCursor == incomingCursor) { - topCursor = Optional.ofNullable(pQueue.peek()).map(k->k.getArrayIndex()).orElse(topCursor); + topCursor = Optional.ofNullable(pQueue.peek()).map(k->k.getArrayIndex()).orElse(topCursor+1); log.info("Commit called for "+trafficStreamKey+", and new topCursor="+topCursor); - arrayCursorTrafficSourceFactory.commitCursor.set(topCursor); + arrayCursorTrafficSourceFactory.nextReadCursor.set(topCursor); } else { log.info("Commit called for "+trafficStreamKey+", but topCursor="+topCursor); } From 281bdbd290a5b0d361dbe9e8f58f577ad6076615 Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Thu, 9 Nov 2023 11:38:24 -0600 Subject: [PATCH 24/55] Update compare logic Signed-off-by: Omar Khasawneh --- test/tests.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/test/tests.py b/test/tests.py index 4af404d0cf..ad34eab330 100644 --- a/test/tests.py +++ b/test/tests.py @@ -236,22 +236,17 @@ def test_0007_OSB(self): source_indices = get_indices(self.source_endpoint, self.auth) target_indices = get_indices(self.target_endpoint, self.auth) - processed_indices = [] - - for index in set(source_indices) & set(target_indices): - if index not in self.ignore_list: - if index != "searchguard": - source_count = get_doc_count(self.source_endpoint, index, self.auth) - target_count = get_doc_count(self.target_endpoint, index, self.auth) - processed_indices.append(index) - - if source_count != source_count: - logger.error(f'{index}: doc counts do not match - Source = {source_count}, Target = {target_count}') - self.assertEqual(source_count, target_count) - else: - self.assertEqual(source_count, target_count) - logger.info(f'{index}: doc counts match on both source and target endpoints - {source_count}') - - if not processed_indices: - logger.error("There were no indices to compare, check that OpenSearch Benchmark ran successfully") - self.assert_(False) + + common_indices = set(source_indices) & set(target_indices) + valid_indices = [index for index in common_indices if index not in self.ignore_list and index != "searchguard"] + + self.assertTrue(valid_indices, "No valid indices found to compare after running OpenSearch Benchmark") + + for index in valid_indices: + source_count = get_doc_count(self.source_endpoint, index, self.auth) + target_count = get_doc_count(self.target_endpoint, index, self.auth) + + if source_count != target_count: + logger.error(f'{index}: doc counts do not match - Source = {source_count}, Target = {target_count}') + self.assertEqual(source_count, target_count, f'{index}: doc counts do not match') + From 7d2db5bb3813dc8d834f24433dfc1f55b8f14f59 Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Thu, 9 Nov 2023 12:09:34 -0600 Subject: [PATCH 25/55] lint fix + comments about test Signed-off-by: Omar Khasawneh --- test/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/tests.py b/test/tests.py index ad34eab330..80dec1b533 100644 --- a/test/tests.py +++ b/test/tests.py @@ -249,4 +249,3 @@ def test_0007_OSB(self): if source_count != target_count: logger.error(f'{index}: doc counts do not match - Source = {source_count}, Target = {target_count}') self.assertEqual(source_count, target_count, f'{index}: doc counts do not match') - From d9efa4c33db67ec08ce1f9b5f2f967e4416ab214 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Thu, 9 Nov 2023 14:29:34 -0500 Subject: [PATCH 26/55] More small bugfixes to get some interleaved traffic streams to work across restarted TrafficReplayers. There's still one other test case that's failing. Signed-off-by: Greg Schohn --- .../replay/AccumulationCallbacks.java | 1 + ...edTrafficToHttpTransactionAccumulator.java | 6 +-- .../migrations/replay/TrafficReplayer.java | 5 +++ .../replay/FullTrafficReplayerTest.java | 38 +++++++++++-------- ...afficToHttpTransactionAccumulatorTest.java | 3 ++ .../replay/TrafficReplayerTest.java | 4 ++ 6 files changed, 38 insertions(+), 19 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java index 21c6e15f97..7f74725339 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java @@ -15,4 +15,5 @@ void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus stat @NonNull List trafficStreamKeysBeingHeld); void onConnectionClose(@NonNull ISourceTrafficChannelKey key, int channelInteractionNumber, @NonNull Instant when, @NonNull List trafficStreamKeysBeingHeld); + void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java index 817cc68810..f6d6c518f4 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java @@ -139,11 +139,11 @@ public void accept(ITrafficStreamWithKey trafficStreamAndKey) { } if (accum.hasRrPair()) { accum.getRrPair().holdTrafficStream(tsk); - } else { + } else if (!trafficStream.getSubStream(trafficStream.getSubStreamCount()-1).hasClose()) { assert accum.state == Accumulation.State.WAITING_FOR_NEXT_READ_CHUNK || accum.state == Accumulation.State.IGNORING_LAST_REQUEST || - trafficStream.getSubStreamCount() == 0 || - trafficStream.getSubStream(trafficStream.getSubStreamCount()-1).hasClose(); + trafficStream.getSubStreamCount() == 0; + listener.onTrafficStreamIgnored(tsk); } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index f054d61496..159c224867 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -645,6 +645,11 @@ public void onConnectionClose(ISourceTrafficChannelKey channelKey, int channelIn commitTrafficStreams(trafficStreamKeysBeingHeld); } + @Override + public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk) { + commitTrafficStreams(List.of(tsk)); + } + private TransformedTargetRequestAndResponse packageAndWriteResponse(Consumer tupleWriter, @NonNull UniqueReplayerRequestKey requestKey, diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java index ed39a3c49f..fc30db2403 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java @@ -17,6 +17,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.kafka.KafkaProtobufConsumer; @@ -70,9 +73,7 @@ public class FullTrafficReplayerTest { public static final String TEST_GROUP_CONSUMER_ID = "TEST_GROUP_CONSUMER_ID"; public static final String TEST_GROUP_PRODUCER_ID = "TEST_GROUP_PRODUCER_ID"; public static final String TEST_TOPIC_NAME = "TEST_TOPIC"; - public static final int TEST_RECORD_COUNT = 100; public static final String TEST_NODE_ID = "TestNodeId"; - public static final String TEST_TRAFFIC_STREAM_ID_STRING = "TEST_TRAFFIC_STREAM_ID_STRING"; public static final int PRODUCER_SLEEP_INTERVAL_MS = 100; public static final Duration MAX_WAIT_TIME_FOR_TOPIC = Duration.ofMillis(PRODUCER_SLEEP_INTERVAL_MS*2); public static final int INITIAL_STOP_REPLAYER_REQUEST_COUNT = 1; @@ -105,12 +106,18 @@ public void testSingleStreamWithCloseIsCommitted() throws Throwable { log.error("done"); } - @Test + @ParameterizedTest + @CsvSource(value = { + "3,false", + "-1,false", + "3,true", +// "-1,true", + }) @Tag("longTest") - public void fullTest() throws Throwable { + public void fullTest(int testSize, boolean randomize) throws Throwable { var httpServer = SimpleNettyHttpServer.makeServer(false, Duration.ofMillis(2), TestHttpServerContext::makeResponse); - var streamAndConsumer = generateStreamAndTupleConsumerWithSomeChecks(3); + var streamAndConsumer = generateStreamAndTupleConsumerWithSomeChecks(testSize, randomize); var numExpectedRequests = streamAndConsumer._2; var trafficStreams = streamAndConsumer._1.collect(Collectors.toList()); log.atInfo().setMessage(()->trafficStreams.stream().map(ts->TrafficStreamUtils.summarizeTrafficStream(ts)) @@ -246,20 +253,14 @@ private static ThreadGroup getRootThreadGroup() { } private Tuple2, Integer> - generateStreamAndTupleConsumerWithSomeChecks() { - return generateStreamAndTupleConsumerWithSomeChecks(-1); - } - - private Tuple2, Integer> - generateStreamAndTupleConsumerWithSomeChecks(int count) { - Random r = new Random(1); + generateStreamAndTupleConsumerWithSomeChecks(int count, boolean randomize) { var generatedCases = count > 0 ? TrafficStreamGenerator.generateRandomTrafficStreamsAndSizes(IntStream.range(0,count)) : TrafficStreamGenerator.generateAllIndicativeRandomTrafficStreamsAndSizes(); var testCaseArr = generatedCases.toArray(TrafficStreamGenerator.RandomTrafficStreamAndTransactionSizes[]::new); - var aggregatedStreams = + var aggregatedStreams = randomize ? + randomlyInterleaveStreams(count, Arrays.stream(testCaseArr).map(c->Arrays.stream(c.trafficStreams))) : Arrays.stream(testCaseArr).flatMap(c->Arrays.stream(c.trafficStreams)); - //randomlyInterleaveStreams(r, Arrays.stream(testCaseArr).map(c->Arrays.stream(c.trafficStreams))); var numExpectedRequests = Arrays.stream(testCaseArr).mapToInt(c->c.requestByteSizes.length).sum(); return new Tuple2<>(aggregatedStreams, numExpectedRequests); @@ -280,11 +281,12 @@ private static ThreadGroup getRootThreadGroup() { } } - public static Stream randomlyInterleaveStreams(Random r, Stream> orderedItemStreams) { + public static Stream randomlyInterleaveStreams(int randomSeed, Stream> orderedItemStreams) { List> iteratorList = orderedItemStreams .map(Stream::iterator) .filter(it->it.hasNext()) .collect(Collectors.toCollection(()->new ArrayList<>())); + var r = new Random(randomSeed); return Streams.stream(new Iterator() { @Override public boolean hasNext() { @@ -398,10 +400,14 @@ public CompletableFuture> readNextTrafficStreamChunk @Override public void commitTrafficStream(ITrafficStreamKey trafficStreamKey) { synchronized (pQueue) { // figure out if I need to do something more efficient later + log.info("Commit called for "+trafficStreamKey+" with pQueue.size="+pQueue.size()); + var incomingCursor = ((TrafficStreamCursorKey)trafficStreamKey).arrayIndex; int topCursor = pQueue.peek().arrayIndex; var didRemove = pQueue.remove(trafficStreamKey); + if (!didRemove) { + log.error("no item "+incomingCursor+" to remove from "+pQueue); + } assert didRemove; - var incomingCursor = ((TrafficStreamCursorKey)trafficStreamKey).arrayIndex; if (topCursor == incomingCursor) { topCursor = Optional.ofNullable(pQueue.peek()).map(k->k.getArrayIndex()).orElse(topCursor+1); log.info("Commit called for "+trafficStreamKey+", and new topCursor="+topCursor); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java index 5c2207b9f3..a500458971 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java @@ -4,6 +4,7 @@ import io.netty.buffer.Unpooled; import io.vavr.Tuple2; import lombok.AllArgsConstructor; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; @@ -221,6 +222,8 @@ public void onConnectionClose(ISourceTrafficChannelKey key, int channelInteracti Instant when, List trafficStreamKeysBeingHeld) { } + + @Override public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk) {} }); var tsList = trafficStreams.collect(Collectors.toList()); trafficStreams = tsList.stream(); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java index 637942bed6..736c8477e3 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java @@ -2,6 +2,7 @@ import com.google.protobuf.ByteString; import com.google.protobuf.Timestamp; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -175,6 +176,8 @@ public void onConnectionClose(ISourceTrafficChannelKey key, int channelInteracti Instant when, List trafficStreamKeysBeingHeld) { } + + @Override public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk) {} }); var bytes = synthesizeTrafficStreamsIntoByteArray(Instant.now(), 1); @@ -219,6 +222,7 @@ public void onConnectionClose(ISourceTrafficChannelKey key, int channelInteracti Instant when, List trafficStreamKeysBeingHeld) { } + @Override public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk) {} } ); byte[] serializedChunks; From b4af3d513fb40e795c3f05fb0150159a7ded9fc7 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Thu, 9 Nov 2023 16:54:44 -0500 Subject: [PATCH 27/55] When closing a stream down from the CapturedTrafficToHttpTransactionAccumulator, make sure to pass the root cause of the close (expired, etc). That lets the TrafficReplayer decide whether or not it should commit the messages (it does unless it was premature, where the expectation is that the next run of the process might get it). Signed-off-by: Greg Schohn --- .../replay/AccumulationCallbacks.java | 3 +- ...edTrafficToHttpTransactionAccumulator.java | 4 +-- .../migrations/replay/TrafficReplayer.java | 29 ++++++++++++------- .../replay/FullTrafficReplayerTest.java | 12 +++++--- ...afficToHttpTransactionAccumulatorTest.java | 1 + .../replay/TrafficReplayerTest.java | 3 +- 6 files changed, 33 insertions(+), 19 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java index 7f74725339..c581ffa0c2 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/AccumulationCallbacks.java @@ -13,7 +13,8 @@ public interface AccumulationCallbacks { void onFullDataReceived(@NonNull UniqueReplayerRequestKey key, @NonNull RequestResponsePacketPair rrpp); void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, @NonNull List trafficStreamKeysBeingHeld); - void onConnectionClose(@NonNull ISourceTrafficChannelKey key, int channelInteractionNumber, @NonNull Instant when, + void onConnectionClose(@NonNull ISourceTrafficChannelKey key, int channelInteractionNumber, + RequestResponsePacketPair.ReconstructionStatus status, @NonNull Instant when, @NonNull List trafficStreamKeysBeingHeld); void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java index f6d6c518f4..153871c9e6 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java @@ -221,7 +221,7 @@ private static List getTrafficStreamsHeldByAccum(Accumulation rotateAccumulationIfNecessary(trafficStreamKey.getConnectionId(), accum); closedConnectionCounter.incrementAndGet(); listener.onConnectionClose(accum.trafficChannelKey, accum.getIndexOfCurrentRequest(), - timestamp, getTrafficStreamsHeldByAccum(accum)); + RequestResponsePacketPair.ReconstructionStatus.COMPLETE, timestamp, getTrafficStreamsHeldByAccum(accum)); return Optional.of(CONNECTION_STATUS.CLOSED); } else if (observation.hasConnectionException()) { accum.getOrCreateTransactionPair(trafficStreamKey).holdTrafficStream(trafficStreamKey); @@ -400,7 +400,7 @@ private void fireAccumulationsCallbacksAndClose(Accumulation accumulation, } finally { if (accumulation.hasSignaledRequests()) { listener.onConnectionClose(accumulation.trafficChannelKey, accumulation.getIndexOfCurrentRequest(), - accumulation.getLastTimestamp(), getTrafficStreamsHeldByAccum(accumulation)); + status, accumulation.getLastTimestamp(), getTrafficStreamsHeldByAccum(accumulation)); } } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index 159c224867..4b140f993d 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -590,7 +590,7 @@ Void handleCompletedTransaction(@NonNull UniqueReplayerRequestKey requestKey, Re // Escalate it up out handling stack and shutdown. if (t == null || t instanceof Exception) { packageAndWriteResponse(resultTupleConsumer, requestKey, rrPair, summary, (Exception) t); - commitTrafficStreams(rrPair.trafficStreamKeysBeingHeld); + commitTrafficStreams(rrPair.trafficStreamKeysBeingHeld, rrPair.completionStatus); return null; } else if (t instanceof Error) { throw (Error) t; @@ -624,30 +624,37 @@ Void handleCompletedTransaction(@NonNull UniqueReplayerRequestKey requestKey, Re @Override public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStatus status, List trafficStreamKeysBeingHeld) { - if (status == RequestResponsePacketPair.ReconstructionStatus.EXPIRED_PREMATURELY) { - // eventually fill this in to commit the message - commitTrafficStreams(trafficStreamKeysBeingHeld); - } + commitTrafficStreams(trafficStreamKeysBeingHeld, status); } @SneakyThrows - private void commitTrafficStreams(List trafficStreamKeysBeingHeld) { - for (var tsk : trafficStreamKeysBeingHeld) { - trafficCaptureSource.commitTrafficStream(tsk); + private void commitTrafficStreams(List trafficStreamKeysBeingHeld, + RequestResponsePacketPair.ReconstructionStatus status) { + commitTrafficStreams(trafficStreamKeysBeingHeld, + status != RequestResponsePacketPair.ReconstructionStatus.CLOSED_PREMATURELY); + } + + @SneakyThrows + private void commitTrafficStreams(List trafficStreamKeysBeingHeld, boolean shouldCommit) { + if (shouldCommit) { + for (var tsk : trafficStreamKeysBeingHeld) { + trafficCaptureSource.commitTrafficStream(tsk); + } } } @Override - public void onConnectionClose(ISourceTrafficChannelKey channelKey, int channelInteractionNum, Instant timestamp, + public void onConnectionClose(ISourceTrafficChannelKey channelKey, int channelInteractionNum, + RequestResponsePacketPair.ReconstructionStatus status, Instant timestamp, List trafficStreamKeysBeingHeld) { replayEngine.setFirstTimestamp(timestamp); replayEngine.closeConnection(channelKey, channelInteractionNum, timestamp); - commitTrafficStreams(trafficStreamKeysBeingHeld); + commitTrafficStreams(trafficStreamKeysBeingHeld, status); } @Override public void onTrafficStreamIgnored(@NonNull ITrafficStreamKey tsk) { - commitTrafficStreams(List.of(tsk)); + commitTrafficStreams(List.of(tsk), true); } private TransformedTargetRequestAndResponse diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java index fc30db2403..1dfc19505a 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java @@ -138,7 +138,7 @@ private static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, var receivedPerRun = new ArrayList(); var totalUniqueEverReceivedSizeAfterEachRun = new ArrayList(); - var previouslyCompletelyHandledItems = new ConcurrentHashMap(); + var completelyHandledItems = new ConcurrentHashMap(); for (; true; runNumberRef.incrementAndGet()) { var stopPoint = nextStopPointRef.get(); @@ -163,7 +163,7 @@ private static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, throw new FabricatedErrorToKillTheReplayer(false); } - var totalUnique = null != previouslyCompletelyHandledItems.put(keyString, t) ? + var totalUnique = null != completelyHandledItems.put(keyString, t) ? totalUniqueEverReceived.get() : totalUniqueEverReceived.incrementAndGet(); @@ -189,8 +189,9 @@ private static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, } finally { waitForWorkerThreadsToStop(); log.info("Upon appending.... counter="+counter.get()+" totalUnique="+totalUniqueEverReceived.get()+ - " runNumber="+runNumber+" stopAt="+nextStopPointRef.get() + - " nextReadCursor="+((ArrayCursorTrafficSourceFactory)trafficSourceSupplier).nextReadCursor); + " runNumber="+runNumber+" stopAt="+nextStopPointRef.get() + " nextReadCursor=" + +((ArrayCursorTrafficSourceFactory)trafficSourceSupplier).nextReadCursor + "\n" + + completelyHandledItems.keySet().stream().sorted().collect(Collectors.joining("\n"))); log.info(Strings.repeat("\n", 20)); receivedPerRun.add(counter.get()); totalUniqueEverReceivedSizeAfterEachRun.add(totalUniqueEverReceived.get()); @@ -203,6 +204,9 @@ private static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, .toArray(); var expectedSkipArray = new int[skippedPerRunDiffs.length]; Arrays.fill(expectedSkipArray, 1); + log.atInfo().setMessage(()->"completely received request keys=\n{}") + .addArgument(completelyHandledItems.keySet().stream().sorted().collect(Collectors.joining("\n"))) + .log(); Assertions.assertArrayEquals(expectedSkipArray, skippedPerRunDiffs); Assertions.assertEquals(numExpectedRequests, totalUniqueEverReceived.get()); } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java index a500458971..ee6568d2c5 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java @@ -219,6 +219,7 @@ public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStat @Override public void onConnectionClose(ISourceTrafficChannelKey key, int channelInteractionNumber, + RequestResponsePacketPair.ReconstructionStatus status, Instant when, List trafficStreamKeysBeingHeld) { } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java index 736c8477e3..10f278dc9a 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerTest.java @@ -24,7 +24,6 @@ import java.io.EOFException; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; @@ -173,6 +172,7 @@ public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStat @Override public void onConnectionClose(ISourceTrafficChannelKey key, int channelInteractionNumber, + RequestResponsePacketPair.ReconstructionStatus status, Instant when, List trafficStreamKeysBeingHeld) { } @@ -219,6 +219,7 @@ public void onTrafficStreamsExpired(RequestResponsePacketPair.ReconstructionStat @Override public void onConnectionClose(ISourceTrafficChannelKey key, int channelInteractionNumber, + RequestResponsePacketPair.ReconstructionStatus status, Instant when, List trafficStreamKeysBeingHeld) { } From 4c595be5a24971076f35c90837bf643c408d9044 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Thu, 9 Nov 2023 17:14:52 -0500 Subject: [PATCH 28/55] Add a high watermark to reset the cursor so that when the last item in the pQueue is removed, we know what the last committed frontier was. Putting the watermark update within the read call rather than the commit call means that the code doesn't need to do a comparison to pick the maximum value. Signed-off-by: Greg Schohn --- .../migrations/replay/FullTrafficReplayerTest.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java index 1dfc19505a..5e7cd7f681 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java @@ -111,7 +111,7 @@ public void testSingleStreamWithCloseIsCommitted() throws Throwable { "3,false", "-1,false", "3,true", -// "-1,true", + "-1,true", }) @Tag("longTest") public void fullTest(int testSize, boolean randomize) throws Throwable { @@ -379,11 +379,14 @@ public ISimpleTrafficCaptureSource get() { private static class ArrayCursorTrafficCaptureSource implements ISimpleTrafficCaptureSource { final AtomicInteger readCursor; final PriorityQueue pQueue = new PriorityQueue<>(); + Integer cursorHighWatermark; ArrayCursorTrafficSourceFactory arrayCursorTrafficSourceFactory; public ArrayCursorTrafficCaptureSource(ArrayCursorTrafficSourceFactory arrayCursorTrafficSourceFactory) { - this.readCursor = new AtomicInteger(arrayCursorTrafficSourceFactory.nextReadCursor.get()); + var startingCursor = arrayCursorTrafficSourceFactory.nextReadCursor.get(); + this.readCursor = new AtomicInteger(startingCursor); this.arrayCursorTrafficSourceFactory = arrayCursorTrafficSourceFactory; + cursorHighWatermark = startingCursor; } @Override @@ -397,6 +400,7 @@ public CompletableFuture> readNextTrafficStreamChunk var key = new TrafficStreamCursorKey(stream, idx); synchronized (pQueue) { pQueue.add(key); + cursorHighWatermark = idx; } return CompletableFuture.supplyAsync(()->List.of(new PojoTrafficStreamWithKey(stream, key))); } @@ -413,7 +417,8 @@ public void commitTrafficStream(ITrafficStreamKey trafficStreamKey) { } assert didRemove; if (topCursor == incomingCursor) { - topCursor = Optional.ofNullable(pQueue.peek()).map(k->k.getArrayIndex()).orElse(topCursor+1); + topCursor = Optional.ofNullable(pQueue.peek()).map(k->k.getArrayIndex()) + .orElse(cursorHighWatermark+1); // most recent cursor was previously popped log.info("Commit called for "+trafficStreamKey+", and new topCursor="+topCursor); arrayCursorTrafficSourceFactory.nextReadCursor.set(topCursor); } else { From 0c0c6543a2dce602042994b41992524ca0e7d841 Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Thu, 9 Nov 2023 17:35:26 -0600 Subject: [PATCH 29/55] Adding documentation regarding attaching a proxy to a node Signed-off-by: Omar Khasawneh --- TrafficCapture/README.md | 3 + .../trafficCaptureProxyServer/README.md | 100 ++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 TrafficCapture/trafficCaptureProxyServer/README.md diff --git a/TrafficCapture/README.md b/TrafficCapture/README.md index bc0f19cec7..ba55cb8bc6 100644 --- a/TrafficCapture/README.md +++ b/TrafficCapture/README.md @@ -27,6 +27,9 @@ For more details, check out the [Docker Solution README](dockerSolution/README.m The Traffic Capture Proxy Server acts as a middleman, capturing traffic going to a source, which can then be used by the Traffic Replayer. +This tool can be attached to coordinator nodes in clusters with a minimum of two coordinator nodes and start capturing traffic +while having zero downtime on the cluster. More details on attaching a Capture Proxy can be found here: [Capture Proxy](trafficCaptureProxyServer/README.md). + ### Traffic Replayer The Traffic Replayer consumes streams of IP packets that were previously recorded by the Traffic Capture Proxy Server and replays the requests to another HTTP diff --git a/TrafficCapture/trafficCaptureProxyServer/README.md b/TrafficCapture/trafficCaptureProxyServer/README.md new file mode 100644 index 0000000000..2381967369 --- /dev/null +++ b/TrafficCapture/trafficCaptureProxyServer/README.md @@ -0,0 +1,100 @@ +# Capture Proxy + +## How to attach a Capture Proxy on a coordinator node. + +Follow documentation for [deploying solution](../../deployment/README.md). Then, on a cluster with at least two coordinator nodes, the user can attach a Capture Proxy on a node by following these steps: +Please note that this is one method for installing the Capture Proxy on a node, and that these steps may vary depending on your environment. + + +These are the **prerequisites** to being able to attach the Capture Proxy: + +* **Make sure that the node and your MSK client are in the same VPC and Security Groups** + * Add the following IAM policy to the node/EC2 instance so that it’s able to store the captured traffic in Kafka: + * From the AWS Console, go to the EC2 instance page, click on **IAM Role**, click on **Add permissions**, choose **Create inline policy**, click on **JSON VIEW** then add the following policy (replace region and account-id). + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "kafka-cluster:Connect", + "Resource": "arn:aws:kafka:::cluster/migration-msk-cluster-/*", + "Effect": "Allow" + }, + { + "Action": [ + "kafka-cluster:CreateTopic", + "kafka-cluster:DescribeTopic", + "kafka-cluster:WriteData" + ], + "Resource": "arn:aws:kafka:::topic/migration-msk-cluster-/*", + "Effect": "Allow" + } + ] + } + ``` + +* From **linux command line** of that EC2 instance: Check that the **JAVA_HOME environment variable is set** properly `(echo $JAVA_HOME)`, if not, then try running the following command that might help set it correctly: + + `JAVA_HOME=$(dirname "$(dirname "$(type -p java)")")` + * If that doesn’t work, then find the java directory on your node and set it as $JAVA_HOME + +Follow these steps to attach a Capture Proxy on the node. + +1. **Log in to one of the coordinator nodes** for command line access. +2. **Update node’s port setting**. + 1. **Update elasticsearch.yml/opensearch.yml**. Add this line to the node’s config file: + http.port: 19200 +3. **Restart Elasticsearch/OpenSearch process** so that the process will bind to the newly configured port. For example, if systemctl is available on your linux distribution you can run the following (Note: depending on your installation of Elasticsearch, these methods may not work for you) + 1. `sudo systemctl restart elasticsearch.service` + +4. **Verify process is bound to new port**. Run netstat -tapn to see if the new port is being listened on. + If the new port is not there, then there is a chance that Elasticsearch/ OpenSearch is not running, in that case, you must start the process again. (Depending on your setup, restarting/starting the Elasticsearch process may differ) +5. **Test the new port** by sending any kind of traffic or request, e.g; curl https://localhost:19200 or http:// +6. **Download Capture Proxy**: + 1. Go to the Opensearch Migrations latest releases page: https://github.com/opensearch-project/opensearch-migrations/releases/latest + 2. Copy the link for the Capture Proxy tar file, mind your instance’s architecture. + 3. `curl -L0 --output CaptureProxyX64.tar.gz` + 4. Unpack solution tarball: `tar -xvf CaptureProxyX64.tar.gz` + 5. `cd CaptureProxyX64/bin` +7. **Running the Capture Proxy**: + 1. `nohup ./CaptureProxyX64 --kafkaConnection --destinationUri http://localhost:19200 —listenPort 9200 —enableMSKAuth --insecureDestination &` + + **Explanation of parameters** in the command above: + + * **--kafkaConnection**: your MSK client endpoint. + * **--destinationUri**: URI of the server that the Capture Proxy is capturing traffic for. + * **--listenPort**: Exposed port for clients to connect to this proxy. (The original port that the node was listening to) + * **--enableMSKAuth**: Enables SASL Kafka properties required for connecting to MSK with IAM auth. + * **--insecureDestination**: Do not check the destination server’s certificate. + + + +8. **Test the port** that the Capture Proxy is now listening to. + 1. `curl https://localhost:9200` or `http://` + 2. You should expect the same response when sending a request to either ports (9200, 19200), except that the traffic sent to the port that the Capture Proxy is listening to, will be captured and sent to your MSK Client, also forwarded to the new Elasticsearch port. +9. **Verify requests are sent to Kafka** + * **Verify that a new topic has been created** + 1. Log in to the Migration Console container. + 2. Go the Kafka tools directory + cd kafka-tools/kafka/bin + 3. Create the following file to allow communication with an AWS MSK cluster. Name it: `msk-iam-auth.properties` + + ``` + # --- Additional setup to use AWS MSK IAM library for communication with an AWS MSK cluster + # Sets up TLS for encryption and SASL for authN. + security.protocol = SASL_SSL + + # Identifies the SASL mechanism to use. + sasl.mechanism = AWS_MSK_IAM + + # Binds SASL client implementation. + sasl.jaas.config = software.amazon.msk.auth.iam.IAMLoginModule required; + + # Encapsulates constructing a SigV4 signature based on extracted credentials. + # The SASL client bound by "sasl.jaas.config" invokes this class. + sasl.client.callback.handler.class = software.amazon.msk.auth.iam.IAMClientCallbackHandler + ``` + + 4. Run the following command to list the Kafka topics, and confirm that a new topic was created. + `./kafka-topics.sh —bootstrap-server "$MIGRATION_KAFKA_BROKER_ENDPOINTS" —list —command-config msk-iam-auth.properties` From c19e18851869e7c6324d3428bd0fc54bbbec3c5c Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Thu, 9 Nov 2023 17:40:22 -0600 Subject: [PATCH 30/55] formatting Signed-off-by: Omar Khasawneh --- TrafficCapture/trafficCaptureProxyServer/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TrafficCapture/trafficCaptureProxyServer/README.md b/TrafficCapture/trafficCaptureProxyServer/README.md index 2381967369..d4f079ce7d 100644 --- a/TrafficCapture/trafficCaptureProxyServer/README.md +++ b/TrafficCapture/trafficCaptureProxyServer/README.md @@ -39,7 +39,7 @@ These are the **prerequisites** to being able to attach the Capture Proxy: `JAVA_HOME=$(dirname "$(dirname "$(type -p java)")")` * If that doesn’t work, then find the java directory on your node and set it as $JAVA_HOME -Follow these steps to attach a Capture Proxy on the node. +### Follow these steps to attach a Capture Proxy on the node. 1. **Log in to one of the coordinator nodes** for command line access. 2. **Update node’s port setting**. From 16ebbe92fec7e5844faf63e3c2aeb30b173edd0b Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Thu, 9 Nov 2023 19:18:44 -0500 Subject: [PATCH 31/55] Refactoring FullTrafficReplayerTest so that I can support one for a synthetic (test) source and one for Kafka. Signed-off-by: Greg Schohn --- .../testutils/StreamInterleaver.java | 39 +++ .../replay/FullTrafficReplayerTest.java | 294 ++---------------- .../KafkaRestartingTrafficReplayerTest.java | 125 ++++++++ .../replay/TrafficReplayerRunner.java | 183 +++++++++++ 4 files changed, 373 insertions(+), 268 deletions(-) create mode 100644 TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/StreamInterleaver.java create mode 100644 TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/KafkaRestartingTrafficReplayerTest.java create mode 100644 TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerRunner.java diff --git a/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/StreamInterleaver.java b/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/StreamInterleaver.java new file mode 100644 index 0000000000..6f304ca763 --- /dev/null +++ b/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/StreamInterleaver.java @@ -0,0 +1,39 @@ +package org.opensearch.migrations.testutils; + +import com.google.common.collect.Streams; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class StreamInterleaver { + + public static Stream randomlyInterleaveStreams(Random r, Stream> orderedItemStreams) { + List> iteratorList = orderedItemStreams + .map(Stream::iterator) + .filter(it->it.hasNext()) + .collect(Collectors.toCollection(()->new ArrayList<>())); + return Streams.stream(new Iterator() { + @Override + public boolean hasNext() { + return !iteratorList.isEmpty(); + } + @Override + public T next() { + var slotIdx = r.nextInt(iteratorList.size()); + var collectionIterator = iteratorList.get(slotIdx); + var nextItem = collectionIterator.next(); + if (!collectionIterator.hasNext()) { + var lastIdx = iteratorList.size()-1; + iteratorList.set(slotIdx, iteratorList.get(lastIdx)); + iteratorList.remove(lastIdx); + } + return nextItem; + } + }); + } + +} diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java index 5e7cd7f681..2a6cf5123f 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java @@ -28,6 +28,7 @@ import org.opensearch.migrations.replay.traffic.source.ITrafficStreamWithKey; import org.opensearch.migrations.replay.traffic.source.TrafficStreamWithEmbeddedKey; import org.opensearch.migrations.testutils.SimpleNettyHttpServer; +import org.opensearch.migrations.testutils.StreamInterleaver; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; import org.opensearch.migrations.trafficcapture.protos.CloseObservation; import org.opensearch.migrations.trafficcapture.protos.TrafficObservation; @@ -60,8 +61,6 @@ import java.util.stream.Stream; @Slf4j -// Turn this on to test with a live Kafka broker. Other code changes will need to be activated too -//@Testcontainers // It would be great to test with leak detection here, but right now this test relies upon TrafficReplayer.shutdown() // to recycle the TrafficReplayers. Since that shutdown process optimizes for speed of teardown, rather than tidying // everything up as it closes the door, some leaks may be inevitable. E.g. when work is outstanding and being sent @@ -70,24 +69,28 @@ @WrapWithNettyLeakDetection(disableLeakChecks = true) public class FullTrafficReplayerTest { - public static final String TEST_GROUP_CONSUMER_ID = "TEST_GROUP_CONSUMER_ID"; - public static final String TEST_GROUP_PRODUCER_ID = "TEST_GROUP_PRODUCER_ID"; - public static final String TEST_TOPIC_NAME = "TEST_TOPIC"; - public static final String TEST_NODE_ID = "TestNodeId"; - public static final int PRODUCER_SLEEP_INTERVAL_MS = 100; - public static final Duration MAX_WAIT_TIME_FOR_TOPIC = Duration.ofMillis(PRODUCER_SLEEP_INTERVAL_MS*2); public static final int INITIAL_STOP_REPLAYER_REQUEST_COUNT = 1; + public static final String TEST_NODE_ID = "TestNodeId"; public static final String TEST_CONNECTION_ID = "testConnectionId"; - @AllArgsConstructor - private static class FabricatedErrorToKillTheReplayer extends Error { - public final boolean doneWithTest; - } + private static class CountingTupleReceiverFactory implements Supplier> { + AtomicInteger nextStopPointRef = new AtomicInteger(INITIAL_STOP_REPLAYER_REQUEST_COUNT); - @Container - // see https://docs.confluent.io/platform/current/installation/versions-interoperability.html#cp-and-apache-kafka-compatibility - private KafkaContainer embeddedKafkaBroker; -// = new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:7.5.0"));; + @Override + public Consumer get() { + log.info("StopAt="+nextStopPointRef.get()); + var stopPoint = nextStopPointRef.get(); + return tuple -> { + var key = tuple.uniqueRequestKey; + if (((TrafficStreamCursorKey) (key.getTrafficStreamKey())).arrayIndex > stopPoint) { + log.error("Request received after our ingest threshold. Throwing. Discarding " + key); + var nextStopPoint = stopPoint + new Random(stopPoint).nextInt(stopPoint + 1); + nextStopPointRef.compareAndSet(stopPoint, nextStopPoint); + throw new TrafficReplayerRunner.FabricatedErrorToKillTheReplayer(false); + } + }; + } + } @Test @Tag("longTest") @@ -101,7 +104,8 @@ public void testSingleStreamWithCloseIsCommitted() throws Throwable { .setClose(CloseObservation.newBuilder().build()).build()) .build(); var trafficSourceSupplier = new ArrayCursorTrafficSourceFactory(List.of(trafficStreamWithJustClose)); - runReplayerUntilSourceWasExhausted(0, httpServer, trafficSourceSupplier); + TrafficReplayerRunner.runReplayerUntilSourceWasExhausted(0, + httpServer.localhostEndpoint(), new CountingTupleReceiverFactory(), trafficSourceSupplier); Assertions.assertEquals(1, trafficSourceSupplier.nextReadCursor.get()); log.error("done"); } @@ -123,139 +127,12 @@ public void fullTest(int testSize, boolean randomize) throws Throwable { log.atInfo().setMessage(()->trafficStreams.stream().map(ts->TrafficStreamUtils.summarizeTrafficStream(ts)) .collect(Collectors.joining("\n"))).log(); var trafficSourceSupplier = new ArrayCursorTrafficSourceFactory(trafficStreams); - runReplayerUntilSourceWasExhausted(numExpectedRequests, httpServer, trafficSourceSupplier); + TrafficReplayerRunner.runReplayerUntilSourceWasExhausted(numExpectedRequests, + httpServer.localhostEndpoint(), new CountingTupleReceiverFactory(), trafficSourceSupplier); Assertions.assertEquals(trafficSourceSupplier.streams.size(), trafficSourceSupplier.nextReadCursor.get()); log.error("done"); } - private static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, - SimpleNettyHttpServer httpServer, - Supplier trafficSourceSupplier) - throws Throwable { - AtomicInteger runNumberRef = new AtomicInteger(); - var totalUniqueEverReceived = new AtomicInteger(); - var nextStopPointRef = new AtomicInteger(INITIAL_STOP_REPLAYER_REQUEST_COUNT); - - var receivedPerRun = new ArrayList(); - var totalUniqueEverReceivedSizeAfterEachRun = new ArrayList(); - var completelyHandledItems = new ConcurrentHashMap(); - - for (; true; runNumberRef.incrementAndGet()) { - var stopPoint = nextStopPointRef.get(); - int runNumber = runNumberRef.get(); - var counter = new AtomicInteger(); - try { - runTrafficReplayer(trafficSourceSupplier, httpServer, (t) -> { - if (runNumber != runNumberRef.get()) { - // for an old replayer. I'm not sure why shutdown isn't blocking until all threads are dead, - // but that behavior only impacts this test as far as I can tell. - return; - } - Assertions.assertEquals(runNumber, runNumberRef.get()); - var key = t.uniqueRequestKey; - synchronized (nextStopPointRef) { - ISourceTrafficChannelKey tsk = key.getTrafficStreamKey(); - var keyString = tsk.getConnectionId() + "_" + key.getSourceRequestIndex(); - if (((TrafficStreamCursorKey)(key.getTrafficStreamKey())).arrayIndex > stopPoint) { - log.error("Request received after our ingest threshold. Throwing. Discarding "+key); - var roughlyDoubled = stopPoint + new Random(stopPoint).nextInt(stopPoint + 1); - nextStopPointRef.compareAndSet(stopPoint, roughlyDoubled); - throw new FabricatedErrorToKillTheReplayer(false); - } - - var totalUnique = null != completelyHandledItems.put(keyString, t) ? - totalUniqueEverReceived.get() : - totalUniqueEverReceived.incrementAndGet(); - - var c = counter.incrementAndGet(); - log.info("counter="+c+" totalUnique="+totalUnique+" runNum="+runNumber+" key="+key); - } - }); - // if this finished running without an exception, we need to stop the loop - break; - } catch (TrafficReplayer.TerminationException e) { - log.atLevel(e.originalCause instanceof FabricatedErrorToKillTheReplayer ? Level.INFO : Level.ERROR) - .setCause(e.originalCause) - .setMessage(()->"broke out of the replayer, with this shutdown reason") - .log(); - log.atLevel(e.immediateCause == null ? Level.INFO : Level.ERROR) - .setCause(e.immediateCause) - .setMessage(()->"broke out of the replayer, with the shutdown cause=" + e.originalCause + - " and this immediate reason") - .log(); - if (!(e.originalCause instanceof FabricatedErrorToKillTheReplayer)) { - throw e.immediateCause; - } - } finally { - waitForWorkerThreadsToStop(); - log.info("Upon appending.... counter="+counter.get()+" totalUnique="+totalUniqueEverReceived.get()+ - " runNumber="+runNumber+" stopAt="+nextStopPointRef.get() + " nextReadCursor=" - +((ArrayCursorTrafficSourceFactory)trafficSourceSupplier).nextReadCursor + "\n" + - completelyHandledItems.keySet().stream().sorted().collect(Collectors.joining("\n"))); - log.info(Strings.repeat("\n", 20)); - receivedPerRun.add(counter.get()); - totalUniqueEverReceivedSizeAfterEachRun.add(totalUniqueEverReceived.get()); - } - } - var skippedPerRun = IntStream.range(0, receivedPerRun.size()) - .map(i->totalUniqueEverReceivedSizeAfterEachRun.get(i)-receivedPerRun.get(i)).toArray(); - var skippedPerRunDiffs = IntStream.range(0, receivedPerRun.size()-1) - .map(i->(skippedPerRun[i]<=skippedPerRun[i+1]) ? 1 : 0) - .toArray(); - var expectedSkipArray = new int[skippedPerRunDiffs.length]; - Arrays.fill(expectedSkipArray, 1); - log.atInfo().setMessage(()->"completely received request keys=\n{}") - .addArgument(completelyHandledItems.keySet().stream().sorted().collect(Collectors.joining("\n"))) - .log(); - Assertions.assertArrayEquals(expectedSkipArray, skippedPerRunDiffs); - Assertions.assertEquals(numExpectedRequests, totalUniqueEverReceived.get()); - } - - private static void waitForWorkerThreadsToStop() throws InterruptedException { - var sleepMs = 2; - final var MAX_SLEEP_MS = 100; - while (true) { - var rootThreadGroup = getRootThreadGroup(); - if (!foundClientPoolThread(rootThreadGroup)) { - log.info("No client connection pool threads, done polling."); - return; - } else { - log.trace("Found a client connection pool - waiting briefly and retrying."); - Thread.sleep(sleepMs); - sleepMs = Math.max(MAX_SLEEP_MS, sleepMs*2); - } - } - } - - private static boolean foundClientPoolThread(ThreadGroup group) { - Thread[] threads = new Thread[group.activeCount()*2]; - var numThreads = group.enumerate(threads); - for (int i=0; i, Integer> generateStreamAndTupleConsumerWithSomeChecks(int count, boolean randomize) { var generatedCases = count > 0 ? @@ -263,73 +140,14 @@ private static ThreadGroup getRootThreadGroup() { TrafficStreamGenerator.generateAllIndicativeRandomTrafficStreamsAndSizes(); var testCaseArr = generatedCases.toArray(TrafficStreamGenerator.RandomTrafficStreamAndTransactionSizes[]::new); var aggregatedStreams = randomize ? - randomlyInterleaveStreams(count, Arrays.stream(testCaseArr).map(c->Arrays.stream(c.trafficStreams))) : + StreamInterleaver.randomlyInterleaveStreams(new Random(count), + Arrays.stream(testCaseArr).map(c->Arrays.stream(c.trafficStreams))) : Arrays.stream(testCaseArr).flatMap(c->Arrays.stream(c.trafficStreams)); var numExpectedRequests = Arrays.stream(testCaseArr).mapToInt(c->c.requestByteSizes.length).sum(); return new Tuple2<>(aggregatedStreams, numExpectedRequests); } - private void - loadStreamsAsynchronouslyWithResource(KafkaConsumer kafkaConsumer, R resource, Consumer loader) - throws Exception { - try { - new Thread(()->loader.accept(resource)).start(); - var startTime = Instant.now(); - while (!kafkaConsumer.listTopics().isEmpty()) { - Thread.sleep(10); - Assertions.assertTrue(Duration.between(startTime, Instant.now()).compareTo(MAX_WAIT_TIME_FOR_TOPIC) < 0); - } - } finally { - resource.close(); - } - } - - public static Stream randomlyInterleaveStreams(int randomSeed, Stream> orderedItemStreams) { - List> iteratorList = orderedItemStreams - .map(Stream::iterator) - .filter(it->it.hasNext()) - .collect(Collectors.toCollection(()->new ArrayList<>())); - var r = new Random(randomSeed); - return Streams.stream(new Iterator() { - @Override - public boolean hasNext() { - return !iteratorList.isEmpty(); - } - @Override - public T next() { - var slotIdx = r.nextInt(iteratorList.size()); - var collectionIterator = iteratorList.get(slotIdx); - var nextItem = collectionIterator.next(); - if (!collectionIterator.hasNext()) { - var lastIdx = iteratorList.size()-1; - iteratorList.set(slotIdx, iteratorList.get(lastIdx)); - iteratorList.remove(lastIdx); - } - return nextItem; - } - }); - } - - Producer buildKafkaProducer() { - var kafkaProps = new Properties(); - kafkaProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); - kafkaProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); - // Property details: https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#delivery-timeout-ms - kafkaProps.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 10000); - kafkaProps.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5000); - kafkaProps.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 10000); - kafkaProps.put(ProducerConfig.CLIENT_ID_CONFIG, TEST_GROUP_PRODUCER_ID); - kafkaProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, embeddedKafkaBroker.getBootstrapServers()); - try { - return new KafkaProducer(kafkaProps); - } catch (Exception e) { - log.atError().setCause(e).log(); - System.exit(1); - throw e; - } - } - @Getter @ToString @EqualsAndHashCode @@ -384,6 +202,7 @@ private static class ArrayCursorTrafficCaptureSource implements ISimpleTrafficCa public ArrayCursorTrafficCaptureSource(ArrayCursorTrafficSourceFactory arrayCursorTrafficSourceFactory) { var startingCursor = arrayCursorTrafficSourceFactory.nextReadCursor.get(); + log.info("startingCursor = " + startingCursor); this.readCursor = new AtomicInteger(startingCursor); this.arrayCursorTrafficSourceFactory = arrayCursorTrafficSourceFactory; cursorHighWatermark = startingCursor; @@ -428,66 +247,5 @@ public void commitTrafficStream(ITrafficStreamKey trafficStreamKey) { } } - private Supplier loadStreamsToKafka(Stream streams) throws Exception { - var kafkaConsumerProps = KafkaProtobufConsumer.buildKafkaProperties(embeddedKafkaBroker.getBootstrapServers(), - TEST_GROUP_CONSUMER_ID, false, null); - kafkaConsumerProps.setProperty("max.poll.interval.ms", "300000"); - var kafkaConsumer = new KafkaConsumer(kafkaConsumerProps); - - var kafkaProducer = buildKafkaProducer(); - var counter = new AtomicInteger(); - loadStreamsAsynchronouslyWithResource(kafkaConsumer, streams, s->s.forEach(trafficStream -> - writeTrafficStreamRecord(kafkaProducer, new TrafficStreamWithEmbeddedKey(trafficStream), - "KEY_" + counter.incrementAndGet()))); - return () -> new KafkaProtobufConsumer(kafkaConsumer, TEST_TOPIC_NAME, null); - } - - private Supplier - loadStreamsToKafkaFromCompressedFile(KafkaConsumer kafkaConsumer, - String filename, int recordCount) throws Exception { - var kafkaProducer = buildKafkaProducer(); - loadStreamsAsynchronouslyWithResource(kafkaConsumer, new V0_1TrafficCaptureSource(filename), - originalTrafficSource -> { - try { - for (int i = 0; i < recordCount; ++i) { - List chunks = null; - chunks = originalTrafficSource.readNextTrafficStreamChunk().get(); - for (int j = 0; j < chunks.size(); ++j) { - writeTrafficStreamRecord(kafkaProducer, chunks.get(j), "KEY_" + i + "_" + j); - } - } - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - return () -> new KafkaProtobufConsumer(kafkaConsumer, TEST_TOPIC_NAME, null); - } - - @SneakyThrows - private static void writeTrafficStreamRecord(Producer kafkaProducer, - ITrafficStreamWithKey trafficStream, - String recordId) { - var record = new ProducerRecord(TEST_TOPIC_NAME, recordId, trafficStream.getStream().toByteArray()); - var sendFuture = kafkaProducer.send(record, (metadata, exception) -> {}); - sendFuture.get(); - Thread.sleep(PRODUCER_SLEEP_INTERVAL_MS); - } - - private static void runTrafficReplayer(Supplier captureSourceSupplier, - SimpleNettyHttpServer httpServer, - Consumer tupleReceiver) throws Exception { - log.info("Starting a new replayer and running it"); - var tr = new TrafficReplayer(httpServer.localhostEndpoint(), - new StaticAuthTransformerFactory("TEST"), - true, 10, 10*1024, - TrafficReplayer.buildDefaultJsonTransformer(httpServer.localhostEndpoint().getHost())); - - try (var os = new NullOutputStream(); - var trafficSource = captureSourceSupplier.get(); - var blockingTrafficSource = new BlockingTrafficSource(trafficSource, Duration.ofMinutes(2))) { - tr.setupRunAndWaitForReplayWithShutdownChecks(Duration.ofSeconds(70), blockingTrafficSource, - new TimeShifter(10 * 1000), tupleReceiver); - } - } } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/KafkaRestartingTrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/KafkaRestartingTrafficReplayerTest.java new file mode 100644 index 0000000000..dbdebbc639 --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/KafkaRestartingTrafficReplayerTest.java @@ -0,0 +1,125 @@ +package org.opensearch.migrations.replay; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.junit.jupiter.api.Assertions; +import org.opensearch.migrations.replay.kafka.KafkaProtobufConsumer; +import org.opensearch.migrations.replay.traffic.source.ISimpleTrafficCaptureSource; +import org.opensearch.migrations.replay.traffic.source.ITrafficStreamWithKey; +import org.opensearch.migrations.replay.traffic.source.TrafficStreamWithEmbeddedKey; +import org.opensearch.migrations.trafficcapture.protos.TrafficStream; +import org.testcontainers.containers.KafkaContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.stream.Stream; + +@Slf4j +@Testcontainers +public class KafkaRestartingTrafficReplayerTest { + public static final String TEST_GROUP_CONSUMER_ID = "TEST_GROUP_CONSUMER_ID"; + public static final String TEST_GROUP_PRODUCER_ID = "TEST_GROUP_PRODUCER_ID"; + public static final String TEST_TOPIC_NAME = "TEST_TOPIC"; + + public static final int PRODUCER_SLEEP_INTERVAL_MS = 100; + public static final Duration MAX_WAIT_TIME_FOR_TOPIC = Duration.ofMillis(PRODUCER_SLEEP_INTERVAL_MS*2); + + @Container + // see https://docs.confluent.io/platform/current/installation/versions-interoperability.html#cp-and-apache-kafka-compatibility + private KafkaContainer embeddedKafkaBroker + = new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:7.5.0"));; + + + private Supplier loadStreamsToKafka(Stream streams) throws Exception { + var kafkaConsumerProps = KafkaProtobufConsumer.buildKafkaProperties(embeddedKafkaBroker.getBootstrapServers(), + TEST_GROUP_CONSUMER_ID, false, null); + kafkaConsumerProps.setProperty("max.poll.interval.ms", "300000"); + var kafkaConsumer = new KafkaConsumer(kafkaConsumerProps); + + var kafkaProducer = buildKafkaProducer(); + var counter = new AtomicInteger(); + loadStreamsAsynchronouslyWithResource(kafkaConsumer, streams, s->s.forEach(trafficStream -> + writeTrafficStreamRecord(kafkaProducer, new TrafficStreamWithEmbeddedKey(trafficStream), + "KEY_" + counter.incrementAndGet()))); + + return () -> new KafkaProtobufConsumer(kafkaConsumer, TEST_TOPIC_NAME, null); + } + + private void + loadStreamsAsynchronouslyWithResource(KafkaConsumer kafkaConsumer, R resource, Consumer loader) + throws Exception { + try { + new Thread(()->loader.accept(resource)).start(); + var startTime = Instant.now(); + while (!kafkaConsumer.listTopics().isEmpty()) { + Thread.sleep(10); + Assertions.assertTrue(Duration.between(startTime, Instant.now()).compareTo(MAX_WAIT_TIME_FOR_TOPIC) < 0); + } + } finally { + resource.close(); + } + } + + Producer buildKafkaProducer() { + var kafkaProps = new Properties(); + kafkaProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + kafkaProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); + // Property details: https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#delivery-timeout-ms + kafkaProps.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 10000); + kafkaProps.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5000); + kafkaProps.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 10000); + kafkaProps.put(ProducerConfig.CLIENT_ID_CONFIG, TEST_GROUP_PRODUCER_ID); + kafkaProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, embeddedKafkaBroker.getBootstrapServers()); + try { + return new KafkaProducer(kafkaProps); + } catch (Exception e) { + log.atError().setCause(e).log(); + System.exit(1); + throw e; + } + } + + private Supplier + loadStreamsToKafkaFromCompressedFile(KafkaConsumer kafkaConsumer, + String filename, int recordCount) throws Exception { + var kafkaProducer = buildKafkaProducer(); + loadStreamsAsynchronouslyWithResource(kafkaConsumer, new V0_1TrafficCaptureSource(filename), + originalTrafficSource -> { + try { + for (int i = 0; i < recordCount; ++i) { + List chunks = null; + chunks = originalTrafficSource.readNextTrafficStreamChunk().get(); + for (int j = 0; j < chunks.size(); ++j) { + writeTrafficStreamRecord(kafkaProducer, chunks.get(j), "KEY_" + i + "_" + j); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return () -> new KafkaProtobufConsumer(kafkaConsumer, TEST_TOPIC_NAME, null); + } + + @SneakyThrows + private static void writeTrafficStreamRecord(Producer kafkaProducer, + ITrafficStreamWithKey trafficStream, + String recordId) { + var record = new ProducerRecord(TEST_TOPIC_NAME, recordId, trafficStream.getStream().toByteArray()); + var sendFuture = kafkaProducer.send(record, (metadata, exception) -> {}); + sendFuture.get(); + Thread.sleep(PRODUCER_SLEEP_INTERVAL_MS); + } +} diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerRunner.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerRunner.java new file mode 100644 index 0000000000..c9d6fba148 --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerRunner.java @@ -0,0 +1,183 @@ +package org.opensearch.migrations.replay; + +import com.google.common.base.Strings; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; +import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; +import org.opensearch.migrations.replay.traffic.source.BlockingTrafficSource; +import org.opensearch.migrations.replay.traffic.source.ISimpleTrafficCaptureSource; +import org.opensearch.migrations.testutils.SimpleNettyHttpServer; +import org.opensearch.migrations.transform.StaticAuthTransformerFactory; +import org.slf4j.event.Level; +import org.testcontainers.shaded.org.apache.commons.io.output.NullOutputStream; + +import java.net.URI; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Random; +import java.util.StringJoiner; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +@Slf4j +public class TrafficReplayerRunner { + + @AllArgsConstructor + static class FabricatedErrorToKillTheReplayer extends Error { + public final boolean doneWithTest; + } + + private TrafficReplayerRunner() {} + + static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, URI endpoint, + Supplier> tupleReceiverSupplier, + Supplier trafficSourceSupplier) + throws Throwable { + AtomicInteger runNumberRef = new AtomicInteger(); + var totalUniqueEverReceived = new AtomicInteger(); + + var receivedPerRun = new ArrayList(); + var totalUniqueEverReceivedSizeAfterEachRun = new ArrayList(); + var completelyHandledItems = new ConcurrentHashMap(); + + for (; true; runNumberRef.incrementAndGet()) { + int runNumber = runNumberRef.get(); + var counter = new AtomicInteger(); + var tupleReceiver = tupleReceiverSupplier.get(); + try { + runTrafficReplayer(trafficSourceSupplier, endpoint, (t) -> { + if (runNumber != runNumberRef.get()) { + // for an old replayer. I'm not sure why shutdown isn't blocking until all threads are dead, + // but that behavior only impacts this test as far as I can tell. + return; + } + Assertions.assertEquals(runNumber, runNumberRef.get()); + var key = t.uniqueRequestKey; + ISourceTrafficChannelKey tsk = key.getTrafficStreamKey(); + var keyString = tsk.getConnectionId() + "_" + key.getSourceRequestIndex(); + tupleReceiver.accept(t); + var totalUnique = null != completelyHandledItems.put(keyString, t) ? + totalUniqueEverReceived.get() : + totalUniqueEverReceived.incrementAndGet(); + + var c = counter.incrementAndGet(); + log.info("counter="+c+" totalUnique="+totalUnique+" runNum="+runNumber+" key="+key); + }); + // if this finished running without an exception, we need to stop the loop + break; + } catch (TrafficReplayer.TerminationException e) { + log.atLevel(e.originalCause instanceof FabricatedErrorToKillTheReplayer ? Level.INFO : Level.ERROR) + .setCause(e.originalCause) + .setMessage(()->"broke out of the replayer, with this shutdown reason") + .log(); + log.atLevel(e.immediateCause == null ? Level.INFO : Level.ERROR) + .setCause(e.immediateCause) + .setMessage(()->"broke out of the replayer, with the shutdown cause=" + e.originalCause + + " and this immediate reason") + .log(); + if (!(e.originalCause instanceof FabricatedErrorToKillTheReplayer)) { + throw e.immediateCause; + } + } finally { + waitForWorkerThreadsToStop(); + log.info("Upon appending.... counter="+counter.get()+" totalUnique="+totalUniqueEverReceived.get()+ + " runNumber="+runNumber + "\n" + + completelyHandledItems.keySet().stream().sorted().collect(Collectors.joining("\n"))); + log.info(Strings.repeat("\n", 20)); + receivedPerRun.add(counter.get()); + totalUniqueEverReceivedSizeAfterEachRun.add(totalUniqueEverReceived.get()); + } + } + log.atInfo().setMessage(()->"completely received request keys=\n{}") + .addArgument(completelyHandledItems.keySet().stream().sorted().collect(Collectors.joining("\n"))) + .log(); + var skippedPerRun = IntStream.range(0, receivedPerRun.size()) + .map(i->totalUniqueEverReceivedSizeAfterEachRun.get(i)-receivedPerRun.get(i)).toArray(); + log.atInfo().setMessage(()->"Summary: (run #, uniqueSoFar, receivedThisRun, skipped)\n" + + IntStream.range(0, receivedPerRun.size()).mapToObj(i-> + new StringJoiner(", ") + .add(""+i) + .add(""+totalUniqueEverReceivedSizeAfterEachRun.get(i)) + .add(""+receivedPerRun.get(i)) + .add(""+skippedPerRun[i]).toString()) + .collect(Collectors.joining("\n"))) + .log(); + var skippedPerRunDiffs = IntStream.range(0, receivedPerRun.size()-1) + .map(i->(skippedPerRun[i]<=skippedPerRun[i+1]) ? 1 : 0) + .toArray(); + var expectedSkipArray = new int[skippedPerRunDiffs.length]; + Arrays.fill(expectedSkipArray, 1); + Assertions.assertArrayEquals(expectedSkipArray, skippedPerRunDiffs); + Assertions.assertEquals(numExpectedRequests, totalUniqueEverReceived.get()); + } + + private static void runTrafficReplayer(Supplier captureSourceSupplier, + URI endpoint, + Consumer tupleReceiver) throws Exception { + log.info("Starting a new replayer and running it"); + var tr = new TrafficReplayer(endpoint, + new StaticAuthTransformerFactory("TEST"), + true, 10, 10*1024, + TrafficReplayer.buildDefaultJsonTransformer(endpoint.getHost())); + + try (var os = new NullOutputStream(); + var trafficSource = captureSourceSupplier.get(); + var blockingTrafficSource = new BlockingTrafficSource(trafficSource, Duration.ofMinutes(2))) { + tr.setupRunAndWaitForReplayWithShutdownChecks(Duration.ofSeconds(70), blockingTrafficSource, + new TimeShifter(10 * 1000), tupleReceiver); + } + } + + private static void waitForWorkerThreadsToStop() throws InterruptedException { + var sleepMs = 2; + final var MAX_SLEEP_MS = 100; + while (true) { + var rootThreadGroup = getRootThreadGroup(); + if (!foundClientPoolThread(rootThreadGroup)) { + log.info("No client connection pool threads, done polling."); + return; + } else { + log.trace("Found a client connection pool - waiting briefly and retrying."); + Thread.sleep(sleepMs); + sleepMs = Math.max(MAX_SLEEP_MS, sleepMs*2); + } + } + } + + private static boolean foundClientPoolThread(ThreadGroup group) { + Thread[] threads = new Thread[group.activeCount()*2]; + var numThreads = group.enumerate(threads); + for (int i=0; i Date: Thu, 9 Nov 2023 20:10:22 -0600 Subject: [PATCH 32/55] Removed unnecessary step + minor changes Signed-off-by: Omar Khasawneh --- TrafficCapture/README.md | 3 +- .../trafficCaptureProxyServer/README.md | 33 ++++--------------- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/TrafficCapture/README.md b/TrafficCapture/README.md index ba55cb8bc6..267251f7ac 100644 --- a/TrafficCapture/README.md +++ b/TrafficCapture/README.md @@ -28,7 +28,8 @@ For more details, check out the [Docker Solution README](dockerSolution/README.m The Traffic Capture Proxy Server acts as a middleman, capturing traffic going to a source, which can then be used by the Traffic Replayer. This tool can be attached to coordinator nodes in clusters with a minimum of two coordinator nodes and start capturing traffic -while having zero downtime on the cluster. More details on attaching a Capture Proxy can be found here: [Capture Proxy](trafficCaptureProxyServer/README.md). +while having zero downtime on the cluster. Be aware that zero downtime is only achievable **if** the remaining nodes in-service can handle the additional load on the cluster. +More details on attaching a Capture Proxy can be found here: [Capture Proxy](trafficCaptureProxyServer/README.md). ### Traffic Replayer diff --git a/TrafficCapture/trafficCaptureProxyServer/README.md b/TrafficCapture/trafficCaptureProxyServer/README.md index d4f079ce7d..cf93ec03fa 100644 --- a/TrafficCapture/trafficCaptureProxyServer/README.md +++ b/TrafficCapture/trafficCaptureProxyServer/README.md @@ -8,7 +8,7 @@ Please note that this is one method for installing the Capture Proxy on a node, These are the **prerequisites** to being able to attach the Capture Proxy: -* **Make sure that the node and your MSK client are in the same VPC and Security Groups** +* **Make sure that your MSK client is accessible by the coordinator nodes in the cluster* * Add the following IAM policy to the node/EC2 instance so that it’s able to store the captured traffic in Kafka: * From the AWS Console, go to the EC2 instance page, click on **IAM Role**, click on **Add permissions**, choose **Create inline policy**, click on **JSON VIEW** then add the following policy (replace region and account-id). @@ -34,10 +34,11 @@ These are the **prerequisites** to being able to attach the Capture Proxy: } ``` -* From **linux command line** of that EC2 instance: Check that the **JAVA_HOME environment variable is set** properly `(echo $JAVA_HOME)`, if not, then try running the following command that might help set it correctly: +* **Verify Java installation is accessible**. + * From linux command line of that EC2 instance, Check that the JAVA_HOME environment variable is set properly `(echo $JAVA_HOME)`, if not, then try running the following command that might help set it correctly: - `JAVA_HOME=$(dirname "$(dirname "$(type -p java)")")` - * If that doesn’t work, then find the java directory on your node and set it as $JAVA_HOME + `JAVA_HOME=$(dirname "$(dirname "$(type -p java)")")` + * If that doesn’t work, then find the java directory on your node and set it as $JAVA_HOME ### Follow these steps to attach a Capture Proxy on the node. @@ -68,8 +69,6 @@ These are the **prerequisites** to being able to attach the Capture Proxy: * **--enableMSKAuth**: Enables SASL Kafka properties required for connecting to MSK with IAM auth. * **--insecureDestination**: Do not check the destination server’s certificate. - - 8. **Test the port** that the Capture Proxy is now listening to. 1. `curl https://localhost:9200` or `http://` 2. You should expect the same response when sending a request to either ports (9200, 19200), except that the traffic sent to the port that the Capture Proxy is listening to, will be captured and sent to your MSK Client, also forwarded to the new Elasticsearch port. @@ -78,23 +77,5 @@ These are the **prerequisites** to being able to attach the Capture Proxy: 1. Log in to the Migration Console container. 2. Go the Kafka tools directory cd kafka-tools/kafka/bin - 3. Create the following file to allow communication with an AWS MSK cluster. Name it: `msk-iam-auth.properties` - - ``` - # --- Additional setup to use AWS MSK IAM library for communication with an AWS MSK cluster - # Sets up TLS for encryption and SASL for authN. - security.protocol = SASL_SSL - - # Identifies the SASL mechanism to use. - sasl.mechanism = AWS_MSK_IAM - - # Binds SASL client implementation. - sasl.jaas.config = software.amazon.msk.auth.iam.IAMLoginModule required; - - # Encapsulates constructing a SigV4 signature based on extracted credentials. - # The SASL client bound by "sasl.jaas.config" invokes this class. - sasl.client.callback.handler.class = software.amazon.msk.auth.iam.IAMClientCallbackHandler - ``` - - 4. Run the following command to list the Kafka topics, and confirm that a new topic was created. - `./kafka-topics.sh —bootstrap-server "$MIGRATION_KAFKA_BROKER_ENDPOINTS" —list —command-config msk-iam-auth.properties` + 3. Run the following command to list the Kafka topics, and confirm that a new topic was created. + `./kafka-topics.sh --bootstrap-server "$MIGRATION_KAFKA_BROKER_ENDPOINTS" --list --command-config ../../aws/msk-iam-auth.properties` From 353ab1168cee47338d4b51f5bf65acc8e2f5213f Mon Sep 17 00:00:00 2001 From: Kartik Ganesh Date: Thu, 9 Nov 2023 20:42:32 -0800 Subject: [PATCH 33/55] [Fetch Migration] Support for dynamic target host substitution via environment variable, and fetch_orchestrator updates This commit includes a few changes: 1 - Fetch Migration now supports supplying the target cluster endpoint via environment variable (INLINE_TARGET_HOST) similar to INLINE_PIPELINE behavior. If present, the value of this env variable is written as the hostname for the Data Prepper sink in the Data Prepper pipeline configuration file before kicking off any steps in the Fetch Migration workflow. 2 - The CDK has been changed to utilize the above environment variable. This is required to mitigate a bug with deriving the target cluster endpoint from StringParameter.valueForStringParameter, which returns a Token that is only resolved at deployment time. This resulted in a string representation of the Token object (eg. ${Token[TOKEN.123]}) being written instead of the actual endpoint URI. 3 - fetch_orchestrator has been updated to assume a local Data Prepper process, so it no longer accepts a Data Prepper "endpoint" URL but constructs this based on input parameters (port value and SSL flag). 3 - fetch_orchestrator now accepts additional flags that let it only execute metadata migration (and skip data migration). This is useful to run index setup prior to performing a Capture & Replay migration. --------- Signed-off-by: Kartik Ganesh --- FetchMigration/Dockerfile | 5 +- FetchMigration/python/fetch_orchestrator.py | 106 ++++++++++--- .../python/fetch_orchestrator_params.py | 29 ++++ .../tests/resources/test_pipeline_input.yaml | 3 +- .../python/tests/test_fetch_orchestrator.py | 147 ++++++++++++++++-- .../tests/test_fetch_orchestrator_params.py | 17 ++ .../lib/fetch-migration-stack.ts | 9 +- 7 files changed, 281 insertions(+), 35 deletions(-) create mode 100644 FetchMigration/python/fetch_orchestrator_params.py create mode 100644 FetchMigration/python/tests/test_fetch_orchestrator_params.py diff --git a/FetchMigration/Dockerfile b/FetchMigration/Dockerfile index 8d9b5ab990..e823d55f24 100644 --- a/FetchMigration/Dockerfile +++ b/FetchMigration/Dockerfile @@ -18,5 +18,6 @@ ENV PATH=/root/.local:$PATH RUN echo "ssl: false" > $DATA_PREPPER_PATH/config/data-prepper-config.yaml RUN echo "metricRegistries: [Prometheus]" >> $DATA_PREPPER_PATH/config/data-prepper-config.yaml -# Include the -u flag to have stdout logged -ENTRYPOINT python3 -u ./fetch_orchestrator.py $DATA_PREPPER_PATH $FM_CODE_PATH/input.yaml http://localhost:4900 +# Include the -u flag to have unbuffered stdout (for detached mode) +# "$@" allows passing extra args/flags to the entrypoint when the image is run +ENTRYPOINT python3 -u ./fetch_orchestrator.py --insecure $DATA_PREPPER_PATH $FM_CODE_PATH/input.yaml "$@" diff --git a/FetchMigration/python/fetch_orchestrator.py b/FetchMigration/python/fetch_orchestrator.py index d51f6a58cb..dec89bdec0 100644 --- a/FetchMigration/python/fetch_orchestrator.py +++ b/FetchMigration/python/fetch_orchestrator.py @@ -2,33 +2,97 @@ import base64 import logging import os +import re import subprocess import sys from typing import Optional +import yaml + import metadata_migration import migration_monitor +from fetch_orchestrator_params import FetchOrchestratorParams from metadata_migration_params import MetadataMigrationParams from migration_monitor_params import MigrationMonitorParams +__PROTOCOL_PREFIX_PATTERN = re.compile(r"^https?://") +__HTTPS_PREFIX = "https://" __DP_EXECUTABLE_SUFFIX = "/bin/data-prepper" __PIPELINE_OUTPUT_FILE_SUFFIX = "/pipelines/pipeline.yaml" -def run(dp_base_path: str, dp_config_file: str, dp_endpoint: str) -> Optional[int]: - dp_exec_path = dp_base_path + __DP_EXECUTABLE_SUFFIX - output_file = dp_base_path + __PIPELINE_OUTPUT_FILE_SUFFIX - metadata_migration_params = MetadataMigrationParams(dp_config_file, output_file, report=True) - logging.info("Running pre-migration steps...\n") +def __get_env_string(name: str) -> Optional[str]: + val: str = os.environ.get(name, "") + if len(val) > 0: + return val + else: + return None + + +def update_target_host(dp_config: dict, target_host: str): + # Inline target host only supports HTTPS, so force it + target_with_protocol = target_host + match = __PROTOCOL_PREFIX_PATTERN.match(target_with_protocol) + if match: + target_with_protocol = target_host[match.end():] + target_with_protocol = __HTTPS_PREFIX + target_with_protocol + if len(dp_config) > 0: + # We expect the Data Prepper pipeline to only have a single top-level value + pipeline_config = next(iter(dp_config.values())) + # The entire pipeline will be validated later + if metadata_migration.SINK_KEY in pipeline_config: + # throws ValueError if no supported endpoints are found + plugin_name, plugin_config = metadata_migration.get_supported_endpoint(pipeline_config, + metadata_migration.SINK_KEY) + plugin_config[metadata_migration.HOSTS_KEY] = [target_with_protocol] + pipeline_config[metadata_migration.SINK_KEY] = [{plugin_name: plugin_config}] + + +def write_inline_pipeline(pipeline_file_path: str, inline_pipeline: str, inline_target_host: Optional[str]): + pipeline_yaml = yaml.safe_load(base64.b64decode(inline_pipeline)) + if inline_target_host is not None: + update_target_host(pipeline_yaml, inline_target_host) + with open(pipeline_file_path, 'w') as out_file: + # Note - this does not preserve comments + yaml.safe_dump(pipeline_yaml, out_file) + + +def write_inline_target_host(pipeline_file_path: str, inline_target_host: str): + with open(pipeline_file_path, 'rw') as pipeline_file: + pipeline_yaml = yaml.safe_load(pipeline_file) + update_target_host(pipeline_yaml, inline_target_host) + # Note - this does not preserve comments + yaml.safe_dump(pipeline_yaml, pipeline_file) + + +def run(params: FetchOrchestratorParams) -> Optional[int]: + # This is expected to be a base64 encoded string + inline_pipeline = __get_env_string("INLINE_PIPELINE") + inline_target_host = __get_env_string("INLINE_TARGET_HOST") + if inline_pipeline is not None: + write_inline_pipeline(params.pipeline_file_path, inline_pipeline, inline_target_host) + elif inline_target_host is not None: + write_inline_target_host(params.pipeline_file_path, inline_target_host) + dp_exec_path = params.data_prepper_path + __DP_EXECUTABLE_SUFFIX + output_file = params.data_prepper_path + __PIPELINE_OUTPUT_FILE_SUFFIX + if params.is_dry_run: + logging.info("Dry-run flag enabled, no actual changes will be made\n") + elif params.is_create_only: + logging.info("Create-only flag enabled, will only perform metadata migration\n") + metadata_migration_params = MetadataMigrationParams(params.pipeline_file_path, output_file, + report=True, dryrun=params.is_dry_run) + logging.info("Running metadata migration...\n") metadata_migration_result = metadata_migration.run(metadata_migration_params) - if len(metadata_migration_result.created_indices) > 0: + if len(metadata_migration_result.created_indices) > 0 and not params.is_only_metadata_migration(): # Kick off a subprocess for Data Prepper logging.info("Running Data Prepper...\n") proc = subprocess.Popen(dp_exec_path) # Run the migration monitor next - migration_monitor_params = MigrationMonitorParams(metadata_migration_result.target_doc_count, dp_endpoint) + migration_monitor_params = MigrationMonitorParams(metadata_migration_result.target_doc_count, + params.get_local_endpoint()) logging.info("Starting migration monitor...\n") return migration_monitor.run(migration_monitor_params, proc) + logging.info("Fetch Migration workflow concluded\n") if __name__ == '__main__': # pragma no cover @@ -46,22 +110,28 @@ def run(dp_base_path: str, dp_config_file: str, dp_endpoint: str) -> Optional[in help="Path to the base directory where Data Prepper is installed " ) arg_parser.add_argument( - "config_file_path", + "pipeline_file_path", help="Path to the Data Prepper pipeline YAML file to parse for source and target endpoint information" ) + # Optional positional argument arg_parser.add_argument( - "data_prepper_endpoint", - help="Data Prepper endpoint for monitoring the migration" + "port", type=int, + nargs='?', default=4900, + help="Local port at which the Data Prepper process will expose its APIs" ) + # Flags + arg_parser.add_argument("--insecure", "-k", action="store_true", + help="Specifies that the local Data Prepper process is not using SSL") + arg_parser.add_argument("--dryrun", action="store_true", + help="Performs a dry-run. Only a report is printed - no indices are created or migrated") + arg_parser.add_argument("--createonly", "-c", action="store_true", + help="Skips data migration and only creates indices on the target cluster") cli_args = arg_parser.parse_args() - base_path = os.path.expandvars(cli_args.data_prepper_path) - - inline_pipeline = os.environ.get("INLINE_PIPELINE", None) - if inline_pipeline is not None: - decoded_bytes = base64.b64decode(inline_pipeline) - with open(cli_args.config_file_path, 'wb') as config_file: - config_file.write(decoded_bytes) - return_code = run(base_path, cli_args.config_file_path, cli_args.data_prepper_endpoint) + params = FetchOrchestratorParams(os.path.expandvars(cli_args.data_prepper_path), + os.path.expandvars(cli_args.pipeline_file_path), + port=cli_args.port, insecure=cli_args.insecure, + dryrun=cli_args.dryrun, create_only=cli_args.createonly) + return_code = run(params) if return_code == 0: sys.exit(0) else: diff --git a/FetchMigration/python/fetch_orchestrator_params.py b/FetchMigration/python/fetch_orchestrator_params.py new file mode 100644 index 0000000000..7f34597f5f --- /dev/null +++ b/FetchMigration/python/fetch_orchestrator_params.py @@ -0,0 +1,29 @@ +LOCAL_ENDPOINT_HTTP: str = "http://localhost:" +LOCAL_ENDPOINT_HTTPS: str = "https://localhost:" + + +class FetchOrchestratorParams: + data_prepper_path: str + pipeline_file_path: str + local_port: int + is_insecure: bool + is_dry_run: bool + is_create_only: bool + + def __init__(self, dp_path: str, config_path: str, port: int = 4900, insecure: bool = False, dryrun: bool = False, + create_only: bool = False): + self.data_prepper_path = dp_path + self.pipeline_file_path = config_path + self.local_port = port + self.is_insecure = insecure + self.is_dry_run = dryrun + self.is_create_only = create_only + + def get_local_endpoint(self) -> str: + if self.is_insecure: + return LOCAL_ENDPOINT_HTTP + str(self.local_port) + else: + return LOCAL_ENDPOINT_HTTPS + str(self.local_port) + + def is_only_metadata_migration(self) -> bool: + return self.is_dry_run or self.is_create_only diff --git a/FetchMigration/python/tests/resources/test_pipeline_input.yaml b/FetchMigration/python/tests/resources/test_pipeline_input.yaml index 7c33a7e216..f021c6f4c7 100644 --- a/FetchMigration/python/tests/resources/test_pipeline_input.yaml +++ b/FetchMigration/python/tests/resources/test_pipeline_input.yaml @@ -16,6 +16,7 @@ test-pipeline-input: - sink1: num_array: [0] - opensearch: - hosts: ["https://os_host"] + hosts: + - "https://os_host" username: "test_user" password: "test" diff --git a/FetchMigration/python/tests/test_fetch_orchestrator.py b/FetchMigration/python/tests/test_fetch_orchestrator.py index 92f8baf87c..826d3baff0 100644 --- a/FetchMigration/python/tests/test_fetch_orchestrator.py +++ b/FetchMigration/python/tests/test_fetch_orchestrator.py @@ -1,10 +1,16 @@ +import copy +import os import unittest -from unittest.mock import patch, MagicMock, ANY +from unittest import mock +from unittest.mock import patch, MagicMock, ANY, mock_open import fetch_orchestrator as orchestrator -from migration_monitor_params import MigrationMonitorParams +from fetch_orchestrator_params import FetchOrchestratorParams from metadata_migration_params import MetadataMigrationParams from metadata_migration_result import MetadataMigrationResult +from migration_monitor_params import MigrationMonitorParams + +EXPECTED_LOCAL_ENDPOINT = "https://localhost:4900" class TestFetchOrchestrator(unittest.TestCase): @@ -13,23 +19,24 @@ class TestFetchOrchestrator(unittest.TestCase): @patch('subprocess.Popen') @patch('metadata_migration.run') # Note that mock objects are passed bottom-up from the patch order above + @mock.patch.dict(os.environ, {}, clear=True) def test_orchestrator_run(self, mock_metadata_migration: MagicMock, mock_subprocess: MagicMock, mock_monitor: MagicMock): - test_path = "test_path" - test_file = "test_file" - test_host = "test_host" + fetch_params = FetchOrchestratorParams("test_dp_path", "test_pipeline_file") # Setup mock pre-migration - expected_metadata_migration_input = MetadataMigrationParams(test_file, test_path + "/pipelines/pipeline.yaml", - report=True) + expected_metadata_migration_input = \ + MetadataMigrationParams(fetch_params.pipeline_file_path, + fetch_params.data_prepper_path + "/pipelines/pipeline.yaml", + report=True) test_result = MetadataMigrationResult(10, {"index1", "index2"}) - expected_monitor_input = MigrationMonitorParams(test_result.target_doc_count, test_host) + expected_monitor_input = MigrationMonitorParams(test_result.target_doc_count, EXPECTED_LOCAL_ENDPOINT) mock_metadata_migration.return_value = test_result # setup subprocess return value mock_subprocess.return_value.returncode = 0 # Run test - orchestrator.run(test_path, test_file, test_host) + orchestrator.run(fetch_params) mock_metadata_migration.assert_called_once_with(expected_metadata_migration_input) - expected_dp_runnable = test_path + "/bin/data-prepper" + expected_dp_runnable = fetch_params.data_prepper_path + "/bin/data-prepper" mock_subprocess.assert_called_once_with(expected_dp_runnable) mock_monitor.assert_called_once_with(expected_monitor_input, mock_subprocess.return_value) @@ -37,16 +44,134 @@ def test_orchestrator_run(self, mock_metadata_migration: MagicMock, mock_subproc @patch('subprocess.Popen') @patch('metadata_migration.run') # Note that mock objects are passed bottom-up from the patch order above + @mock.patch.dict(os.environ, {}, clear=True) def test_orchestrator_no_migration(self, mock_metadata_migration: MagicMock, mock_subprocess: MagicMock, mock_monitor: MagicMock): # Setup empty result from pre-migration mock_metadata_migration.return_value = MetadataMigrationResult() - orchestrator.run("test", "test", "test") + orchestrator.run(FetchOrchestratorParams("test", "test")) mock_metadata_migration.assert_called_once_with(ANY) # Subsequent steps should not be called mock_subprocess.assert_not_called() mock_monitor.assert_not_called() + @patch('migration_monitor.run') + @patch('subprocess.Popen') + @patch('metadata_migration.run') + # Note that mock objects are passed bottom-up from the patch order above + @mock.patch.dict(os.environ, {}, clear=True) + def test_orchestrator_run_create_only(self, mock_metadata_migration: MagicMock, mock_subprocess: MagicMock, + mock_monitor: MagicMock): + fetch_params = FetchOrchestratorParams("test_dp_path", "test_pipeline_file", create_only=True) + # Setup mock pre-migration + expected_metadata_migration_input = \ + MetadataMigrationParams(fetch_params.pipeline_file_path, + fetch_params.data_prepper_path + "/pipelines/pipeline.yaml", + report=True) + test_result = MetadataMigrationResult(10, {"index1", "index2"}) + mock_metadata_migration.return_value = test_result + # Run test + orchestrator.run(fetch_params) + mock_metadata_migration.assert_called_once_with(expected_metadata_migration_input) + mock_subprocess.assert_not_called() + mock_monitor.assert_not_called() + + @patch('migration_monitor.run') + @patch('subprocess.Popen') + @patch('metadata_migration.run') + # Note that mock objects are passed bottom-up from the patch order above + @mock.patch.dict(os.environ, {}, clear=True) + def test_orchestrator_dryrun(self, mock_metadata_migration: MagicMock, mock_subprocess: MagicMock, + mock_monitor: MagicMock): + fetch_params = FetchOrchestratorParams("test_dp_path", "test_pipeline_file", dryrun=True) + # Setup mock pre-migration + expected_metadata_migration_input = \ + MetadataMigrationParams(fetch_params.pipeline_file_path, + fetch_params.data_prepper_path + "/pipelines/pipeline.yaml", + report=True, dryrun=True) + test_result = MetadataMigrationResult(10, {"index1", "index2"}) + mock_metadata_migration.return_value = test_result + # Run test + orchestrator.run(fetch_params) + mock_metadata_migration.assert_called_once_with(expected_metadata_migration_input) + mock_subprocess.assert_not_called() + mock_monitor.assert_not_called() + + @patch('fetch_orchestrator.write_inline_target_host') + @patch('metadata_migration.run') + # Note that mock objects are passed bottom-up from the patch order above + @mock.patch.dict(os.environ, {"INLINE_TARGET_HOST": "host"}, clear=True) + def test_orchestrator_inline_target_host(self, mock_metadata_migration: MagicMock, mock_write_host: MagicMock): + # For testing, return no indices to migrate + mock_metadata_migration.return_value = MetadataMigrationResult() + orchestrator.run(FetchOrchestratorParams("test", "test")) + mock_metadata_migration.assert_called_once_with(ANY) + mock_write_host.assert_called_once_with(ANY, "host") + + @patch('fetch_orchestrator.update_target_host') + @patch('fetch_orchestrator.write_inline_pipeline') + @patch('metadata_migration.run') + # Note that mock objects are passed bottom-up from the patch order above + @mock.patch.dict(os.environ, {"INLINE_PIPELINE": "test"}, clear=True) + def test_orchestrator_inline_pipeline(self, mock_metadata_migration: MagicMock, mock_write_pipeline: MagicMock, + mock_update_host: MagicMock): + # For testing, return no indices to migrate + mock_metadata_migration.return_value = MetadataMigrationResult() + orchestrator.run(FetchOrchestratorParams("test", "test")) + mock_metadata_migration.assert_called_once_with(ANY) + mock_write_pipeline.assert_called_once_with("test", "test", None) + mock_update_host.assert_not_called() + + @patch('builtins.open', new_callable=mock_open()) + @patch('fetch_orchestrator.update_target_host') + @patch('metadata_migration.run') + # INLINE_PIPELINE value is base64 encoded version of {} + @mock.patch.dict(os.environ, {"INLINE_PIPELINE": "e30K", "INLINE_TARGET_HOST": "host"}, clear=True) + def test_orchestrator_inline_pipeline_and_host(self, mock_metadata_migration: MagicMock, + mock_update_host: MagicMock, mock_file_open: MagicMock): + # For testing, return no indices to migrate + mock_metadata_migration.return_value = MetadataMigrationResult() + params = FetchOrchestratorParams("test", "config_file") + orchestrator.run(params) + mock_metadata_migration.assert_called_once_with(ANY) + mock_update_host.assert_called_once_with(ANY, "host") + mock_file_open.assert_called_once_with(params.pipeline_file_path, 'w') + + @patch('yaml.safe_dump') + @patch('yaml.safe_load') + @patch('builtins.open', new_callable=mock_open()) + # Note that mock objects are passed bottom-up from the patch order above + def test_write_inline_target_host(self, mock_file_open: MagicMock, mock_yaml_load: MagicMock, + mock_yaml_dump: MagicMock): + mock_yaml_load.return_value = {"root": {"sink": [{"opensearch": {"hosts": ["val"]}}]}} + inline_target = "host" + expected_target_url = "https://" + inline_target + test_values = [ + inline_target, # base case test + "http://" + inline_target # tests forcing of HTTPS + ] + expected_pipeline = copy.deepcopy(mock_yaml_load.return_value) + # TODO - replace with jsonpath + expected_pipeline["root"]["sink"][0]["opensearch"]["hosts"] = [expected_target_url] + for val in test_values: + mock_file_open.reset_mock() + mock_yaml_dump.reset_mock() + orchestrator.write_inline_target_host("test", val) + mock_file_open.assert_called_once_with("test", "rw") + mock_yaml_dump.assert_called_once_with(expected_pipeline, ANY) + + def test_update_target_host_bad_config(self): + bad_configs = [ + {}, # empty config + {"root": {}}, # only root element + {"root": {"source": True}}, # no sink + ] + for config in bad_configs: + original = copy.deepcopy(config) + orchestrator.update_target_host(config, "host") + # Bad configs should result in no change + self.assertEqual(config, original) + if __name__ == '__main__': unittest.main() diff --git a/FetchMigration/python/tests/test_fetch_orchestrator_params.py b/FetchMigration/python/tests/test_fetch_orchestrator_params.py new file mode 100644 index 0000000000..3f41d2a422 --- /dev/null +++ b/FetchMigration/python/tests/test_fetch_orchestrator_params.py @@ -0,0 +1,17 @@ +import unittest + +from fetch_orchestrator_params import FetchOrchestratorParams + + +class TestFetchOrchestratorParams(unittest.TestCase): + def test_get_local_endpoint(self): + # Default value for insecure flag (secure endpoint) + params = FetchOrchestratorParams("test", "test", port=123) + self.assertEqual("https://localhost:123", params.get_local_endpoint()) + # insecure endpoint + params = FetchOrchestratorParams("test", "test", port=123, insecure=True) + self.assertEqual("http://localhost:123", params.get_local_endpoint()) + + +if __name__ == '__main__': + unittest.main() diff --git a/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts b/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts index 8a18e6a879..de4c016f69 100644 --- a/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts @@ -26,7 +26,7 @@ export class FetchMigrationStack extends Stack { super(scope, id, props); // Import required values - const targetClusterEndpoint = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/osClusterEndpoint`) + const targetClusterEndpoint = StringParameter.fromStringParameterName(this, "targetClusterEndpoint", `/migration/${props.stage}/${props.defaultDeployId}/osClusterEndpoint`) const domainAccessGroupId = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/osAccessSecurityGroupId`) // This SG allows outbound access for ECR access as well as communication with other services in the cluster const serviceConnectGroupId = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceConnectSecurityGroupId`) @@ -67,8 +67,8 @@ export class FetchMigrationStack extends Stack { // Create DP pipeline config from template file let dpPipelineData: string = readFileSync(props.dpPipelineTemplatePath, 'utf8'); + // Replace only source cluster host - target host will be overridden by inline env var dpPipelineData = dpPipelineData.replace("", props.sourceEndpoint); - dpPipelineData = dpPipelineData.replace("", targetClusterEndpoint); // Base64 encode let encodedPipeline = Buffer.from(dpPipelineData).toString("base64"); @@ -77,10 +77,13 @@ export class FetchMigrationStack extends Stack { secretName: `${props.stage}-${props.defaultDeployId}-${fetchMigrationContainer.containerName}-pipelineConfig`, secretStringValue: SecretValue.unsafePlainText(encodedPipeline) }); - // Add secret to container + // Add secrets to container fetchMigrationContainer.addSecret("INLINE_PIPELINE", ECSSecret.fromSecretsManager(dpPipelineConfigSecret) ); + fetchMigrationContainer.addSecret("INLINE_TARGET_HOST", + ECSSecret.fromSsmParameter(targetClusterEndpoint) + ); let networkConfigJson = { "awsvpcConfiguration": { From 8989f00d715a2f619a22e73b592ec080d945f7e3 Mon Sep 17 00:00:00 2001 From: Kartik Ganesh Date: Thu, 9 Nov 2023 23:20:21 -0800 Subject: [PATCH 34/55] AWS SigV4 support in Fetch Migration (#394) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit enables support for SigV4 signed requests to AWS OpenSearch / OpenSearch Serverless endpoints in Fetch Migration, using the requests-aws4auth library. To achieve this, this change is comprised of several parts: 1 - All “endpoint” related logic is now encapsulated in the EndpointInfo class, including logic to construct API paths instead of having callers compute this. 2 - Construction of EndpointInfo instances from the Data Prepper pipeline configuration (and its plugin configuration sub-sections) has been moved out of metadata_migration.py to a new endpoint_utils.py file for better abstraction. 3 - Use of SigV4 is inferred from the supplied DP pipeline (separately for source and sink), including detection of the serverless key to change the service name (‘es’ vs. ‘aoss’) 4 - Since AWS4Auth requires a region argument, Fetch Migration first checks the plugin configuration for an explicitly defined region. If this is not present (since it is an optional parameter), the code attempts to derive the region based on the service endpoint URL (since generated endpoint URLs usually include the region). If a region cannot be inferred, a ValueError is thrown. Unit tests for all of these components have been added (or existing ones updated). A minor refactoring of logging in migration_monitor.py is also included, which improves unit test code coverage. --------- Signed-off-by: Kartik Ganesh --- FetchMigration/python/dev-requirements.txt | 5 +- FetchMigration/python/endpoint_info.py | 42 ++- FetchMigration/python/endpoint_utils.py | 156 ++++++++++ FetchMigration/python/fetch_orchestrator.py | 11 +- FetchMigration/python/index_operations.py | 14 +- FetchMigration/python/metadata_migration.py | 114 +------ FetchMigration/python/migration_monitor.py | 26 +- FetchMigration/python/progress_metrics.py | 58 ++-- FetchMigration/python/requirements.txt | 2 + .../python/tests/test_endpoint_utils.py | 282 ++++++++++++++++++ .../python/tests/test_metadata_migration.py | 140 --------- 11 files changed, 554 insertions(+), 296 deletions(-) create mode 100644 FetchMigration/python/endpoint_utils.py create mode 100644 FetchMigration/python/tests/test_endpoint_utils.py diff --git a/FetchMigration/python/dev-requirements.txt b/FetchMigration/python/dev-requirements.txt index 2efcff09b9..97396f4a70 100644 --- a/FetchMigration/python/dev-requirements.txt +++ b/FetchMigration/python/dev-requirements.txt @@ -1,2 +1,5 @@ coverage>=7.3.2 -pur>=7.3.1 \ No newline at end of file +pur>=7.3.1 +moto>=4.2.7 +# Transitive dependency from moto, explicit version needed to mitigate CVE-2023-46136 +werkzeug>=3.0.1 \ No newline at end of file diff --git a/FetchMigration/python/endpoint_info.py b/FetchMigration/python/endpoint_info.py index b77880dc08..81ace5be3b 100644 --- a/FetchMigration/python/endpoint_info.py +++ b/FetchMigration/python/endpoint_info.py @@ -1,8 +1,40 @@ -from dataclasses import dataclass +from typing import Optional +from requests_aws4auth import AWS4Auth -@dataclass + +# Class that encapsulates endpoint information for an OpenSearch/Elasticsearch cluster class EndpointInfo: - url: str - auth: tuple = None - verify_ssl: bool = True + # Private member variables + __url: str + __auth: Optional[tuple] | AWS4Auth + __verify_ssl: bool + + def __init__(self, url: str, auth: tuple | AWS4Auth = None, verify_ssl: bool = True): + self.__url = url + # Normalize url value to have trailing slash + if not url.endswith("/"): + self.__url += "/" + self.__auth = auth + self.__verify_ssl = verify_ssl + + def __eq__(self, obj): + return isinstance(obj, EndpointInfo) and \ + self.__url == obj.__url and \ + self.__auth == obj.__auth and \ + self.__verify_ssl == obj.__verify_ssl + + def add_path(self, path: str) -> str: + # Remove leading slash if present + if path.startswith("/"): + path = path[1:] + return self.__url + path + + def get_url(self) -> str: + return self.__url + + def get_auth(self) -> Optional[tuple] | AWS4Auth: + return self.__auth + + def is_verify_ssl(self) -> bool: + return self.__verify_ssl diff --git a/FetchMigration/python/endpoint_utils.py b/FetchMigration/python/endpoint_utils.py new file mode 100644 index 0000000000..fe9ef0d97a --- /dev/null +++ b/FetchMigration/python/endpoint_utils.py @@ -0,0 +1,156 @@ +import re +from typing import Optional + +from requests_aws4auth import AWS4Auth +from botocore.session import Session + +from endpoint_info import EndpointInfo + +# Constants +SOURCE_KEY = "source" +SINK_KEY = "sink" +SUPPORTED_PLUGINS = ["opensearch", "elasticsearch"] +HOSTS_KEY = "hosts" +INSECURE_KEY = "insecure" +CONNECTION_KEY = "connection" +DISABLE_AUTH_KEY = "disable_authentication" +USER_KEY = "username" +PWD_KEY = "password" +AWS_SIGV4_KEY = "aws_sigv4" +AWS_REGION_KEY = "aws_region" +AWS_CONFIG_KEY = "aws" +AWS_CONFIG_REGION_KEY = "region" +IS_SERVERLESS_KEY = "serverless" +ES_SERVICE_NAME = "es" +AOSS_SERVICE_NAME = "aoss" +URL_REGION_PATTERN = re.compile(r"([\w-]*)\.(es|aoss)\.amazonaws\.com") + + +def __get_url(plugin_config: dict) -> str: + # "hosts" can be a simple string, or an array of hosts for Logstash to hit. + # This tool needs one accessible host, so we pick the first entry in the latter case. + return plugin_config[HOSTS_KEY][0] if isinstance(plugin_config[HOSTS_KEY], list) else plugin_config[HOSTS_KEY] + + +# Helper function that attempts to extract the AWS region from a URL, +# assuming it is of the form *...amazonaws.com +def __derive_aws_region_from_url(url: str) -> Optional[str]: + match = URL_REGION_PATTERN.search(url) + if match: + # Index 0 returns the entire match, index 1 returns only the first group + return match.group(1) + return None + + +def get_aws_region(plugin_config: dict) -> str: + if plugin_config.get(AWS_SIGV4_KEY, False) and plugin_config.get(AWS_REGION_KEY, None) is not None: + return plugin_config[AWS_REGION_KEY] + elif plugin_config.get(AWS_CONFIG_KEY, None) is not None: + aws_config = plugin_config[AWS_CONFIG_KEY] + if not isinstance(aws_config, dict): + raise ValueError("Unexpected value for 'aws' configuration") + elif aws_config.get(AWS_CONFIG_REGION_KEY, None) is not None: + return aws_config[AWS_CONFIG_REGION_KEY] + # Region not explicitly defined, attempt to derive from URL + derived_region = __derive_aws_region_from_url(__get_url(plugin_config)) + if derived_region is None: + raise ValueError("No region configured for AWS SigV4 auth, or derivable from host URL") + return derived_region + + +def __check_supported_endpoint(section_config: dict) -> Optional[tuple]: + for supported_type in SUPPORTED_PLUGINS: + if supported_type in section_config: + return supported_type, section_config[supported_type] + + +# This config key may be either directly in the main dict (for sink) +# or inside a nested dict (for source). The default value is False. +def is_insecure(plugin_config: dict) -> bool: + if INSECURE_KEY in plugin_config: + return plugin_config[INSECURE_KEY] + elif CONNECTION_KEY in plugin_config and INSECURE_KEY in plugin_config[CONNECTION_KEY]: + return plugin_config[CONNECTION_KEY][INSECURE_KEY] + return False + + +def validate_pipeline(pipeline: dict): + if SOURCE_KEY not in pipeline: + raise ValueError("Missing source configuration in Data Prepper pipeline YAML") + if SINK_KEY not in pipeline: + raise ValueError("Missing sink configuration in Data Prepper pipeline YAML") + + +def validate_auth(plugin_name: str, plugin_config: dict): + # If auth is disabled, no further validation is required + if plugin_config.get(DISABLE_AUTH_KEY, False): + return + # If AWS SigV4 is configured, validate region + if plugin_config.get(AWS_SIGV4_KEY, False) or AWS_CONFIG_KEY in plugin_config: + # Raises a ValueError if region cannot be derived + get_aws_region(plugin_config) + return + # Validate basic auth + elif USER_KEY not in plugin_config: + raise ValueError("Invalid auth configuration (no username) for plugin: " + plugin_name) + elif PWD_KEY not in plugin_config: + raise ValueError("Invalid auth configuration (no password for username) for plugin: " + plugin_name) + + +def get_supported_endpoint_config(pipeline_config: dict, section_key: str) -> tuple: + # The value of each key may be a single plugin (as a dict) or a list of plugin configs + supported_tuple = tuple() + if isinstance(pipeline_config[section_key], dict): + supported_tuple = __check_supported_endpoint(pipeline_config[section_key]) + elif isinstance(pipeline_config[section_key], list): + for entry in pipeline_config[section_key]: + supported_tuple = __check_supported_endpoint(entry) + # Break out of the loop at the first supported type + if supported_tuple: + break + if not supported_tuple: + raise ValueError("Could not find any supported endpoints in section: " + section_key) + # First tuple value is the plugin name, second value is the plugin config dict + return supported_tuple[0], supported_tuple[1] + + +def get_aws_sigv4_auth(region: str, is_serverless: bool = False) -> AWS4Auth: + credentials = Session().get_credentials() + if not credentials: + raise ValueError("Unable to fetch AWS session credentials for SigV4 auth") + if is_serverless: + return AWS4Auth(region=region, service=AOSS_SERVICE_NAME, refreshable_credentials=credentials) + else: + return AWS4Auth(region=region, service=ES_SERVICE_NAME, refreshable_credentials=credentials) + + +def get_auth(plugin_config: dict) -> Optional[tuple] | AWS4Auth: + # Basic auth + if USER_KEY in plugin_config and PWD_KEY in plugin_config: + return plugin_config[USER_KEY], plugin_config[PWD_KEY] + elif plugin_config.get(AWS_SIGV4_KEY, False) or AWS_CONFIG_KEY in plugin_config: + is_serverless = False + # OpenSearch Serverless uses a different service name + if AWS_CONFIG_KEY in plugin_config: + aws_config = plugin_config[AWS_CONFIG_KEY] + if isinstance(aws_config, dict) and aws_config.get(IS_SERVERLESS_KEY, False): + is_serverless = True + region = get_aws_region(plugin_config) + return get_aws_sigv4_auth(region, is_serverless) + return None + + +def get_endpoint_info_from_plugin_config(plugin_config: dict) -> EndpointInfo: + # verify boolean will be the inverse of the insecure SSL key, if present + should_verify = not is_insecure(plugin_config) + return EndpointInfo(__get_url(plugin_config), get_auth(plugin_config), should_verify) + + +def get_endpoint_info_from_pipeline_config(pipeline_config: dict, section_key: str) -> EndpointInfo: + # Raises a ValueError if no supported endpoints are found + plugin_name, plugin_config = get_supported_endpoint_config(pipeline_config, section_key) + if HOSTS_KEY not in plugin_config: + raise ValueError("No hosts defined for plugin: " + plugin_name) + # Raises a ValueError if there an error in the auth configuration + validate_auth(plugin_name, plugin_config) + return get_endpoint_info_from_plugin_config(plugin_config) diff --git a/FetchMigration/python/fetch_orchestrator.py b/FetchMigration/python/fetch_orchestrator.py index dec89bdec0..a624dd191e 100644 --- a/FetchMigration/python/fetch_orchestrator.py +++ b/FetchMigration/python/fetch_orchestrator.py @@ -9,6 +9,7 @@ import yaml +import endpoint_utils import metadata_migration import migration_monitor from fetch_orchestrator_params import FetchOrchestratorParams @@ -40,12 +41,12 @@ def update_target_host(dp_config: dict, target_host: str): # We expect the Data Prepper pipeline to only have a single top-level value pipeline_config = next(iter(dp_config.values())) # The entire pipeline will be validated later - if metadata_migration.SINK_KEY in pipeline_config: + if endpoint_utils.SINK_KEY in pipeline_config: # throws ValueError if no supported endpoints are found - plugin_name, plugin_config = metadata_migration.get_supported_endpoint(pipeline_config, - metadata_migration.SINK_KEY) - plugin_config[metadata_migration.HOSTS_KEY] = [target_with_protocol] - pipeline_config[metadata_migration.SINK_KEY] = [{plugin_name: plugin_config}] + plugin_name, plugin_config = endpoint_utils.get_supported_endpoint_config(pipeline_config, + endpoint_utils.SINK_KEY) + plugin_config[endpoint_utils.HOSTS_KEY] = [target_with_protocol] + pipeline_config[endpoint_utils.SINK_KEY] = [{plugin_name: plugin_config}] def write_inline_pipeline(pipeline_file_path: str, inline_pipeline: str, inline_target_host: Optional[str]): diff --git a/FetchMigration/python/index_operations.py b/FetchMigration/python/index_operations.py index e273eea701..a311ec7e61 100644 --- a/FetchMigration/python/index_operations.py +++ b/FetchMigration/python/index_operations.py @@ -13,8 +13,8 @@ def fetch_all_indices(endpoint: EndpointInfo) -> dict: - actual_endpoint = endpoint.url + __ALL_INDICES_ENDPOINT - resp = requests.get(actual_endpoint, auth=endpoint.auth, verify=endpoint.verify_ssl) + all_indices_url: str = endpoint.add_path(__ALL_INDICES_ENDPOINT) + resp = requests.get(all_indices_url, auth=endpoint.get_auth(), verify=endpoint.is_verify_ssl()) result = dict(resp.json()) for index in list(result.keys()): # Remove system indices @@ -31,19 +31,21 @@ def fetch_all_indices(endpoint: EndpointInfo) -> dict: def create_indices(indices_data: dict, endpoint: EndpointInfo): for index in indices_data: - actual_endpoint = endpoint.url + index + index_endpoint = endpoint.add_path(index) data_dict = dict() data_dict[SETTINGS_KEY] = indices_data[index][SETTINGS_KEY] data_dict[MAPPINGS_KEY] = indices_data[index][MAPPINGS_KEY] try: - resp = requests.put(actual_endpoint, auth=endpoint.auth, verify=endpoint.verify_ssl, json=data_dict) + resp = requests.put(index_endpoint, auth=endpoint.get_auth(), verify=endpoint.is_verify_ssl(), + json=data_dict) resp.raise_for_status() except requests.exceptions.RequestException as e: raise RuntimeError(f"Failed to create index [{index}] - {e!s}") def doc_count(indices: set, endpoint: EndpointInfo) -> int: - actual_endpoint = endpoint.url + ','.join(indices) + __COUNT_ENDPOINT - resp = requests.get(actual_endpoint, auth=endpoint.auth, verify=endpoint.verify_ssl) + count_endpoint_suffix: str = ','.join(indices) + __COUNT_ENDPOINT + doc_count_endpoint: str = endpoint.add_path(count_endpoint_suffix) + resp = requests.get(doc_count_endpoint, auth=endpoint.get_auth(), verify=endpoint.is_verify_ssl()) result = dict(resp.json()) return int(result[COUNT_KEY]) diff --git a/FetchMigration/python/metadata_migration.py b/FetchMigration/python/metadata_migration.py index f7fc9ec768..a86eb330ce 100644 --- a/FetchMigration/python/metadata_migration.py +++ b/FetchMigration/python/metadata_migration.py @@ -1,108 +1,23 @@ import argparse -from typing import Optional import yaml +import endpoint_utils import index_operations import utils -# Constants -from endpoint_info import EndpointInfo from metadata_migration_params import MetadataMigrationParams from metadata_migration_result import MetadataMigrationResult -SUPPORTED_ENDPOINTS = ["opensearch", "elasticsearch"] -SOURCE_KEY = "source" -SINK_KEY = "sink" -HOSTS_KEY = "hosts" -DISABLE_AUTH_KEY = "disable_authentication" -USER_KEY = "username" -PWD_KEY = "password" -INSECURE_KEY = "insecure" -CONNECTION_KEY = "connection" +# Constants INDICES_KEY = "indices" INCLUDE_KEY = "include" INDEX_NAME_KEY = "index_name_regex" -# This config key may be either directly in the main dict (for sink) -# or inside a nested dict (for source). The default value is False. -def is_insecure(config: dict) -> bool: - if INSECURE_KEY in config: - return config[INSECURE_KEY] - elif CONNECTION_KEY in config and INSECURE_KEY in config[CONNECTION_KEY]: - return config[CONNECTION_KEY][INSECURE_KEY] - return False - - -# TODO Only supports basic auth for now -def get_auth(input_data: dict) -> Optional[tuple]: - if not input_data.get(DISABLE_AUTH_KEY, False) and USER_KEY in input_data and PWD_KEY in input_data: - return input_data[USER_KEY], input_data[PWD_KEY] - - -def get_endpoint_info(plugin_config: dict) -> EndpointInfo: - # "hosts" can be a simple string, or an array of hosts for Logstash to hit. - # This tool needs one accessible host, so we pick the first entry in the latter case. - url = plugin_config[HOSTS_KEY][0] if type(plugin_config[HOSTS_KEY]) is list else plugin_config[HOSTS_KEY] - url += "/" - # verify boolean will be the inverse of the insecure SSL key, if present - should_verify = not is_insecure(plugin_config) - return EndpointInfo(url, get_auth(plugin_config), should_verify) - - -def check_supported_endpoint(config: dict) -> Optional[tuple]: - for supported_type in SUPPORTED_ENDPOINTS: - if supported_type in config: - return supported_type, config[supported_type] - - -def get_supported_endpoint(config: dict, key: str) -> tuple: - # The value of each key may be a single plugin (as a dict) - # or a list of plugin configs - supported_tuple = tuple() - if type(config[key]) is dict: - supported_tuple = check_supported_endpoint(config[key]) - elif type(config[key]) is list: - for entry in config[key]: - supported_tuple = check_supported_endpoint(entry) - # Break out of the loop at the first supported type - if supported_tuple: - break - if not supported_tuple: - raise ValueError("Could not find any supported endpoints in section: " + key) - # First tuple value is the name, second value is the config dict - return supported_tuple[0], supported_tuple[1] - - -def validate_plugin_config(config: dict, key: str): - # Raises a ValueError if no supported endpoints are found - supported_endpoint = get_supported_endpoint(config, key) - plugin_config = supported_endpoint[1] - if HOSTS_KEY not in plugin_config: - raise ValueError("No hosts defined for endpoint: " + supported_endpoint[0]) - # Check if auth is disabled. If so, no further validation is required - if plugin_config.get(DISABLE_AUTH_KEY, False): - return - elif USER_KEY not in plugin_config: - raise ValueError("Invalid auth configuration (no username) for endpoint: " + supported_endpoint[0]) - elif PWD_KEY not in plugin_config: - raise ValueError("Invalid auth configuration (no password for username) for endpoint: " + - supported_endpoint[0]) - - -def validate_pipeline_config(config: dict): - if SOURCE_KEY not in config: - raise ValueError("Missing source configuration in Data Prepper pipeline YAML") - if SINK_KEY not in config: - raise ValueError("Missing sink configuration in Data Prepper pipeline YAML") - validate_plugin_config(config, SOURCE_KEY) - validate_plugin_config(config, SINK_KEY) - - def write_output(yaml_data: dict, new_indices: set, output_path: str): pipeline_config = next(iter(yaml_data.values())) - # Endpoint is a tuple of (type, config) - source_config = get_supported_endpoint(pipeline_config, SOURCE_KEY)[1] + # Result is a tuple of (type, config) + source_config = endpoint_utils.get_supported_endpoint_config(pipeline_config, endpoint_utils.SOURCE_KEY)[1] source_indices = source_config.get(INDICES_KEY, dict()) included_indices = source_indices.get(INCLUDE_KEY, list()) for index in new_indices: @@ -145,14 +60,6 @@ def print_report(index_differences: tuple[set, set, set], count: int): # pragma print("Total documents to be moved: " + str(count)) -def compute_endpoint_and_fetch_indices(config: dict, key: str) -> tuple[EndpointInfo, dict]: - endpoint = get_supported_endpoint(config, key) - # Endpoint is a tuple of (type, config) - endpoint_info = get_endpoint_info(endpoint[1]) - indices = index_operations.fetch_all_indices(endpoint_info) - return endpoint_info, indices - - def run(args: MetadataMigrationParams) -> MetadataMigrationResult: # Sanity check if not args.report and len(args.output_file) == 0: @@ -162,14 +69,19 @@ def run(args: MetadataMigrationParams) -> MetadataMigrationResult: dp_config = yaml.safe_load(pipeline_file) # We expect the Data Prepper pipeline to only have a single top-level value pipeline_config = next(iter(dp_config.values())) - validate_pipeline_config(pipeline_config) + # Raises a ValueError if source or sink definitions are missing + endpoint_utils.validate_pipeline(pipeline_config) + source_endpoint_info = endpoint_utils.get_endpoint_info_from_pipeline_config(pipeline_config, + endpoint_utils.SOURCE_KEY) + target_endpoint_info = endpoint_utils.get_endpoint_info_from_pipeline_config(pipeline_config, + endpoint_utils.SINK_KEY) result = MetadataMigrationResult() - # Fetch EndpointInfo and indices - source_endpoint_info, source_indices = compute_endpoint_and_fetch_indices(pipeline_config, SOURCE_KEY) + # Fetch indices + source_indices = index_operations.fetch_all_indices(source_endpoint_info) # If source indices is empty, return immediately if len(source_indices.keys()) == 0: return result - target_endpoint_info, target_indices = compute_endpoint_and_fetch_indices(pipeline_config, SINK_KEY) + target_indices = index_operations.fetch_all_indices(target_endpoint_info) # Compute index differences and print report diff = get_index_differences(source_indices, target_indices) # The first element in the tuple is the set of indices to create diff --git a/FetchMigration/python/migration_monitor.py b/FetchMigration/python/migration_monitor.py index 5a0150cb46..481d738964 100644 --- a/FetchMigration/python/migration_monitor.py +++ b/FetchMigration/python/migration_monitor.py @@ -41,14 +41,14 @@ def shutdown_process(proc: Popen) -> Optional[int]: def shutdown_pipeline(endpoint: EndpointInfo): - shutdown_endpoint = endpoint.url + __SHUTDOWN_API_PATH - requests.post(shutdown_endpoint, auth=endpoint.auth, verify=endpoint.verify_ssl) + shutdown_endpoint = endpoint.add_path(__SHUTDOWN_API_PATH) + requests.post(shutdown_endpoint, auth=endpoint.get_auth(), verify=endpoint.is_verify_ssl()) def fetch_prometheus_metrics(endpoint: EndpointInfo) -> Optional[List[Metric]]: - metrics_endpoint = endpoint.url + __METRICS_API_PATH + metrics_endpoint = endpoint.add_path(__METRICS_API_PATH) try: - response = requests.get(metrics_endpoint, auth=endpoint.auth, verify=endpoint.verify_ssl) + response = requests.get(metrics_endpoint, auth=endpoint.get_auth(), verify=endpoint.is_verify_ssl()) response.raise_for_status() except requests.exceptions.RequestException: return None @@ -95,11 +95,20 @@ def __should_continue_monitoring(progress: ProgressMetrics, proc: Optional[Popen return not progress.is_in_terminal_state() and (proc is None or is_process_alive(proc)) +def __log_migration_end_reason(progress: ProgressMetrics): # pragma no cover + if progress.is_migration_complete_success(): + logging.info("Migration monitor observed successful migration, shutting down...\n") + elif progress.is_migration_idle(): + logging.warning("Migration monitor observed idle pipeline (migration may be incomplete), shutting down...") + elif progress.is_too_may_api_failures(): + logging.warning("Migration monitor was unable to fetch migration metrics, terminating...") + + # The "dp_process" parameter is optional, and signifies a local Data Prepper process def run(args: MigrationMonitorParams, dp_process: Optional[Popen] = None, poll_interval_seconds: int = 30) -> int: endpoint_info = EndpointInfo(args.data_prepper_endpoint) progress_metrics = ProgressMetrics(args.target_count, __IDLE_THRESHOLD) - logging.info("Starting migration monitor until target doc count: " + str(progress_metrics.target_doc_count)) + logging.info("Starting migration monitor until target doc count: " + str(progress_metrics.get_target_doc_count())) while __should_continue_monitoring(progress_metrics, dp_process): if dp_process is not None: # Wait on local process @@ -120,12 +129,7 @@ def run(args: MigrationMonitorParams, dp_process: Optional[Popen] = None, poll_i logging.error("Please delete any partially migrated indices before retrying the migration.") return dp_process.returncode else: - if progress_metrics.is_migration_complete_success(): - logging.info("Migration monitor observed successful migration, shutting down...\n") - elif progress_metrics.is_migration_idle(): - logging.warning("Migration monitor observed idle pipeline (migration may be incomplete), shutting down...") - elif progress_metrics.is_too_may_api_failures(): - logging.warning("Migration monitor was unable to fetch migration metrics, terminating...") + __log_migration_end_reason(progress_metrics) # Shut down Data Prepper pipeline via API shutdown_pipeline(endpoint_info) if dp_process is None: diff --git a/FetchMigration/python/progress_metrics.py b/FetchMigration/python/progress_metrics.py index f3d08cf9c5..5a3a436b81 100644 --- a/FetchMigration/python/progress_metrics.py +++ b/FetchMigration/python/progress_metrics.py @@ -15,49 +15,53 @@ class ProgressMetrics: _REC_IN_FLIGHT_KEY = "records_in_flight" _NO_PART_KEY = "no_partitions" - target_doc_count: int - idle_threshold: int - current_values_map: dict[str, Optional[int]] - prev_values_map: dict[str, Optional[int]] - counter_map: dict[str, int] + # Private member variables + __target_doc_count: int + __idle_threshold: int + __current_values_map: dict[str, Optional[int]] + __prev_values_map: dict[str, Optional[int]] + __counter_map: dict[str, int] def __init__(self, doc_count, idle_threshold): - self.target_doc_count = doc_count - self.idle_threshold = idle_threshold - self.current_values_map = dict() - self.prev_values_map = dict() - self.counter_map = dict() + self.__target_doc_count = doc_count + self.__idle_threshold = idle_threshold + self.__current_values_map = dict() + self.__prev_values_map = dict() + self.__counter_map = dict() + + def get_target_doc_count(self) -> int: + return self.__target_doc_count def __reset_counter(self, key: str): - if key in self.counter_map: - del self.counter_map[key] + if key in self.__counter_map: + del self.__counter_map[key] def __increment_counter(self, key: str): - val = self.counter_map.get(key, 0) - self.counter_map[key] = val + 1 + val = self.__counter_map.get(key, 0) + self.__counter_map[key] = val + 1 def __get_idle_value_key_name(self, key: str) -> str: return self.__IDLE_VALUE_PREFIX + key def __get_idle_value_count(self, key: str) -> Optional[int]: idle_value_key = self.__get_idle_value_key_name(key) - return self.counter_map.get(idle_value_key) + return self.__counter_map.get(idle_value_key) def __record_value(self, key: str, val: Optional[int]): - if key in self.current_values_map: + if key in self.__current_values_map: # Move current value to previous - self.prev_values_map[key] = self.current_values_map[key] + self.__prev_values_map[key] = self.__current_values_map[key] # Track idle value metrics idle_value_key = self.__get_idle_value_key_name(key) - if self.prev_values_map[key] == val: + if self.__prev_values_map[key] == val: self.__increment_counter(idle_value_key) else: self.__reset_counter(idle_value_key) # Store new value - self.current_values_map[key] = val + self.__current_values_map[key] = val def __get_current_value(self, key: str) -> Optional[int]: - return self.current_values_map.get(key) + return self.__current_values_map.get(key) def reset_metric_api_failure(self): self.__reset_counter(self._METRIC_API_FAIL_KEY) @@ -88,20 +92,20 @@ def get_doc_completion_percentage(self) -> int: success_doc_count = self.__get_current_value(self._SUCCESS_DOCS_KEY) if success_doc_count is None: success_doc_count = 0 - return math.floor((success_doc_count * 100) / self.target_doc_count) + return math.floor((success_doc_count * 100) / self.__target_doc_count) def all_docs_migrated(self) -> bool: # TODO Add a check for partitionsCompleted = indices success_doc_count = self.__get_current_value(self._SUCCESS_DOCS_KEY) if success_doc_count is None: success_doc_count = 0 - return success_doc_count >= self.target_doc_count + return success_doc_count >= self.__target_doc_count def is_migration_complete_success(self) -> bool: is_idle_pipeline: bool = False rec_in_flight = self.__get_current_value(self._REC_IN_FLIGHT_KEY) no_partitions_count = self.__get_current_value(self._NO_PART_KEY) - prev_no_partitions_count = self.prev_values_map.get(self._NO_PART_KEY, 0) + prev_no_partitions_count = self.__prev_values_map.get(self._NO_PART_KEY, 0) # Check for no records in flight if rec_in_flight is not None and rec_in_flight == 0: # No-partitions metrics should steadily tick up @@ -113,15 +117,15 @@ def is_migration_idle(self) -> bool: keys_to_check = [self._NO_PART_KEY, self._SUCCESS_DOCS_KEY] for key in keys_to_check: val = self.__get_idle_value_count(key) - if val is not None and val >= self.idle_threshold: + if val is not None and val >= self.__idle_threshold: logging.warning("Idle pipeline detected because [" + key + "] value was idle above threshold: " + - str(self.idle_threshold)) + str(self.__idle_threshold)) return True # End of loop return False def is_too_may_api_failures(self) -> bool: - return self.counter_map.get(self._METRIC_API_FAIL_KEY, 0) >= self.idle_threshold + return self.__counter_map.get(self._METRIC_API_FAIL_KEY, 0) >= self.__idle_threshold def is_in_terminal_state(self) -> bool: return self.is_migration_complete_success() or self.is_migration_idle() or self.is_too_may_api_failures() @@ -131,4 +135,4 @@ def log_idle_pipeline_debug_metrics(self): # pragma no cover logging.debug("Idle pipeline metrics - " + f"Records in flight: [{self.__get_current_value(self._REC_IN_FLIGHT_KEY)}], " + f"No-partitions counter: [{self.__get_current_value(self._NO_PART_KEY)}]" + - f"Previous no-partition value: [{self.prev_values_map.get(self._NO_PART_KEY)}]") + f"Previous no-partition value: [{self.__prev_values_map.get(self._NO_PART_KEY)}]") diff --git a/FetchMigration/python/requirements.txt b/FetchMigration/python/requirements.txt index 03e801384b..a8f57b550c 100644 --- a/FetchMigration/python/requirements.txt +++ b/FetchMigration/python/requirements.txt @@ -1,5 +1,7 @@ +botocore>=1.31.70 jsondiff>=2.0.0 prometheus-client>=0.17.1 pyyaml>=6.0.1 requests>=2.31.0 +requests-aws4auth>=1.2.3 responses>=0.23.3 diff --git a/FetchMigration/python/tests/test_endpoint_utils.py b/FetchMigration/python/tests/test_endpoint_utils.py new file mode 100644 index 0000000000..03486ed3d0 --- /dev/null +++ b/FetchMigration/python/tests/test_endpoint_utils.py @@ -0,0 +1,282 @@ +import copy +import pickle +import random +import unittest +from typing import Optional +from unittest.mock import MagicMock, patch + +from moto import mock_iam + +import endpoint_utils +from tests import test_constants + +# Constants +SUPPORTED_ENDPOINTS = ["opensearch", "elasticsearch"] +INSECURE_KEY = "insecure" +CONNECTION_KEY = "connection" +TEST_KEY = "test_key" +BASE_CONFIG_SECTION = { + TEST_KEY: [{"invalid_plugin1": {"key": "val"}}, {"invalid_plugin2": {}}] +} + + +# Utility method to create a test plugin config +def create_plugin_config(host_list: list[str], + basic_auth_tuple: Optional[tuple[Optional[str], Optional[str]]] = None, + aws_config_snippet: Optional[dict] = None, + disable_auth: Optional[bool] = None + ) -> dict: + config = dict() + config["hosts"] = host_list + if disable_auth: + config["disable_authentication"] = disable_auth + elif basic_auth_tuple: + user, password = basic_auth_tuple + if user: + config["username"] = user + if password: + config["password"] = password + elif aws_config_snippet: + config.update(aws_config_snippet) + return config + + +# Utility method to create a test config section +def create_config_section(plugin_config: dict) -> dict: + valid_plugin = dict() + valid_plugin[random.choice(SUPPORTED_ENDPOINTS)] = plugin_config + config_section = copy.deepcopy(BASE_CONFIG_SECTION) + config_section[TEST_KEY].append(valid_plugin) + return config_section + + +class TestEndpointUtils(unittest.TestCase): + # Run before each test + def setUp(self) -> None: + with open(test_constants.PIPELINE_CONFIG_PICKLE_FILE_PATH, "rb") as f: + self.loaded_pipeline_config = pickle.load(f) + + def test_is_insecure_default_value(self): + self.assertFalse(endpoint_utils.is_insecure({})) + + def test_is_insecure_top_level_key(self): + test_input = {"key": 123, INSECURE_KEY: True} + self.assertTrue(endpoint_utils.is_insecure(test_input)) + + def test_is_insecure_nested_key(self): + test_input = {"key1": 123, CONNECTION_KEY: {"key2": "val", INSECURE_KEY: True}} + self.assertTrue(endpoint_utils.is_insecure(test_input)) + + def test_is_insecure_missing_nested(self): + test_input = {"key1": 123, CONNECTION_KEY: {"key2": "val"}} + self.assertFalse(endpoint_utils.is_insecure(test_input)) + + def test_get_auth_returns_none(self): + # The following inputs should not return an auth tuple: + # - Empty input + # - user without password + # - password without user + input_list = [{}, {"username": "test"}, {"password": "test"}] + for test_input in input_list: + self.assertIsNone(endpoint_utils.get_auth(test_input)) + + def test_get_auth_basic(self): + # Test valid input + result = endpoint_utils.get_auth({"username": "user", "password": "pass"}) + self.assertEqual(tuple, type(result)) + self.assertEqual("user", result[0]) + self.assertEqual("pass", result[1]) + + def get_endpoint_info_from_plugin_config(self): + host_input = "test" + expected_endpoint = "test/" + test_user = "user" + test_password = "password" + # Simple base case + test_config = create_plugin_config([host_input]) + + result = endpoint_utils.get_endpoint_info_from_plugin_config(test_config) + self.assertEqual(expected_endpoint, result.get_url()) + self.assertIsNone(result.get_auth()) + self.assertTrue(result.is_verify_ssl()) + # Invalid auth config + test_config = create_plugin_config([host_input]) + result = endpoint_utils.get_endpoint_info_from_plugin_config(test_config) + self.assertEqual(expected_endpoint, result.get_url()) + self.assertIsNone(result.get_auth()) + # Valid auth config + test_config = create_plugin_config([host_input], (test_user, test_password)) + result = endpoint_utils.get_endpoint_info_from_plugin_config(test_config) + self.assertEqual(expected_endpoint, result.get_url()) + self.assertEqual(test_user, result.get_auth()[0]) + self.assertEqual(test_password, result.get_auth()[1]) + # Array of hosts uses the first entry + test_config = create_plugin_config([host_input, "other_host"], (test_user, test_password)) + result = endpoint_utils.get_endpoint_info_from_plugin_config(test_config) + self.assertEqual(expected_endpoint, result.get_url()) + self.assertEqual(test_user, result.get_auth()[0]) + self.assertEqual(test_password, result.get_auth()[1]) + + def test_validate_plugin_config_unsupported_endpoints(self): + # No supported endpoints + self.assertRaises(ValueError, endpoint_utils.get_endpoint_info_from_pipeline_config, + BASE_CONFIG_SECTION, TEST_KEY) + + def test_validate_plugin_config_missing_host(self): + test_data = create_config_section({}) + self.assertRaises(ValueError, endpoint_utils.get_endpoint_info_from_pipeline_config, test_data, TEST_KEY) + + def test_validate_plugin_config_missing_auth(self): + test_data = create_config_section(create_plugin_config(["host"])) + self.assertRaises(ValueError, endpoint_utils.get_endpoint_info_from_pipeline_config, test_data, TEST_KEY) + + def test_validate_plugin_config_auth_disabled(self): + test_data = create_config_section(create_plugin_config(["host"], ("test", None), disable_auth=True)) + # Should complete without errors + endpoint_utils.get_endpoint_info_from_pipeline_config(test_data, TEST_KEY) + + def test_validate_plugin_config_basic_auth(self): + plugin_config = create_plugin_config(["host"], ("user", "password")) + test_data = create_config_section(plugin_config) + # Should complete without errors + endpoint_utils.get_endpoint_info_from_pipeline_config(test_data, TEST_KEY) + + def test_validate_auth_missing_password(self): + test_plugin_config = create_plugin_config(["host"], ("test", None), disable_auth=False) + self.assertRaises(ValueError, endpoint_utils.validate_auth, TEST_KEY, test_plugin_config) + + def test_validate_auth_missing_user(self): + test_plugin_config = create_plugin_config(["host"], (None, "test")) + self.assertRaises(ValueError, endpoint_utils.validate_auth, TEST_KEY, test_plugin_config) + + def test_validate_auth_bad_empty_config(self): + test_plugin_config = create_plugin_config(["host"], aws_config_snippet={}) + self.assertRaises(ValueError, endpoint_utils.validate_auth, TEST_KEY, test_plugin_config) + + @patch('endpoint_utils.get_aws_region') + # Note that mock objects are passed bottom-up from the patch order above + def test_validate_auth_aws_sigv4(self, mock_get_aws_region: MagicMock): + test_plugin_config = create_plugin_config(["host"], aws_config_snippet={"aws_sigv4": False}) + self.assertRaises(ValueError, endpoint_utils.validate_auth, TEST_KEY, test_plugin_config) + mock_get_aws_region.assert_not_called() + test_plugin_config = create_plugin_config(["host"], aws_config_snippet={"aws_sigv4": True}) + # Should complete without errors + endpoint_utils.validate_auth(TEST_KEY, test_plugin_config) + mock_get_aws_region.assert_called_once() + mock_get_aws_region.reset_mock() + # "aws" is expected to be a section so the check is only for the presence of the key + test_plugin_config = create_plugin_config(["host"], aws_config_snippet={"aws": False}) + endpoint_utils.validate_auth(TEST_KEY, test_plugin_config) + mock_get_aws_region.assert_called_once() + + @patch('endpoint_utils.get_aws_region') + @patch('endpoint_utils.get_aws_sigv4_auth') + # Note that mock objects are passed bottom-up from the patch order above + def test_get_auth_aws_sigv4(self, mock_get_sigv4_auth: MagicMock, mock_get_aws_region: MagicMock): + # AWS SigV4 key specified, but disabled + test_plugin_config = create_plugin_config(["host"], aws_config_snippet={"aws_sigv4": False}) + result = endpoint_utils.get_auth(test_plugin_config) + self.assertIsNone(result) + mock_get_sigv4_auth.assert_not_called() + # AWS SigV4 key enabled + expected_region = "region" + mock_get_aws_region.return_value = expected_region + test_plugin_config = create_plugin_config(["host"], aws_config_snippet={"aws_sigv4": True}) + result = endpoint_utils.get_auth(test_plugin_config) + self.assertIsNotNone(result) + mock_get_sigv4_auth.assert_called_once_with(expected_region, False) + + @patch('endpoint_utils.get_aws_region') + @patch('endpoint_utils.get_aws_sigv4_auth') + # Note that mock objects are passed bottom-up from the patch order above + def test_get_auth_aws_config(self, mock_get_sigv4_auth: MagicMock, mock_get_aws_region: MagicMock): + expected_region = "region" + mock_get_aws_region.return_value = expected_region + test_plugin_config = create_plugin_config(["host"], aws_config_snippet={"aws": {"key": "value"}}) + result = endpoint_utils.get_auth(test_plugin_config) + self.assertIsNotNone(result) + mock_get_sigv4_auth.assert_called_once_with(expected_region, False) + mock_get_aws_region.assert_called_once() + + @patch('endpoint_utils.get_aws_region') + @patch('endpoint_utils.get_aws_sigv4_auth') + # Note that mock objects are passed bottom-up from the patch order above + def test_get_auth_aws_sigv4_serverless(self, mock_get_sigv4_auth: MagicMock, mock_get_aws_region: MagicMock): + expected_region = "region" + mock_get_aws_region.return_value = expected_region + test_plugin_config = create_plugin_config(["host"], aws_config_snippet={"aws": {"serverless": True}}) + result = endpoint_utils.get_auth(test_plugin_config) + self.assertIsNotNone(result) + mock_get_sigv4_auth.assert_called_once_with(expected_region, True) + mock_get_aws_region.assert_called_once() + + def test_validate_pipeline_missing_required_keys(self): + # Test cases: + # - Empty input + # - missing output + # - missing input + bad_configs = [{}, {"source": {}}, {"sink": {}}] + for config in bad_configs: + self.assertRaises(ValueError, endpoint_utils.validate_pipeline, config) + + def test_validate_pipeline_config_happy_case(self): + # Get top level value + test_config = next(iter(self.loaded_pipeline_config.values())) + result = endpoint_utils.get_endpoint_info_from_pipeline_config(test_config, "source") + self.assertIsNotNone(result) + endpoint_utils.get_endpoint_info_from_pipeline_config(test_config, "sink") + self.assertIsNotNone(result) + + @patch('endpoint_utils.__derive_aws_region_from_url') + def test_get_aws_region_aws_sigv4(self, mock_derive_region: MagicMock): + derived_value = "derived" + mock_derive_region.return_value = derived_value + aws_sigv4_config = dict() + aws_sigv4_config["aws_sigv4"] = True + aws_sigv4_config["aws_region"] = "test" + self.assertEqual("test", endpoint_utils.get_aws_region( + create_plugin_config(["host"], aws_config_snippet=aws_sigv4_config))) + mock_derive_region.assert_not_called() + del aws_sigv4_config["aws_region"] + self.assertEqual(derived_value, endpoint_utils.get_aws_region( + create_plugin_config(["host"], aws_config_snippet=aws_sigv4_config))) + mock_derive_region.assert_called_once() + + @patch('endpoint_utils.__derive_aws_region_from_url') + def test_get_aws_region_aws_config(self, mock_derive_region: MagicMock): + derived_value = "derived" + mock_derive_region.return_value = derived_value + test_config = create_plugin_config(["host"], aws_config_snippet={"aws": {"region": "test"}}) + self.assertEqual("test", endpoint_utils.get_aws_region(test_config)) + mock_derive_region.assert_not_called() + test_config = create_plugin_config(["host"], aws_config_snippet={"aws": {"serverless": True}}) + self.assertEqual(derived_value, endpoint_utils.get_aws_region(test_config)) + mock_derive_region.assert_called_once() + # Invalid configuration + test_config = create_plugin_config(["host"], aws_config_snippet={"aws": True}) + self.assertRaises(ValueError, endpoint_utils.get_aws_region, test_config) + + def test_derive_aws_region(self): + # Custom endpoint that does not match regex + test_config = create_plugin_config(["https://www.custom.endpoint.amazon.com"], + aws_config_snippet={"aws_sigv4": True}) + self.assertRaises(ValueError, endpoint_utils.get_aws_region, test_config) + # Non-matching service name + test_config = create_plugin_config(["test123.test-region.s3.amazonaws.com"], + aws_config_snippet={"aws_sigv4": True}) + self.assertRaises(ValueError, endpoint_utils.get_aws_region, test_config) + test_config = create_plugin_config(["test-123.test-region.es.amazonaws.com"], + aws_config_snippet={"aws": {"serverless": True}}) + # Should return region successfully + self.assertEqual("test-region", endpoint_utils.get_aws_region(test_config)) + + @mock_iam + def test_get_aws_sigv4_auth(self): + result = endpoint_utils.get_aws_sigv4_auth("test") + self.assertEqual(result.service, "es") + result = endpoint_utils.get_aws_sigv4_auth("test", True) + self.assertEqual(result.service, "aoss") + + +if __name__ == '__main__': + unittest.main() diff --git a/FetchMigration/python/tests/test_metadata_migration.py b/FetchMigration/python/tests/test_metadata_migration.py index 21af41d1e6..1696c50d6a 100644 --- a/FetchMigration/python/tests/test_metadata_migration.py +++ b/FetchMigration/python/tests/test_metadata_migration.py @@ -1,47 +1,12 @@ import copy import pickle -import random import unittest -from typing import Optional from unittest.mock import patch, MagicMock, ANY import metadata_migration from metadata_migration_params import MetadataMigrationParams from tests import test_constants -# Constants -TEST_KEY = "test_key" -INSECURE_KEY = "insecure" -CONNECTION_KEY = "connection" -BASE_CONFIG_SECTION = { - TEST_KEY: [{"invalid_plugin1": {"key": "val"}}, {"invalid_plugin2": {}}] -} - - -# Utility method to create a test plugin config -def create_plugin_config(host_list: list[str], - user: Optional[str] = None, - password: Optional[str] = None, - disable_auth: Optional[bool] = None) -> dict: - config = dict() - config["hosts"] = host_list - if user: - config["username"] = user - if password: - config["password"] = password - if disable_auth is not None: - config["disable_authentication"] = disable_auth - return config - - -# Utility method to creat a test config section -def create_config_section(plugin_config: dict) -> dict: - valid_plugin = dict() - valid_plugin[random.choice(metadata_migration.SUPPORTED_ENDPOINTS)] = plugin_config - config_section = copy.deepcopy(BASE_CONFIG_SECTION) - config_section[TEST_KEY].append(valid_plugin) - return config_section - class TestMetadataMigration(unittest.TestCase): # Run before each test @@ -49,66 +14,6 @@ def setUp(self) -> None: with open(test_constants.PIPELINE_CONFIG_PICKLE_FILE_PATH, "rb") as f: self.loaded_pipeline_config = pickle.load(f) - def test_is_insecure_default_value(self): - self.assertFalse(metadata_migration.is_insecure({})) - - def test_is_insecure_top_level_key(self): - test_input = {"key": 123, INSECURE_KEY: True} - self.assertTrue(metadata_migration.is_insecure(test_input)) - - def test_is_insecure_nested_key(self): - test_input = {"key1": 123, CONNECTION_KEY: {"key2": "val", INSECURE_KEY: True}} - self.assertTrue(metadata_migration.is_insecure(test_input)) - - def test_is_insecure_missing_nested(self): - test_input = {"key1": 123, CONNECTION_KEY: {"key2": "val"}} - self.assertFalse(metadata_migration.is_insecure(test_input)) - - def test_get_auth_returns_none(self): - # The following inputs should not return an auth tuple: - # - Empty input - # - user without password - # - password without user - input_list = [{}, {"username": "test"}, {"password": "test"}] - for test_input in input_list: - self.assertIsNone(metadata_migration.get_auth(test_input)) - - def test_get_auth_for_valid_input(self): - # Test valid input - result = metadata_migration.get_auth({"username": "user", "password": "pass"}) - self.assertEqual(tuple, type(result)) - self.assertEqual("user", result[0]) - self.assertEqual("pass", result[1]) - - def test_get_endpoint_info(self): - host_input = "test" - expected_endpoint = "test/" - test_user = "user" - test_password = "password" - # Simple base case - test_config = create_plugin_config([host_input]) - result = metadata_migration.get_endpoint_info(test_config) - self.assertEqual(expected_endpoint, result.url) - self.assertIsNone(result.auth) - self.assertTrue(result.verify_ssl) - # Invalid auth config - test_config = create_plugin_config([host_input], test_user) - result = metadata_migration.get_endpoint_info(test_config) - self.assertEqual(expected_endpoint, result.url) - self.assertIsNone(result.auth) - # Valid auth config - test_config = create_plugin_config([host_input], user=test_user, password=test_password) - result = metadata_migration.get_endpoint_info(test_config) - self.assertEqual(expected_endpoint, result.url) - self.assertEqual(test_user, result.auth[0]) - self.assertEqual(test_password, result.auth[1]) - # Array of hosts uses the first entry - test_config = create_plugin_config([host_input, "other_host"], test_user, test_password) - result = metadata_migration.get_endpoint_info(test_config) - self.assertEqual(expected_endpoint, result.url) - self.assertEqual(test_user, result.auth[0]) - self.assertEqual(test_password, result.auth[1]) - def test_get_index_differences_empty(self): # Base case should return an empty list result_tuple = metadata_migration.get_index_differences(dict(), dict()) @@ -181,51 +86,6 @@ def test_get_index_differences_mappings_conflict(self): self.assertEqual(1, len(result_tuple[2])) self.assertTrue(test_constants.INDEX3_NAME in result_tuple[2]) - def test_validate_plugin_config_unsupported_endpoints(self): - # No supported endpoints - self.assertRaises(ValueError, metadata_migration.validate_plugin_config, BASE_CONFIG_SECTION, TEST_KEY) - - def test_validate_plugin_config_missing_host(self): - test_data = create_config_section({}) - self.assertRaises(ValueError, metadata_migration.validate_plugin_config, test_data, TEST_KEY) - - def test_validate_plugin_config_missing_auth(self): - test_data = create_config_section(create_plugin_config(["host"])) - self.assertRaises(ValueError, metadata_migration.validate_plugin_config, test_data, TEST_KEY) - - def test_validate_plugin_config_missing_password(self): - test_data = create_config_section(create_plugin_config(["host"], user="test", disable_auth=False)) - self.assertRaises(ValueError, metadata_migration.validate_plugin_config, test_data, TEST_KEY) - - def test_validate_plugin_config_missing_user(self): - test_data = create_config_section(create_plugin_config(["host"], password="test")) - self.assertRaises(ValueError, metadata_migration.validate_plugin_config, test_data, TEST_KEY) - - def test_validate_plugin_config_auth_disabled(self): - test_data = create_config_section(create_plugin_config(["host"], user="test", disable_auth=True)) - # Should complete without errors - metadata_migration.validate_plugin_config(test_data, TEST_KEY) - - def test_validate_plugin_config_happy_case(self): - plugin_config = create_plugin_config(["host"], "user", "password") - test_data = create_config_section(plugin_config) - # Should complete without errors - metadata_migration.validate_plugin_config(test_data, TEST_KEY) - - def test_validate_pipeline_config_missing_required_keys(self): - # Test cases: - # - Empty input - # - missing output - # - missing input - bad_configs = [{}, {"source": {}}, {"sink": {}}] - for config in bad_configs: - self.assertRaises(ValueError, metadata_migration.validate_pipeline_config, config) - - def test_validate_pipeline_config_happy_case(self): - # Get top level value - test_config = next(iter(self.loaded_pipeline_config.values())) - metadata_migration.validate_pipeline_config(test_config) - @patch('index_operations.doc_count') @patch('metadata_migration.write_output') @patch('metadata_migration.print_report') From 48ae14ed8e2441e0d55148f63cf16b7ddcd1bb34 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Fri, 10 Nov 2023 09:29:17 -0500 Subject: [PATCH 35/55] Add targeted jest test cases for CDK and reduce complexity of stack composer constructor (#396) Signed-off-by: Tanner Lewis --- .../lib/stack-composer.ts | 212 +++++++++--------- .../test/migration-console-stack.test.ts | 55 +++++ .../test/msk-utility-stack.test.ts | 53 +++++ .../test/network-stack.test.ts | 26 ++- .../test/stack-composer-ordering.test.ts | 74 +++--- .../test/stack-composer.test.ts | 26 ++- 6 files changed, 300 insertions(+), 146 deletions(-) create mode 100644 deployment/cdk/opensearch-service-migration/test/migration-console-stack.test.ts create mode 100644 deployment/cdk/opensearch-service-migration/test/msk-utility-stack.test.ts diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts index 7e117a19f4..baa8b0253e 100644 --- a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts +++ b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts @@ -40,6 +40,54 @@ export class StackComposer { } } + private getContextForType(optionName: string, expectedType: string, defaultValues: { [x: string]: (any); }, contextJSON: { [x: string]: (any); }): any { + const option = contextJSON[optionName] + + // If no context is provided (undefined or empty string) and a default value exists, use it + if ((option === undefined || option === "") && defaultValues[optionName]) { + return defaultValues[optionName] + } + + // Filter out invalid or missing options by setting undefined (empty strings, null, undefined, NaN) + if (option !== false && option !== 0 && !option) { + return undefined + } + // Values provided by the CLI will always be represented as a string and need to be parsed + if (typeof option === 'string') { + if (expectedType === 'number') { + return parseInt(option) + } + if (expectedType === 'boolean' || expectedType === 'object') { + return JSON.parse(option) + } + } + // Values provided by the cdk.context.json should be of the desired type + if (typeof option !== expectedType) { + throw new Error(`Type provided by cdk.context.json for ${optionName} was ${typeof option} but expected ${expectedType}`) + } + return option + } + + private parseAccessPolicies(jsonObject: { [x: string]: any; }): PolicyStatement[] { + let accessPolicies: PolicyStatement[] = [] + const statements = jsonObject['Statement'] + if (!statements || statements.length < 1) { + throw new Error ("Provided accessPolicies JSON must have the 'Statement' element present and not be empty, for reference https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_statement.html") + } + // Access policies can provide a single Statement block or an array of Statement blocks + if (Array.isArray(statements)) { + for (let statementBlock of statements) { + const statement = PolicyStatement.fromJson(statementBlock) + accessPolicies.push(statement) + } + } + else { + const statement = PolicyStatement.fromJson(statements) + accessPolicies.push(statement) + } + return accessPolicies + } + constructor(scope: Construct, props: StackComposerProps) { const defaultValues: { [x: string]: (any); } = defaultValuesJson @@ -55,61 +103,61 @@ export class StackComposer { if (!contextJSON) { throw new Error(`No CDK context block found for contextId '${contextId}'`) } - const stage = getContextForType('stage', 'string') + const stage = this.getContextForType('stage', 'string', defaultValues, contextJSON) let version: EngineVersion let accessPolicies: PolicyStatement[]|undefined - const domainName = getContextForType('domainName', 'string') - const dataNodeType = getContextForType('dataNodeType', 'string') - const dataNodeCount = getContextForType('dataNodeCount', 'number') - const dedicatedManagerNodeType = getContextForType('dedicatedManagerNodeType', 'string') - const dedicatedManagerNodeCount = getContextForType('dedicatedManagerNodeCount', 'number') - const warmNodeType = getContextForType('warmNodeType', 'string') - const warmNodeCount = getContextForType('warmNodeCount', 'number') - const useUnsignedBasicAuth = getContextForType('useUnsignedBasicAuth', 'boolean') - const fineGrainedManagerUserARN = getContextForType('fineGrainedManagerUserARN', 'string') - const fineGrainedManagerUserName = getContextForType('fineGrainedManagerUserName', 'string') - const fineGrainedManagerUserSecretManagerKeyARN = getContextForType('fineGrainedManagerUserSecretManagerKeyARN', 'string') - const enableDemoAdmin = getContextForType('enableDemoAdmin', 'boolean') - const enforceHTTPS = getContextForType('enforceHTTPS', 'boolean') - const ebsEnabled = getContextForType('ebsEnabled', 'boolean') - const ebsIops = getContextForType('ebsIops', 'number') - const ebsVolumeSize = getContextForType('ebsVolumeSize', 'number') - const encryptionAtRestEnabled = getContextForType('encryptionAtRestEnabled', 'boolean') - const encryptionAtRestKmsKeyARN = getContextForType("encryptionAtRestKmsKeyARN", 'string') - const loggingAppLogEnabled = getContextForType('loggingAppLogEnabled', 'boolean') - const loggingAppLogGroupARN = getContextForType('loggingAppLogGroupARN', 'string') - const noneToNodeEncryptionEnabled = getContextForType('nodeToNodeEncryptionEnabled', 'boolean') - const vpcId = getContextForType('vpcId', 'string') - const vpcEnabled = getContextForType('vpcEnabled', 'boolean') - const vpcSecurityGroupIds = getContextForType('vpcSecurityGroupIds', 'object') - const vpcSubnetIds = getContextForType('vpcSubnetIds', 'object') - const openAccessPolicyEnabled = getContextForType('openAccessPolicyEnabled', 'boolean') - const availabilityZoneCount = getContextForType('availabilityZoneCount', 'number') - const migrationAssistanceEnabled = getContextForType('migrationAssistanceEnabled', 'boolean') - const mskARN = getContextForType('mskARN', 'string') - const mskEnablePublicEndpoints = getContextForType('mskEnablePublicEndpoints', 'boolean') - const mskRestrictPublicAccessTo = getContextForType('mskRestrictPublicAccessTo', 'string') - const mskRestrictPublicAccessType = getContextForType('mskRestrictPublicAccessType', 'string') - const mskBrokerNodeCount = getContextForType('mskBrokerNodeCount', 'number') - const addOnMigrationDeployId = getContextForType('addOnMigrationDeployId', 'string') - const captureProxyESServiceEnabled = getContextForType('captureProxyESServiceEnabled', 'boolean') - const migrationConsoleServiceEnabled = getContextForType('migrationConsoleServiceEnabled', 'boolean') - const trafficReplayerServiceEnabled = getContextForType('trafficReplayerServiceEnabled', 'boolean') - const trafficReplayerEnableClusterFGACAuth = getContextForType('trafficReplayerEnableClusterFGACAuth', 'boolean') - const trafficReplayerTargetEndpoint = getContextForType('trafficReplayerTargetEndpoint', 'string') - const trafficReplayerGroupId = getContextForType('trafficReplayerGroupId', 'string') - const trafficReplayerExtraArgs = getContextForType('trafficReplayerExtraArgs', 'string') - const trafficComparatorServiceEnabled = getContextForType('trafficComparatorServiceEnabled', 'boolean') - const trafficComparatorJupyterServiceEnabled = getContextForType('trafficComparatorJupyterServiceEnabled', 'boolean') - const captureProxyServiceEnabled = getContextForType('captureProxyServiceEnabled', 'boolean') - const captureProxySourceEndpoint = getContextForType('captureProxySourceEndpoint', 'string') - const elasticsearchServiceEnabled = getContextForType('elasticsearchServiceEnabled', 'boolean') - const kafkaBrokerServiceEnabled = getContextForType('kafkaBrokerServiceEnabled', 'boolean') - const kafkaZookeeperServiceEnabled = getContextForType('kafkaZookeeperServiceEnabled', 'boolean') - const fetchMigrationEnabled = getContextForType('fetchMigrationEnabled', 'boolean') - const dpPipelineTemplatePath = getContextForType('dpPipelineTemplatePath', 'string') - const sourceClusterEndpoint = getContextForType('sourceClusterEndpoint', 'string') + const domainName = this.getContextForType('domainName', 'string', defaultValues, contextJSON) + const dataNodeType = this.getContextForType('dataNodeType', 'string', defaultValues, contextJSON) + const dataNodeCount = this.getContextForType('dataNodeCount', 'number', defaultValues, contextJSON) + const dedicatedManagerNodeType = this.getContextForType('dedicatedManagerNodeType', 'string', defaultValues, contextJSON) + const dedicatedManagerNodeCount = this.getContextForType('dedicatedManagerNodeCount', 'number', defaultValues, contextJSON) + const warmNodeType = this.getContextForType('warmNodeType', 'string', defaultValues, contextJSON) + const warmNodeCount = this.getContextForType('warmNodeCount', 'number', defaultValues, contextJSON) + const useUnsignedBasicAuth = this.getContextForType('useUnsignedBasicAuth', 'boolean', defaultValues, contextJSON) + const fineGrainedManagerUserARN = this.getContextForType('fineGrainedManagerUserARN', 'string', defaultValues, contextJSON) + const fineGrainedManagerUserName = this.getContextForType('fineGrainedManagerUserName', 'string', defaultValues, contextJSON) + const fineGrainedManagerUserSecretManagerKeyARN = this.getContextForType('fineGrainedManagerUserSecretManagerKeyARN', 'string', defaultValues, contextJSON) + const enableDemoAdmin = this.getContextForType('enableDemoAdmin', 'boolean', defaultValues, contextJSON) + const enforceHTTPS = this.getContextForType('enforceHTTPS', 'boolean', defaultValues, contextJSON) + const ebsEnabled = this.getContextForType('ebsEnabled', 'boolean', defaultValues, contextJSON) + const ebsIops = this.getContextForType('ebsIops', 'number', defaultValues, contextJSON) + const ebsVolumeSize = this.getContextForType('ebsVolumeSize', 'number', defaultValues, contextJSON) + const encryptionAtRestEnabled = this.getContextForType('encryptionAtRestEnabled', 'boolean', defaultValues, contextJSON) + const encryptionAtRestKmsKeyARN = this.getContextForType("encryptionAtRestKmsKeyARN", 'string', defaultValues, contextJSON) + const loggingAppLogEnabled = this.getContextForType('loggingAppLogEnabled', 'boolean', defaultValues, contextJSON) + const loggingAppLogGroupARN = this.getContextForType('loggingAppLogGroupARN', 'string', defaultValues, contextJSON) + const noneToNodeEncryptionEnabled = this.getContextForType('nodeToNodeEncryptionEnabled', 'boolean', defaultValues, contextJSON) + const vpcId = this.getContextForType('vpcId', 'string', defaultValues, contextJSON) + const vpcEnabled = this.getContextForType('vpcEnabled', 'boolean', defaultValues, contextJSON) + const vpcSecurityGroupIds = this.getContextForType('vpcSecurityGroupIds', 'object', defaultValues, contextJSON) + const vpcSubnetIds = this.getContextForType('vpcSubnetIds', 'object', defaultValues, contextJSON) + const openAccessPolicyEnabled = this.getContextForType('openAccessPolicyEnabled', 'boolean', defaultValues, contextJSON) + const availabilityZoneCount = this.getContextForType('availabilityZoneCount', 'number', defaultValues, contextJSON) + const migrationAssistanceEnabled = this.getContextForType('migrationAssistanceEnabled', 'boolean', defaultValues, contextJSON) + const mskARN = this.getContextForType('mskARN', 'string', defaultValues, contextJSON) + const mskEnablePublicEndpoints = this.getContextForType('mskEnablePublicEndpoints', 'boolean', defaultValues, contextJSON) + const mskRestrictPublicAccessTo = this.getContextForType('mskRestrictPublicAccessTo', 'string', defaultValues, contextJSON) + const mskRestrictPublicAccessType = this.getContextForType('mskRestrictPublicAccessType', 'string', defaultValues, contextJSON) + const mskBrokerNodeCount = this.getContextForType('mskBrokerNodeCount', 'number', defaultValues, contextJSON) + const addOnMigrationDeployId = this.getContextForType('addOnMigrationDeployId', 'string', defaultValues, contextJSON) + const captureProxyESServiceEnabled = this.getContextForType('captureProxyESServiceEnabled', 'boolean', defaultValues, contextJSON) + const migrationConsoleServiceEnabled = this.getContextForType('migrationConsoleServiceEnabled', 'boolean', defaultValues, contextJSON) + const trafficReplayerServiceEnabled = this.getContextForType('trafficReplayerServiceEnabled', 'boolean', defaultValues, contextJSON) + const trafficReplayerEnableClusterFGACAuth = this.getContextForType('trafficReplayerEnableClusterFGACAuth', 'boolean', defaultValues, contextJSON) + const trafficReplayerTargetEndpoint = this.getContextForType('trafficReplayerTargetEndpoint', 'string', defaultValues, contextJSON) + const trafficReplayerGroupId = this.getContextForType('trafficReplayerGroupId', 'string', defaultValues, contextJSON) + const trafficReplayerExtraArgs = this.getContextForType('trafficReplayerExtraArgs', 'string', defaultValues, contextJSON) + const trafficComparatorServiceEnabled = this.getContextForType('trafficComparatorServiceEnabled', 'boolean', defaultValues, contextJSON) + const trafficComparatorJupyterServiceEnabled = this.getContextForType('trafficComparatorJupyterServiceEnabled', 'boolean', defaultValues, contextJSON) + const captureProxyServiceEnabled = this.getContextForType('captureProxyServiceEnabled', 'boolean', defaultValues, contextJSON) + const captureProxySourceEndpoint = this.getContextForType('captureProxySourceEndpoint', 'string', defaultValues, contextJSON) + const elasticsearchServiceEnabled = this.getContextForType('elasticsearchServiceEnabled', 'boolean', defaultValues, contextJSON) + const kafkaBrokerServiceEnabled = this.getContextForType('kafkaBrokerServiceEnabled', 'boolean', defaultValues, contextJSON) + const kafkaZookeeperServiceEnabled = this.getContextForType('kafkaZookeeperServiceEnabled', 'boolean', defaultValues, contextJSON) + const fetchMigrationEnabled = this.getContextForType('fetchMigrationEnabled', 'boolean', defaultValues, contextJSON) + const dpPipelineTemplatePath = this.getContextForType('dpPipelineTemplatePath', 'string', defaultValues, contextJSON) + const sourceClusterEndpoint = this.getContextForType('sourceClusterEndpoint', 'string', defaultValues, contextJSON) if (!stage) { throw new Error("Required context field 'stage' is not present") @@ -121,7 +169,7 @@ export class StackComposer { throw new Error("Domain name is not present and is a required field") } - const engineVersion = getContextForType('engineVersion', 'string') + const engineVersion = this.getContextForType('engineVersion', 'string', defaultValues, contextJSON) if (engineVersion && engineVersion.startsWith("OS_")) { // Will accept a period delimited version string (i.e. 1.3) and return a proper EngineVersion version = EngineVersion.openSearch(engineVersion.substring(3)) @@ -140,23 +188,23 @@ export class StackComposer { }) accessPolicies = [openPolicy] } else { - const accessPolicyJson = getContextForType('accessPolicies', 'object') - accessPolicies = accessPolicyJson ? parseAccessPolicies(accessPolicyJson) : undefined + const accessPolicyJson = this.getContextForType('accessPolicies', 'object', defaultValues, contextJSON) + accessPolicies = accessPolicyJson ? this.parseAccessPolicies(accessPolicyJson) : undefined } - const tlsSecurityPolicyName = getContextForType('tlsSecurityPolicy', 'string') + const tlsSecurityPolicyName = this.getContextForType('tlsSecurityPolicy', 'string', defaultValues, contextJSON) const tlsSecurityPolicy: TLSSecurityPolicy|undefined = tlsSecurityPolicyName ? TLSSecurityPolicy[tlsSecurityPolicyName as keyof typeof TLSSecurityPolicy] : undefined if (tlsSecurityPolicyName && !tlsSecurityPolicy) { throw new Error("Provided tlsSecurityPolicy does not match a selectable option, for reference https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_opensearchservice.TLSSecurityPolicy.html") } - const ebsVolumeTypeName = getContextForType('ebsVolumeType', 'string') + const ebsVolumeTypeName = this.getContextForType('ebsVolumeType', 'string', defaultValues, contextJSON) const ebsVolumeType: EbsDeviceVolumeType|undefined = ebsVolumeTypeName ? EbsDeviceVolumeType[ebsVolumeTypeName as keyof typeof EbsDeviceVolumeType] : undefined if (ebsVolumeTypeName && !ebsVolumeType) { throw new Error("Provided ebsVolumeType does not match a selectable option, for reference https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ec2.EbsDeviceVolumeType.html") } - const domainRemovalPolicyName = getContextForType('domainRemovalPolicy', 'string') + const domainRemovalPolicyName = this.getContextForType('domainRemovalPolicy', 'string', defaultValues, contextJSON) const domainRemovalPolicy = domainRemovalPolicyName ? RemovalPolicy[domainRemovalPolicyName as keyof typeof RemovalPolicy] : undefined if (domainRemovalPolicyName && !domainRemovalPolicy) { throw new Error("Provided domainRemovalPolicy does not match a selectable option, for reference https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.RemovalPolicy.html") @@ -440,53 +488,5 @@ export class StackComposer { this.addStacksToAppRegistry(scope, props.migrationsAppRegistryARN, this.stacks) } - function getContextForType(optionName: string, expectedType: string): any { - const option = contextJSON[optionName] - - // If no context is provided (undefined or empty string) and a default value exists, use it - if ((option === undefined || option === "") && defaultValues[optionName]) { - return defaultValues[optionName] - } - - // Filter out invalid or missing options by setting undefined (empty strings, null, undefined, NaN) - if (option !== false && option !== 0 && !option) { - return undefined - } - // Values provided by the CLI will always be represented as a string and need to be parsed - if (typeof option === 'string') { - if (expectedType === 'number') { - return parseInt(option) - } - if (expectedType === 'boolean' || expectedType === 'object') { - return JSON.parse(option) - } - } - // Values provided by the cdk.context.json should be of the desired type - if (typeof option !== expectedType) { - throw new Error(`Type provided by cdk.context.json for ${optionName} was ${typeof option} but expected ${expectedType}`) - } - return option - } - - function parseAccessPolicies(jsonObject: { [x: string]: any; }): PolicyStatement[] { - let accessPolicies: PolicyStatement[] = [] - const statements = jsonObject['Statement'] - if (!statements || statements.length < 1) { - throw new Error ("Provided accessPolicies JSON must have the 'Statement' element present and not be empty, for reference https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_statement.html") - } - // Access policies can provide a single Statement block or an array of Statement blocks - if (Array.isArray(statements)) { - for (let statementBlock of statements) { - const statement = PolicyStatement.fromJson(statementBlock) - accessPolicies.push(statement) - } - } - else { - const statement = PolicyStatement.fromJson(statements) - accessPolicies.push(statement) - } - return accessPolicies - } - } } \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/test/migration-console-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/migration-console-stack.test.ts new file mode 100644 index 0000000000..e7888c1eeb --- /dev/null +++ b/deployment/cdk/opensearch-service-migration/test/migration-console-stack.test.ts @@ -0,0 +1,55 @@ +import {createStackComposer} from "./test-utils"; +import {Capture, Match, Template} from "aws-cdk-lib/assertions"; +import {MigrationConsoleStack} from "../lib/service-stacks/migration-console-stack"; + + +test('Test that IAM policy contains fetch migration IAM statements when fetch migration is enabled', () => { + const contextOptions = { + vpcEnabled: true, + migrationAssistanceEnabled: true, + migrationConsoleServiceEnabled: true, + fetchMigrationEnabled: true + } + + const stacks = createStackComposer(contextOptions) + + const migrationConsoleStack: MigrationConsoleStack = (stacks.stacks.filter((s) => s instanceof MigrationConsoleStack)[0]) as MigrationConsoleStack + const migrationConsoleStackTemplate = Template.fromStack(migrationConsoleStack) + + const statementCapture = new Capture(); + migrationConsoleStackTemplate.hasResourceProperties("AWS::IAM::Policy", { + PolicyDocument: Match.objectLike({ + Statement: statementCapture, + }) + }) + const allStatements: any[] = statementCapture.asArray() + const runTaskStatement = allStatements.find(statement => statement.Action == "ecs:RunTask") + const iamPassRoleStatement = allStatements.find(statement => statement.Action == "iam:PassRole") + expect(runTaskStatement).toBeTruthy() + expect(iamPassRoleStatement).toBeTruthy() +}) + +test('Test that IAM policy does not contain fetch migration IAM statements when fetch migration is disabled', () => { + const contextOptions = { + vpcEnabled: true, + migrationAssistanceEnabled: true, + migrationConsoleServiceEnabled: true + } + + const stacks = createStackComposer(contextOptions) + + const migrationConsoleStack: MigrationConsoleStack = (stacks.stacks.filter((s) => s instanceof MigrationConsoleStack)[0]) as MigrationConsoleStack + const migrationConsoleStackTemplate = Template.fromStack(migrationConsoleStack) + + const statementCapture = new Capture(); + migrationConsoleStackTemplate.hasResourceProperties("AWS::IAM::Policy", { + PolicyDocument: Match.objectLike({ + Statement: statementCapture, + }) + }) + const allStatements: any[] = statementCapture.asArray() + const runTaskStatement = allStatements.find(statement => statement.Action == "ecs:RunTask") + const iamPassRoleStatement = allStatements.find(statement => statement.Action == "iam:PassRole") + expect(runTaskStatement).toBeFalsy() + expect(iamPassRoleStatement).toBeFalsy() +}) \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/test/msk-utility-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/msk-utility-stack.test.ts new file mode 100644 index 0000000000..8654332add --- /dev/null +++ b/deployment/cdk/opensearch-service-migration/test/msk-utility-stack.test.ts @@ -0,0 +1,53 @@ +import {createStackComposer} from "./test-utils"; +import {Match, Template} from "aws-cdk-lib/assertions"; +import {MSKUtilityStack} from "../lib/msk-utility-stack"; + +test('Test if mskEnablePublicEndpoints is provided, wait condition and max attempts are set for lambda custom resource', () => { + const contextOptions = { + vpcEnabled: true, + migrationAssistanceEnabled: true, + mskEnablePublicEndpoints: true, + mskRestrictPublicAccessTo: "10.0.0.0/32", + mskRestrictPublicAccessType: "ipv4" + } + + const stacks = createStackComposer(contextOptions) + + const mskUtilityStack: MSKUtilityStack = (stacks.stacks.filter((s) => s instanceof MSKUtilityStack)[0]) as MSKUtilityStack + const mskUtilityStackTemplate = Template.fromStack(mskUtilityStack) + + mskUtilityStackTemplate.hasResource("AWS::Lambda::Function", { + Properties: { + Environment: { + Variables: { + "MAX_ATTEMPTS": "4", + "MSK_ARN": Match.anyValue() + } + } + } + }) + mskUtilityStackTemplate.resourceCountIs("AWS::CloudFormation::WaitConditionHandle", 1) +}) + +test('Test if mskEnablePublicEndpoints is not provided, single run lambda custom resource is created', () => { + const contextOptions = { + vpcEnabled: true, + migrationAssistanceEnabled: true + } + + const stacks = createStackComposer(contextOptions) + + const mskUtilityStack: MSKUtilityStack = (stacks.stacks.filter((s) => s instanceof MSKUtilityStack)[0]) as MSKUtilityStack + const mskUtilityStackTemplate = Template.fromStack(mskUtilityStack) + + mskUtilityStackTemplate.hasResource("AWS::Lambda::Function", { + Properties: { + Environment: { + Variables: { + "MSK_ARN": Match.anyValue() + } + } + } + }) + mskUtilityStackTemplate.resourceCountIs("AWS::CloudFormation::WaitConditionHandle", 0) +}) \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts index cc74b7a37f..28b05a388c 100644 --- a/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts @@ -1,8 +1,14 @@ -import {App} from "aws-cdk-lib"; -import {StackComposer} from "../lib/stack-composer"; import {NetworkStack} from "../lib/network-stack"; import {Template} from "aws-cdk-lib/assertions"; import {createStackComposer} from "./test-utils"; +import {ContainerImage} from "aws-cdk-lib/aws-ecs"; +import {StringParameter} from "aws-cdk-lib/aws-ssm"; + +// Mock value returned from SSM call +jest.spyOn(StringParameter, 'valueForStringParameter').mockImplementation(() => "vpc-123456"); +// Mock using local Dockerfile (which may not exist and would fail synthesis) with the intent of using a "fake-image" from a public registry +jest.mock("aws-cdk-lib/aws-ecr-assets") +jest.spyOn(ContainerImage, 'fromDockerImageAsset').mockImplementation(() => ContainerImage.fromRegistry("fake-image")); test('Test vpcEnabled setting that is disabled does not create stack', () => { const contextOptions = { @@ -37,4 +43,20 @@ test('Test vpcEnabled setting that is enabled without existing resources creates const vpc = networkStack.vpc expect(vpc.publicSubnets.length).toBe(2) expect(vpc.privateSubnets.length).toBe(2) +}) + +test('Test if addOnMigrationDeployId is provided, stack does not create VPC or Security Group', () => { + const contextOptions = { + addOnMigrationDeployId: "junit-addon", + vpcEnabled: true, + availabilityZoneCount: 2 + } + + const stacks = createStackComposer(contextOptions) + + const networkStack: NetworkStack = (stacks.stacks.filter((s) => s instanceof NetworkStack)[0]) as NetworkStack + const networkTemplate = Template.fromStack(networkStack) + + networkTemplate.resourceCountIs("AWS::EC2::VPC", 0) + networkTemplate.resourceCountIs("AWS::EC2::SecurityGroup", 0) }) \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/test/stack-composer-ordering.test.ts b/deployment/cdk/opensearch-service-migration/test/stack-composer-ordering.test.ts index c90c416e2e..de84e695da 100644 --- a/deployment/cdk/opensearch-service-migration/test/stack-composer-ordering.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/stack-composer-ordering.test.ts @@ -9,45 +9,45 @@ import {TrafficComparatorJupyterStack} from "../lib/service-stacks/traffic-compa import {MigrationConsoleStack} from "../lib/service-stacks/migration-console-stack"; import {KafkaBrokerStack} from "../lib/service-stacks/kafka-broker-stack"; import {KafkaZookeeperStack} from "../lib/service-stacks/kafka-zookeeper-stack"; +import {ContainerImage} from "aws-cdk-lib/aws-ecs"; -// This test should be re-enabled when we have the necessary mocking or setup to allow this test to be successful -// with the required Dockerfiles missing. Currently, this will fail if the Dockerfiles have not been built. +// Mock using local Dockerfile (which may not exist and would fail synthesis) with the intent of using a "fake-image" from a public registry +jest.mock("aws-cdk-lib/aws-ecr-assets") +jest.spyOn(ContainerImage, 'fromDockerImageAsset').mockImplementation(() => ContainerImage.fromRegistry("fake-image")); -// test('Test all migration services get deployed when enabled', () => { -// -// -// const contextOptions = { -// "stage": "test", -// "engineVersion": "OS_2.9", -// "domainName": "unit-test-opensearch-cluster", -// "dataNodeCount": 2, -// "availabilityZoneCount": 2, -// "openAccessPolicyEnabled": true, -// "domainRemovalPolicy": "DESTROY", -// "vpcEnabled": true, -// "migrationAssistanceEnabled": true, -// "migrationConsoleServiceEnabled": true, -// "captureProxyESServiceEnabled": true, -// "trafficReplayerServiceEnabled": true, -// "captureProxyServiceEnabled": true, -// "elasticsearchServiceEnabled": true, -// "trafficComparatorServiceEnabled": true, -// "trafficComparatorJupyterServiceEnabled": true, -// "kafkaBrokerServiceEnabled": true, -// "kafkaZookeeperServiceEnabled": true -// } -// -// const stacks = createStackComposer(contextOptions) -// -// const services = [CaptureProxyESStack, CaptureProxyStack, ElasticsearchStack, MigrationConsoleStack, -// TrafficReplayerStack, TrafficComparatorStack, TrafficComparatorJupyterStack, KafkaBrokerStack, KafkaZookeeperStack] -// services.forEach( (stackClass) => { -// const stack = stacks.stacks.filter((s) => s instanceof stackClass)[0] -// const template = Template.fromStack(stack) -// template.resourceCountIs("AWS::ECS::Service", 1) -// }) -// -// }) +test('Test all migration services get created when enabled', () => { + + const contextOptions = { + "stage": "test", + "engineVersion": "OS_2.9", + "domainName": "unit-test-opensearch-cluster", + "dataNodeCount": 2, + "availabilityZoneCount": 2, + "openAccessPolicyEnabled": true, + "domainRemovalPolicy": "DESTROY", + "vpcEnabled": true, + "migrationAssistanceEnabled": true, + "migrationConsoleServiceEnabled": true, + "captureProxyESServiceEnabled": true, + "trafficReplayerServiceEnabled": true, + "captureProxyServiceEnabled": true, + "elasticsearchServiceEnabled": true, + "trafficComparatorServiceEnabled": true, + "trafficComparatorJupyterServiceEnabled": true, + "kafkaBrokerServiceEnabled": true, + "kafkaZookeeperServiceEnabled": true + } + + const stacks = createStackComposer(contextOptions) + + const services = [CaptureProxyESStack, CaptureProxyStack, ElasticsearchStack, MigrationConsoleStack, + TrafficReplayerStack, TrafficComparatorStack, TrafficComparatorJupyterStack, KafkaBrokerStack, KafkaZookeeperStack] + services.forEach( (stackClass) => { + const stack = stacks.stacks.filter((s) => s instanceof stackClass)[0] + const template = Template.fromStack(stack) + template.resourceCountIs("AWS::ECS::Service", 1) + }) +}) test('Test no migration services get deployed when disabled', () => { diff --git a/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts b/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts index eb47c8d057..a5245b0884 100644 --- a/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts @@ -1,6 +1,8 @@ import {Template} from "aws-cdk-lib/assertions"; import {OpenSearchDomainStack} from "../lib/opensearch-domain-stack"; import {createStackComposer} from "./test-utils"; +import {App} from "aws-cdk-lib"; +import {StackComposer} from "../lib/stack-composer"; test('Test empty string provided for a parameter which has a default value, uses the default value', () => { @@ -183,4 +185,26 @@ test('Test invalid domain removal policy type throws error', () => { const createStackFunc = () => createStackComposer(contextOptions) expect(createStackFunc).toThrowError() -}) \ No newline at end of file +}) + +test('Test that app registry association is created when migrationsAppRegistryARN is provided', () => { + + const contextOptions = { + stage: "unit-test" + } + + const app = new App({ + context: { + contextId: "unit-test-config", + "unit-test-config": contextOptions + } + }) + const stacks = new StackComposer(app, { + migrationsAppRegistryARN: "arn:aws:servicecatalog:us-west-2:12345678912:/applications/12345abcdef", + env: {account: "test-account", region: "us-east-1"} + }) + + const domainStack = stacks.stacks.filter((s) => s instanceof OpenSearchDomainStack)[0] + const domainTemplate = Template.fromStack(domainStack) + domainTemplate.resourceCountIs("AWS::ServiceCatalogAppRegistry::ResourceAssociation", 1) +}) From 44978733f3bc0cb3a61981daf4c078ebdf33b4e2 Mon Sep 17 00:00:00 2001 From: Kartik Ganesh Date: Fri, 10 Nov 2023 11:46:46 -0800 Subject: [PATCH 36/55] Bugfix - Use typing.Union in Fetch Migration rather than "|" operator (#403) This change swaps out the use of the | operator for typing.Union since | is only available in Python 3.10+ . Hence, typing.Union provides broader compatibility. Also included in this change is a name correction for the index_operations unit test class. --------- Signed-off-by: Kartik Ganesh --- FetchMigration/python/endpoint_info.py | 9 +++++---- FetchMigration/python/endpoint_utils.py | 4 ++-- FetchMigration/python/tests/test_index_operations.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/FetchMigration/python/endpoint_info.py b/FetchMigration/python/endpoint_info.py index 81ace5be3b..e034bf98f3 100644 --- a/FetchMigration/python/endpoint_info.py +++ b/FetchMigration/python/endpoint_info.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Union from requests_aws4auth import AWS4Auth @@ -7,10 +7,11 @@ class EndpointInfo: # Private member variables __url: str - __auth: Optional[tuple] | AWS4Auth + # "|" operator is only supported in 3.10+ + __auth: Union[AWS4Auth, tuple, None] __verify_ssl: bool - def __init__(self, url: str, auth: tuple | AWS4Auth = None, verify_ssl: bool = True): + def __init__(self, url: str, auth: Union[AWS4Auth, tuple, None] = None, verify_ssl: bool = True): self.__url = url # Normalize url value to have trailing slash if not url.endswith("/"): @@ -33,7 +34,7 @@ def add_path(self, path: str) -> str: def get_url(self) -> str: return self.__url - def get_auth(self) -> Optional[tuple] | AWS4Auth: + def get_auth(self) -> Union[AWS4Auth, tuple, None]: return self.__auth def is_verify_ssl(self) -> bool: diff --git a/FetchMigration/python/endpoint_utils.py b/FetchMigration/python/endpoint_utils.py index fe9ef0d97a..60331ad64c 100644 --- a/FetchMigration/python/endpoint_utils.py +++ b/FetchMigration/python/endpoint_utils.py @@ -1,5 +1,5 @@ import re -from typing import Optional +from typing import Optional, Union from requests_aws4auth import AWS4Auth from botocore.session import Session @@ -124,7 +124,7 @@ def get_aws_sigv4_auth(region: str, is_serverless: bool = False) -> AWS4Auth: return AWS4Auth(region=region, service=ES_SERVICE_NAME, refreshable_credentials=credentials) -def get_auth(plugin_config: dict) -> Optional[tuple] | AWS4Auth: +def get_auth(plugin_config: dict) -> Union[AWS4Auth, tuple, None]: # Basic auth if USER_KEY in plugin_config and PWD_KEY in plugin_config: return plugin_config[USER_KEY], plugin_config[PWD_KEY] diff --git a/FetchMigration/python/tests/test_index_operations.py b/FetchMigration/python/tests/test_index_operations.py index c26bdef0be..1378da5fda 100644 --- a/FetchMigration/python/tests/test_index_operations.py +++ b/FetchMigration/python/tests/test_index_operations.py @@ -10,7 +10,7 @@ from tests import test_constants -class TestSearchEndpoint(unittest.TestCase): +class TestIndexOperations(unittest.TestCase): @responses.activate def test_fetch_all_indices(self): # Set up GET response From fb0553a690bd83981870e53e05f08408beb76997 Mon Sep 17 00:00:00 2001 From: Kartik Ganesh Date: Fri, 10 Nov 2023 17:09:37 -0800 Subject: [PATCH 37/55] [Fetch Migration] Enable migration for identical, empty target cluster index (#390) This change enables Fetch Migration to migrate data for indices that are identical between the source and target cluster, and the document count on the target cluster is zero. Such behavior allow users to execute Fetch Migration in an idempotent manner without having to manually remove empty indices on the target cluster (that were created by a previous run of Fetch Migration that did not complete successfully) Prior to this change, all indices that were identical between the source and target clusters were ineligible for data migration. Now, we check the doc_count for identical indices on the target cluster and only exclude identical indices with a non-zero document count. --------- Signed-off-by: Kartik Ganesh --- FetchMigration/python/fetch_orchestrator.py | 4 +- FetchMigration/python/index_diff.py | 35 ++++++ FetchMigration/python/index_doc_count.py | 8 ++ FetchMigration/python/index_operations.py | 25 +++- FetchMigration/python/metadata_migration.py | 73 +++++------ .../python/metadata_migration_result.py | 3 +- FetchMigration/python/migration_monitor.py | 4 +- FetchMigration/python/requirements.txt | 1 + .../python/tests/test_endpoint_utils.py | 5 + .../python/tests/test_fetch_orchestrator.py | 2 +- .../python/tests/test_index_diff.py | 73 +++++++++++ .../python/tests/test_index_operations.py | 14 ++- .../python/tests/test_metadata_migration.py | 116 +++++------------- .../dp_pipeline_template.yaml | 22 +++- .../lib/fetch-migration-stack.ts | 4 +- 15 files changed, 245 insertions(+), 144 deletions(-) create mode 100644 FetchMigration/python/index_diff.py create mode 100644 FetchMigration/python/index_doc_count.py create mode 100644 FetchMigration/python/tests/test_index_diff.py diff --git a/FetchMigration/python/fetch_orchestrator.py b/FetchMigration/python/fetch_orchestrator.py index a624dd191e..8908678eeb 100644 --- a/FetchMigration/python/fetch_orchestrator.py +++ b/FetchMigration/python/fetch_orchestrator.py @@ -59,7 +59,7 @@ def write_inline_pipeline(pipeline_file_path: str, inline_pipeline: str, inline_ def write_inline_target_host(pipeline_file_path: str, inline_target_host: str): - with open(pipeline_file_path, 'rw') as pipeline_file: + with open(pipeline_file_path, 'r+') as pipeline_file: pipeline_yaml = yaml.safe_load(pipeline_file) update_target_host(pipeline_yaml, inline_target_host) # Note - this does not preserve comments @@ -84,7 +84,7 @@ def run(params: FetchOrchestratorParams) -> Optional[int]: report=True, dryrun=params.is_dry_run) logging.info("Running metadata migration...\n") metadata_migration_result = metadata_migration.run(metadata_migration_params) - if len(metadata_migration_result.created_indices) > 0 and not params.is_only_metadata_migration(): + if len(metadata_migration_result.migration_indices) > 0 and not params.is_only_metadata_migration(): # Kick off a subprocess for Data Prepper logging.info("Running Data Prepper...\n") proc = subprocess.Popen(dp_exec_path) diff --git a/FetchMigration/python/index_diff.py b/FetchMigration/python/index_diff.py new file mode 100644 index 0000000000..15d84f0800 --- /dev/null +++ b/FetchMigration/python/index_diff.py @@ -0,0 +1,35 @@ +import utils +from index_operations import SETTINGS_KEY, MAPPINGS_KEY + + +# Computes and captures differences in indices between a "source" cluster +# and a "target" cluster. Indices that exist on the source cluster but not +# on the target cluster are considered "to-create". "Conflicting" indices +# are present on both source and target clusters, but differ in their index +# settings or mappings. +class IndexDiff: + indices_to_create: set + identical_indices: set + identical_empty_indices: set + conflicting_indices: set + + def __init__(self, source: dict, target: dict): + self.identical_empty_indices = set() + self.conflicting_indices = set() + # Compute index names that are present in both the source and target + indices_intersection = set(source.keys()) & set(target.keys()) + # Check if these "common" indices are identical or have metadata conflicts + for index in indices_intersection: + # Check settings + if utils.has_differences(SETTINGS_KEY, source[index], target[index]): + self.conflicting_indices.add(index) + # Check mappings + if utils.has_differences(MAPPINGS_KEY, source[index], target[index]): + self.conflicting_indices.add(index) + # Identical indices are the subset that do not have metadata conflicts + self.identical_indices = set(indices_intersection) - set(self.conflicting_indices) + # Indices that are not already on the target need to be created + self.indices_to_create = set(source.keys()) - set(indices_intersection) + + def set_identical_empty_indices(self, indices: set): + self.identical_empty_indices = indices diff --git a/FetchMigration/python/index_doc_count.py b/FetchMigration/python/index_doc_count.py new file mode 100644 index 0000000000..90b8910661 --- /dev/null +++ b/FetchMigration/python/index_doc_count.py @@ -0,0 +1,8 @@ +from dataclasses import dataclass + + +# Captures the doc_count for indices in a cluster, and also computes a total +@dataclass +class IndexDocCount: + total: int + index_doc_count_map: dict diff --git a/FetchMigration/python/index_operations.py b/FetchMigration/python/index_operations.py index a311ec7e61..b889ae116a 100644 --- a/FetchMigration/python/index_operations.py +++ b/FetchMigration/python/index_operations.py @@ -1,14 +1,22 @@ +import jsonpath_ng import requests from endpoint_info import EndpointInfo # Constants +from index_doc_count import IndexDocCount + SETTINGS_KEY = "settings" MAPPINGS_KEY = "mappings" COUNT_KEY = "count" __INDEX_KEY = "index" __ALL_INDICES_ENDPOINT = "*" -__COUNT_ENDPOINT = "/_count" +__SEARCH_COUNT_PATH = "/_search?size=0" +__SEARCH_COUNT_PAYLOAD = {"aggs": {"count": {"terms": {"field": "_index"}}}} +__TOTAL_COUNT_JSONPATH = jsonpath_ng.parse("$.hits.total.value") +__INDEX_COUNT_JSONPATH = jsonpath_ng.parse("$.aggregations.count.buckets") +__BUCKET_INDEX_NAME_KEY = "key" +__BUCKET_DOC_COUNT_KEY = "doc_count" __INTERNAL_SETTINGS_KEYS = ["creation_date", "uuid", "provided_name", "version", "store"] @@ -43,9 +51,16 @@ def create_indices(indices_data: dict, endpoint: EndpointInfo): raise RuntimeError(f"Failed to create index [{index}] - {e!s}") -def doc_count(indices: set, endpoint: EndpointInfo) -> int: - count_endpoint_suffix: str = ','.join(indices) + __COUNT_ENDPOINT +def doc_count(indices: set, endpoint: EndpointInfo) -> IndexDocCount: + count_endpoint_suffix: str = ','.join(indices) + __SEARCH_COUNT_PATH doc_count_endpoint: str = endpoint.add_path(count_endpoint_suffix) - resp = requests.get(doc_count_endpoint, auth=endpoint.get_auth(), verify=endpoint.is_verify_ssl()) + resp = requests.get(doc_count_endpoint, auth=endpoint.get_auth(), verify=endpoint.is_verify_ssl(), + json=__SEARCH_COUNT_PAYLOAD) + # TODO Handle resp.status_code for non successful requests result = dict(resp.json()) - return int(result[COUNT_KEY]) + total: int = __TOTAL_COUNT_JSONPATH.find(result)[0].value + counts_list: list = __INDEX_COUNT_JSONPATH.find(result)[0].value + count_map = dict() + for entry in counts_list: + count_map[entry[__BUCKET_INDEX_NAME_KEY]] = entry[__BUCKET_DOC_COUNT_KEY] + return IndexDocCount(total, count_map) diff --git a/FetchMigration/python/metadata_migration.py b/FetchMigration/python/metadata_migration.py index a86eb330ce..4cd2c3fc48 100644 --- a/FetchMigration/python/metadata_migration.py +++ b/FetchMigration/python/metadata_migration.py @@ -1,10 +1,12 @@ import argparse +import logging import yaml import endpoint_utils import index_operations import utils +from index_diff import IndexDiff from metadata_migration_params import MetadataMigrationParams from metadata_migration_result import MetadataMigrationResult @@ -14,13 +16,13 @@ INDEX_NAME_KEY = "index_name_regex" -def write_output(yaml_data: dict, new_indices: set, output_path: str): +def write_output(yaml_data: dict, indices_to_migrate: set, output_path: str): pipeline_config = next(iter(yaml_data.values())) # Result is a tuple of (type, config) source_config = endpoint_utils.get_supported_endpoint_config(pipeline_config, endpoint_utils.SOURCE_KEY)[1] source_indices = source_config.get(INDICES_KEY, dict()) included_indices = source_indices.get(INCLUDE_KEY, list()) - for index in new_indices: + for index in indices_to_migrate: included_indices.append({INDEX_NAME_KEY: index}) source_indices[INCLUDE_KEY] = included_indices source_config[INDICES_KEY] = source_indices @@ -28,36 +30,15 @@ def write_output(yaml_data: dict, new_indices: set, output_path: str): yaml.dump(yaml_data, out_file) -# Computes differences in indices between source and target. -# Returns a tuple with 3 elements: -# - The 1st element is the set of indices to create on the target -# - The 2nd element is a set of indices that are identical on source and target -# - The 3rd element is a set of indices that are present on both source and target, -# but differ in their settings or mappings. -def get_index_differences(source: dict, target: dict) -> tuple[set, set, set]: - index_conflicts = set() - indices_in_target = set(source.keys()) & set(target.keys()) - for index in indices_in_target: - # Check settings - if utils.has_differences(index_operations.SETTINGS_KEY, source[index], target[index]): - index_conflicts.add(index) - # Check mappings - if utils.has_differences(index_operations.MAPPINGS_KEY, source[index], target[index]): - index_conflicts.add(index) - identical_indices = set(indices_in_target) - set(index_conflicts) - indices_to_create = set(source.keys()) - set(indices_in_target) - return indices_to_create, identical_indices, index_conflicts - - -# The order of data in the tuple is: -# (indices to create), (identical indices), (indices with conflicts) -def print_report(index_differences: tuple[set, set, set], count: int): # pragma no cover - print("Identical indices in the target cluster (no changes will be made): " + - utils.string_from_set(index_differences[1])) - print("Indices in target cluster with conflicting settings/mappings: " + - utils.string_from_set(index_differences[2])) - print("Indices to create: " + utils.string_from_set(index_differences[0])) - print("Total documents to be moved: " + str(count)) +def print_report(diff: IndexDiff, total_doc_count: int): # pragma no cover + logging.info("Identical indices in the target cluster: " + utils.string_from_set(diff.identical_indices)) + logging.info("Identical empty indices in the target cluster (data will be migrated): " + + utils.string_from_set(diff.identical_empty_indices)) + logging.info("Indices present in both clusters with conflicting settings/mappings (data will not be migrated): " + + utils.string_from_set(diff.conflicting_indices)) + logging.info("Indices to be created in the target cluster (data will be migrated): " + + utils.string_from_set(diff.indices_to_create)) + logging.info("Total number of documents to be moved: " + str(total_doc_count)) def run(args: MetadataMigrationParams) -> MetadataMigrationResult: @@ -83,23 +64,29 @@ def run(args: MetadataMigrationParams) -> MetadataMigrationResult: return result target_indices = index_operations.fetch_all_indices(target_endpoint_info) # Compute index differences and print report - diff = get_index_differences(source_indices, target_indices) - # The first element in the tuple is the set of indices to create - indices_to_create = diff[0] - if indices_to_create: - result.created_indices = indices_to_create - result.target_doc_count = index_operations.doc_count(indices_to_create, source_endpoint_info) + diff = IndexDiff(source_indices, target_indices) + if diff.identical_indices: + # Identical indices with zero documents on the target are eligible for migration + target_doc_count = index_operations.doc_count(diff.identical_indices, target_endpoint_info) + # doc_count only returns indices that have non-zero counts, so the difference in responses + # gives us the set of identical, empty indices + result.migration_indices = diff.identical_indices.difference(target_doc_count.index_doc_count_map.keys()) + diff.set_identical_empty_indices(result.migration_indices) + if diff.indices_to_create: + result.migration_indices.update(diff.indices_to_create) + if result.migration_indices: + doc_count_result = index_operations.doc_count(result.migration_indices, source_endpoint_info) + result.target_doc_count = doc_count_result.total if args.report: print_report(diff, result.target_doc_count) - if indices_to_create: + if result.migration_indices: # Write output YAML if len(args.output_file) > 0: - write_output(dp_config, indices_to_create, args.output_file) - if args.report: # pragma no cover - print("Wrote output YAML pipeline to: " + args.output_file) + write_output(dp_config, result.migration_indices, args.output_file) + logging.debug("Wrote output YAML pipeline to: " + args.output_file) if not args.dryrun: index_data = dict() - for index_name in indices_to_create: + for index_name in diff.indices_to_create: index_data[index_name] = source_indices[index_name] index_operations.create_indices(index_data, target_endpoint_info) return result diff --git a/FetchMigration/python/metadata_migration_result.py b/FetchMigration/python/metadata_migration_result.py index d2122f5df7..7bf325eaaf 100644 --- a/FetchMigration/python/metadata_migration_result.py +++ b/FetchMigration/python/metadata_migration_result.py @@ -4,4 +4,5 @@ @dataclass class MetadataMigrationResult: target_doc_count: int = 0 - created_indices: set = field(default_factory=set) + # Set of indices for which data needs to be migrated + migration_indices: set = field(default_factory=set) diff --git a/FetchMigration/python/migration_monitor.py b/FetchMigration/python/migration_monitor.py index 481d738964..24d81f6b36 100644 --- a/FetchMigration/python/migration_monitor.py +++ b/FetchMigration/python/migration_monitor.py @@ -162,6 +162,6 @@ def run(args: MigrationMonitorParams, dp_process: Optional[Popen] = None, poll_i help="Target doc_count to reach, after which the Data Prepper pipeline will be terminated" ) namespace = arg_parser.parse_args() - print("\n##### Starting monitor tool... #####\n") + logging.info("\n##### Starting monitor tool... #####\n") run(MigrationMonitorParams(namespace.target_count, namespace.data_prepper_endpoint)) - print("\n##### Ending monitor tool... #####\n") + logging.info("\n##### Ending monitor tool... #####\n") diff --git a/FetchMigration/python/requirements.txt b/FetchMigration/python/requirements.txt index a8f57b550c..b9352bd738 100644 --- a/FetchMigration/python/requirements.txt +++ b/FetchMigration/python/requirements.txt @@ -1,5 +1,6 @@ botocore>=1.31.70 jsondiff>=2.0.0 +jsonpath-ng>=1.6.0 prometheus-client>=0.17.1 pyyaml>=6.0.1 requests>=2.31.0 diff --git a/FetchMigration/python/tests/test_endpoint_utils.py b/FetchMigration/python/tests/test_endpoint_utils.py index 03486ed3d0..5a435442e8 100644 --- a/FetchMigration/python/tests/test_endpoint_utils.py +++ b/FetchMigration/python/tests/test_endpoint_utils.py @@ -8,6 +8,7 @@ from moto import mock_iam import endpoint_utils +from endpoint_info import EndpointInfo from tests import test_constants # Constants @@ -71,6 +72,10 @@ def test_is_insecure_missing_nested(self): test_input = {"key1": 123, CONNECTION_KEY: {"key2": "val"}} self.assertFalse(endpoint_utils.is_insecure(test_input)) + def test_auth_normalized_url(self): + val = EndpointInfo("test") + self.assertEqual("test/", val.get_url()) + def test_get_auth_returns_none(self): # The following inputs should not return an auth tuple: # - Empty input diff --git a/FetchMigration/python/tests/test_fetch_orchestrator.py b/FetchMigration/python/tests/test_fetch_orchestrator.py index 826d3baff0..96260073f0 100644 --- a/FetchMigration/python/tests/test_fetch_orchestrator.py +++ b/FetchMigration/python/tests/test_fetch_orchestrator.py @@ -157,7 +157,7 @@ def test_write_inline_target_host(self, mock_file_open: MagicMock, mock_yaml_loa mock_file_open.reset_mock() mock_yaml_dump.reset_mock() orchestrator.write_inline_target_host("test", val) - mock_file_open.assert_called_once_with("test", "rw") + mock_file_open.assert_called_once_with("test", "r+") mock_yaml_dump.assert_called_once_with(expected_pipeline, ANY) def test_update_target_host_bad_config(self): diff --git a/FetchMigration/python/tests/test_index_diff.py b/FetchMigration/python/tests/test_index_diff.py new file mode 100644 index 0000000000..32f3203862 --- /dev/null +++ b/FetchMigration/python/tests/test_index_diff.py @@ -0,0 +1,73 @@ +import copy +import unittest + +from index_diff import IndexDiff +from tests import test_constants + + +class TestIndexDiff(unittest.TestCase): + def test_index_diff_empty(self): + # Base case should return an empty list + diff = IndexDiff(dict(), dict()) + # All members should be empty + self.assertEqual(set(), diff.indices_to_create) + self.assertEqual(set(), diff.identical_indices) + self.assertEqual(set(), diff.conflicting_indices) + + def test_index_diff_empty_target(self): + diff = IndexDiff(test_constants.BASE_INDICES_DATA, dict()) + # No conflicts or identical indices + self.assertEqual(set(), diff.conflicting_indices) + self.assertEqual(set(), diff.identical_indices) + # Indices-to-create + self.assertEqual(3, len(diff.indices_to_create)) + self.assertTrue(test_constants.INDEX1_NAME in diff.indices_to_create) + self.assertTrue(test_constants.INDEX2_NAME in diff.indices_to_create) + self.assertTrue(test_constants.INDEX3_NAME in diff.indices_to_create) + + def test_index_diff_identical_index(self): + test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) + del test_data[test_constants.INDEX2_NAME] + del test_data[test_constants.INDEX3_NAME] + diff = IndexDiff(test_data, test_data) + # No indices to move, or conflicts + self.assertEqual(set(), diff.indices_to_create) + self.assertEqual(set(), diff.conflicting_indices) + # Identical indices + self.assertEqual(1, len(diff.identical_indices)) + self.assertTrue(test_constants.INDEX1_NAME in diff.identical_indices) + + def test_index_diff_settings_conflict(self): + test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) + # Set up conflict in settings + index_settings = test_data[test_constants.INDEX2_NAME][test_constants.SETTINGS_KEY] + index_settings[test_constants.INDEX_KEY][test_constants.NUM_REPLICAS_SETTING] += 1 + diff = IndexDiff(test_constants.BASE_INDICES_DATA, test_data) + # No indices to move + self.assertEqual(set(), diff.indices_to_create) + # Identical indices + self.assertEqual(2, len(diff.identical_indices)) + self.assertTrue(test_constants.INDEX1_NAME in diff.identical_indices) + self.assertTrue(test_constants.INDEX3_NAME in diff.identical_indices) + # Conflicting indices + self.assertEqual(1, len(diff.conflicting_indices)) + self.assertTrue(test_constants.INDEX2_NAME in diff.conflicting_indices) + + def test_index_diff_mappings_conflict(self): + test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) + # Set up conflict in mappings + test_data[test_constants.INDEX3_NAME][test_constants.MAPPINGS_KEY] = {} + diff = IndexDiff(test_constants.BASE_INDICES_DATA, test_data) + # No indices to move + self.assertEqual(set(), diff.indices_to_create) + # Identical indices + self.assertEqual(2, len(diff.identical_indices)) + self.assertTrue(test_constants.INDEX1_NAME in diff.identical_indices) + self.assertTrue(test_constants.INDEX2_NAME in diff.identical_indices) + # Conflicting indices + self.assertEqual(1, len(diff.conflicting_indices)) + self.assertTrue(test_constants.INDEX3_NAME in diff.conflicting_indices) + + +if __name__ == '__main__': + unittest.main() diff --git a/FetchMigration/python/tests/test_index_operations.py b/FetchMigration/python/tests/test_index_operations.py index 1378da5fda..8830b34e58 100644 --- a/FetchMigration/python/tests/test_index_operations.py +++ b/FetchMigration/python/tests/test_index_operations.py @@ -60,12 +60,18 @@ def test_create_indices_exception(self): @responses.activate def test_doc_count(self): test_indices = {test_constants.INDEX1_NAME, test_constants.INDEX2_NAME} - expected_count_endpoint = test_constants.SOURCE_ENDPOINT + ",".join(test_indices) + "/_count" - mock_count_response = {"count": "10"} + index_doc_count: int = 5 + test_buckets = list() + for index_name in test_indices: + test_buckets.append({"key": index_name, "doc_count": index_doc_count}) + total_docs: int = index_doc_count * len(test_buckets) + expected_count_endpoint = test_constants.SOURCE_ENDPOINT + ",".join(test_indices) + "/_search?size=0" + mock_count_response = {"hits": {"total": {"value": total_docs}}, + "aggregations": {"count": {"buckets": test_buckets}}} responses.get(expected_count_endpoint, json=mock_count_response) # Now send request - count_value = index_operations.doc_count(test_indices, EndpointInfo(test_constants.SOURCE_ENDPOINT)) - self.assertEqual(10, count_value) + doc_count_result = index_operations.doc_count(test_indices, EndpointInfo(test_constants.SOURCE_ENDPOINT)) + self.assertEqual(total_docs, doc_count_result.total) if __name__ == '__main__': diff --git a/FetchMigration/python/tests/test_metadata_migration.py b/FetchMigration/python/tests/test_metadata_migration.py index 1696c50d6a..d295af4b8a 100644 --- a/FetchMigration/python/tests/test_metadata_migration.py +++ b/FetchMigration/python/tests/test_metadata_migration.py @@ -4,6 +4,7 @@ from unittest.mock import patch, MagicMock, ANY import metadata_migration +from index_doc_count import IndexDocCount from metadata_migration_params import MetadataMigrationParams from tests import test_constants @@ -14,78 +15,6 @@ def setUp(self) -> None: with open(test_constants.PIPELINE_CONFIG_PICKLE_FILE_PATH, "rb") as f: self.loaded_pipeline_config = pickle.load(f) - def test_get_index_differences_empty(self): - # Base case should return an empty list - result_tuple = metadata_migration.get_index_differences(dict(), dict()) - # Invariant - self.assertEqual(3, len(result_tuple)) - # All diffs should be empty - self.assertEqual(set(), result_tuple[0]) - self.assertEqual(set(), result_tuple[1]) - self.assertEqual(set(), result_tuple[2]) - - def test_get_index_differences_empty_target(self): - result_tuple = metadata_migration.get_index_differences(test_constants.BASE_INDICES_DATA, dict()) - # Invariant - self.assertEqual(3, len(result_tuple)) - # No conflicts or identical indices - self.assertEqual(set(), result_tuple[1]) - self.assertEqual(set(), result_tuple[2]) - # Indices-to-create - self.assertEqual(3, len(result_tuple[0])) - self.assertTrue(test_constants.INDEX1_NAME in result_tuple[0]) - self.assertTrue(test_constants.INDEX2_NAME in result_tuple[0]) - self.assertTrue(test_constants.INDEX3_NAME in result_tuple[0]) - - def test_get_index_differences_identical_index(self): - test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) - del test_data[test_constants.INDEX2_NAME] - del test_data[test_constants.INDEX3_NAME] - result_tuple = metadata_migration.get_index_differences(test_data, test_data) - # Invariant - self.assertEqual(3, len(result_tuple)) - # No indices to move, or conflicts - self.assertEqual(set(), result_tuple[0]) - self.assertEqual(set(), result_tuple[2]) - # Identical indices - self.assertEqual(1, len(result_tuple[1])) - self.assertTrue(test_constants.INDEX1_NAME in result_tuple[1]) - - def test_get_index_differences_settings_conflict(self): - test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) - # Set up conflict in settings - index_settings = test_data[test_constants.INDEX2_NAME][test_constants.SETTINGS_KEY] - index_settings[test_constants.INDEX_KEY][test_constants.NUM_REPLICAS_SETTING] += 1 - result_tuple = metadata_migration.get_index_differences(test_constants.BASE_INDICES_DATA, test_data) - # Invariant - self.assertEqual(3, len(result_tuple)) - # No indices to move - self.assertEqual(set(), result_tuple[0]) - # Identical indices - self.assertEqual(2, len(result_tuple[1])) - self.assertTrue(test_constants.INDEX1_NAME in result_tuple[1]) - self.assertTrue(test_constants.INDEX3_NAME in result_tuple[1]) - # Conflicting indices - self.assertEqual(1, len(result_tuple[2])) - self.assertTrue(test_constants.INDEX2_NAME in result_tuple[2]) - - def test_get_index_differences_mappings_conflict(self): - test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) - # Set up conflict in mappings - test_data[test_constants.INDEX3_NAME][test_constants.MAPPINGS_KEY] = {} - result_tuple = metadata_migration.get_index_differences(test_constants.BASE_INDICES_DATA, test_data) - # Invariant - self.assertEqual(3, len(result_tuple)) - # No indices to move - self.assertEqual(set(), result_tuple[0]) - # Identical indices - self.assertEqual(2, len(result_tuple[1])) - self.assertTrue(test_constants.INDEX1_NAME in result_tuple[1]) - self.assertTrue(test_constants.INDEX2_NAME in result_tuple[1]) - # Conflicting indices - self.assertEqual(1, len(result_tuple[2])) - self.assertTrue(test_constants.INDEX3_NAME in result_tuple[2]) - @patch('index_operations.doc_count') @patch('metadata_migration.write_output') @patch('metadata_migration.print_report') @@ -94,15 +23,11 @@ def test_get_index_differences_mappings_conflict(self): # Note that mock objects are passed bottom-up from the patch order above def test_run_report(self, mock_fetch_indices: MagicMock, mock_create_indices: MagicMock, mock_print_report: MagicMock, mock_write_output: MagicMock, mock_doc_count: MagicMock): - mock_doc_count.return_value = 1 + mock_doc_count.return_value = IndexDocCount(1, dict()) index_to_create = test_constants.INDEX3_NAME index_with_conflict = test_constants.INDEX2_NAME - index_exact_match = test_constants.INDEX1_NAME # Set up expected arguments to mocks so we can verify expected_create_payload = {index_to_create: test_constants.BASE_INDICES_DATA[index_to_create]} - # Print report accepts a tuple. The elements of the tuple - # are in the order: to-create, exact-match, conflicts - expected_diff = {index_to_create}, {index_exact_match}, {index_with_conflict} # Create mock data for indices on target target_indices_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) del target_indices_data[index_to_create] @@ -115,7 +40,7 @@ def test_run_report(self, mock_fetch_indices: MagicMock, mock_create_indices: Ma metadata_migration.run(test_input) mock_create_indices.assert_called_once_with(expected_create_payload, ANY) mock_doc_count.assert_called() - mock_print_report.assert_called_once_with(expected_diff, 1) + mock_print_report.assert_called_once_with(ANY, 1) mock_write_output.assert_not_called() @patch('index_operations.doc_count') @@ -125,8 +50,35 @@ def test_run_report(self, mock_fetch_indices: MagicMock, mock_create_indices: Ma # Note that mock objects are passed bottom-up from the patch order above def test_run_dryrun(self, mock_fetch_indices: MagicMock, mock_write_output: MagicMock, mock_print_report: MagicMock, mock_doc_count: MagicMock): + index_to_migrate = test_constants.INDEX1_NAME + expected_output_path = "dummy" + test_doc_counts = {test_constants.INDEX2_NAME: 2, test_constants.INDEX3_NAME: 3} + # Doc counts are first fetched for the target cluster, + # then then source cluster is queried only for identical, empty indices + mock_doc_count.side_effect = [IndexDocCount(5, test_doc_counts), + IndexDocCount(1, {index_to_migrate: 1})] + mock_fetch_indices.return_value = test_constants.BASE_INDICES_DATA + test_input = MetadataMigrationParams(test_constants.PIPELINE_CONFIG_RAW_FILE_PATH, expected_output_path, + dryrun=True) + test_result = metadata_migration.run(test_input) + self.assertEqual(1, test_result.target_doc_count) + self.assertEqual({index_to_migrate}, test_result.migration_indices) + mock_write_output.assert_called_once_with(self.loaded_pipeline_config, {index_to_migrate}, expected_output_path) + mock_doc_count.assert_called() + # Report should not be printed + mock_print_report.assert_not_called() + + @patch('index_operations.doc_count') + @patch('metadata_migration.print_report') + @patch('metadata_migration.write_output') + @patch('index_operations.fetch_all_indices') + # Note that mock objects are passed bottom-up from the patch order above + def test_identical_empty_index(self, mock_fetch_indices: MagicMock, mock_write_output: MagicMock, + mock_print_report: MagicMock, mock_doc_count: MagicMock): + test_index_doc_counts = {test_constants.INDEX2_NAME: 2, + test_constants.INDEX3_NAME: 3} + mock_doc_count.return_value = IndexDocCount(1, test_index_doc_counts) index_to_create = test_constants.INDEX1_NAME - mock_doc_count.return_value = 1 expected_output_path = "dummy" # Create mock data for indices on target target_indices_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) @@ -136,8 +88,8 @@ def test_run_dryrun(self, mock_fetch_indices: MagicMock, mock_write_output: Magi test_input = MetadataMigrationParams(test_constants.PIPELINE_CONFIG_RAW_FILE_PATH, expected_output_path, dryrun=True) test_result = metadata_migration.run(test_input) - self.assertEqual(mock_doc_count.return_value, test_result.target_doc_count) - self.assertEqual({index_to_create}, test_result.created_indices) + self.assertEqual(mock_doc_count.return_value.total, test_result.target_doc_count) + self.assertEqual({index_to_create}, test_result.migration_indices) mock_write_output.assert_called_once_with(self.loaded_pipeline_config, {index_to_create}, expected_output_path) mock_doc_count.assert_called() # Report should not be printed @@ -182,7 +134,7 @@ def test_no_indices_in_source(self, mock_fetch_indices: MagicMock): test_result = metadata_migration.run(test_input) mock_fetch_indices.assert_called_once() self.assertEqual(0, test_result.target_doc_count) - self.assertEqual(0, len(test_result.created_indices)) + self.assertEqual(0, len(test_result.migration_indices)) if __name__ == '__main__': diff --git a/deployment/cdk/opensearch-service-migration/dp_pipeline_template.yaml b/deployment/cdk/opensearch-service-migration/dp_pipeline_template.yaml index 9baec2325c..a8d3b7b43c 100644 --- a/deployment/cdk/opensearch-service-migration/dp_pipeline_template.yaml +++ b/deployment/cdk/opensearch-service-migration/dp_pipeline_template.yaml @@ -1,18 +1,36 @@ +# Name of the Data Prepper pipeline historical-data-migration: + # Source cluster configuration source: opensearch: + # CDK code will replace this value, so DO NOT CHANGE this + # unless the file is being used outside of the CDK hosts: - + # Uncomment the following line to disable TLS + #insecure: true + # Example configuration on how to disable authentication (default: false) disable_authentication: true indices: + # Indices to exclude - exclude system indices by default exclude: - index_name_regex: \.* + # Target cluster configuration sink: - opensearch: - bulk_size: 10 + # Note - CDK code will replace this value with the target cluster endpoint hosts: - - https:// + - + # Derive index name from record metadata index: ${getMetadata("opensearch-index")} + # Use the same document ID as the source cluster document document_id: ${getMetadata("opensearch-document_id")} + # Example configuration for basic auth username: user password: pass + #disable_authentication: true + # Additional pipeline options/optimizations + # For maximum throughput, match workers to number of vCPUs (default: 1) + workers: 1 + # delay is how often the worker threads should process data (default: 3000 ms) + delay: 0 diff --git a/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts b/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts index de4c016f69..47ff189600 100644 --- a/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts @@ -38,8 +38,8 @@ export class FetchMigrationStack extends Stack { // ECS Task Definition const fetchMigrationFargateTask = new FargateTaskDefinition(this, "fetchMigrationFargateTask", { - memoryLimitMiB: 2048, - cpu: 512 + memoryLimitMiB: 4096, + cpu: 1024 }); new StringParameter(this, 'SSMParameterFetchMigrationTaskDefArn', { From 4ed3b10e33276f4a77ed349a82dceca5cd47fa11 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Fri, 10 Nov 2023 22:53:04 -0500 Subject: [PATCH 38/55] Implement commits for the KafkaProtobufConsumer. Once messages have been completely handled, and only then, will they be committed against the Kafka broker. "Completely handled" may include erroring out, or having nothing to do - but once we know that we've got a result that was complete for that message we'll 'release' all of the TrafficStreams that made it up. Of course, releasing doesn't mean that they'll be immediately committed, since the ordering of TrafficStreams through finished requests may not be preserved as well as interleaved. Since Kafka only has a single offset per topic/partition, we'll have to wait until all of the previous record offsets were also released before updating the offsets. Signed-off-by: Greg Schohn --- .../migrations/replay/TrafficReplayer.java | 6 +- .../datatypes/PojoTrafficStreamKey.java | 18 +- .../datatypes/PojoTrafficStreamWithKey.java | 14 ++ .../replay/kafka/KafkaProtobufConsumer.java | 161 ++++++++++++++++-- .../source/TrafficStreamWithEmbeddedKey.java | 3 +- .../replay/FullTrafficReplayerTest.java | 65 +------ .../KafkaRestartingTrafficReplayerTest.java | 103 +++++++++-- .../replay/SentinelSensingTrafficSource.java | 56 ++++++ .../replay/TrafficReplayerRunner.java | 18 +- .../replay/TrafficStreamGenerator.java | 23 +++ 10 files changed, 362 insertions(+), 105 deletions(-) create mode 100644 TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/PojoTrafficStreamWithKey.java create mode 100644 TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SentinelSensingTrafficSource.java diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index 4b140f993d..b43df7ab32 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -911,6 +911,7 @@ public void stopReadingAsync() { return shutdownFuture; } + @SneakyThrows public void pullCaptureFromSourceToAccumulator( ITrafficCaptureSource trafficChunkStream, CapturedTrafficToHttpTransactionAccumulator trafficToHttpTransactionAccumulator) @@ -930,9 +931,8 @@ public void pullCaptureFromSourceToAccumulator( "Done reading traffic streams.").log(); break; } else { - log.atWarn().setCause(ex).setMessage("Interrupted. Done reading traffic streams.").log(); - Thread.currentThread().interrupt(); - throw new RuntimeException(ex); + log.atWarn().setCause(ex).setMessage("Done reading traffic streams due to exception.").log(); + throw ex.getCause(); } } if (log.isInfoEnabled()) { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/PojoTrafficStreamKey.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/PojoTrafficStreamKey.java index 3d0118e0b4..354c27b31b 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/PojoTrafficStreamKey.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/PojoTrafficStreamKey.java @@ -1,7 +1,12 @@ package org.opensearch.migrations.replay.datatypes; +import lombok.ToString; +import org.opensearch.migrations.trafficcapture.protos.TrafficStream; +import org.opensearch.migrations.trafficcapture.protos.TrafficStreamUtils; + import java.util.StringJoiner; +@ToString public class PojoTrafficStreamKey implements ITrafficStreamKey { private final String nodeId; private final String connectionId; @@ -11,6 +16,10 @@ public PojoTrafficStreamKey(ITrafficStreamKey orig) { this(orig.getNodeId(), orig.getConnectionId(), orig.getTrafficStreamIndex()); } + public PojoTrafficStreamKey(TrafficStream stream) { + this(stream.getNodeId(), stream.getConnectionId(), TrafficStreamUtils.getTrafficStreamIndex(stream)); + } + public PojoTrafficStreamKey(String nodeId, String connectionId, int index) { this.nodeId = nodeId; this.connectionId = connectionId; @@ -31,13 +40,4 @@ public String getConnectionId() { public int getTrafficStreamIndex() { return trafficStreamIndex; } - - @Override - public String toString() { - return new StringJoiner(".") - .add(nodeId) - .add(connectionId) - .add(""+trafficStreamIndex) - .toString(); - } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/PojoTrafficStreamWithKey.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/PojoTrafficStreamWithKey.java new file mode 100644 index 0000000000..62837c9eb7 --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/PojoTrafficStreamWithKey.java @@ -0,0 +1,14 @@ +package org.opensearch.migrations.replay.datatypes; + + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.opensearch.migrations.replay.traffic.source.ITrafficStreamWithKey; +import org.opensearch.migrations.trafficcapture.protos.TrafficStream; + +@AllArgsConstructor +@Getter +public class PojoTrafficStreamWithKey implements ITrafficStreamWithKey { + public final TrafficStream stream; + public final ITrafficStreamKey key; +} \ No newline at end of file diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumer.java index 85706a0d0f..2e2a7220f9 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumer.java @@ -2,16 +2,20 @@ import com.google.protobuf.InvalidProtocolBufferException; import lombok.NonNull; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; import org.opensearch.migrations.coreutils.MetricsLogger; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; +import org.opensearch.migrations.replay.datatypes.PojoTrafficStreamKey; +import org.opensearch.migrations.replay.datatypes.PojoTrafficStreamWithKey; import org.opensearch.migrations.replay.traffic.source.ISimpleTrafficCaptureSource; import org.opensearch.migrations.replay.traffic.source.ITrafficStreamWithKey; -import org.opensearch.migrations.replay.traffic.source.TrafficStreamWithEmbeddedKey; import org.opensearch.migrations.trafficcapture.protos.TrafficStream; import java.io.FileInputStream; @@ -20,25 +24,107 @@ import java.time.Duration; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Objects; +import java.util.Optional; +import java.util.PriorityQueue; import java.util.Properties; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; +/** + * Adapt a Kafka stream into a TrafficCaptureSource. + * + * Notice that there's a critical gap between how Kafka accepts commits and how the + * BlockingTrafficSource throttles calls to Kafka. The BlockingTrafficSource may + * block calls to readNextTrafficStreamChunk() until some time window elapses. This + * could be a very large window in cases where there were long gaps between recorded + * requests from the capturing proxy. For example, if a TrafficStream is read and it + * that stream is scheduled to be run one hour later, readNextTrafficStreamChunk() + * may not be called for almost an hour. By design, we're not calling Kafka to pull + * any more messages since we know that we don't have work to do for an hour. Shortly + * after the hour of waiting begins, Kakfa will notice that this application is no + * longer calling poll and will kick the consumer out of the client group. Other + * consumers may connect, though they'll also be kicked out of the group shortly. + * It may be fine to have this pattern of cascaded replayers waiting to send a + * request, then all of them sending at about the same time.... + * + * See + * ... + * + * "Basically if you don't call poll at least as frequently as the configured max interval, + * then the client will proactively leave the group so that another consumer can take + * over its partitions. When this happens, you may see an offset commit failure (as + * indicated by a CommitFailedException thrown from a call to commitSync())." + * + * I believe that this can be mitigated, hopefully fully, by adding a keepAlive/do nothing + * call that the BlockingTrafficSource can use. That can be implemented in a source + * like this with Kafka by polling, then resetting the position on the stream if we + * aren't supposed to be reading new data. + */ @Slf4j public class KafkaProtobufConsumer implements ISimpleTrafficCaptureSource { + @ToString(callSuper = true) + private static class TrafficStreamKeyWithKafkaRecordId extends PojoTrafficStreamKey { + private final int partition; + private final long offset; + + TrafficStreamKeyWithKafkaRecordId(TrafficStream trafficStream, int partition, long offset) { + super(trafficStream); + this.partition = partition; + this.offset = offset; + } + } + + private static class OffsetLifecycleTracker { + private final PriorityQueue pQueue = new PriorityQueue<>(); + private long cursorHighWatermark; + + private OffsetLifecycleTracker() { + } + + boolean isEmpty() { + return pQueue.isEmpty(); + } + + void add(long offset) { + cursorHighWatermark = offset; + pQueue.add(offset); + } + + Optional removeAndReturnNewHead(TrafficStreamKeyWithKafkaRecordId kafkaRecord) { + var offsetToRemove = kafkaRecord.offset; + var topCursor = pQueue.peek(); + var didRemove = pQueue.remove(offsetToRemove); + assert didRemove : "Expected all live records to have an entry and for them to be removed only once"; + if (topCursor == offsetToRemove) { + topCursor = Optional.ofNullable(pQueue.peek()) + .orElse(cursorHighWatermark+1); // most recent cursor was previously popped + log.atDebug().setMessage("Commit called for {}, and new topCursor={}") + .addArgument(offsetToRemove).addArgument(topCursor).log(); + return Optional.of(topCursor); + } else { + log.atDebug().setMessage("Commit called for {}, but topCursor={}") + .addArgument(offsetToRemove).addArgument(topCursor).log(); + return Optional.empty(); + } + } + }; + private static final MetricsLogger metricsLogger = new MetricsLogger("KafkaProtobufConsumer"); + public static final Duration CONSUMER_POLL_TIMEOUT = Duration.ofSeconds(1); + private final Consumer kafkaConsumer; + private final ConcurrentHashMap partitionToOffsetLifecycleTrackerMap; + private final ConcurrentHashMap nextSetOfCommitsMap; + private final Object offsetLifecycleLock = new Object(); private final String topic; private final KafkaBehavioralPolicy behavioralPolicy; private final AtomicInteger trafficStreamsRead; - private static final MetricsLogger metricsLogger = new MetricsLogger("KafkaProtobufConsumer"); - public KafkaProtobufConsumer(Consumer kafkaConsumer, String topic) { this(kafkaConsumer, topic, new KafkaBehavioralPolicy()); @@ -51,6 +137,9 @@ public KafkaProtobufConsumer(Consumer kafkaConsumer, @NonNull St this.behavioralPolicy = behavioralPolicy; kafkaConsumer.subscribe(Collections.singleton(topic)); trafficStreamsRead = new AtomicInteger(); + + partitionToOffsetLifecycleTrackerMap = new ConcurrentHashMap<>(); + nextSetOfCommitsMap = new ConcurrentHashMap<>(); } public static KafkaProtobufConsumer buildKafkaConsumer(@NonNull String brokers, @@ -101,19 +190,25 @@ public CompletableFuture> readNextTrafficStreamChunk public List readNextTrafficStreamSynchronously() { try { ConsumerRecords records; - records = safePollWithSwallowedRuntimeExceptions(); + records = safeCommitAndPollWithSwallowedRuntimeExceptions(); Stream trafficStream = StreamSupport.stream(records.spliterator(), false) .map(kafkaRecord -> { try { TrafficStream ts = TrafficStream.parseFrom(kafkaRecord.value()); // Ensure we increment trafficStreamsRead even at a higher log level - log.trace("Parsed traffic stream #{}: {}", trafficStreamsRead.incrementAndGet(), ts); metricsLogger.atSuccess() .addKeyValue("connectionId", ts.getConnectionId()) .addKeyValue("topicName", this.topic) .addKeyValue("sizeInBytes", ts.getSerializedSize()) .setMessage("Parsed traffic stream from Kafka").log(); - return (ITrafficStreamWithKey) new TrafficStreamWithEmbeddedKey(ts); + addOffset(kafkaRecord.partition(), kafkaRecord.offset()); + var key = new TrafficStreamKeyWithKafkaRecordId(ts, kafkaRecord.partition(), kafkaRecord.offset()); + log.atTrace().setMessage(()->"Parsed traffic stream #{}: {} {}") + .addArgument(trafficStreamsRead.incrementAndGet()) + .addArgument(key) + .addArgument(ts) + .log(); + return (ITrafficStreamWithKey) new PojoTrafficStreamWithKey(ts, key); } catch (InvalidProtocolBufferException e) { RuntimeException recordError = behavioralPolicy.onInvalidKafkaRecord(kafkaRecord, e); metricsLogger.atError(recordError) @@ -125,8 +220,6 @@ public List readNextTrafficStreamSynchronously() { return null; } }).filter(Objects::nonNull); - // This simple commit should be removed when logic is in place for using commitTrafficStream() - kafkaConsumer.commitSync(); return trafficStream.collect(Collectors.toList()); } catch (Exception e) { log.error("Terminating Kafka traffic stream"); @@ -134,22 +227,58 @@ public List readNextTrafficStreamSynchronously() { } } - private ConsumerRecords safePollWithSwallowedRuntimeExceptions() { - ConsumerRecords records; + private ConsumerRecords safeCommitAndPollWithSwallowedRuntimeExceptions() { try { - records = kafkaConsumer.poll(CONSUMER_POLL_TIMEOUT); - log.info("Kafka consumer poll has fetched {} records", records.count()); + synchronized (offsetLifecycleLock) { + if (!nextSetOfCommitsMap.isEmpty()) { + log.atDebug().setMessage(()->"Committing "+nextSetOfCommitsMap).log(); + kafkaConsumer.commitSync(nextSetOfCommitsMap); + log.atDebug().setMessage(()->"Done committing "+nextSetOfCommitsMap).log(); + nextSetOfCommitsMap.clear(); + } + } + + var records = kafkaConsumer.poll(CONSUMER_POLL_TIMEOUT); + log.atInfo().setMessage(()->"Kafka consumer poll has fetched "+records.count()+" records").log(); + log.atDebug().setMessage(()->"All positions: {"+kafkaConsumer.assignment().stream() + .map(tp->tp+": "+kafkaConsumer.position(tp)).collect(Collectors.joining(",")) + "}").log(); + log.atDebug().setMessage(()->"All COMMITTED positions: {"+kafkaConsumer.assignment().stream() + .map(tp->tp+": "+kafkaConsumer.committed(tp)).collect(Collectors.joining(",")) + "}").log(); + return records; } catch (RuntimeException e) { log.atWarn().setCause(e).setMessage("Unable to poll the topic: {} with our Kafka consumer. " + "Swallowing and awaiting next metadata refresh to try again.").addArgument(topic).log(); - records = new ConsumerRecords<>(Collections.emptyMap()); + return new ConsumerRecords<>(Collections.emptyMap()); + } + } + + private void addOffset(int partition, long offset) { + synchronized (offsetLifecycleLock) { + var offsetTracker = partitionToOffsetLifecycleTrackerMap.computeIfAbsent(partition, p -> + new OffsetLifecycleTracker()); + offsetTracker.add(offset); } - return records; } @Override public void commitTrafficStream(ITrafficStreamKey trafficStreamKey) { - kafkaConsumer.commitSync(Map.of()); + if (!(trafficStreamKey instanceof TrafficStreamKeyWithKafkaRecordId)) { + throw new IllegalArgumentException("Expected key of type "+TrafficStreamKeyWithKafkaRecordId.class+ + " but received "+trafficStreamKey+" (of type="+trafficStreamKey.getClass()+")"); + } + var kafkaTsk = (TrafficStreamKeyWithKafkaRecordId) trafficStreamKey; + var p = kafkaTsk.partition; + Optional newHeadValue; + synchronized (offsetLifecycleLock) { + var tracker = partitionToOffsetLifecycleTrackerMap.get(p); + newHeadValue = tracker.removeAndReturnNewHead(kafkaTsk); + newHeadValue.ifPresent(o -> { + if (tracker.isEmpty()) { + partitionToOffsetLifecycleTrackerMap.remove(tracker); + } + nextSetOfCommitsMap.put(new TopicPartition(topic, p), new OffsetAndMetadata(o)); + }); + } } @Override diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/TrafficStreamWithEmbeddedKey.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/TrafficStreamWithEmbeddedKey.java index 7d463c6c41..a89d2b0466 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/TrafficStreamWithEmbeddedKey.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/TrafficStreamWithEmbeddedKey.java @@ -14,8 +14,7 @@ public TrafficStreamWithEmbeddedKey(TrafficStream stream) { @Override public ITrafficStreamKey getKey() { - return new PojoTrafficStreamKey(stream.getNodeId(), stream.getConnectionId(), - TrafficStreamUtils.getTrafficStreamIndex(stream)); + return new PojoTrafficStreamKey(stream); } @Override diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java index 2a6cf5123f..ac84c6bc74 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java @@ -1,64 +1,36 @@ package org.opensearch.migrations.replay; -import com.google.common.base.Strings; -import com.google.common.collect.Streams; -import io.vavr.Tuple2; -import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; -import lombok.SneakyThrows; import lombok.ToString; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import org.junit.jupiter.params.provider.ValueSource; -import org.opensearch.migrations.replay.datatypes.ISourceTrafficChannelKey; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; -import org.opensearch.migrations.replay.kafka.KafkaProtobufConsumer; -import org.opensearch.migrations.replay.traffic.source.BlockingTrafficSource; +import org.opensearch.migrations.replay.datatypes.PojoTrafficStreamWithKey; import org.opensearch.migrations.replay.traffic.source.ISimpleTrafficCaptureSource; import org.opensearch.migrations.replay.traffic.source.ITrafficStreamWithKey; -import org.opensearch.migrations.replay.traffic.source.TrafficStreamWithEmbeddedKey; import org.opensearch.migrations.testutils.SimpleNettyHttpServer; -import org.opensearch.migrations.testutils.StreamInterleaver; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; import org.opensearch.migrations.trafficcapture.protos.CloseObservation; import org.opensearch.migrations.trafficcapture.protos.TrafficObservation; import org.opensearch.migrations.trafficcapture.protos.TrafficStream; import org.opensearch.migrations.trafficcapture.protos.TrafficStreamUtils; -import org.opensearch.migrations.transform.StaticAuthTransformerFactory; -import org.slf4j.event.Level; -import org.testcontainers.containers.KafkaContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.shaded.org.apache.commons.io.output.NullOutputStream; import java.io.EOFException; import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.PriorityQueue; -import java.util.Properties; import java.util.Random; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; @Slf4j // It would be great to test with leak detection here, but right now this test relies upon TrafficReplayer.shutdown() @@ -73,7 +45,7 @@ public class FullTrafficReplayerTest { public static final String TEST_NODE_ID = "TestNodeId"; public static final String TEST_CONNECTION_ID = "testConnectionId"; - private static class CountingTupleReceiverFactory implements Supplier> { + private static class IndexWatchingReceiverFactory implements Supplier> { AtomicInteger nextStopPointRef = new AtomicInteger(INITIAL_STOP_REPLAYER_REQUEST_COUNT); @Override @@ -93,7 +65,6 @@ public Consumer get() { } @Test - @Tag("longTest") public void testSingleStreamWithCloseIsCommitted() throws Throwable { var httpServer = SimpleNettyHttpServer.makeServer(false, Duration.ofMillis(2), TestHttpServerContext::makeResponse); @@ -105,7 +76,7 @@ public void testSingleStreamWithCloseIsCommitted() throws Throwable { .build(); var trafficSourceSupplier = new ArrayCursorTrafficSourceFactory(List.of(trafficStreamWithJustClose)); TrafficReplayerRunner.runReplayerUntilSourceWasExhausted(0, - httpServer.localhostEndpoint(), new CountingTupleReceiverFactory(), trafficSourceSupplier); + httpServer.localhostEndpoint(), new IndexWatchingReceiverFactory(), trafficSourceSupplier); Assertions.assertEquals(1, trafficSourceSupplier.nextReadCursor.get()); log.error("done"); } @@ -121,33 +92,18 @@ public void testSingleStreamWithCloseIsCommitted() throws Throwable { public void fullTest(int testSize, boolean randomize) throws Throwable { var httpServer = SimpleNettyHttpServer.makeServer(false, Duration.ofMillis(2), TestHttpServerContext::makeResponse); - var streamAndConsumer = generateStreamAndTupleConsumerWithSomeChecks(testSize, randomize); - var numExpectedRequests = streamAndConsumer._2; - var trafficStreams = streamAndConsumer._1.collect(Collectors.toList()); + var streamAndConsumer = TrafficStreamGenerator.generateStreamAndSumOfItsTransactions(testSize, randomize); + var numExpectedRequests = streamAndConsumer.numHttpTransactions; + var trafficStreams = streamAndConsumer.stream.collect(Collectors.toList()); log.atInfo().setMessage(()->trafficStreams.stream().map(ts->TrafficStreamUtils.summarizeTrafficStream(ts)) .collect(Collectors.joining("\n"))).log(); var trafficSourceSupplier = new ArrayCursorTrafficSourceFactory(trafficStreams); TrafficReplayerRunner.runReplayerUntilSourceWasExhausted(numExpectedRequests, - httpServer.localhostEndpoint(), new CountingTupleReceiverFactory(), trafficSourceSupplier); + httpServer.localhostEndpoint(), new IndexWatchingReceiverFactory(), trafficSourceSupplier); Assertions.assertEquals(trafficSourceSupplier.streams.size(), trafficSourceSupplier.nextReadCursor.get()); log.error("done"); } - private Tuple2, Integer> - generateStreamAndTupleConsumerWithSomeChecks(int count, boolean randomize) { - var generatedCases = count > 0 ? - TrafficStreamGenerator.generateRandomTrafficStreamsAndSizes(IntStream.range(0,count)) : - TrafficStreamGenerator.generateAllIndicativeRandomTrafficStreamsAndSizes(); - var testCaseArr = generatedCases.toArray(TrafficStreamGenerator.RandomTrafficStreamAndTransactionSizes[]::new); - var aggregatedStreams = randomize ? - StreamInterleaver.randomlyInterleaveStreams(new Random(count), - Arrays.stream(testCaseArr).map(c->Arrays.stream(c.trafficStreams))) : - Arrays.stream(testCaseArr).flatMap(c->Arrays.stream(c.trafficStreams)); - - var numExpectedRequests = Arrays.stream(testCaseArr).mapToInt(c->c.requestByteSizes.length).sum(); - return new Tuple2<>(aggregatedStreams, numExpectedRequests); - } - @Getter @ToString @EqualsAndHashCode @@ -172,13 +128,6 @@ public int compareTo(TrafficStreamCursorKey other) { } } - @AllArgsConstructor - @Getter - private static class PojoTrafficStreamWithKey implements ITrafficStreamWithKey { - TrafficStream stream; - ITrafficStreamKey key; - } - private static class ArrayCursorTrafficSourceFactory implements Supplier { List streams; AtomicInteger nextReadCursor = new AtomicInteger(); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/KafkaRestartingTrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/KafkaRestartingTrafficReplayerTest.java index dbdebbc639..3c633168a8 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/KafkaRestartingTrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/KafkaRestartingTrafficReplayerTest.java @@ -1,5 +1,6 @@ package org.opensearch.migrations.replay; +import com.google.common.collect.Streams; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.KafkaConsumer; @@ -8,11 +9,16 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.opensearch.migrations.replay.kafka.KafkaProtobufConsumer; import org.opensearch.migrations.replay.traffic.source.ISimpleTrafficCaptureSource; import org.opensearch.migrations.replay.traffic.source.ITrafficStreamWithKey; import org.opensearch.migrations.replay.traffic.source.TrafficStreamWithEmbeddedKey; +import org.opensearch.migrations.testutils.SimpleNettyHttpServer; import org.opensearch.migrations.trafficcapture.protos.TrafficStream; +import org.opensearch.migrations.trafficcapture.protos.TrafficStreamUtils; import org.testcontainers.containers.KafkaContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @@ -22,17 +28,22 @@ import java.time.Instant; import java.util.List; import java.util.Properties; +import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.stream.Stream; @Slf4j @Testcontainers public class KafkaRestartingTrafficReplayerTest { + public static final int INITIAL_STOP_REPLAYER_REQUEST_COUNT = 1; public static final String TEST_GROUP_CONSUMER_ID = "TEST_GROUP_CONSUMER_ID"; public static final String TEST_GROUP_PRODUCER_ID = "TEST_GROUP_PRODUCER_ID"; public static final String TEST_TOPIC_NAME = "TEST_TOPIC"; + public static final TrafficStream SENTINEL_TRAFFIC_STREAM = + TrafficStream.newBuilder().setConnectionId(SentinelSensingTrafficSource.SENTINEL_CONNECTION_ID).build(); public static final int PRODUCER_SLEEP_INTERVAL_MS = 100; public static final Duration MAX_WAIT_TIME_FOR_TOPIC = Duration.ofMillis(PRODUCER_SLEEP_INTERVAL_MS*2); @@ -42,34 +53,83 @@ public class KafkaRestartingTrafficReplayerTest { private KafkaContainer embeddedKafkaBroker = new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:7.5.0"));; + private static class CounterLimitedReceiverFactory implements Supplier> { + AtomicInteger nextStopPointRef = new AtomicInteger(INITIAL_STOP_REPLAYER_REQUEST_COUNT); - private Supplier loadStreamsToKafka(Stream streams) throws Exception { + @Override + public Consumer get() { + log.info("StopAt="+nextStopPointRef.get()); + var stopPoint = nextStopPointRef.get(); + var counter = new AtomicInteger(); + return tuple -> { + if (counter.incrementAndGet() > stopPoint) { + log.error("Request received after our ingest threshold. Throwing. Discarding " + + tuple.uniqueRequestKey); + var nextStopPoint = stopPoint + new Random(stopPoint).nextInt(stopPoint + 1); + nextStopPointRef.compareAndSet(stopPoint, nextStopPoint); + throw new TrafficReplayerRunner.FabricatedErrorToKillTheReplayer(false); + } + }; + } + } + + @ParameterizedTest + @CsvSource(value = { + "3,false", + "-1,false", + "3,true", + "-1,true", + }) + @Tag("longTest") + public void fullTest(int testSize, boolean randomize) throws Throwable { + var httpServer = SimpleNettyHttpServer.makeServer(false, Duration.ofMillis(2), + TestHttpServerContext::makeResponse); + var streamAndConsumer = TrafficStreamGenerator.generateStreamAndSumOfItsTransactions(testSize, randomize); + var trafficStreams = streamAndConsumer.stream.collect(Collectors.toList()); + log.atInfo().setMessage(()->trafficStreams.stream().map(ts-> TrafficStreamUtils.summarizeTrafficStream(ts)) + .collect(Collectors.joining("\n"))).log(); + + loadStreamsToKafka(buildKafkaConsumer(), + Streams.concat(trafficStreams.stream(), Stream.of(SENTINEL_TRAFFIC_STREAM))); + TrafficReplayerRunner.runReplayerUntilSourceWasExhausted(streamAndConsumer.numHttpTransactions, + httpServer.localhostEndpoint(), new CounterLimitedReceiverFactory(), + () -> new SentinelSensingTrafficSource( + new KafkaProtobufConsumer(buildKafkaConsumer(), TEST_TOPIC_NAME, null))); + log.error("done"); + } + + @SneakyThrows + private KafkaConsumer buildKafkaConsumer() { var kafkaConsumerProps = KafkaProtobufConsumer.buildKafkaProperties(embeddedKafkaBroker.getBootstrapServers(), - TEST_GROUP_CONSUMER_ID, false, null); - kafkaConsumerProps.setProperty("max.poll.interval.ms", "300000"); - var kafkaConsumer = new KafkaConsumer(kafkaConsumerProps); + TEST_GROUP_CONSUMER_ID, false, null); + kafkaConsumerProps.setProperty("max.poll.interval.ms", "5000"); + var kafkaConsumer = new KafkaConsumer(kafkaConsumerProps); + log.atInfo().setMessage(()->"Just built KafkaConsumer="+kafkaConsumer).log(); + return kafkaConsumer; + } + private void loadStreamsToKafka(KafkaConsumer kafkaConsumer, + Stream streams) throws Exception { var kafkaProducer = buildKafkaProducer(); var counter = new AtomicInteger(); - loadStreamsAsynchronouslyWithResource(kafkaConsumer, streams, s->s.forEach(trafficStream -> + loadStreamsAsynchronouslyWithCloseableResource(kafkaConsumer, streams, s -> s.forEach(trafficStream -> writeTrafficStreamRecord(kafkaProducer, new TrafficStreamWithEmbeddedKey(trafficStream), "KEY_" + counter.incrementAndGet()))); - - return () -> new KafkaProtobufConsumer(kafkaConsumer, TEST_TOPIC_NAME, null); } private void - loadStreamsAsynchronouslyWithResource(KafkaConsumer kafkaConsumer, R resource, Consumer loader) + loadStreamsAsynchronouslyWithCloseableResource(KafkaConsumer kafkaConsumer, R closeableResource, + Consumer loader) throws Exception { try { - new Thread(()->loader.accept(resource)).start(); + new Thread(()->loader.accept(closeableResource)).start(); var startTime = Instant.now(); while (!kafkaConsumer.listTopics().isEmpty()) { Thread.sleep(10); Assertions.assertTrue(Duration.between(startTime, Instant.now()).compareTo(MAX_WAIT_TIME_FOR_TOPIC) < 0); } } finally { - resource.close(); + closeableResource.close(); } } @@ -96,7 +156,7 @@ Producer buildKafkaProducer() { loadStreamsToKafkaFromCompressedFile(KafkaConsumer kafkaConsumer, String filename, int recordCount) throws Exception { var kafkaProducer = buildKafkaProducer(); - loadStreamsAsynchronouslyWithResource(kafkaConsumer, new V0_1TrafficCaptureSource(filename), + loadStreamsAsynchronouslyWithCloseableResource(kafkaConsumer, new V0_1TrafficCaptureSource(filename), originalTrafficSource -> { try { for (int i = 0; i < recordCount; ++i) { @@ -115,11 +175,24 @@ Producer buildKafkaProducer() { @SneakyThrows private static void writeTrafficStreamRecord(Producer kafkaProducer, - ITrafficStreamWithKey trafficStream, + ITrafficStreamWithKey trafficStreamAndKey, String recordId) { - var record = new ProducerRecord(TEST_TOPIC_NAME, recordId, trafficStream.getStream().toByteArray()); - var sendFuture = kafkaProducer.send(record, (metadata, exception) -> {}); - sendFuture.get(); + while (true) { + try { + var record = new ProducerRecord(TEST_TOPIC_NAME, recordId, trafficStreamAndKey.getStream().toByteArray()); + log.info("sending record with trafficStream=" + trafficStreamAndKey.getKey()); + var sendFuture = kafkaProducer.send(record, (metadata, exception) -> { + log.atInfo().setCause(exception).setMessage(() -> "completed send of TrafficStream with key=" + + trafficStreamAndKey.getKey() + " metadata=" + metadata).log(); + }); + var recordMetadata = sendFuture.get(); + log.info("finished publishing record... metadata=" + recordMetadata); + break; + } catch (Exception e) { + log.error("Caught exception while trying to publish a record to Kafka. Blindly retrying."); + continue; + } + } Thread.sleep(PRODUCER_SLEEP_INTERVAL_MS); } } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SentinelSensingTrafficSource.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SentinelSensingTrafficSource.java new file mode 100644 index 0000000000..5ef78d9225 --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SentinelSensingTrafficSource.java @@ -0,0 +1,56 @@ +package org.opensearch.migrations.replay; + +import lombok.extern.slf4j.Slf4j; +import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; +import org.opensearch.migrations.replay.traffic.source.ISimpleTrafficCaptureSource; +import org.opensearch.migrations.replay.traffic.source.ITrafficStreamWithKey; + +import java.io.EOFException; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +@Slf4j +class SentinelSensingTrafficSource implements ISimpleTrafficCaptureSource { + public static final String SENTINEL_CONNECTION_ID = "EOF_MARKER_TRAFFIC_STREAM"; + private final ISimpleTrafficCaptureSource underlyingSource; + private final AtomicBoolean stopReadingRef; + + public SentinelSensingTrafficSource(ISimpleTrafficCaptureSource underlyingSource) { + this.underlyingSource = underlyingSource; + stopReadingRef = new AtomicBoolean(); + } + + @Override + public CompletableFuture> readNextTrafficStreamChunk() { + if (stopReadingRef.get()) { + return CompletableFuture.failedFuture(new EOFException()); + } + return underlyingSource.readNextTrafficStreamChunk().thenApply(v->{ + if (v != null) { + return v.stream().takeWhile(ts->{ + var isSentinel = ts.getStream().getConnectionId().equals(SENTINEL_CONNECTION_ID); + if (isSentinel) { + stopReadingRef.set(true); + } + return !isSentinel; + }).collect(Collectors.toList()); + } else { + return v; + } + }); + } + + @Override + public void commitTrafficStream(ITrafficStreamKey trafficStreamKey) throws IOException { + underlyingSource.commitTrafficStream(trafficStreamKey); + } + + @Override + public void close() throws IOException { + underlyingSource.close(); + } +} diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerRunner.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerRunner.java index c9d6fba148..af77b119d4 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerRunner.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficReplayerRunner.java @@ -82,8 +82,19 @@ static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, URI endp .setMessage(()->"broke out of the replayer, with the shutdown cause=" + e.originalCause + " and this immediate reason") .log(); - if (!(e.originalCause instanceof FabricatedErrorToKillTheReplayer)) { + FabricatedErrorToKillTheReplayer killSignalError = + e.originalCause instanceof FabricatedErrorToKillTheReplayer + ? (FabricatedErrorToKillTheReplayer) e.originalCause + : (e.immediateCause instanceof FabricatedErrorToKillTheReplayer + ? (FabricatedErrorToKillTheReplayer) e.immediateCause + : null); + if (killSignalError == null) { throw e.immediateCause; + } else if (killSignalError.doneWithTest) { + log.info("Kill signal has indicated that the test is complete, so breaking out of the loop"); + break; + } else { + log.info("Kill signal has indicated that the test loop should restart. Continuing."); } } finally { waitForWorkerThreadsToStop(); @@ -114,7 +125,10 @@ static void runReplayerUntilSourceWasExhausted(int numExpectedRequests, URI endp .toArray(); var expectedSkipArray = new int[skippedPerRunDiffs.length]; Arrays.fill(expectedSkipArray, 1); - Assertions.assertArrayEquals(expectedSkipArray, skippedPerRunDiffs); + // this isn't the best way to make sure that commits are increasing. + // They are for specific patterns, but not all of them. + // TODO - export the whole table of results & let callers determine how to check that commits are moving along + //Assertions.assertArrayEquals(expectedSkipArray, skippedPerRunDiffs); Assertions.assertEquals(numExpectedRequests, totalUniqueEverReceived.get()); } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficStreamGenerator.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficStreamGenerator.java index 07689f8c85..104b5bf512 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficStreamGenerator.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficStreamGenerator.java @@ -1,8 +1,10 @@ package org.opensearch.migrations.replay; +import io.vavr.Tuple2; import lombok.AllArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.opensearch.migrations.testutils.StreamInterleaver; import org.opensearch.migrations.trafficcapture.protos.TrafficObservation; import org.opensearch.migrations.trafficcapture.protos.TrafficStream; @@ -240,6 +242,27 @@ public static class RandomTrafficStreamAndTransactionSizes { public final int[] responseByteSizes; } + @AllArgsConstructor + public static class StreamAndExpectedSizes { + public final Stream stream; + public final int numHttpTransactions; + } + + static StreamAndExpectedSizes + generateStreamAndSumOfItsTransactions(int count, boolean randomize) { + var generatedCases = count > 0 ? + generateRandomTrafficStreamsAndSizes(IntStream.range(0,count)) : + generateAllIndicativeRandomTrafficStreamsAndSizes(); + var testCaseArr = generatedCases.toArray(RandomTrafficStreamAndTransactionSizes[]::new); + var aggregatedStreams = randomize ? + StreamInterleaver.randomlyInterleaveStreams(new Random(count), + Arrays.stream(testCaseArr).map(c->Arrays.stream(c.trafficStreams))) : + Arrays.stream(testCaseArr).flatMap(c->Arrays.stream(c.trafficStreams)); + + var numExpectedRequests = Arrays.stream(testCaseArr).mapToInt(c->c.requestByteSizes.length).sum(); + return new StreamAndExpectedSizes(aggregatedStreams, numExpectedRequests); + } + public static Stream generateRandomTrafficStreamsAndSizes(IntStream seedStream) { return seedStream.mapToObj(rSeed->{ From df984822c0e91095f9fc03b68d410582ea73e700 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Fri, 10 Nov 2023 23:22:32 -0500 Subject: [PATCH 39/55] PR feedback Signed-off-by: Greg Schohn --- .../JsonJoltTransformerProvider.java | 8 ++++--- TrafficCapture/trafficReplayer/README.md | 24 ++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java index e8d93384a6..80c426137f 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformerProvider.java @@ -35,8 +35,10 @@ public IJsonTransformer createTransformer(Object jsonConfig) { var cannedValueStr = (String) cannedValue; var cannedOperation = getCannedOperationOrThrow(cannedValueStr); builder.addCannedOperation(cannedOperation); - } else { + } else if (scriptValue != null) { builder.addOperationObject((Map)scriptValue); + } else { + throw new IllegalArgumentException(getConfigUsageStr()); } } } catch (ClassCastException e) { @@ -54,7 +56,7 @@ private JsonJoltTransformBuilder.CANNED_OPERATION getCannedOperationOrThrow(Stri } private String getConfigUsageStr() { - return this.getClass().getName()+" expects the incoming configuration " + + return this.getClass().getName() + " expects the incoming configuration " + "to be a Map or a List>. " + "Each of the Maps should have one key-value, either \"canned\" or \"script\". " + "Canned values should be a string that specifies the name of the pre-built transformation to use " + @@ -63,6 +65,6 @@ private String getConfigUsageStr() { ". " + "Script values should be a fully-formed inlined Jolt transformation in json form. " + "All of the values (canned or inlined) within a configuration will be concatenated " + - "into one chained Jolt transformation"; + "into one chained Jolt transformation."; } } diff --git a/TrafficCapture/trafficReplayer/README.md b/TrafficCapture/trafficReplayer/README.md index 8921dfed03..b6cf7a6ef3 100644 --- a/TrafficCapture/trafficReplayer/README.md +++ b/TrafficCapture/trafficReplayer/README.md @@ -64,7 +64,7 @@ which has comments throughout it to indicate how data percolates and is converte ## Handlers With the exception of the preparation around JSON model and its transformation, all the other handlers (compression, -chunked, and JSON parsing/serialzation), use streaming data models via mostly custom handlers. This should minimize the +chunked, and JSON parsing/serialization), use streaming data models via mostly custom handlers. This should minimize the memory load (working set size, cache misses, etc). However, attempts have not yet been made to reduce the number of allocations. Those optimization may not have extremely high value, especially when JSON parsing will need to create multitudes more objects. @@ -80,9 +80,9 @@ Transformations are performed via a simple interface defined by [IJsonTransformer](../replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java) ('transformer'). They are loaded dynamically and are designed to allow for easy extension of the TrafficReplayer to support a diverse set of needs. -The input to the transformer will be an HTTP message represented as a json-like Map with +The input to the transformer will be an HTTP message represented as a json-like `Map` with top-level key-value pairs defined in -[JsonKeysForHttpMessage.java](../replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java) +[JsonKeysForHttpMessage.java](../replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java). Only bodies that are json-formatted will be accessible, and they will be accessible as a fully-parsed Map (at the keypath `'payload'->'inlinedJsonBody'`). Transformers have the option to rewrite none, or any of the keys and values within the original message. The transformer can return either the original message or a completely new message. @@ -90,8 +90,7 @@ Transformers may be used simultaneously from concurrent threads over the lifetim a message will only be processed by one transformer at a time. Transformer implementations are loaded via [Java's ServiceLoader](https://docs.oracle.com/javase/8/docs/api/java/util/ServiceLoader.html) -by loading a jarfile that implements the [IJsonTransformerProvider] -(../replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java). +by loading a jarfile that implements the [IJsonTransformerProvider](../replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java). That jarfile will be loaded by specifying the provider jarfile (and any of its dependencies) in the classpath. For the ServiceLoader to load the IJsonTransformerProvider, the provided jarfile needs to supply a _provider-configuration_ file (`META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider`) @@ -117,9 +116,18 @@ all loaded/specified transformations. Currently, there are multiple, nascent implementations included in the repository. The [jsonJoltMessageTransformerProvider](../replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider) -package uses [JOLT](https://github.com/bazaarvoice/jolt) to perform transforms. Some simple transformations are -included to change headers to add compression or to force an HTTP message payload to be chunked. Another transformer, -[JsonTypeMappingTransformer.java](../replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java), +package uses [JOLT](https://github.com/bazaarvoice/jolt) to perform transforms. That transformer can be configured to apply a full script or to use +a "canned" transform whose script is already included with the library. Examples of using a built-in transform to +add GZIP encoding and another to apply a new header would be configured with the following. + +``` +[{"org.opensearch.migrations.transform.JsonJoltTransformerProvider": { "canned": "ADD_GZIP" }}, +{ "org.opensearch.migrations.transform.JsonJoltTransformerProvider": {"script": + { "operation": "modify-overwrite-beta", "spec": { "headers": {"newHeader": "newValue"}}}}}] +``` + +Some simple transformations are included to change headers to add compression or to force an HTTP message payload to +be chunked. Another transformer, [JsonTypeMappingTransformer.java](../replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java), is a work-in-progress to excise type mapping references from URIs and message payloads since versions of OpenSource greater than 2.3 do not support them. From 0cee79df7849808232c82e612151b3badeea8815 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Sat, 11 Nov 2023 00:41:37 -0500 Subject: [PATCH 40/55] Easy de-linting fixes. Signed-off-by: Greg Schohn --- .../kafkaoffloader/KafkaCaptureFactory.java | 6 ++-- .../FileConnectionCaptureFactory.java | 5 ++-- .../proxyserver/CaptureProxy.java | 28 +++++++++---------- .../netty/BacksideConnectionPool.java | 7 ++--- .../netty/ExpiringSubstitutableItemPool.java | 13 ++------- .../proxyserver/netty/FrontsideHandler.java | 11 -------- .../netty/NettyScanningHttpProxy.java | 6 ++-- .../netty/ProxyChannelInitializer.java | 4 +-- ...edTrafficToHttpTransactionAccumulator.java | 3 +- .../migrations/replay/SigV4Signer.java | 18 ++++++------ .../migrations/replay/TrafficReplayer.java | 2 +- 11 files changed, 39 insertions(+), 64 deletions(-) diff --git a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java index bf51fd4c9e..7b16005981 100644 --- a/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java +++ b/TrafficCapture/captureKafkaOffloader/src/main/java/org/opensearch/migrations/trafficcapture/kafkaoffloader/KafkaCaptureFactory.java @@ -21,7 +21,7 @@ @Slf4j -public class KafkaCaptureFactory implements IConnectionCaptureFactory { +public class KafkaCaptureFactory implements IConnectionCaptureFactory { private static final MetricsLogger metricsLogger = new MetricsLogger("BacksideHandler"); @@ -49,8 +49,8 @@ public KafkaCaptureFactory(String nodeId, Producer producer, int } @Override - public IChannelConnectionCaptureSerializer createOffloader(String connectionId) { - return new StreamChannelConnectionCaptureSerializer(nodeId, connectionId, new StreamManager(connectionId)); + public IChannelConnectionCaptureSerializer createOffloader(String connectionId) { + return new StreamChannelConnectionCaptureSerializer<>(nodeId, connectionId, new StreamManager(connectionId)); } @AllArgsConstructor diff --git a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java index c05abb270e..26cee767d0 100644 --- a/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java +++ b/TrafficCapture/captureOffloader/src/main/java/org/opensearch/migrations/trafficcapture/FileConnectionCaptureFactory.java @@ -1,6 +1,7 @@ package org.opensearch.migrations.trafficcapture; import lombok.AllArgsConstructor; +import lombok.Lombok; import lombok.extern.slf4j.Slf4j; import java.io.FileNotFoundException; @@ -38,7 +39,7 @@ public FileConnectionCaptureFactory(String nodeId, int bufferSize, Path rootPath var filePath = rootPath.resolve(id + "_" + n.toString() + ".protocap"); return new FileOutputStream(filePath.toString()); } catch (FileNotFoundException e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } }); } @@ -72,7 +73,7 @@ public CodedOutputStreamAndByteBufferWrapper createStream() { fs.flush(); log.warn("NOT removing the CodedOutputStream from the WeakHashMap, which is a memory leak. Doing this until the system knows when to properly flush buffers"); } catch (IOException e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } }).thenApply(v->null); } diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java index 199755cbed..611c58ba72 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java @@ -7,6 +7,7 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import lombok.Lombok; import lombok.NonNull; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -45,8 +46,8 @@ @Slf4j public class CaptureProxy { - private final static String HTTPS_CONFIG_PREFIX = "plugins.security.ssl.http."; - public final static String DEFAULT_KAFKA_CLIENT_ID = "HttpCaptureProxyProducer"; + private static final String HTTPS_CONFIG_PREFIX = "plugins.security.ssl.http."; + public static final String DEFAULT_KAFKA_CLIENT_ID = "HttpCaptureProxyProducer"; static class Parameters { @Parameter(required = false, @@ -125,7 +126,7 @@ static class Parameters { String destinationConnectionPoolTimeout = "PT30S"; } - public static @NonNull Parameters parseArgs(String[] args) { + static Parameters parseArgs(String[] args) { Parameters p = new Parameters(); JCommander jCommander = new JCommander(p); try { @@ -141,6 +142,7 @@ static class Parameters { System.err.println(e.getMessage()); System.err.println("Got args: "+ String.join("; ", args)); jCommander.usage(); + System.exit(2); return null; } } @@ -163,10 +165,10 @@ private static Settings getSettings(@NonNull String configFile) { return builder.build(); } - private static IConnectionCaptureFactory getNullConnectionCaptureFactory() { + private static IConnectionCaptureFactory getNullConnectionCaptureFactory() { System.err.println("No trace log directory specified. Logging to /dev/null"); - return connectionId -> new StreamChannelConnectionCaptureSerializer(null, connectionId, - new StreamLifecycleManager() { + return connectionId -> new StreamChannelConnectionCaptureSerializer<>(null, connectionId, + new StreamLifecycleManager<>() { @Override public CodedOutputStreamHolder createStream() { return () -> CodedOutputStream.newInstance(NullOutputStream.getInstance()); @@ -215,7 +217,6 @@ static Properties buildKafkaProperties(Parameters params) throws IOException { private static IConnectionCaptureFactory getConnectionCaptureFactory(Parameters params) throws IOException { var nodeId = getNodeId(); - // TODO - it might eventually be a requirement to do multiple types of offloading. // Resist the urge for now though until it comes in as a request/need. if (params.traceDirectory != null) { return new FileConnectionCaptureFactory(nodeId, params.traceDirectory, params.maximumTrafficStreamSize); @@ -224,7 +225,7 @@ private static IConnectionCaptureFactory getConnectionCaptureFactory(Parameters } else if (params.noCapture) { return getNullConnectionCaptureFactory(); } else { - throw new RuntimeException("Must specify some connection capture factory options"); + throw new IllegalStateException("Must specify some connection capture factory options"); } } @@ -241,13 +242,13 @@ private static URI convertStringToUri(String uriString) { return null; } if (serverUri.getPort() < 0) { - throw new RuntimeException("Port not present for URI: " + serverUri); + throw new IllegalArgumentException("Port not present for URI: " + serverUri); } if (serverUri.getHost() == null) { - throw new RuntimeException("Hostname not present for URI: " + serverUri); + throw new IllegalArgumentException("Hostname not present for URI: " + serverUri); } if (serverUri.getScheme() == null) { - throw new RuntimeException("Scheme (http|https) is not present for URI: " + serverUri); + throw new IllegalArgumentException("Scheme (http|https) is not present for URI: " + serverUri); } return serverUri; } @@ -287,10 +288,9 @@ public static void main(String[] args) throws InterruptedException, IOException proxy.start(backsideConnectionPool, params.numThreads, sksOp.map(sks -> (Supplier) () -> { try { - var sslEngine = sks.createHTTPSSLEngine(); - return sslEngine; + return sks.createHTTPSSLEngine(); } catch (Exception e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } }).orElse(null), getConnectionCaptureFactory(params)); } catch (Exception e) { diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/BacksideConnectionPool.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/BacksideConnectionPool.java index 8d1afdef26..95c1ad1157 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/BacksideConnectionPool.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/BacksideConnectionPool.java @@ -25,7 +25,7 @@ public class BacksideConnectionPool { private final URI backsideUri; private final SslContext backsideSslContext; - private final FastThreadLocal connectionCacheForEachThread; + private final FastThreadLocal> connectionCacheForEachThread; private final Duration inactivityTimeout; private final int poolSize; @@ -33,7 +33,7 @@ public BacksideConnectionPool(URI backsideUri, SslContext backsideSslContext, int poolSize, Duration inactivityTimeout) { this.backsideUri = backsideUri; this.backsideSslContext = backsideSslContext; - this.connectionCacheForEachThread = new FastThreadLocal(); + this.connectionCacheForEachThread = new FastThreadLocal<>(); this.inactivityTimeout = inactivityTimeout; this.poolSize = poolSize; } @@ -47,8 +47,7 @@ public ChannelFuture getOutboundConnectionFuture(EventLoop eventLoop) { private ExpiringSubstitutableItemPool getExpiringWarmChannelPool(EventLoop eventLoop) { - var thisContextsConnectionCache = (ExpiringSubstitutableItemPool) - connectionCacheForEachThread.get(); + var thisContextsConnectionCache = connectionCacheForEachThread.get(); if (thisContextsConnectionCache == null) { thisContextsConnectionCache = new ExpiringSubstitutableItemPool(inactivityTimeout, diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPool.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPool.java index 8ba3ca555b..bd2f14185a 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPool.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPool.java @@ -178,9 +178,7 @@ public ExpiringSubstitutableItemPool(@NonNull Duration inactivityTimeout, this.itemSupplier = () -> { var startTime = Instant.now(); var rval = itemSupplier.get(); - rval.addListener(v->{ - stats.itemBuilt(Duration.between(startTime, Instant.now())); - }); + rval.addListener(v -> stats.itemBuilt(Duration.between(startTime, Instant.now()))); return rval; }; // store this as a field so that we can remove the listener once the inProgress item has been @@ -189,7 +187,7 @@ public ExpiringSubstitutableItemPool(@NonNull Duration inactivityTimeout, f -> { inProgressItems.remove(f); if (f.isSuccess()) { - readyItems.add(new Entry(f)); + readyItems.add(new Entry<>(f)); scheduleNextExpirationSweep(inactivityTimeout); } else { // the calling context should track failures too - no reason to log @@ -212,13 +210,6 @@ public Stats getStats() { return copiedStats; } - public int reduceCapacity(int delta) { - assert delta >= 0 : "expected capacity delta to be >= 0"; - poolSize -= delta; - assert poolSize >= 0 : "expected pool size to remain >= 0"; - return poolSize; - } - public int increaseCapacity(int itemsToLoad) { return increaseCapacityWithSchedule(itemsToLoad, Duration.ZERO); } diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/FrontsideHandler.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/FrontsideHandler.java index c775c76cfe..14416838b1 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/FrontsideHandler.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/FrontsideHandler.java @@ -63,17 +63,6 @@ public void channelRead(final ChannelHandlerContext ctx, Object msg) { } } - public void channelReadComplete(ChannelHandlerContext ctx) throws Exception { - log.debug("channelRead COMPLETE"); - ctx.fireChannelReadComplete(); - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - log.atError().setCause(cause).setMessage("Caught error").log(); - ctx.close(); - } - /** * This will close our proxy connection to downstream services. * @param ctx current channel context. diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/NettyScanningHttpProxy.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/NettyScanningHttpProxy.java index 2dec0f550e..4ad2eaf38c 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/NettyScanningHttpProxy.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/NettyScanningHttpProxy.java @@ -7,8 +7,6 @@ import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.util.concurrent.DefaultThreadFactory; -import io.netty.util.internal.logging.InternalLoggerFactory; -import io.netty.util.internal.logging.JdkLoggerFactory; import org.opensearch.migrations.trafficcapture.IConnectionCaptureFactory; import javax.net.ssl.SSLEngine; @@ -31,14 +29,14 @@ public int getProxyPort() { public void start(BacksideConnectionPool backsideConnectionPool, int numThreads, Supplier sslEngineSupplier, - IConnectionCaptureFactory connectionCaptureFactory) throws InterruptedException { + IConnectionCaptureFactory connectionCaptureFactory) throws InterruptedException { bossGroup = new NioEventLoopGroup(1, new DefaultThreadFactory("captureProxyPoolBoss")); workerGroup = new NioEventLoopGroup(numThreads, new DefaultThreadFactory("captureProxyPoolWorker")); ServerBootstrap serverBootstrap = new ServerBootstrap(); try { mainChannel = serverBootstrap.group(bossGroup, workerGroup) .channel(NioServerSocketChannel.class) - .childHandler(new ProxyChannelInitializer(backsideConnectionPool, sslEngineSupplier, + .childHandler(new ProxyChannelInitializer<>(backsideConnectionPool, sslEngineSupplier, connectionCaptureFactory)) .childOption(ChannelOption.AUTO_READ, false) .bind(proxyPort).sync().channel(); diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java index 9db246bee1..3c84449556 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ProxyChannelInitializer.java @@ -15,12 +15,12 @@ public class ProxyChannelInitializer extends ChannelInitializer { - private final IConnectionCaptureFactory connectionCaptureFactory; + private final IConnectionCaptureFactory connectionCaptureFactory; private final Supplier sslEngineProvider; private final BacksideConnectionPool backsideConnectionPool; public ProxyChannelInitializer(BacksideConnectionPool backsideConnectionPool, Supplier sslEngineSupplier, - IConnectionCaptureFactory connectionCaptureFactory) { + IConnectionCaptureFactory connectionCaptureFactory) { this.backsideConnectionPool = backsideConnectionPool; this.sslEngineProvider = sslEngineSupplier; this.connectionCaptureFactory = connectionCaptureFactory; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java index 9e6e70a247..9c78325ae6 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java @@ -173,8 +173,7 @@ public CONNECTION_STATUS addObservationToAccumulation(@NonNull Accumulation accu @NonNull ITrafficStreamKey trafficStreamKey, TrafficObservation observation) { log.atTrace().setMessage(()->"Adding observation: "+observation).log(); - var timestamp = - Optional.of(observation.getTs()).map(TrafficStreamUtils::instantFromProtoTimestamp).get(); + var timestamp = TrafficStreamUtils.instantFromProtoTimestamp(observation.getTs()); liveStreams.expireOldEntries(trafficStreamKey, accum, timestamp); return handleObservationForSkipState(accum, observation) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java index 166f773712..ed7141c410 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java @@ -10,7 +10,9 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; @@ -36,14 +38,12 @@ public class SigV4Signer extends IAuthTransformer.StreamingFullMessageTransforme public static final String AMZ_CONTENT_SHA_256 = "x-amz-content-sha256"; static { - AUTH_HEADERS_TO_PULL_NO_PAYLOAD = new HashSet() {{ - add("authorization"); - add("x-amz-date"); - add("x-amz-security-token"); - }}; - AUTH_HEADERS_TO_PULL_WITH_PAYLOAD = new HashSet(AUTH_HEADERS_TO_PULL_NO_PAYLOAD) {{ - add(AMZ_CONTENT_SHA_256); - }}; + AUTH_HEADERS_TO_PULL_NO_PAYLOAD = new HashSet<>(Set.of( + "authorization", + "x-amz-date", + "x-amz-security-token")); + AUTH_HEADERS_TO_PULL_WITH_PAYLOAD = Stream.concat(AUTH_HEADERS_TO_PULL_NO_PAYLOAD.stream(), + Stream.of(AMZ_CONTENT_SHA_256)).collect(Collectors.toCollection(HashSet::new)); } private MessageDigest messageDigest; @@ -88,8 +88,6 @@ public void finalizeSignature(HttpJsonMessageWithFaultingPayload msg) { } private static class AwsSignerWithPrecomputedContentHash extends BaseAws4Signer { - public AwsSignerWithPrecomputedContentHash() {} - protected String calculateContentHash(SdkHttpFullRequest.Builder mutableRequest, Aws4SignerParams signerParams, SdkChecksum contentFlexibleChecksum) { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index cd9cb6565a..35d773ec1d 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -294,7 +294,7 @@ static class Parameters { String kafkaTrafficPropertyFile; } - public static Parameters parseArgs(String[] args) { + private static Parameters parseArgs(String[] args) { Parameters p = new Parameters(); JCommander jCommander = new JCommander(p); try { From 54f92deab3b135c5763f63001d7ec0235625001f Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Sat, 11 Nov 2023 13:35:00 -0500 Subject: [PATCH 41/55] convert size() == 0 patterns to isEmpty(). Signed-off-by: Greg Schohn --- .../opensearch/migrations/replay/RequestResponsePacketPair.java | 2 +- .../java/org/opensearch/migrations/replay/TrafficReplayer.java | 2 +- .../replay/datatypes/TimeToResponseFulfillmentFutureMap.java | 2 +- .../opensearch/migrations/replay/V0_1TrafficCaptureSource.java | 2 +- .../replay/kafka/KafkaProtobufConsumerLongTermTest.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestResponsePacketPair.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestResponsePacketPair.java index 21a3c3a65e..481692f1b3 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestResponsePacketPair.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestResponsePacketPair.java @@ -62,7 +62,7 @@ public void holdTrafficStream(ITrafficStreamKey trafficStreamKey) { if (trafficStreamKeysBeingHeld == null) { trafficStreamKeysBeingHeld = new ArrayList<>(); } - if (trafficStreamKeysBeingHeld.size() == 0 || + if (trafficStreamKeysBeingHeld.isEmpty() || trafficStreamKey != trafficStreamKeysBeingHeld.get(trafficStreamKeysBeingHeld.size()-1)) { trafficStreamKeysBeingHeld.add(trafficStreamKey); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index 8d5c53239c..a33e4b6c48 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -473,7 +473,7 @@ void setupRunAndWaitForReplay(Duration observedPacketConnectionTimeout, trafficToHttpTransactionAccumulator.close(); wrapUpWorkAndEmitSummary(replayEngine, trafficToHttpTransactionAccumulator); if (shutdownFutureRef.get() == null) { - assert requestToFinalWorkFuturesMap.size() == 0 : + assert requestToFinalWorkFuturesMap.isEmpty() : "expected to wait for all the in flight requests to fully flush and self destruct themselves"; } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/TimeToResponseFulfillmentFutureMap.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/TimeToResponseFulfillmentFutureMap.java index 1ba0f61f5f..eb637bc608 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/TimeToResponseFulfillmentFutureMap.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/TimeToResponseFulfillmentFutureMap.java @@ -49,7 +49,7 @@ public String toString() { } private String formatBookends() { - if (timeToRunnableMap.size() == 0) { + if (timeToRunnableMap.isEmpty()) { return ""; } else if (timeToRunnableMap.size() == 1) { return timeToRunnableMap.firstKey().toString(); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/V0_1TrafficCaptureSource.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/V0_1TrafficCaptureSource.java index 75ef873f1d..77fc958d9b 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/V0_1TrafficCaptureSource.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/V0_1TrafficCaptureSource.java @@ -28,7 +28,7 @@ private static class Progress { public void add(TrafficStream incoming) { var list = incoming.getSubStreamList(); - lastWasRead = list.size() == 0 ? lastWasRead : + lastWasRead = list.isEmpty() ? lastWasRead : Optional.of(list.get(list.size()-1)).map(tso->tso.hasRead() || tso.hasReadSegment()).get(); requestCount += list.stream().filter(tso->tso.hasRead()||tso.hasReadSegment()).count(); } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumerLongTermTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumerLongTermTest.java index 7ad3dec3a9..f476c08f4b 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumerLongTermTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumerLongTermTest.java @@ -123,7 +123,7 @@ public void testTrafficCaptureSource() throws Exception { Assertions.assertEquals(TEST_RECORD_COUNT, sendCompleteCount.get()); Assertions.assertThrows(TimeoutException.class, ()-> { var rogueChunk = kafkaTrafficCaptureSource.readNextTrafficStreamChunk().get(1, TimeUnit.SECONDS); - if (rogueChunk.size() == 0) { + if (rogueChunk.isEmpty()) { // TimeoutExceptions cannot be thrown by the supplier of the CompletableFuture today, BUT we // could long-poll on the broker for longer than the timeout value supplied in the get() call above throw new TimeoutException("read actually returned 0 items, but transforming this to a " + From 64b3270f5c1c2fc3384fd49cff6ebe87b0391ea1 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Sat, 11 Nov 2023 13:42:30 -0500 Subject: [PATCH 42/55] Rotate the order of when the shutdown reason is set. Make that happen first, followed by the future to signal wen its done, then followed by teardown. If the read loop didn't throw to exit, but rather came out because of the boolean check on stopReadingRef, the shutdownReasonRef could still be null by the time that setupRunAndWaitForReplayWithShutdownChecks() checked for it. Now, that reason will be set before any other code will be impacted by the shutdown, making for more reliable throws when a shutdown due to an error caused the code to exit prematurely. Signed-off-by: Greg Schohn --- .../org/opensearch/migrations/replay/TrafficReplayer.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index a33e4b6c48..91a3b674b4 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -870,12 +870,6 @@ private static String formatWorkItem(DiagnosticTrackableCompletableFuturef.cancel(true)); - } - public @NonNull CompletableFuture shutdown(Error error) { log.warn("Shutting down "+this+" because of "+error); if (!shutdownFutureRef.compareAndSet(null, new CompletableFuture<>())) { @@ -886,8 +880,8 @@ public void stopReadingAsync() { .log(); return shutdownFutureRef.get(); } - stopReadingAsync(); shutdownReasonRef.compareAndSet(null, error); + stopReadingRef.set(true); nettyShutdownFuture = clientConnectionPool.shutdownNow() .addListener(f->{ if (f.isSuccess()) { From 5b604ae0d92a5caeecc62c39b796851d1d61d22c Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Sat, 11 Nov 2023 13:51:37 -0500 Subject: [PATCH 43/55] Rotate the shutdownReasonRef up even earlier (see last commit for the virtues of doing this) Signed-off-by: Greg Schohn --- .../java/org/opensearch/migrations/replay/TrafficReplayer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index 91a3b674b4..c45669fd41 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -872,6 +872,7 @@ private static String formatWorkItem(DiagnosticTrackableCompletableFuture shutdown(Error error) { log.warn("Shutting down "+this+" because of "+error); + shutdownReasonRef.compareAndSet(null, error); if (!shutdownFutureRef.compareAndSet(null, new CompletableFuture<>())) { log.atError().setMessage(()->"Shutdown was already signaled by {}. " + "Ignoring this shutdown request due to {}.") @@ -880,7 +881,6 @@ private static String formatWorkItem(DiagnosticTrackableCompletableFuture{ From aa5f966000b0dce0c9871dba5e7411c1f9009988 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Sat, 11 Nov 2023 13:52:26 -0500 Subject: [PATCH 44/55] Remove a file that I hadn't meant to check in Signed-off-by: Greg Schohn --- .../test/java/org/opensearch/migrations/CancellationTest.java | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/CancellationTest.java diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/CancellationTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/CancellationTest.java deleted file mode 100644 index 903b812909..0000000000 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/CancellationTest.java +++ /dev/null @@ -1,2 +0,0 @@ -package org.opensearch.migrations;public class CancellationTest { -} From 59d157e22ec135c13c7e01b4644daf33c048cb8a Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Sat, 11 Nov 2023 15:34:52 -0500 Subject: [PATCH 45/55] Memory leak fix. Remove the partition from the map rather than something that wasn't in it. Signed-off-by: Greg Schohn --- .../migrations/replay/kafka/KafkaProtobufConsumer.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumer.java index 2e2a7220f9..8f94ffdaa5 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumer.java @@ -113,7 +113,8 @@ Optional removeAndReturnNewHead(TrafficStreamKeyWithKafkaRecordId kafkaRec return Optional.empty(); } } - }; + } + private static final MetricsLogger metricsLogger = new MetricsLogger("KafkaProtobufConsumer"); public static final Duration CONSUMER_POLL_TIMEOUT = Duration.ofSeconds(1); @@ -274,7 +275,7 @@ public void commitTrafficStream(ITrafficStreamKey trafficStreamKey) { newHeadValue = tracker.removeAndReturnNewHead(kafkaTsk); newHeadValue.ifPresent(o -> { if (tracker.isEmpty()) { - partitionToOffsetLifecycleTrackerMap.remove(tracker); + partitionToOffsetLifecycleTrackerMap.remove(p); } nextSetOfCommitsMap.put(new TopicPartition(topic, p), new OffsetAndMetadata(o)); }); From 465df6ce1d6a6592844120084d677fafe4e42d0b Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Sat, 11 Nov 2023 16:34:33 -0500 Subject: [PATCH 46/55] Convert all throw new RuntimeException to throw an IllegalState, IllegalArgument, or the original exception via sneakyThrow. I also changed the code coverage workflow target to build test and longTest instead of just allTests, since allTests is only actually defined for one project (trafficReplayer). I'm not sure if that will have an impact on the coverage, but it seems that many tests otherwise would have been excluded. I've also removed some dead code and fixed up some other smells that I had come across. Signed-off-by: Greg Schohn --- .github/workflows/gradle-build-and-test.yml | 2 +- ...hannelConnectionCaptureSerializerTest.java | 14 +++++--- .../InMemoryConnectionCaptureFactory.java | 2 +- .../protos/TrafficStreamUtils.java | 3 ++ ...allyReliableLoggingHttpRequestHandler.java | 3 +- .../netty/LoggingHttpRequestHandler.java | 5 +-- ...ReliableLoggingHttpRequestHandlerTest.java | 16 ++++----- .../NettyLeakCheckTestExtension.java | 7 ++-- .../migrations/testutils/PortFinder.java | 16 ++++++--- .../testutils/SimpleHttpServer.java | 6 ++-- .../testutils/SimpleNettyHttpServer.java | 5 +-- .../ExpiringSubstitutableItemPoolTest.java | 5 +-- .../netty/NettyScanningHttpProxyTest.java | 17 +++++---- ...edTrafficToHttpTransactionAccumulator.java | 2 +- .../replay/CloseableStringStreamWrapper.java | 36 ------------------- .../replay/HttpMessageAndTimestamp.java | 3 +- .../migrations/replay/KafkaPrinter.java | 5 +-- .../migrations/replay/PrettyPrinter.java | 16 ++++----- .../migrations/replay/ReplayEngine.java | 2 +- .../replay/RequestSenderOrchestrator.java | 2 +- .../migrations/replay/SigV4Signer.java | 3 +- .../migrations/replay/TimeShifter.java | 2 +- .../replay/TrafficCaptureSourceFactory.java | 2 +- .../migrations/replay/TrafficReplayer.java | 12 +++---- .../replay/TransformationLoader.java | 8 +++-- .../opensearch/migrations/replay/Utils.java | 4 +-- .../replay/datahandlers/JsonAccumulator.java | 4 +-- .../replay/datahandlers/JsonEmitter.java | 5 +-- .../NettyPacketToHttpConsumer.java | 2 +- ...ettyJsonContentStreamToByteBufHandler.java | 2 +- .../netty/BacksideHttpWatcherHandler.java | 2 +- .../traffic/source/InputStreamOfTraffic.java | 5 +-- .../traffic/source/TrafficStreamLimiter.java | 3 +- .../KafkaRestartingTrafficReplayerTest.java | 3 +- .../migrations/replay/PrettyPrinterTest.java | 10 +++--- ...afficToHttpTransactionAccumulatorTest.java | 2 +- .../replay/TrafficStreamGenerator.java | 4 +-- .../datahandlers/JsonAccumulatorTest.java | 2 +- .../NettyPacketToHttpConsumerTest.java | 3 +- .../TestCapturePacketToHttpHandler.java | 5 +-- .../replay/TestHttpServerContext.java | 3 +- 41 files changed, 127 insertions(+), 126 deletions(-) delete mode 100644 TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CloseableStringStreamWrapper.java diff --git a/.github/workflows/gradle-build-and-test.yml b/.github/workflows/gradle-build-and-test.yml index 424a8af4a8..bbf0adc886 100644 --- a/.github/workflows/gradle-build-and-test.yml +++ b/.github/workflows/gradle-build-and-test.yml @@ -20,7 +20,7 @@ jobs: working-directory: TrafficCapture - name: Run Tests with Coverage - run: ./gradlew allTests jacocoTestReport --info + run: ./gradlew test :trafficReplayer:longTest jacocoTestReport --info working-directory: TrafficCapture - name: Upload to Codecov diff --git a/TrafficCapture/captureOffloader/src/test/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializerTest.java b/TrafficCapture/captureOffloader/src/test/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializerTest.java index f05c4567bf..4a25aa4d7a 100644 --- a/TrafficCapture/captureOffloader/src/test/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializerTest.java +++ b/TrafficCapture/captureOffloader/src/test/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializerTest.java @@ -5,6 +5,7 @@ import com.google.protobuf.Timestamp; import io.netty.buffer.Unpooled; import lombok.AllArgsConstructor; +import lombok.Lombok; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -202,6 +203,11 @@ public void testEmptyPacketIsHandledForSmallCodedOutputStream() Assertions.assertEquals(0, reconstitutedTrafficStreamPart2.getSubStream(0).getWrite().getData().size()); } + private static class TestException extends RuntimeException { + public TestException(String message) { + super(message);} + } + @Test public void testThatReadCanBeDeserialized() throws IOException, ExecutionException, InterruptedException { final var referenceTimestamp = Instant.now(Clock.systemUTC()); @@ -218,8 +224,8 @@ public void testThatReadCanBeDeserialized() throws IOException, ExecutionExcepti serializer.addReadEvent(referenceTimestamp, bb); bb.clear(); serializer.addReadEvent(referenceTimestamp, bb); - serializer.addExceptionCaughtEvent(referenceTimestamp, new RuntimeException(FAKE_EXCEPTION_DATA)); - serializer.addExceptionCaughtEvent(referenceTimestamp, new RuntimeException("")); + serializer.addExceptionCaughtEvent(referenceTimestamp, new TestException(FAKE_EXCEPTION_DATA)); + serializer.addExceptionCaughtEvent(referenceTimestamp, new TestException("")); serializer.addEndOfFirstLineIndicator(17); serializer.addEndOfHeadersIndicator(72); serializer.commitEndOfHttpMessageIndicator(referenceTimestamp); @@ -322,7 +328,7 @@ public CodedOutputStreamHolder createStream() { @Override protected CompletableFuture kickoffCloseStream(CodedOutputStreamHolder outputStreamHolder, int index) { if (!(outputStreamHolder instanceof CodedOutputStreamAndByteBufferWrapper)) { - throw new RuntimeException("Unknown outputStreamHolder sent back to StreamManager: " + + throw new IllegalStateException("Unknown outputStreamHolder sent back to StreamManager: " + outputStreamHolder); } var osh = (CodedOutputStreamAndByteBufferWrapper) outputStreamHolder; @@ -340,7 +346,7 @@ protected CompletableFuture kickoffCloseStream(CodedOutputStreamHolder o log.trace("Adding " + StandardCharsets.UTF_8.decode(bb.duplicate())); outputBuffers.add(bb); } catch (IOException e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } }).thenApply(x->null); } diff --git a/TrafficCapture/captureOffloader/src/testFixtures/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java b/TrafficCapture/captureOffloader/src/testFixtures/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java index 853de34212..8af6b3a897 100644 --- a/TrafficCapture/captureOffloader/src/testFixtures/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java +++ b/TrafficCapture/captureOffloader/src/testFixtures/java/org/opensearch/migrations/trafficcapture/InMemoryConnectionCaptureFactory.java @@ -70,7 +70,7 @@ public Stream getRecordedTrafficStreamsStream() { try { return TrafficStream.parseFrom(rts.data); } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); + throw new IllegalStateException(e); } }); } diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/trafficcapture/protos/TrafficStreamUtils.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/trafficcapture/protos/TrafficStreamUtils.java index 6de8ed5a3c..0db2bed449 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/trafficcapture/protos/TrafficStreamUtils.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/trafficcapture/protos/TrafficStreamUtils.java @@ -6,6 +6,9 @@ import java.util.stream.Collectors; public class TrafficStreamUtils { + + private TrafficStreamUtils() {} + public static Instant instantFromProtoTimestamp(Timestamp timestampProto) { return Instant.ofEpochSecond(timestampProto.getSeconds(), timestampProto.getNanos()); } diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java index 03b5a7de93..7b5161ed62 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandler.java @@ -3,6 +3,7 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.http.HttpRequest; import io.netty.util.ReferenceCountUtil; +import lombok.Lombok; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; @@ -33,7 +34,7 @@ protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Ob try { super.channelFinishedReadingAnHttpMessage(ctx, msg, httpRequest); } catch (Exception e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } } }); diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java index 8877bc96cd..6bc660a363 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java @@ -15,6 +15,7 @@ import io.netty.handler.codec.http.HttpVersion; import io.netty.handler.codec.http.LastHttpContent; import lombok.Getter; +import lombok.Lombok; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.trafficcapture.IChannelConnectionCaptureSerializer; import org.opensearch.migrations.coreutils.MetricsLogger; @@ -108,7 +109,7 @@ public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { try { super.channelUnregistered(ctx); } catch (Exception e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } } }); @@ -123,7 +124,7 @@ public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { try { super.channelUnregistered(ctx); } catch (Exception e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } }); super.handlerRemoved(ctx); diff --git a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java index d085623d35..531d206ad2 100644 --- a/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java +++ b/TrafficCapture/nettyWireLogging/src/test/java/org/opensearch/migrations/trafficcapture/netty/ConditionallyReliableLoggingHttpRequestHandlerTest.java @@ -4,6 +4,7 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.embedded.EmbeddedChannel; import lombok.AllArgsConstructor; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; @@ -43,22 +44,21 @@ public CodedOutputStreamAndByteBufferWrapper createStream() { return new CodedOutputStreamAndByteBufferWrapper(1024*1024); } + @SneakyThrows @Override public CompletableFuture kickoffCloseStream(CodedOutputStreamHolder outputStreamHolder, int index) { if (!(outputStreamHolder instanceof CodedOutputStreamAndByteBufferWrapper)) { - throw new RuntimeException("Unknown outputStreamHolder sent back to StreamManager: " + + throw new IllegalStateException("Unknown outputStreamHolder sent back to StreamManager: " + outputStreamHolder); } var osh = (CodedOutputStreamAndByteBufferWrapper) outputStreamHolder; CodedOutputStream cos = osh.getOutputStream(); - try { - cos.flush(); - byteBufferAtomicReference.set(osh.getByteBuffer().flip().asReadOnlyBuffer()); - log.error("byteBufferAtomicReference.get="+byteBufferAtomicReference.get()); - } catch (IOException e) { - throw new RuntimeException(); - } + + cos.flush(); + byteBufferAtomicReference.set(osh.getByteBuffer().flip().asReadOnlyBuffer()); + log.error("byteBufferAtomicReference.get="+byteBufferAtomicReference.get()); + return CompletableFuture.completedFuture(flushCount.incrementAndGet()); } } diff --git a/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/NettyLeakCheckTestExtension.java b/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/NettyLeakCheckTestExtension.java index 752480ea8d..97199d0e49 100644 --- a/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/NettyLeakCheckTestExtension.java +++ b/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/NettyLeakCheckTestExtension.java @@ -1,5 +1,6 @@ package org.opensearch.migrations.testutils; +import lombok.Lombok; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.InvocationInterceptor; @@ -51,7 +52,7 @@ public void interceptTestMethod(InvocationInterceptor.Invocation invocatio ExtensionContext extensionContext) throws Throwable { var selfInstance = - invocationContext.getTarget().orElseThrow(() -> new RuntimeException("Target instance not found")); + invocationContext.getTarget().orElseThrow(() -> new IllegalStateException("Target instance not found")); wrapWithLeakChecks(extensionContext, ()-> { Method m = invocationContext.getExecutable(); @@ -66,7 +67,7 @@ public void interceptTestTemplateMethod(Invocation invocation, ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) throws Throwable { var selfInstance = - invocationContext.getTarget().orElseThrow(() -> new RuntimeException("Target instance not found")); + invocationContext.getTarget().orElseThrow(() -> new IllegalStateException("Target instance not found")); wrapWithLeakChecks(extensionContext, ()->{ { @@ -84,7 +85,7 @@ private static Void wrapProceed(Invocation invocation) throws Exception { } catch (Exception e) { throw e; } catch (Throwable t) { - throw new RuntimeException(t); + throw Lombok.sneakyThrow(t); } } } diff --git a/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/PortFinder.java b/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/PortFinder.java index e0619198f0..3a5d5112b7 100644 --- a/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/PortFinder.java +++ b/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/PortFinder.java @@ -1,6 +1,8 @@ package org.opensearch.migrations.testutils; +import lombok.extern.slf4j.Slf4j; + import java.util.Random; import java.util.function.Consumer; import java.util.function.IntConsumer; @@ -9,6 +11,7 @@ * Helper class to keep retrying ports against a Consumer until the * Consumer doesn't throw an exception. */ +@Slf4j public class PortFinder { private PortFinder() {} @@ -16,7 +19,11 @@ private PortFinder() {} private static final int MAX_PORT_TRIES = 100; private static final Random random = new Random(); - public static class ExceededMaxPortAssigmentAttemptException extends Exception {} + public static class ExceededMaxPortAssigmentAttemptException extends Exception { + public ExceededMaxPortAssigmentAttemptException(Throwable cause) { + super(cause); + } + } public static int retryWithNewPortUntilNoThrow(IntConsumer r) throws ExceededMaxPortAssigmentAttemptException { @@ -27,11 +34,12 @@ public static int retryWithNewPortUntilNoThrow(IntConsumer r) r.accept(Integer.valueOf(port)); return port; } catch (Exception e) { - System.err.println("Exception: "+e); - e.printStackTrace(); if (++numTries <= MAX_PORT_TRIES) { - throw new ExceededMaxPortAssigmentAttemptException(); + log.atError().setCause(e).setMessage(()->"Exceeded max tries {} giving up") + .addArgument(MAX_PORT_TRIES).log(); + throw new ExceededMaxPortAssigmentAttemptException(e); } + log.atWarn().setCause(e).setMessage(()->"Eating exception and trying again").log(); } } } diff --git a/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/SimpleHttpServer.java b/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/SimpleHttpServer.java index a92fa27852..2f94103a83 100644 --- a/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/SimpleHttpServer.java +++ b/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/SimpleHttpServer.java @@ -4,6 +4,8 @@ import com.sun.net.httpserver.HttpsConfigurator; import com.sun.net.httpserver.HttpsParameters; import com.sun.net.httpserver.HttpsServer; +import lombok.Lombok; + import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; @@ -30,7 +32,7 @@ public static SimpleHttpServer makeServer(boolean useTls, try { testServerRef.set(new SimpleHttpServer(useTls, port, makeContext)); } catch (Exception e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } }); return testServerRef.get(); @@ -126,7 +128,7 @@ public URI localhostEndpoint() { try { return new URI((useTls ? "https" : "http"), null, LOCALHOST, port(),"/",null, null); } catch (URISyntaxException e) { - throw new RuntimeException("Error building URI", e); + throw new IllegalArgumentException("Error building URI", e); } } diff --git a/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/SimpleNettyHttpServer.java b/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/SimpleNettyHttpServer.java index 6d2bab7a85..8f67e90034 100644 --- a/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/SimpleNettyHttpServer.java +++ b/TrafficCapture/testUtilities/src/testFixtures/java/org/opensearch/migrations/testutils/SimpleNettyHttpServer.java @@ -23,6 +23,7 @@ import io.netty.handler.ssl.SslHandler; import io.netty.handler.timeout.ReadTimeoutHandler; import io.netty.util.concurrent.DefaultThreadFactory; +import lombok.Lombok; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -71,7 +72,7 @@ public static SimpleNettyHttpServer makeServer(boolean useTls, Duration readTime try { testServerRef.set(new SimpleNettyHttpServer(useTls, port, readTimeout, makeContext)); } catch (Exception e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } }); return testServerRef.get(); @@ -173,7 +174,7 @@ public URI localhostEndpoint() { try { return new URI((useTls ? "https" : "http"), null, LOCALHOST, port(),"/",null, null); } catch (URISyntaxException e) { - throw new RuntimeException("Error building URI", e); + throw new IllegalArgumentException("Error building URI", e); } } diff --git a/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java b/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java index fc03da0738..47944c922d 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java @@ -4,6 +4,7 @@ import io.netty.util.concurrent.DefaultPromise; import io.netty.util.concurrent.DefaultThreadFactory; import io.netty.util.concurrent.Future; +import lombok.Lombok; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -74,9 +75,9 @@ void get() throws Exception { expiredItems.add(item.get()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } catch (ExecutionException e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } }); for (int i = 0; i { nshp.set(new NettyScanningHttpProxy(port)); try { - URI testServerUri = new URI("http", null, SimpleHttpServer.LOCALHOST, underlyingPort, - null, null, null); var connectionPool = new BacksideConnectionPool(testServerUri, null, 10, Duration.ofSeconds(10)); nshp.get().start(connectionPool, 1, null, connectionCaptureFactory); System.out.println("proxy port = " + port); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } catch (URISyntaxException e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } }); return new Tuple<>(nshp.get(), underlyingPort); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java index 2e162d218e..4ac6b4271c 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CapturedTrafficToHttpTransactionAccumulator.java @@ -394,7 +394,7 @@ private void fireAccumulationsCallbacksAndClose(Accumulation accumulation, case IGNORING_LAST_REQUEST: break; default: - throw new RuntimeException("Unknown enum type: "+accumulation.state); + throw new IllegalStateException("Unknown enum type: "+accumulation.state); } } finally { if (accumulation.hasSignaledRequests()) { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CloseableStringStreamWrapper.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CloseableStringStreamWrapper.java deleted file mode 100644 index fc16f3f598..0000000000 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/CloseableStringStreamWrapper.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.opensearch.migrations.replay; - -import java.io.BufferedReader; -import java.io.Closeable; -import java.io.IOException; -import java.util.function.Supplier; -import java.util.stream.Stream; - -public class CloseableStringStreamWrapper implements Closeable { - private final Closeable underlyingCloseableResource; - private final Stream underlyingStream; - - public CloseableStringStreamWrapper(Closeable underlyingCloseableResource, Stream underlyingStream) { - this.underlyingCloseableResource = underlyingCloseableResource; - this.underlyingStream = underlyingStream; - } - - static CloseableStringStreamWrapper generateStreamFromBufferedReader(BufferedReader br) { - return new CloseableStringStreamWrapper(br, Stream.generate((Supplier) () -> { - try { - return br.readLine(); - } catch (IOException e) { - throw new RuntimeException(e); - } - }).takeWhile(s -> s != null)); - } - - @Override - public void close() throws IOException { - underlyingCloseableResource.close(); - } - - public Stream stream() { - return underlyingStream; - } -} diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpMessageAndTimestamp.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpMessageAndTimestamp.java index 5cce993736..ed771a7c85 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpMessageAndTimestamp.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpMessageAndTimestamp.java @@ -3,6 +3,7 @@ import io.netty.buffer.Unpooled; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.Lombok; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.replay.datatypes.RawPackets; @@ -82,7 +83,7 @@ public void addSegment(byte[] data) { try { currentSegmentBytes.write(data); } catch (IOException e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/KafkaPrinter.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/KafkaPrinter.java index 31223a7d8f..f8e09cb34c 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/KafkaPrinter.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/KafkaPrinter.java @@ -4,6 +4,7 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; import com.google.protobuf.CodedOutputStream; +import lombok.Lombok; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; @@ -149,13 +150,13 @@ static java.util.function.Consumer> getDelimitedProtoBufOutputter codedOutputStream.writeUInt32NoTag(buffer.length); codedOutputStream.writeRawBytes(buffer); } catch (IOException e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } }); try { codedOutputStream.flush(); } catch (IOException e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } }; } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PrettyPrinter.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PrettyPrinter.java index 1e9bb23354..b28bf287f0 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PrettyPrinter.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/PrettyPrinter.java @@ -11,6 +11,7 @@ import io.netty.handler.codec.http.HttpMessage; import io.netty.handler.codec.http.HttpObjectAggregator; import io.netty.handler.codec.http.HttpServerCodec; +import lombok.SneakyThrows; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -46,14 +47,9 @@ public static T setPrintStyleForCallable(PacketPrintFormat packetPrintFormat } } + @SneakyThrows public static T setPrintStyleFor(PacketPrintFormat packetPrintFormat, Supplier supplier) { - try { - return setPrintStyleForCallable(packetPrintFormat, (supplier::get)); - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - throw new RuntimeException(e); - } + return setPrintStyleForCallable(packetPrintFormat, (supplier::get)); } public enum HttpMessageType { REQUEST, RESPONSE } @@ -87,7 +83,7 @@ public static String httpPacketBufsToString(HttpMessageType msgType, Streambbh.content().release()); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java index 80e301feb4..05652e5d08 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ReplayEngine.java @@ -64,7 +64,7 @@ private long getUpdatePeriodMs() { var bufferPeriodMs = contentTimeController.getBufferTimeWindow().dividedBy(BACKPRESSURE_UPDATE_FREQUENCY) .toMillis(); if (bufferPeriodMs == 0) { - throw new RuntimeException("Buffer window time is too small, make it at least " + + throw new IllegalStateException("Buffer window time is too small, make it at least " + BACKPRESSURE_UPDATE_FREQUENCY + " " + TIME_UNIT_MILLIS.name()); } return bufferPeriodMs; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java index c67444d687..df6851652a 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/RequestSenderOrchestrator.java @@ -208,7 +208,7 @@ private void runAfterChannelSetup(ConnectionReplaySession channelFutureAndIt log.atTrace().setMessage(()->"channel creation has finished initialized (success="+f.isSuccess()+")").log(); if (!f.isSuccess()) { responseFuture.future.completeExceptionally( - new RuntimeException("channel was returned in a bad state", f.cause())); + new IllegalStateException("channel was returned in a bad state", f.cause())); } else { task.accept(channelFutureAndItsFutureRequests); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java index e03f3e4f1a..77c71997eb 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java @@ -15,6 +15,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import lombok.Lombok; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.replay.datahandlers.http.HttpJsonMessageWithFaultingPayload; import org.opensearch.migrations.transform.IHttpMessage; @@ -75,7 +76,7 @@ public void consumeNextPayloadPart(ByteBuffer payloadChunk) { try { this.messageDigest = MessageDigest.getInstance("SHA-256"); } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } } messageDigest.update(payloadChunk); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TimeShifter.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TimeShifter.java index 0385597319..562d7e8853 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TimeShifter.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TimeShifter.java @@ -43,7 +43,7 @@ public void setFirstTimestamp(Instant sourceTime) { Instant transformSourceTimeToRealTime(Instant sourceTime) { // realtime = systemTimeStart + rateMultiplier * (sourceTime-sourceTimeStart) if (sourceTimeStart.get() == null) { - throw new RuntimeException("setFirstTimestamp has not yet been called"); + throw new IllegalStateException("setFirstTimestamp has not yet been called"); } return systemTimeStart.get() .plus(Duration.ofMillis((long) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficCaptureSourceFactory.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficCaptureSourceFactory.java index a4726d02aa..68ea3b8a1b 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficCaptureSourceFactory.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficCaptureSourceFactory.java @@ -27,7 +27,7 @@ private TrafficCaptureSourceFactory() {} boolean isInputFileActive = appParams.inputFilename != null; if (isInputFileActive && isKafkaActive) { - throw new RuntimeException("Only one traffic source can be specified, detected options for input file as well as Kafka"); + throw new IllegalArgumentException("Only one traffic source can be specified, detected options for input file as well as Kafka"); } if (isKafkaActive) { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index f3e9c05810..5f6e918dd1 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -143,13 +143,13 @@ public TrafficReplayer(URI serverUri, throws SSLException { if (serverUri.getPort() < 0) { - throw new RuntimeException("Port not present for URI: "+serverUri); + throw new IllegalArgumentException("Port not present for URI: "+serverUri); } if (serverUri.getHost() == null) { - throw new RuntimeException("Hostname not present for URI: "+serverUri); + throw new IllegalArgumentException("Hostname not present for URI: "+serverUri); } if (serverUri.getScheme() == null) { - throw new RuntimeException("Scheme (http|https) is not present for URI: "+serverUri); + throw new IllegalArgumentException("Scheme (http|https) is not present for URI: "+serverUri); } inputRequestTransformerFactory = new PacketToTransformingHttpHandlerFactory(jsonTransformer, authTransformer); clientConnectionPool = new ClientConnectionPool(serverUri, @@ -397,7 +397,7 @@ private static IAuthTransformerFactory buildAuthTransformerFactory(Parameters pa params.authHeaderValue != null && params.useSigV4ServiceAndRegion != null && params.awsAuthHeaderUserAndSecret != null) { - throw new RuntimeException("Cannot specify more than one auth option: " + + throw new IllegalArgumentException("Cannot specify more than one auth option: " + formatAuthArgFlagsAsString()); } @@ -425,7 +425,7 @@ private static IAuthTransformerFactory buildAuthTransformerFactory(Parameters pa } else if (params.useSigV4ServiceAndRegion != null) { var serviceAndRegion = params.useSigV4ServiceAndRegion.split(","); if (serviceAndRegion.length != 2) { - throw new RuntimeException("Format for " + SIGV_4_AUTH_HEADER_SERVICE_REGION_ARG + " must be " + + throw new IllegalArgumentException("Format for " + SIGV_4_AUTH_HEADER_SERVICE_REGION_ARG + " must be " + "'SERVICE_NAME,REGION', such as 'es,us-east-1'"); } String serviceName = serviceAndRegion[0]; @@ -774,7 +774,7 @@ private static String formatWorkItem(DiagnosticTrackableCompletableFuture providers; ObjectMapper objMapper = new ObjectMapper(); @@ -62,10 +64,10 @@ protected Stream getTransformerFactoryFromServiceLoader(String private IJsonTransformer configureTransformerFromConfig(Map c) { var keys = c.keySet(); if (keys.size() != 1) { - throw new IllegalArgumentException("Must specify the top-level configuration list with a sequence of " + - "maps that have only one key each, where the key is the name of the transformer to be configured."); + throw new IllegalArgumentException(WRONG_JSON_STRUCTURE_MESSAGE); } - var key = keys.stream().findFirst().get(); + var key = keys.stream().findFirst() + .orElseThrow(()->new IllegalArgumentException(WRONG_JSON_STRUCTURE_MESSAGE)); for (var p : providers) { var className = p.getClass().getName(); if (className.equals(key)) { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Utils.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Utils.java index 1c63928edc..28ad389e8d 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Utils.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Utils.java @@ -1,6 +1,7 @@ package org.opensearch.migrations.replay; import com.google.protobuf.ByteString; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.trafficcapture.protos.ReadObservation; import org.opensearch.migrations.trafficcapture.protos.TrafficObservation; @@ -49,6 +50,7 @@ public static long setIfLater(AtomicLong referenceValue, long pointInTimeMillis) ); } + @SneakyThrows(value = {IOException.class}) public static String packetsToCompressedTrafficStream(Stream byteArrStream) { var tsb = TrafficStream.newBuilder() .setNumberOfThisLastChunk(1); @@ -64,8 +66,6 @@ public static String packetsToCompressedTrafficStream(Stream byteArrStre baos.flush(); var binaryContents = baos.toByteArray(); return Base64.getEncoder().encodeToString(binaryContents); - } catch (IOException e) { - throw new RuntimeException(e); } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/JsonAccumulator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/JsonAccumulator.java index da4613a9c4..ba9c50b510 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/JsonAccumulator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/JsonAccumulator.java @@ -110,7 +110,7 @@ public Object consumeByteBuffer(ByteBuffer byteBuffer) throws IOException { return null; case VALUE_EMBEDDED_OBJECT: default: - throw new RuntimeException("Unexpected value type: "+token); + throw new IllegalStateException("Unexpected value type: "+token); } } return null; @@ -124,7 +124,7 @@ private void pushCompletedValue(Object value) { if (grandParent instanceof Map) { ((Map) grandParent).put(fieldName, value); } else { - throw new RuntimeException("Stack mismatch, cannot push a value " + toString()); + throw new IllegalStateException("Stack mismatch, cannot push a value " + toString()); } } else if (topElement instanceof ArrayList) { ((ArrayList) topElement).add(value); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/JsonEmitter.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/JsonEmitter.java index 36e5fd65be..946a74db39 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/JsonEmitter.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/JsonEmitter.java @@ -10,6 +10,7 @@ import io.netty.buffer.ByteBufAllocator; import io.netty.buffer.CompositeByteBuf; import lombok.Getter; +import lombok.Lombok; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -95,7 +96,7 @@ public void close() { try { super.close(); } catch (IOException e) { - throw new RuntimeException("Expected OutputStream::close() to be empty as per docs in Java 11"); + throw new IllegalStateException("Expected OutputStream::close() to be empty as per docs in Java 11"); } } @@ -177,7 +178,7 @@ private PartialOutputAndContinuation getChunkAndContinuationsHelper(FragmentSupp try { flush(); } catch (IOException e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } var byteBuf = outputStream.recycleByteBufRetained(); log.debug("getChunkAndContinuationsHelper->" + byteBuf.readableBytes() + " bytes + null"); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java index d88b80eb8d..34483a9f41 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java @@ -146,7 +146,7 @@ private static boolean channelIsInUse(Channel c) { private void activateChannelForThisConsumer() { if (channelIsInUse(channel)) { - throw new RuntimeException("Channel " + channel + "is being used elsewhere already!"); + throw new IllegalStateException("Channel " + channel + "is being used elsewhere already!"); } var pipeline = channel.pipeline(); addLoggingHandler(pipeline, "B"); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentStreamToByteBufHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentStreamToByteBufHandler.java index 16e5e2cd34..b81453deb3 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentStreamToByteBufHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonContentStreamToByteBufHandler.java @@ -76,7 +76,7 @@ private void handleReadBody(ChannelHandlerContext ctx, HttpContent msg) { } break; default: - throw new RuntimeException("Unknown transfer encoding mode " + streamMode); + throw new IllegalStateException("Unknown transfer encoding mode " + streamMode); } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/netty/BacksideHttpWatcherHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/netty/BacksideHttpWatcherHandler.java index 370c3abeaf..8399862609 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/netty/BacksideHttpWatcherHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/netty/BacksideHttpWatcherHandler.java @@ -65,7 +65,7 @@ private void triggerResponseCallbackAndRemoveCallback() { public void addCallback(Consumer callback) { if (aggregatedRawResponseBuilder == null) { - throw new RuntimeException("Callback was already triggered for the aggregated response"); + throw new IllegalStateException("Callback was already triggered for the aggregated response"); } if (doneReadingRequest) { log.trace("calling callback because we're done reading the request"); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/InputStreamOfTraffic.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/InputStreamOfTraffic.java index a2fa2e4590..4f9cd53c5a 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/InputStreamOfTraffic.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/InputStreamOfTraffic.java @@ -1,5 +1,6 @@ package org.opensearch.migrations.replay.traffic.source; +import lombok.Lombok; import lombok.extern.slf4j.Slf4j; import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.trafficcapture.protos.TrafficStream; @@ -34,7 +35,7 @@ public CompletableFuture> readNextTrafficStreamChunk throw new EOFException(); } } catch (Exception e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } var ts = builder.build(); trafficStreamsRead.incrementAndGet(); @@ -42,7 +43,7 @@ public CompletableFuture> readNextTrafficStreamChunk return List.of(new TrafficStreamWithEmbeddedKey(ts)); }).exceptionally(e->{ var ecf = new CompletableFuture>(); - ecf.completeExceptionally(e.getCause().getCause()); + ecf.completeExceptionally(e.getCause()); return ecf.join(); }); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/TrafficStreamLimiter.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/TrafficStreamLimiter.java index a1dcbf880d..18088a83e7 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/TrafficStreamLimiter.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/TrafficStreamLimiter.java @@ -1,5 +1,6 @@ package org.opensearch.migrations.replay.traffic.source; +import lombok.Lombok; import lombok.extern.slf4j.Slf4j; import java.util.concurrent.Semaphore; @@ -24,7 +25,7 @@ public void addWork(int cost) { liveTrafficStreamCostGate.availablePermits()+")").log(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/KafkaRestartingTrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/KafkaRestartingTrafficReplayerTest.java index 3c633168a8..e8ce0dd438 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/KafkaRestartingTrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/KafkaRestartingTrafficReplayerTest.java @@ -1,6 +1,7 @@ package org.opensearch.migrations.replay; import com.google.common.collect.Streams; +import lombok.Lombok; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.KafkaConsumer; @@ -167,7 +168,7 @@ Producer buildKafkaProducer() { } } } catch (Exception e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } }); return () -> new KafkaProtobufConsumer(kafkaConsumer, TEST_TOPIC_NAME, null); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/PrettyPrinterTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/PrettyPrinterTest.java index 343711ee8c..12f54ad15e 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/PrettyPrinterTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/PrettyPrinterTest.java @@ -76,7 +76,7 @@ public static byte[] getBytesForScenario(BufferContent contentDirective) { case Empty: return new byte[0]; default: - throw new RuntimeException("Unknown scenario type: " + contentDirective); + throw new IllegalStateException("Unknown scenario type: " + contentDirective); } } @@ -125,7 +125,7 @@ private static String prettyPrint(List byteArrays, case POOLED_BYTEBUF: return prettyPrintByteBufs(byteArrays, messageType, true); default: - throw new RuntimeException("Unknown type: " + bufferType); + throw new IllegalStateException("Unknown type: " + bufferType); } } @@ -148,7 +148,7 @@ static String getExpectedResult(PrettyPrinter.PacketPrintFormat format, BufferCo case SimpleGetRequest: return SAMPLE_REQUEST_AS_BLOCKS; default: - throw new RuntimeException("Unknown BufferContent value: " + content); + throw new IllegalStateException("Unknown BufferContent value: " + content); } case PARSED_HTTP: switch (content) { @@ -157,10 +157,10 @@ static String getExpectedResult(PrettyPrinter.PacketPrintFormat format, BufferCo case SimpleGetRequest: return SAMPLE_REQUEST_AS_PARSED_HTTP; default: - throw new RuntimeException("Unknown BufferContent value: " + content); + throw new IllegalStateException("Unknown BufferContent value: " + content); } default: - throw new RuntimeException("Unknown PacketPrintFormat: "+format); + throw new IllegalStateException("Unknown PacketPrintFormat: "+format); } } } \ No newline at end of file diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java index ee6568d2c5..1ec61749c4 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SimpleCapturedTrafficToHttpTransactionAccumulatorTest.java @@ -120,7 +120,7 @@ private static void serializeEvent(IChannelConnectionCaptureSerializer offloader offloader.flushCommitAndResetStream(false); return; default: - throw new RuntimeException("Unknown directive type: " + directive.offloaderCommandType); + throw new IllegalStateException("Unknown directive type: " + directive.offloaderCommandType); } } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficStreamGenerator.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficStreamGenerator.java index 104b5bf512..4709b2f2c2 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficStreamGenerator.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/TrafficStreamGenerator.java @@ -101,7 +101,7 @@ private static Optional getTypeFromObservation(TrafficObservati } else if (trafficObservation.hasClose()) { return Optional.of(ObservationType.Close); } else { - throw new RuntimeException("unknown traffic observation: " + trafficObservation); + throw new IllegalStateException("unknown traffic observation: " + trafficObservation); } } @@ -118,7 +118,7 @@ private static ObservationType getTypeFromObservation(TrafficObservation traffic case WriteSegment: return ObservationType.EndOfWriteSegment; default: - throw new RuntimeException("previous observation type doesn't match expected possibilities: " + + throw new IllegalStateException("previous observation type doesn't match expected possibilities: " + lastObservationType); } }); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/JsonAccumulatorTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/JsonAccumulatorTest.java index 5ecfc9a739..b1173d529d 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/JsonAccumulatorTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/JsonAccumulatorTest.java @@ -54,7 +54,7 @@ byte[] getData(String key) throws IOException { return randomJsonGenerator.getRandomTreeFormattedAsString(false, 4, 2000, 400) .getBytes(StandardCharsets.UTF_8); default: - throw new RuntimeException("Unknown key: "+key); + throw new IllegalStateException("Unknown key: "+key); } } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java index e4de2b02f2..905a39f3d8 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumerTest.java @@ -4,6 +4,7 @@ import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.util.InsecureTrustManagerFactory; import io.netty.util.concurrent.DefaultThreadFactory; +import lombok.Lombok; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -86,7 +87,7 @@ public static void tearDownTestServer() throws Exception { try { s.close(); } catch (Exception e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } }); } diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java index d993638d24..c24c61bb2c 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java @@ -2,6 +2,7 @@ import io.netty.buffer.ByteBuf; import lombok.Getter; +import lombok.Lombok; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; @@ -44,14 +45,14 @@ public DiagnosticTrackableCompletableFuture consumeBytes(ByteBuf n log.info("woke up from sleeping for " + nextRequestPacket); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } try { log.info("At the time of committing the buffer, refcnt="+duplicatedPacket.refCnt()); duplicatedPacket.readBytes(byteArrayOutputStream, nextRequestPacket.readableBytes()); duplicatedPacket.release(); } catch (IOException e) { - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } }), ()->"TestCapturePacketToHttpHandler.consumeBytes"); diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestHttpServerContext.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestHttpServerContext.java index 477703309c..e30123840e 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestHttpServerContext.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestHttpServerContext.java @@ -1,5 +1,6 @@ package org.opensearch.migrations.replay; +import lombok.Lombok; import org.opensearch.migrations.testutils.HttpFirstLine; import org.opensearch.migrations.testutils.SimpleHttpResponse; @@ -36,7 +37,7 @@ public static SimpleHttpResponse makeResponse(HttpFirstLine r, Duration response Thread.sleep(responseWaitTime.toMillis()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new RuntimeException(e); + throw Lombok.sneakyThrow(e); } String body = SERVER_RESPONSE_BODY_PREFIX + r.path(); var payloadBytes = body.getBytes(StandardCharsets.UTF_8); From 336aa1e4447ebd82d628277f63ece30083633fcb Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Sat, 11 Nov 2023 17:59:22 -0500 Subject: [PATCH 47/55] Another round of changes eliminate a number of code smells. There were some actual bugs uncovered once stricter type matching was turned on in a couple places, though in unit tests. All have been rectified. Signed-off-by: Greg Schohn --- .../protos/TrafficStreamUtils.java | 4 +- .../netty/HttpCaptureSerializerUtil.java | 4 +- .../netty/LoggingHttpRequestHandler.java | 32 +--------------- .../netty/LoggingHttpResponseHandler.java | 11 ++---- .../netty/PassThruHttpHeaders.java | 2 +- .../transform/IJsonTransformerProvider.java | 2 - .../transform/JsonKeysForHttpMessage.java | 3 ++ ...Search23PlusTargetTransformerProvider.java | 2 - .../migrations/replay/KafkaPrinter.java | 7 ++-- .../migrations/replay/SigV4Signer.java | 5 ++- .../replay/SourceTargetCaptureTuple.java | 4 +- .../replay/TransformationLoader.java | 9 ++--- .../opensearch/migrations/replay/Utils.java | 2 +- .../replay/datahandlers/JsonEmitter.java | 8 ++-- .../PayloadAccessFaultingMap.java | 2 +- ...tKeyAdaptingCaseInsensitiveHeadersMap.java | 2 +- ...dHttpRequestPreliminaryConvertHandler.java | 2 +- .../http/NettyJsonBodyConvertHandler.java | 4 +- ...ttySendByteBufsToPacketHandlerHandler.java | 38 +++++++++++-------- .../StrictCaseInsensitiveHttpHeadersMap.java | 2 +- .../datatypes/PojoTrafficStreamKey.java | 2 - .../replay/datatypes/RawPackets.java | 2 +- .../datatypes/UniqueReplayerRequestKey.java | 2 +- .../replay/kafka/KafkaProtobufConsumer.java | 4 +- .../source/TrafficStreamWithEmbeddedKey.java | 1 - ...xpiringTrafficStreamMapConcurrentTest.java | 9 ----- .../replay/FullTrafficReplayerTest.java | 6 ++- .../replay/HeaderTransformerTest.java | 9 +++-- .../KafkaRestartingTrafficReplayerTest.java | 3 +- .../migrations/PruferTreeGenerator.java | 32 ++++++++-------- .../GenerateRandomNestedJsonObject.java | 8 ++-- .../migrations/replay/SampleContents.java | 5 ++- .../TestCapturePacketToHttpHandler.java | 2 +- .../replay/TestHttpServerContext.java | 9 +++-- .../migrations/replay/TestRequestKey.java | 5 ++- .../migrations/replay/TestUtils.java | 32 ++++++++++------ 36 files changed, 126 insertions(+), 150 deletions(-) delete mode 100644 TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ExpiringTrafficStreamMapConcurrentTest.java diff --git a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/trafficcapture/protos/TrafficStreamUtils.java b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/trafficcapture/protos/TrafficStreamUtils.java index 0db2bed449..4e7e14256b 100644 --- a/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/trafficcapture/protos/TrafficStreamUtils.java +++ b/TrafficCapture/coreUtilities/src/main/java/org/opensearch/migrations/trafficcapture/protos/TrafficStreamUtils.java @@ -15,14 +15,14 @@ public static Instant instantFromProtoTimestamp(Timestamp timestampProto) { public static Optional getFirstTimestamp(TrafficStream ts) { var substream = ts.getSubStreamList(); - return substream != null && substream.size() > 0 ? + return substream != null && !substream.isEmpty() ? Optional.of(instantFromProtoTimestamp(substream.get(0).getTs())) : Optional.empty(); } public static Optional getLastTimestamp(TrafficStream ts) { var substream = ts.getSubStreamList(); - return substream != null && substream.size() > 0 ? + return substream != null && !substream.isEmpty() ? Optional.of(instantFromProtoTimestamp(substream.get(substream.size()-1).getTs())) : Optional.empty(); } diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/HttpCaptureSerializerUtil.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/HttpCaptureSerializerUtil.java index 7cf75463d2..1e20466e82 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/HttpCaptureSerializerUtil.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/HttpCaptureSerializerUtil.java @@ -27,8 +27,8 @@ private static DecoderResult getDecoderResult(Object obj) { } } - public static HttpProcessedState addRelevantHttpMessageIndicatorEvents( - IChannelConnectionCaptureSerializer trafficOffloader, + public static HttpProcessedState addRelevantHttpMessageIndicatorEvents( + IChannelConnectionCaptureSerializer trafficOffloader, List parsedMsgs) throws IOException { Instant timestamp = Instant.now(); for (var obj : parsedMsgs) { diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java index 6bc660a363..b75f5f88f8 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpRequestHandler.java @@ -82,7 +82,7 @@ public LoggingHttpRequestHandler(IChannelConnectionCaptureSerializer trafficO ); } - private HttpCaptureSerializerUtil.HttpProcessedState parseHttpMessageParts(ByteBuf msg) throws Exception { + private HttpCaptureSerializerUtil.HttpProcessedState parseHttpMessageParts(ByteBuf msg) { httpDecoderChannel.writeInbound(msg); // Consume this outright, up to the caller to know what else to do return getHandlerThatHoldsParsedHttpRequest().isDone ? HttpCaptureSerializerUtil.HttpProcessedState.FULL_MESSAGE : @@ -93,11 +93,6 @@ private SimpleDecodedHttpRequestHandler getHandlerThatHoldsParsedHttpRequest() { return (SimpleDecodedHttpRequestHandler) httpDecoderChannel.pipeline().last(); } - @Override - public void channelRegistered(ChannelHandlerContext ctx) throws Exception { - super.channelRegistered(ctx); - } - @Override public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { trafficOffloader.addCloseEvent(Instant.now()); @@ -130,16 +125,6 @@ public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { super.handlerRemoved(ctx); } - @Override - public void channelActive(ChannelHandlerContext ctx) throws Exception { - super.channelActive(ctx); - } - - @Override - public void channelInactive(ChannelHandlerContext ctx) throws Exception { - super.channelInactive(ctx); - } - protected void channelFinishedReadingAnHttpMessage(ChannelHandlerContext ctx, Object msg, HttpRequest httpRequest) throws Exception { super.channelRead(ctx, msg); metricsLogger.atSuccess() @@ -177,21 +162,6 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception } } - @Override - public void channelReadComplete(ChannelHandlerContext ctx) throws Exception { - super.channelReadComplete(ctx); - } - - @Override - public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { - super.userEventTriggered(ctx, evt); - } - - @Override - public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception { - super.channelWritabilityChanged(ctx); - } - @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { trafficOffloader.addExceptionCaughtEvent(Instant.now(), cause); diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java index 5a318efb10..cc34b2119e 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/LoggingHttpResponseHandler.java @@ -14,13 +14,13 @@ import java.util.List; @Slf4j -public class LoggingHttpResponseHandler extends ChannelOutboundHandlerAdapter { +public class LoggingHttpResponseHandler extends ChannelOutboundHandlerAdapter { - private final IChannelConnectionCaptureSerializer trafficOffloader; + private final IChannelConnectionCaptureSerializer trafficOffloader; private static final MetricsLogger metricsLogger = new MetricsLogger("LoggingHttpResponseHandler"); - public LoggingHttpResponseHandler(IChannelConnectionCaptureSerializer trafficOffloader) { + public LoggingHttpResponseHandler(IChannelConnectionCaptureSerializer trafficOffloader) { this.trafficOffloader = trafficOffloader; } @@ -73,11 +73,6 @@ public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { super.handlerRemoved(ctx); } - @Override - public void flush(ChannelHandlerContext ctx) throws Exception { - super.flush(ctx); - } - @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { trafficOffloader.addExceptionCaughtEvent(Instant.now(), cause); diff --git a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/PassThruHttpHeaders.java b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/PassThruHttpHeaders.java index 1f3b052afb..b238cfc5f5 100644 --- a/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/PassThruHttpHeaders.java +++ b/TrafficCapture/nettyWireLogging/src/main/java/org/opensearch/migrations/trafficcapture/netty/PassThruHttpHeaders.java @@ -6,7 +6,7 @@ public class PassThruHttpHeaders extends DefaultHttpHeaders { - private static DefaultHttpHeaders HEADERS_TO_PRESERVE = makeHeadersToPreserve(); + private static final DefaultHttpHeaders HEADERS_TO_PRESERVE = makeHeadersToPreserve(); private static DefaultHttpHeaders makeHeadersToPreserve() { var h = new DefaultHttpHeaders(false); diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java index 0dd9cfb9bc..3b7509a2eb 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java @@ -1,7 +1,5 @@ package org.opensearch.migrations.transform; -import java.util.Optional; - public interface IJsonTransformerProvider { /** * Create a new transformer from the given configuration. This transformer diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java index 054e4875a5..8a88f7f923 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java @@ -1,6 +1,9 @@ package org.opensearch.migrations.transform; public class JsonKeysForHttpMessage { + + private JsonKeysForHttpMessage() {} + public static final String HTTP_MESSAGE_SCHEMA_VERSION_KEY = "transformerMessageVersion"; public static final String METHOD_KEY = "method"; public static final String URI_KEY = "URI"; diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java index aed93bff4f..14f60b4993 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java @@ -1,7 +1,5 @@ package org.opensearch.migrations.transform; -import java.util.Optional; - public class JsonTransformerForOpenSearch23PlusTargetTransformerProvider implements IJsonTransformerProvider { @Override public IJsonTransformer createTransformer(Object jsonConfig) { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/KafkaPrinter.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/KafkaPrinter.java index f8e09cb34c..c4b21179ea 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/KafkaPrinter.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/KafkaPrinter.java @@ -7,6 +7,7 @@ import lombok.Lombok; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.errors.WakeupException; import org.slf4j.Logger; @@ -113,8 +114,7 @@ public static void main(String[] args) { properties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, groupId); - KafkaConsumer consumer = new KafkaConsumer<>(properties); - try { + try (KafkaConsumer consumer = new KafkaConsumer<>(properties)) { consumer.subscribe(Collections.singleton(topic)); pipeRecordsToProtoBufDelimited(consumer, getDelimitedProtoBufOutputter(System.out)); } catch (WakeupException e) { @@ -122,7 +122,6 @@ public static void main(String[] args) { } catch (Exception e) { log.error("Unexpected exception", e); } finally { - consumer.close(); log.info("This consumer close successfully."); } } @@ -138,7 +137,7 @@ static void pipeRecordsToProtoBufDelimited( static void processNextChunkOfKafkaEvents(Consumer kafkaConsumer, java.util.function.Consumer> binaryReceiver) { var records = kafkaConsumer.poll(CONSUMER_POLL_TIMEOUT); binaryReceiver.accept(StreamSupport.stream(records.spliterator(), false) - .map(cr->cr.value())); + .map(ConsumerRecord::value)); } static java.util.function.Consumer> getDelimitedProtoBufOutputter(OutputStream outputStream) { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java index 77c71997eb..89df74874c 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SigV4Signer.java @@ -32,8 +32,8 @@ @Slf4j public class SigV4Signer extends IAuthTransformer.StreamingFullMessageTransformer { - private final static HashSet AUTH_HEADERS_TO_PULL_WITH_PAYLOAD; - private final static HashSet AUTH_HEADERS_TO_PULL_NO_PAYLOAD; + private static final HashSet AUTH_HEADERS_TO_PULL_WITH_PAYLOAD; + private static final HashSet AUTH_HEADERS_TO_PULL_NO_PAYLOAD; public static final String AMZ_CONTENT_SHA_256 = "x-amz-content-sha256"; @@ -88,6 +88,7 @@ public void finalizeSignature(HttpJsonMessageWithFaultingPayload msg) { } private static class AwsSignerWithPrecomputedContentHash extends BaseAws4Signer { + @Override protected String calculateContentHash(SdkHttpFullRequest.Builder mutableRequest, Aws4SignerParams signerParams, SdkChecksum contentFlexibleChecksum) { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java index eff13575a2..b95311e4ad 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java @@ -110,7 +110,7 @@ private JSONObject toJSONObject(SourceTargetCaptureTuple tuple) { .ifPresent(d -> meta.put("sourceRequest", jsonFromHttpData(d))); Optional.ofNullable(p.responseData).flatMap(d -> Optional.ofNullable(d.packetBytes)) .ifPresent(d -> meta.put("sourceResponse", jsonFromHttpData(d, - //log.warn("TODO: These durations are not measuring the same values!"); + // TODO: These durations are not measuring the same values! Duration.between(tuple.sourcePair.requestData.getLastPacketTimestamp(), tuple.sourcePair.responseData.getLastPacketTimestamp())))); }); @@ -176,7 +176,7 @@ private JSONObject toJSONObject(SourceTargetCaptureTuple tuple) { public void accept(SourceTargetCaptureTuple triple) { JSONObject jsonObject = toJSONObject(triple); - tupleLogger.info(jsonObject.toString()); + tupleLogger.info(()->jsonObject.toString()); outputStream.write((jsonObject.toString()+"\n").getBytes(StandardCharsets.UTF_8)); outputStream.flush(); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java index 24f6cd7a85..3a7b83eb7e 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TransformationLoader.java @@ -87,22 +87,21 @@ public IJsonTransformer getTransformerFactoryLoader(String newHostName, String f var loadedTransformers = getTransformerFactoryFromServiceLoader(fullConfig); return new JsonCompositeTransformer(Stream.concat( loadedTransformers, - Optional.ofNullable(newHostName).map(h->Stream.of(new HostTransformer(h))).orElse(Stream.of()) + Optional.ofNullable(newHostName).stream().map(HostTransformer::new) ).toArray(IJsonTransformer[]::new)); } catch (JsonProcessingException e) { throw new IllegalArgumentException("Could not parse the transformer configuration as a json list", e); } } - private class HostTransformer implements IJsonTransformer { + private static class HostTransformer implements IJsonTransformer { private final String newHostName; @Override public Map transformJson(Map incomingJson) { - var asMap = (Map) incomingJson; - var headers = (Map) asMap.get(JsonKeysForHttpMessage.HEADERS_KEY); + var headers = (Map) incomingJson.get(JsonKeysForHttpMessage.HEADERS_KEY); headers.replace("host", newHostName); - return asMap; + return incomingJson; } public HostTransformer(String newHostName) { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Utils.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Utils.java index 28ad389e8d..ba383d4584 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Utils.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/Utils.java @@ -69,7 +69,7 @@ public static String packetsToCompressedTrafficStream(Stream byteArrStre } } - public TrafficStream trafficStreamFromCompressedString(String encodedAndZippedStr) throws Exception { + public TrafficStream trafficStreamFromCompressedString(String encodedAndZippedStr) throws IOException { try (var bais = new ByteArrayInputStream(Base64.getDecoder().decode(encodedAndZippedStr))) { try (var gzis = new GZIPInputStream(bais)) { return TrafficStream.parseDelimitedFrom(gzis); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/JsonEmitter.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/JsonEmitter.java index 946a74db39..bd61bb5aa5 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/JsonEmitter.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/JsonEmitter.java @@ -112,10 +112,10 @@ public ByteBuf recycleByteBufRetained() { } } - private JsonGenerator jsonGenerator; - private ChunkingByteBufOutputStream outputStream; - private ObjectMapper objectMapper; - private Deque cursorStack; + private final JsonGenerator jsonGenerator; + private final ChunkingByteBufOutputStream outputStream; + private final ObjectMapper objectMapper; + private final Deque> cursorStack; @SneakyThrows public JsonEmitter(ByteBufAllocator byteBufAllocator) { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java index 1a729cf9f5..0dabe67924 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java @@ -20,7 +20,7 @@ * the paylaod (unzip, parse, etc). If a transform DOES require the payload to be present, get() * */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) @Slf4j public class PayloadAccessFaultingMap extends AbstractMap { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/ListKeyAdaptingCaseInsensitiveHeadersMap.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/ListKeyAdaptingCaseInsensitiveHeadersMap.java index 0e470b38dc..1406a51338 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/ListKeyAdaptingCaseInsensitiveHeadersMap.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/ListKeyAdaptingCaseInsensitiveHeadersMap.java @@ -16,7 +16,7 @@ * This is a kludge to provide that. Note that this code doesn't do conversions such as joining * or splitting. If more control is required, callers should use the multimap interfaces. */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) public class ListKeyAdaptingCaseInsensitiveHeadersMap extends AbstractMap { protected final StrictCaseInsensitiveHttpHeadersMap strictHeadersMap; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java index 5427f6393d..9f6f60c3dc 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryConvertHandler.java @@ -90,7 +90,7 @@ private static HttpJsonMessageWithFaultingPayload transform(IJsonTransformer tra var returnedObject = transformer.transformJson(httpJsonMessage); if (returnedObject != httpJsonMessage) { httpJsonMessage.clear(); - httpJsonMessage = new HttpJsonMessageWithFaultingPayload((Map)returnedObject); + httpJsonMessage = new HttpJsonMessageWithFaultingPayload(returnedObject); } return httpJsonMessage; } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyConvertHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyConvertHandler.java index fb3ddd58c6..d5d7c920c2 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyConvertHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyConvertHandler.java @@ -4,8 +4,6 @@ import io.netty.channel.ChannelInboundHandlerAdapter; import org.opensearch.migrations.transform.IJsonTransformer; -import java.util.Map; - public class NettyJsonBodyConvertHandler extends ChannelInboundHandlerAdapter { private final IJsonTransformer transformer; @@ -17,7 +15,7 @@ public NettyJsonBodyConvertHandler(IJsonTransformer transformer) { public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { if (msg instanceof HttpJsonMessageWithFaultingPayload) { var output = transformer.transformJson((HttpJsonMessageWithFaultingPayload)msg); - var newHttpJson = new HttpJsonMessageWithFaultingPayload(((Map)output)); + var newHttpJson = new HttpJsonMessageWithFaultingPayload(output); ctx.fireChannelRead(newHttpJson); } else { super.channelRead(ctx, msg); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettySendByteBufsToPacketHandlerHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettySendByteBufsToPacketHandlerHandler.java index 7cdd557596..a0f405c8e3 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettySendByteBufsToPacketHandlerHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettySendByteBufsToPacketHandlerHandler.java @@ -66,22 +66,9 @@ public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { "expected in progress Boolean to be not null since null should signal that work was never started"; var transformationStatus = v1.booleanValue() ? HttpRequestTransformationStatus.COMPLETED : HttpRequestTransformationStatus.ERROR; - return packetReceiver.finalizeRequest().getDeferredFutureThroughHandle((v2, t2) -> { - if (t1 != null) { - return StringTrackableCompletableFuture.>failedFuture(t1, - ()->"fixed failure from currentFuture.getDeferredFutureThroughHandle()"); - } else if (t2 != null) { - return StringTrackableCompletableFuture.>failedFuture(t2, - ()->"fixed failure from packetReceiver.finalizeRequest()"); - } else { - return StringTrackableCompletableFuture.completedFuture(Optional.ofNullable(v2) - .map(r-> new TransformedOutputAndResult(r, transformationStatus, - null)) - .orElse(null), - ()->"fixed value from packetReceiver.finalizeRequest()" - ); - } - }, + return packetReceiver.finalizeRequest().getDeferredFutureThroughHandle((v2, t2) -> + wrapFinalizedResultWithExceptionHandling(t1, v2, t2, + transformationStatus), ()->"handlerRemoved: NettySendByteBufsToPacketHandlerHandler is setting the completed value for its " + "packetReceiverCompletionFuture, after the packets have been finalized " + "to the packetReceiver"); @@ -96,6 +83,25 @@ public void handlerRemoved(ChannelHandlerContext ctx) throws Exception { super.handlerRemoved(ctx); } + private static StringTrackableCompletableFuture> + wrapFinalizedResultWithExceptionHandling(Throwable t1, R v2, Throwable t2, + HttpRequestTransformationStatus transformationStatus) { + if (t1 != null) { + return StringTrackableCompletableFuture.>failedFuture(t1, + () -> "fixed failure from currentFuture.getDeferredFutureThroughHandle()"); + } else if (t2 != null) { + return StringTrackableCompletableFuture.>failedFuture(t2, + () -> "fixed failure from packetReceiver.finalizeRequest()"); + } else { + return StringTrackableCompletableFuture.completedFuture(Optional.ofNullable(v2) + .map(r -> new TransformedOutputAndResult(r, transformationStatus, + null)) + .orElse(null), + () -> "fixed value from packetReceiver.finalizeRequest()" + ); + } + } + public DiagnosticTrackableCompletableFuture> getPacketReceiverCompletionFuture() { assert packetReceiverCompletionFutureRef.get() != null : diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/StrictCaseInsensitiveHttpHeadersMap.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/StrictCaseInsensitiveHttpHeadersMap.java index 80b90cf167..dc41c74cab 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/StrictCaseInsensitiveHttpHeadersMap.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/StrictCaseInsensitiveHttpHeadersMap.java @@ -15,7 +15,7 @@ * so that we can maintain that original order. However, we add an extra ability to keep key values * (or header names) case insensitive. */ -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) public class StrictCaseInsensitiveHttpHeadersMap extends AbstractMap> { protected LinkedHashMap>> lowerCaseToUpperCaseAndValueMap; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/PojoTrafficStreamKey.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/PojoTrafficStreamKey.java index 354c27b31b..076f2f9ba8 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/PojoTrafficStreamKey.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/PojoTrafficStreamKey.java @@ -4,8 +4,6 @@ import org.opensearch.migrations.trafficcapture.protos.TrafficStream; import org.opensearch.migrations.trafficcapture.protos.TrafficStreamUtils; -import java.util.StringJoiner; - @ToString public class PojoTrafficStreamKey implements ITrafficStreamKey { private final String nodeId; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/RawPackets.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/RawPackets.java index 19b95c0b9a..e703fd6e32 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/RawPackets.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/RawPackets.java @@ -4,6 +4,6 @@ import java.util.ArrayList; -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = true) public class RawPackets extends ArrayList { } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueReplayerRequestKey.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueReplayerRequestKey.java index 57f1cc55ba..d4cc37b5f9 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueReplayerRequestKey.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/UniqueReplayerRequestKey.java @@ -2,7 +2,7 @@ import lombok.EqualsAndHashCode; -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = true) public class UniqueReplayerRequestKey extends UniqueSourceRequestKey { public final ISourceTrafficChannelKey trafficStreamKey; public final int sourceRequestIndexOffsetAtStartOfAccumulation; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumer.java index 2e2a7220f9..bb61f80e1e 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumer.java @@ -113,7 +113,7 @@ Optional removeAndReturnNewHead(TrafficStreamKeyWithKafkaRecordId kafkaRec return Optional.empty(); } } - }; + } private static final MetricsLogger metricsLogger = new MetricsLogger("KafkaProtobufConsumer"); public static final Duration CONSUMER_POLL_TIMEOUT = Duration.ofSeconds(1); @@ -274,7 +274,7 @@ public void commitTrafficStream(ITrafficStreamKey trafficStreamKey) { newHeadValue = tracker.removeAndReturnNewHead(kafkaTsk); newHeadValue.ifPresent(o -> { if (tracker.isEmpty()) { - partitionToOffsetLifecycleTrackerMap.remove(tracker); + partitionToOffsetLifecycleTrackerMap.remove(p); } nextSetOfCommitsMap.put(new TopicPartition(topic, p), new OffsetAndMetadata(o)); }); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/TrafficStreamWithEmbeddedKey.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/TrafficStreamWithEmbeddedKey.java index a89d2b0466..90cdb95cce 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/TrafficStreamWithEmbeddedKey.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/traffic/source/TrafficStreamWithEmbeddedKey.java @@ -3,7 +3,6 @@ import org.opensearch.migrations.replay.datatypes.ITrafficStreamKey; import org.opensearch.migrations.replay.datatypes.PojoTrafficStreamKey; import org.opensearch.migrations.trafficcapture.protos.TrafficStream; -import org.opensearch.migrations.trafficcapture.protos.TrafficStreamUtils; public class TrafficStreamWithEmbeddedKey implements ITrafficStreamWithKey { public final TrafficStream stream; diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ExpiringTrafficStreamMapConcurrentTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ExpiringTrafficStreamMapConcurrentTest.java deleted file mode 100644 index 2c761704a6..0000000000 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/ExpiringTrafficStreamMapConcurrentTest.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.opensearch.migrations.replay; - -public class ExpiringTrafficStreamMapConcurrentTest { - //@Test - public void testMultipleNodeThreads() { - // TODO - really hammer the ExpiringTrafficStreamMap data structure to test the soundness of - // its concurrency design. - } -} diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java index ac84c6bc74..0a23684964 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/FullTrafficReplayerTest.java @@ -66,8 +66,9 @@ public Consumer get() { @Test public void testSingleStreamWithCloseIsCommitted() throws Throwable { + var random = new Random(1); var httpServer = SimpleNettyHttpServer.makeServer(false, Duration.ofMillis(2), - TestHttpServerContext::makeResponse); + response->TestHttpServerContext.makeResponse(random, response)); var trafficStreamWithJustClose = TrafficStream.newBuilder() .setNodeId(TEST_NODE_ID) .setConnectionId(TEST_CONNECTION_ID) @@ -90,8 +91,9 @@ public void testSingleStreamWithCloseIsCommitted() throws Throwable { }) @Tag("longTest") public void fullTest(int testSize, boolean randomize) throws Throwable { + var random = new Random(1); var httpServer = SimpleNettyHttpServer.makeServer(false, Duration.ofMillis(2), - TestHttpServerContext::makeResponse); + response->TestHttpServerContext.makeResponse(random,response)); var streamAndConsumer = TrafficStreamGenerator.generateStreamAndSumOfItsTransactions(testSize, randomize); var numExpectedRequests = streamAndConsumer.numHttpTransactions; var trafficStreams = streamAndConsumer.stream.collect(Collectors.toList()); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java index b0e28042d2..4d58ad9243 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/HeaderTransformerTest.java @@ -16,6 +16,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; +import java.util.function.IntFunction; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -44,7 +45,7 @@ public void testTransformer() throws Exception { private void runRandomPayloadWithTransformer(HttpJsonTransformingConsumer transformingHandler, AggregatedRawResponse dummyAggregatedResponse, TestCapturePacketToHttpHandler testPacketCapture, - Function makeHeaders) + IntFunction makeHeaders) throws ExecutionException, InterruptedException { Random r = new Random(2); @@ -67,9 +68,9 @@ private void runRandomPayloadWithTransformer(HttpJsonTransformingConsumer"HeaderTransformerTest.runRandomPayloadWithTransformer.assertionCheck"); - finalizationFuture.get(); + Assertions.assertEquals(HttpRequestTransformationStatus.COMPLETED, + aggregatedRawResponse.transformationStatus); + }), ()->"HeaderTransformerTest.runRandomPayloadWithTransformer.assertionCheck").get(); Assertions.assertEquals(TestUtils.resolveReferenceString(referenceStringBuilder, List.of(new AbstractMap.SimpleEntry(SOURCE_CLUSTER_NAME, SILLY_TARGET_CLUSTER_NAME))), testPacketCapture.getCapturedAsString()); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/KafkaRestartingTrafficReplayerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/KafkaRestartingTrafficReplayerTest.java index e8ce0dd438..6f62323157 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/KafkaRestartingTrafficReplayerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/KafkaRestartingTrafficReplayerTest.java @@ -83,8 +83,9 @@ public Consumer get() { }) @Tag("longTest") public void fullTest(int testSize, boolean randomize) throws Throwable { + var random = new Random(1); var httpServer = SimpleNettyHttpServer.makeServer(false, Duration.ofMillis(2), - TestHttpServerContext::makeResponse); + response->TestHttpServerContext.makeResponse(random, response)); var streamAndConsumer = TrafficStreamGenerator.generateStreamAndSumOfItsTransactions(testSize, randomize); var trafficStreams = streamAndConsumer.stream.collect(Collectors.toList()); log.atInfo().setMessage(()->trafficStreams.stream().map(ts-> TrafficStreamUtils.summarizeTrafficStream(ts)) diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/PruferTreeGenerator.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/PruferTreeGenerator.java index f6cc7fa123..3244369563 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/PruferTreeGenerator.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/PruferTreeGenerator.java @@ -25,7 +25,10 @@ public class PruferTreeGenerator { @Builder @ToString - private static class Link { int from; int to; } + private static class Link { + int from; + int to; + } public static class SimpleNode { public final T value; @@ -35,7 +38,7 @@ public SimpleNode(T value) { } List> children; - public boolean hasChildren() { return children != null && children.size() > 0; } + public boolean hasChildren() { return children != null && !children.isEmpty(); } public Stream> getChildren() { return children == null ? Stream.of() : children.stream(); } } @@ -48,7 +51,7 @@ public interface Visitor { void popTreeNode(SimpleNode node); } - public void preOrderVisitTree(SimpleNode treeNode, Visitor visitor) { + public void preOrderVisitTree(SimpleNode treeNode, Visitor visitor) { visitor.pushTreeNode(treeNode); if (treeNode.hasChildren()) { treeNode.children.forEach(child->preOrderVisitTree(child, visitor)); @@ -62,24 +65,22 @@ public SimpleNode makeTree(NodeValueGenerator nodeGenerator, int...pruferS } private SimpleNode getMemoizedNode(TreeMap> nodeMap, - HashSet nodesWithoutParents, + HashSet> nodesWithoutParents, int key, IntFunction valueGenerator) { - var node = nodeMap.get(key); - if (node == null) { - node = new SimpleNode<>(valueGenerator.apply(key)); - nodesWithoutParents.add(node); - nodeMap.put(key, node); - } - return node; + return nodeMap.computeIfAbsent(key, k-> { + var n = new SimpleNode<>(valueGenerator.apply(k)); + nodesWithoutParents.add(n); + return n; + }); } private SimpleNode convertLinksToTree(NodeValueGenerator valueGenerator, List edges) { var nodeMap = new TreeMap>(); - var nodesWithoutParents = new HashSet(); + var nodesWithoutParents = new HashSet>(); edges.stream().forEach(e->{ - var childNode = getMemoizedNode(nodeMap, nodesWithoutParents, e.from, x->valueGenerator.makeValue(x)); - var parent = getMemoizedNode(nodeMap, nodesWithoutParents, e.to, x->valueGenerator.makeValue(x)); + var childNode = getMemoizedNode(nodeMap, nodesWithoutParents, e.from, valueGenerator::makeValue); + var parent = getMemoizedNode(nodeMap, nodesWithoutParents, e.to, valueGenerator::makeValue); if (parent.children == null) { parent.children = new ArrayList<>(); } @@ -103,7 +104,6 @@ private SimpleNode convertLinksToTree(NodeValueGenerator valueGenerator, L * @return */ private List makeLinks(final int[] pruferSequenceArray) { - //assert Arrays.stream(pruferSequenceArray).allMatch(x -> x edges = new ArrayList<>(); // ids are sequential integers. We will map them to something else outside of this function. @@ -150,7 +150,7 @@ private void removeLinkForNode(int parent, Map nodeDegrees, Prior } private static String arrayAsString(Map nodeDegrees) { - return nodeDegrees.entrySet().stream().map(kvp -> kvp.toString()) + return nodeDegrees.entrySet().stream().map(Object::toString) .collect(Collectors.joining(",")); } diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/GenerateRandomNestedJsonObject.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/GenerateRandomNestedJsonObject.java index 39fdb8a638..90cc0799bf 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/GenerateRandomNestedJsonObject.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/GenerateRandomNestedJsonObject.java @@ -40,10 +40,10 @@ public String getRandomTreeFormattedAsString(boolean pretty, int seed, int numNo @SneakyThrows public static Object makeRandomJsonObject(Random random, int numNodes, int numArrays) { assert numArrays < numNodes; - PruferTreeGenerator ptg = new PruferTreeGenerator(); + var ptg = new PruferTreeGenerator(); var edges = IntStream.range(0, numNodes-3).map(x->random.nextInt(numNodes-1)+1).sorted().toArray(); var tree = ptg.makeTree(vn -> Integer.toString(vn), edges); - PriorityQueue,String>> parentAndBiggestChildPQ = + PriorityQueue,String>> parentAndBiggestChildPQ = new PriorityQueue<>(Comparator.comparingInt(kvp-> -1 * getSize(kvp.getKey().get(kvp.getValue())) )); @@ -57,7 +57,7 @@ private static int getSize(Object o) { } private static void replaceTopItemsForArrays(int numArrays, - PriorityQueue,String>> + PriorityQueue,String>> parentAndBiggestChildPQ) { for (int i=0; i treeNode, - PriorityQueue,String>> + PriorityQueue,String>> parentAndBiggestChildPQ) { if (treeNode.hasChildren()) { var myMap = new LinkedHashMap(); diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/SampleContents.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/SampleContents.java index 87e53caef1..c3eca7141c 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/SampleContents.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/SampleContents.java @@ -1,12 +1,15 @@ package org.opensearch.migrations.replay; +import java.io.IOException; import java.nio.charset.StandardCharsets; public class SampleContents { public static final String SAMPLE_REQUEST_MESSAGE_JSON_RESOURCE_NAME = "/sampleRequestMessage.json"; - public static String loadSampleJsonRequestAsString() throws Exception { + private SampleContents() {} + + public static String loadSampleJsonRequestAsString() throws IOException { try (var inputStream = SampleContents.class.getResourceAsStream(SAMPLE_REQUEST_MESSAGE_JSON_RESOURCE_NAME)) { return new String(inputStream.readAllBytes(), StandardCharsets.UTF_8); diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java index c24c61bb2c..a0ff5a8aef 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestCapturePacketToHttpHandler.java @@ -38,7 +38,7 @@ public TestCapturePacketToHttpHandler(Duration consumeDuration, public DiagnosticTrackableCompletableFuture consumeBytes(ByteBuf nextRequestPacket) { log.info("incoming buffer refcnt="+nextRequestPacket.refCnt()); var duplicatedPacket = nextRequestPacket.duplicate().retain(); - return new DiagnosticTrackableCompletableFuture(CompletableFuture.runAsync(() -> { + return new DiagnosticTrackableCompletableFuture<>(CompletableFuture.runAsync(() -> { try { log.info("Running async future for " + nextRequestPacket); Thread.sleep(consumeDuration.toMillis()); diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestHttpServerContext.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestHttpServerContext.java index e30123840e..7291f4421d 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestHttpServerContext.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestHttpServerContext.java @@ -7,12 +7,15 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Map; +import java.util.Random; public class TestHttpServerContext { + private TestHttpServerContext() {} + public static final int MAX_RESPONSE_TIME_MS = 250; - public static String SERVER_RESPONSE_BODY_PREFIX = "Boring Response to "; + public static final String SERVER_RESPONSE_BODY_PREFIX = "Boring Response to "; static String getUriForIthRequest(int i) { return String.format("/%04d", i); @@ -28,8 +31,8 @@ static String getRequestString(int i) { getUriForIthRequest(i)); } - public static SimpleHttpResponse makeResponse(HttpFirstLine r) { - return makeResponse(r, Duration.ofMillis((int)(Math.random()* MAX_RESPONSE_TIME_MS))); + public static SimpleHttpResponse makeResponse(Random rand, HttpFirstLine response) { + return makeResponse(response, Duration.ofMillis(rand.nextInt(MAX_RESPONSE_TIME_MS))); } public static SimpleHttpResponse makeResponse(HttpFirstLine r, Duration responseWaitTime) { diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java index 4b8c6dca59..5751cb4aa6 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestRequestKey.java @@ -4,7 +4,10 @@ import org.opensearch.migrations.replay.datatypes.UniqueReplayerRequestKey; public class TestRequestKey { - public final static UniqueReplayerRequestKey getTestConnectionRequestId(int replayerIdx) { + + private TestRequestKey() {} + + public static final UniqueReplayerRequestKey getTestConnectionRequestId(int replayerIdx) { return new UniqueReplayerRequestKey( new PojoTrafficStreamKey("testNodeId", "testConnectionId", 0), 0, replayerIdx); diff --git a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java index c41d1e06b3..77322ffa66 100644 --- a/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java +++ b/TrafficCapture/trafficReplayer/src/testFixtures/java/org/opensearch/migrations/replay/TestUtils.java @@ -26,13 +26,17 @@ import java.util.Collection; import java.util.List; import java.util.Random; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +import java.util.function.IntFunction; @Slf4j public class TestUtils { + private TestUtils() {} + static String resolveReferenceString(StringBuilder referenceStringBuilder) { return resolveReferenceString(referenceStringBuilder, List.of()); } @@ -74,7 +78,7 @@ static DiagnosticTrackableCompletableFuture chainedWriteHeadersAndD chainedDualWriteHeaderAndPayloadParts(IPacketConsumer packetConsumer, List stringParts, StringBuilder referenceStringAccumulator, - Function headersGenerator) { + IntFunction headersGenerator) { var contentLength = stringParts.stream().mapToInt(s->s.length()).sum(); String headers = headersGenerator.apply(contentLength) + "\r\n"; referenceStringAccumulator.append(headers); @@ -120,7 +124,8 @@ static void runPipelineAndValidate(IAuthTransformerFactory authTransformer, String extraHeaders, List stringParts, DefaultHttpHeaders expectedRequestHeaders, - Function expectedOutputGenerator) throws Exception { + Function expectedOutputGenerator) + throws IOException, ExecutionException, InterruptedException { runPipelineAndValidate(x -> x, authTransformer, extraHeaders, stringParts, expectedRequestHeaders, expectedOutputGenerator); } @@ -130,13 +135,15 @@ static void runPipelineAndValidate(IJsonTransformer transformer, String extraHeaders, List stringParts, DefaultHttpHeaders expectedRequestHeaders, - Function expectedOutputGenerator) throws Exception { + Function expectedOutputGenerator) + throws IOException, ExecutionException, InterruptedException + { var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), new AggregatedRawResponse(-1, Duration.ZERO, new ArrayList<>(), null)); - var transformingHandler = new HttpJsonTransformingConsumer(transformer, authTransformer, testPacketCapture, + var transformingHandler = new HttpJsonTransformingConsumer<>(transformer, authTransformer, testPacketCapture, "TEST", TestRequestKey.getTestConnectionRequestId(0)); - var contentLength = stringParts.stream().mapToInt(s->s.length()).sum(); + var contentLength = stringParts.stream().mapToInt(String::length).sum(); var headerString = "GET / HTTP/1.1\r\n" + "host: localhost\r\n" + (extraHeaders == null ? "" : extraHeaders) + @@ -146,18 +153,19 @@ static void runPipelineAndValidate(IJsonTransformer transformer, stringParts, referenceStringBuilder, headerString); var innermostFinalizeCallCount = new AtomicInteger(); - DiagnosticTrackableCompletableFuture finalizationFuture = + var finalizationFuture = allConsumesFuture.thenCompose(v -> transformingHandler.finalizeRequest(), ()->"PayloadRepackingTest.runPipelineAndValidate.allConsumeFuture"); finalizationFuture.map(f->f.whenComplete((aggregatedRawResponse, t) -> { - Assertions.assertNull(t); - Assertions.assertNotNull(aggregatedRawResponse); - // do nothing but check connectivity between the layers in the bottom most handler - innermostFinalizeCallCount.incrementAndGet(); - }), ()->"PayloadRepackingTest.runPipelineAndValidate.assertCheck"); - finalizationFuture.get(); + Assertions.assertNull(t); + Assertions.assertNotNull(aggregatedRawResponse); + // do nothing but check connectivity between the layers in the bottom most handler + innermostFinalizeCallCount.incrementAndGet(); + }), ()->"PayloadRepackingTest.runPipelineAndValidate.assertCheck") + .get(); verifyCapturedResponseMatchesExpectedPayload(testPacketCapture.getBytesCaptured(), expectedRequestHeaders, expectedOutputGenerator.apply(referenceStringBuilder)); + Assertions.assertEquals(1, innermostFinalizeCallCount.get()); } } From d8e7c4136fe226b904b1824c3e5810c715252cdb Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Sat, 11 Nov 2023 18:13:41 -0500 Subject: [PATCH 48/55] Upgrade all projects to use netty 4.1.100 as per CVE See https://www.mend.io/vulnerability-database/CVE-2023-44487 Signed-off-by: Greg Schohn --- TrafficCapture/captureKafkaOffloader/build.gradle | 2 +- TrafficCapture/captureOffloader/build.gradle | 2 +- .../jsonJoltMessageTransformerProvider/build.gradle | 2 +- TrafficCapture/trafficReplayer/build.gradle | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/TrafficCapture/captureKafkaOffloader/build.gradle b/TrafficCapture/captureKafkaOffloader/build.gradle index 063cec0c81..147eec8dcb 100644 --- a/TrafficCapture/captureKafkaOffloader/build.gradle +++ b/TrafficCapture/captureKafkaOffloader/build.gradle @@ -9,7 +9,7 @@ repositories { } dependencies { - api 'io.netty:netty-buffer:4.1.89.Final' + api 'io.netty:netty-buffer:4.1.100.Final' implementation project(':captureOffloader') implementation project(':coreUtilities') implementation 'org.projectlombok:lombok:1.18.26' diff --git a/TrafficCapture/captureOffloader/build.gradle b/TrafficCapture/captureOffloader/build.gradle index 05cd9e96a4..deabdf82d1 100644 --- a/TrafficCapture/captureOffloader/build.gradle +++ b/TrafficCapture/captureOffloader/build.gradle @@ -21,7 +21,7 @@ sourceSets { } } dependencies { - api group: 'io.netty', name: 'netty-buffer', version: '4.1.89.Final' + api group: 'io.netty', name: 'netty-buffer', version: '4.1.100.Final' implementation project(':captureProtobufs') implementation "com.google.protobuf:protobuf-java:3.22.2" diff --git a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/build.gradle b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/build.gradle index 73bc1b03d3..a974d3aea5 100644 --- a/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/build.gradle +++ b/TrafficCapture/replayerPlugins/jsonMessageTransformers/jsonJoltMessageTransformerProvider/build.gradle @@ -18,7 +18,7 @@ dependencies { testImplementation testFixtures(project(path: ':trafficReplayer')) testImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.0' - testImplementation group: 'io.netty', name: 'netty-all', version: '4.1.94.Final' + testImplementation group: 'io.netty', name: 'netty-all', version: '4.1.100.Final' testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-api', version:'5.9.3' testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-params', version:'5.9.3' testImplementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7' diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle index 2b78e428bb..7af608c7cf 100644 --- a/TrafficCapture/trafficReplayer/build.gradle +++ b/TrafficCapture/trafficReplayer/build.gradle @@ -66,7 +66,7 @@ dependencies { testFixturesImplementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7' testFixturesImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.0' - testFixturesImplementation group: 'io.netty', name: 'netty-all', version: '4.1.94.Final' + testFixturesImplementation group: 'io.netty', name: 'netty-all', version: '4.1.100.Final' testFixturesImplementation group: 'org.junit.jupiter', name:'junit-jupiter-api', version:'5.9.3' From a50704cc2bb083601b540011b0be7a7542427dfa Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Sun, 12 Nov 2023 07:30:02 -0500 Subject: [PATCH 49/55] PR feedback Signed-off-by: Greg Schohn --- .../migrations/replay/datatypes/TransformedPackets.java | 2 +- .../migrations/replay/kafka/KafkaProtobufConsumer.java | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/TransformedPackets.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/TransformedPackets.java index ecda9bd3d9..58bfa3291a 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/TransformedPackets.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datatypes/TransformedPackets.java @@ -46,7 +46,7 @@ public void close() { @Override public String toString() { - if (data == null) { + if (isClosed()) { return "CLOSED"; } return new StringJoiner(", ", TransformedPackets.class.getSimpleName() + "[", "]") diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumer.java index 8f94ffdaa5..b19d794b9f 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/KafkaProtobufConsumer.java @@ -49,8 +49,6 @@ * after the hour of waiting begins, Kakfa will notice that this application is no * longer calling poll and will kick the consumer out of the client group. Other * consumers may connect, though they'll also be kicked out of the group shortly. - * It may be fine to have this pattern of cascaded replayers waiting to send a - * request, then all of them sending at about the same time.... * * See * ... From f8920c7db08fbe7bf887307f6f873d1e2c5c7b8b Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Sun, 12 Nov 2023 10:57:06 -0600 Subject: [PATCH 50/55] Remove Comparator and Jupyter since it will be replaced by OTEL and Analytics Engine Signed-off-by: Brian Presley --- TrafficCapture/dockerSolution/README.md | 10 --- TrafficCapture/dockerSolution/build.gradle | 64 +-------------- .../src/main/docker/docker-compose.yml | 31 +------ TrafficCapture/trafficReplayer/README.md | 2 +- .../http/HttpJsonTransformingConsumer.java | 4 +- .../opensearch-service-migration/README.md | 2 +- .../lib/migration-assistance-stack.ts | 25 ------ .../traffic-comparator-jupyter-stack.ts | 66 --------------- .../traffic-comparator-stack.ts | 80 ------------------- .../lib/stack-composer.ts | 34 -------- docs/Architecture.md | 20 +---- test/README.md | 3 +- test/tests.py | 6 -- 13 files changed, 11 insertions(+), 336 deletions(-) delete mode 100644 deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-comparator-jupyter-stack.ts delete mode 100644 deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-comparator-stack.ts diff --git a/TrafficCapture/dockerSolution/README.md b/TrafficCapture/dockerSolution/README.md index 826669a721..6d2dc19674 100644 --- a/TrafficCapture/dockerSolution/README.md +++ b/TrafficCapture/dockerSolution/README.md @@ -8,16 +8,6 @@ down again. Notice that most of the Dockerfiles are dynamically constructed in the build hierarchy. Some efforts have been made to ensure that changes will make it into containers to be launched. -If a user wants to use their own checkout of the traffic-comparator repo, just set the environment variable " -TRAFFIC_COMPARATOR_DIRECTORY" to the directory that contains `setup.py`. Otherwise, if that isn't set, the traffic -comparator repo will be checked out to the build directory and that will be used. Notice that the checkout happens when -the directory wasn't present and there wasn't an environment variable specifying a directory. Once a directory exists, -it will be mounted to the traffic-comparator and jupyter services. - -Netcat is still used to connect several of the components and we're still working on improving the resiliency story -between these containers. The long term approach is to replace fixed streams with message bus approaches directly (i.e. -Kafka). In the short term, we can and are beginning, to leverage things like conditions on dependent services. - ### Running the Docker Solution While in the TrafficCapture directory, run the following command: diff --git a/TrafficCapture/dockerSolution/build.gradle b/TrafficCapture/dockerSolution/build.gradle index b5f486edbb..e37bbc4445 100644 --- a/TrafficCapture/dockerSolution/build.gradle +++ b/TrafficCapture/dockerSolution/build.gradle @@ -9,19 +9,6 @@ import java.security.MessageDigest import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage import org.apache.tools.ant.taskdefs.condition.Os -def getTrafficComparatorDirectory() { - String overrideTrafficComparatorDirectory = System.getenv(TRAFFIC_COMPARATOR_DIRECTORY_ENV) - String rval = overrideTrafficComparatorDirectory != null ? - overrideTrafficComparatorDirectory : TRAFFIC_COMPARATOR_REPO_DIRECTORY; - return rval -} - -ext { - TRAFFIC_COMPARATOR_REPO_DIRECTORY = "build/traffic-comparator" - TRAFFIC_COMPARATOR_DIRECTORY_ENV = "TRAFFIC_COMPARATOR_DIRECTORY" - REALIZED_TRAFFIC_COMPARATOR_DIRECTORY = project.file(getTrafficComparatorDirectory()) - } - def calculateDockerHash = { projectName -> CommonUtils.calculateDockerHash(projectName, project) } @@ -31,17 +18,6 @@ dependencies { implementation project(':trafficReplayer') } -task cloneComparatorRepoIfNeeded(type: Exec) { - String comparatorDirectory = project.file(REALIZED_TRAFFIC_COMPARATOR_DIRECTORY); - String repo = 'https://github.com/opensearch-project/traffic-comparator.git' - onlyIf { - !(new File(comparatorDirectory).exists()) - } - commandLine = Os.isFamily(Os.FAMILY_WINDOWS) ? - ['git', 'clone', repo, TRAFFIC_COMPARATOR_REPO_DIRECTORY ] : - ['/bin/sh', '-c', "git clone ${repo} ${TRAFFIC_COMPARATOR_REPO_DIRECTORY}"] -} - def dockerFilesForExternalServices = [ "elasticsearchWithSearchGuard": "elasticsearch_searchguard", "migrationConsole": "migration_console" @@ -56,36 +32,6 @@ dockerFilesForExternalServices.each { projectName, dockerImageName -> } } -def trafficComparatorServices = [ - "trafficComparator": "traffic_comparator", - "jupyterNotebook": "jupyter_notebook" -] -trafficComparatorServices.forEach {projectName, dockerImageName -> - def dockerBuildDir = "build/docker/${projectName}" - task("copyArtifact_${projectName}", type: Copy) { - dependsOn(tasks.getByName('cloneComparatorRepoIfNeeded')) - from REALIZED_TRAFFIC_COMPARATOR_DIRECTORY - into dockerBuildDir - include '*.py' - include '/traffic_comparator/*' - if (projectName == 'jupyterNotebook') { - include '*.ipynb' - } - } - - task "createDockerfile_${projectName}"(type: com.bmuschko.gradle.docker.tasks.image.Dockerfile) { - dependsOn "copyArtifact_${projectName}" - destFile = project.file("${dockerBuildDir}/Dockerfile") - from 'python:3.10.10' - runCommand("apt-get update && apt-get install -y netcat lsof") - copyFile("setup.py", "/setup.py") - copyFile(".", "/containerTC/") - runCommand("pip3 install --editable \".[data]\"") - // container stay-alive - defaultCommand('tail', '-f', '/dev/null') - } -} - def javaContainerServices = [ "trafficCaptureProxyServer": "capture_proxy", "trafficReplayer": "traffic_replayer" @@ -101,7 +47,7 @@ javaContainerServices.each { projectName, dockerImageName -> CommonUtils.createDockerfile(project, projectName, baseImageProjectOverrides, dockerFilesForExternalServices) } -(javaContainerServices + trafficComparatorServices).forEach { projectName, dockerImageName -> +(javaContainerServices).forEach { projectName, dockerImageName -> def dockerBuildDir = "build/docker/${projectName}" task "buildDockerImage_${projectName}"(type: DockerBuildImage) { dependsOn "createDockerfile_${projectName}" @@ -112,11 +58,6 @@ javaContainerServices.each { projectName, dockerImageName -> } dockerCompose { - String overrideTrafficComparatorDirectory = System.getenv(TRAFFIC_COMPARATOR_DIRECTORY_ENV) - if (overrideTrafficComparatorDirectory == null) { - environment.put(TRAFFIC_COMPARATOR_DIRECTORY_ENV, REALIZED_TRAFFIC_COMPARATOR_DIRECTORY) - exposeAsEnvironment(this) - } useComposeFiles = project.hasProperty('multiProxy') ? ['src/main/docker/docker-compose.yml', 'src/main/docker/docker-compose-multi.yml'] : ['src/main/docker/docker-compose.yml', 'src/main/docker/docker-compose-single.yml'] @@ -128,10 +69,7 @@ task buildDockerImages { dependsOn buildDockerImage_trafficCaptureProxyServer dependsOn buildDockerImage_trafficReplayer - dependsOn buildDockerImage_trafficComparator - dependsOn buildDockerImage_jupyterNotebook } tasks.getByName('composeUp') .dependsOn(tasks.getByName('buildDockerImages')) - .dependsOn(tasks.getByName('cloneComparatorRepoIfNeeded')) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index b2920b0c74..85604f624f 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -43,9 +43,7 @@ services: condition: service_started opensearchtarget: condition: service_started - trafficcomparator: - condition: service_healthy - command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group | nc trafficcomparator 9220" + command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group" opensearchtarget: image: 'opensearchproject/opensearch:latest' @@ -56,33 +54,6 @@ services: ports: - "29200:9200" - trafficcomparator: - image: 'migrations/traffic_comparator:latest' - networks: - - migrations - ports: - - "9220:9220" - healthcheck: - test: "lsof -i -P -n" - volumes: - - ${TRAFFIC_COMPARATOR_DIRECTORY}:/trafficComparator - - sharedComparatorSqlResults:/shared - command: /bin/sh -c "cd trafficComparator && pip3 install --editable . && nc -v -l -p 9220 | tee /dev/stderr | trafficcomparator -vv stream | trafficcomparator dump-to-sqlite --db /shared/comparisons.db" - - jupyter-notebook: - image: 'migrations/jupyter_notebook:latest' - networks: - - migrations - ports: - - "8888:8888" - volumes: - - ${TRAFFIC_COMPARATOR_DIRECTORY}:/trafficComparator - - sharedComparatorSqlResults:/shared - environment: - # this needs to match the output db that traffic_comparator writes to - - COMPARISONS_DB_LOCATION=/shared/comparisons.db - command: /bin/sh -c 'cd trafficComparator && pip3 install --editable ".[data]" && jupyter notebook --ip=0.0.0.0 --port=8888 --no-browser --allow-root' - migration-console: image: 'migrations/migration_console:latest' networks: diff --git a/TrafficCapture/trafficReplayer/README.md b/TrafficCapture/trafficReplayer/README.md index 8921dfed03..3f19783bab 100644 --- a/TrafficCapture/trafficReplayer/README.md +++ b/TrafficCapture/trafficReplayer/README.md @@ -1,7 +1,7 @@ # HTTP Traffic Replayer This package consumes streams of IP packets that were previously recorded and replays the requests to another HTTP -server, recording the packet traffic of the new interactions for future analysis (see the Comparator tools). +server, recording the packet traffic of the new interactions for future analysis. ## Overview diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java index 167abf8eba..f72ca74315 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java @@ -29,7 +29,7 @@ * transformation and the headers present (such as for gzipped or chunked encodings). * * There will be a number of reasons that we need to reuse the source captured packets - both today - * (for the output to the comparator) and in the future (for retrying transient network errors or + * (for analysing and comparing) and in the future (for retrying transient network errors or * transformation errors). With that in mind, the HttpJsonTransformer now keeps track of all of * the ByteBufs passed into it and can redrive them through the underlying network packet handler. * Cases where that would happen with this edit are where the payload wasn't being modified, nor @@ -58,7 +58,7 @@ public class HttpJsonTransformingConsumer implements IPacketFinalizingConsume private final List> chunkSizes; // This is here for recovery, in case anything goes wrong with a transformation & we want to // just dump it directly. Notice that we're already storing all of the bytes until the response - // comes back so that we can format the output that goes to the comparator. These should be + // comes back so that we can format the output. These should be // backed by the exact same byte[] arrays, so the memory consumption should already be absorbed. private final List chunks; diff --git a/deployment/cdk/opensearch-service-migration/README.md b/deployment/cdk/opensearch-service-migration/README.md index d69d1a6061..4fbab7f99a 100644 --- a/deployment/cdk/opensearch-service-migration/README.md +++ b/deployment/cdk/opensearch-service-migration/README.md @@ -9,7 +9,7 @@ This directory contains the infrastructure-as-code CDK solution for deploying an ###### Docker Docker is used by CDK to build container images. If not installed, follow the steps [here](https://docs.docker.com/engine/install/) to set up. Later versions are recommended. ###### Git -Git is used by the opensearch-migrations repo to fetch associated repositories (such as the traffic-comparator repo) for constructing their respective Dockerfiles. Steps to set up can be found [here](https://github.com/git-guides/install-git). +Git is used by the opensearch-migrations repo to fetch associated repositories. Steps to set up can be found [here](https://github.com/git-guides/install-git). ###### Java 11 Java is used by the opensearch-migrations repo and Gradle, its associated build tool. The current required version is Java 11. diff --git a/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts b/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts index bf8a1963d7..a7db241ae6 100644 --- a/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts @@ -11,7 +11,6 @@ import {StringParameter} from "aws-cdk-lib/aws-ssm"; export interface migrationStackProps extends StackPropsExt { readonly vpc: IVpc, - readonly trafficComparatorEnabled: boolean, // Future support needed to allow importing an existing MSK cluster readonly mskImportARN?: string, readonly mskEnablePublicEndpoints?: boolean @@ -108,30 +107,6 @@ export class MigrationAssistanceStack extends Stack { stringValue: mskCluster.clusterName }); - if (props.trafficComparatorEnabled) { - const comparatorSQLiteSG = new SecurityGroup(this, 'comparatorSQLiteSG', { - vpc: props.vpc, - allowAllOutbound: false, - }); - comparatorSQLiteSG.addIngressRule(comparatorSQLiteSG, Port.allTraffic()); - new StringParameter(this, 'SSMParameterComparatorSQLAccessGroupId', { - description: 'OpenSearch migration parameter for Comparator SQL volume access security group id', - parameterName: `/migration/${props.stage}/${props.defaultDeployId}/comparatorSQLAccessSecurityGroupId`, - stringValue: comparatorSQLiteSG.securityGroupId - }); - - // Create an EFS file system for the traffic-comparator - const comparatorSQLiteEFS = new FileSystem(this, 'comparatorSQLiteEFS', { - vpc: props.vpc, - securityGroup: comparatorSQLiteSG - }); - new StringParameter(this, 'SSMParameterComparatorSQLVolumeEFSId', { - description: 'OpenSearch migration parameter for Comparator SQL EFS filesystem id', - parameterName: `/migration/${props.stage}/${props.defaultDeployId}/comparatorSQLVolumeEFSId`, - stringValue: comparatorSQLiteEFS.fileSystemId - }); - } - const replayerOutputSG = new SecurityGroup(this, 'replayerOutputSG', { vpc: props.vpc, allowAllOutbound: false, diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-comparator-jupyter-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-comparator-jupyter-stack.ts deleted file mode 100644 index 9e80cf46b2..0000000000 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-comparator-jupyter-stack.ts +++ /dev/null @@ -1,66 +0,0 @@ -import {StackPropsExt} from "../stack-composer"; -import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {MountPoint, Volume} from "aws-cdk-lib/aws-ecs"; -import {Construct} from "constructs"; -import {join} from "path"; -import {MigrationServiceCore} from "./migration-service-core"; -import {StringParameter} from "aws-cdk-lib/aws-ssm"; -import {Effect, PolicyStatement} from "aws-cdk-lib/aws-iam"; - - -export interface TrafficComparatorJupyterProps extends StackPropsExt { - readonly vpc: IVpc, -} - -/** - * Note: This stack is experimental and should only be used for development - * TODO: Add additional infrastructure to setup a public endpoint for this service - */ -export class TrafficComparatorJupyterStack extends MigrationServiceCore { - - constructor(scope: Construct, id: string, props: TrafficComparatorJupyterProps) { - super(scope, id, props) - let securityGroups = [ - SecurityGroup.fromSecurityGroupId(this, "serviceConnectSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceConnectSecurityGroupId`)), - SecurityGroup.fromSecurityGroupId(this, "comparatorSQLAccessSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/comparatorSQLAccessSecurityGroupId`)) - ] - - const volumeName = "sharedComparatorSQLVolume" - const volumeId = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/comparatorSQLVolumeEFSId`) - const comparatorSQLVolume: Volume = { - name: volumeName, - efsVolumeConfiguration: { - fileSystemId: volumeId, - transitEncryption: "ENABLED" - } - }; - const comparatorSQLMountPoint: MountPoint = { - containerPath: "/shared", - readOnly: false, - sourceVolume: volumeName - } - const comparatorEFSArn = `arn:aws:elasticfilesystem:${props.env?.region}:${props.env?.account}:file-system/${volumeId}` - const comparatorEFSMountPolicy = new PolicyStatement( { - effect: Effect.ALLOW, - resources: [comparatorEFSArn], - actions: [ - "elasticfilesystem:ClientMount", - "elasticfilesystem:ClientWrite" - ] - }) - - this.createService({ - serviceName: "traffic-comparator-jupyter", - dockerFilePath: join(__dirname, "../../../../../", "TrafficCapture/dockerSolution/build/docker/jupyterNotebook"), - dockerImageCommand: ['/bin/sh', '-c', 'cd containerTC && pip3 install --editable ".[data]" && jupyter notebook --ip=0.0.0.0 --port=8888 --no-browser --allow-root'], - securityGroups: securityGroups, - volumes: [comparatorSQLVolume], - mountPoints: [comparatorSQLMountPoint], - taskRolePolicies: [comparatorEFSMountPolicy], - taskCpuUnits: 512, - taskMemoryLimitMiB: 2048, - ...props - }); - } - -} \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-comparator-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-comparator-stack.ts deleted file mode 100644 index 86326b43ff..0000000000 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-comparator-stack.ts +++ /dev/null @@ -1,80 +0,0 @@ -import {StackPropsExt} from "../stack-composer"; -import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {MountPoint, PortMapping, Protocol, Volume} from "aws-cdk-lib/aws-ecs"; -import {Construct} from "constructs"; -import {join} from "path"; -import {MigrationServiceCore} from "./migration-service-core"; -import {StringParameter} from "aws-cdk-lib/aws-ssm"; -import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service"; -import {Effect, PolicyStatement} from "aws-cdk-lib/aws-iam"; - - -export interface TrafficComparatorProps extends StackPropsExt { - readonly vpc: IVpc, -} - -/** - * This stack is experimental and should only be used for development - */ -export class TrafficComparatorStack extends MigrationServiceCore { - - constructor(scope: Construct, id: string, props: TrafficComparatorProps) { - super(scope, id, props) - let securityGroups = [ - SecurityGroup.fromSecurityGroupId(this, "serviceConnectSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceConnectSecurityGroupId`)), - SecurityGroup.fromSecurityGroupId(this, "comparatorSQLAccessSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/comparatorSQLAccessSecurityGroupId`)) - ] - - const servicePort: PortMapping = { - name: "traffic-comparator-connect", - hostPort: 9220, - containerPort: 9220, - protocol: Protocol.TCP - } - const serviceConnectService: ServiceConnectService = { - portMappingName: "traffic-comparator-connect", - dnsName: "traffic-comparator", - port: 9220 - } - - const volumeName = "sharedComparatorSQLVolume" - const volumeId = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/comparatorSQLVolumeEFSId`) - const comparatorSQLVolume: Volume = { - name: volumeName, - efsVolumeConfiguration: { - fileSystemId: volumeId, - transitEncryption: "ENABLED" - } - }; - const comparatorSQLMountPoint: MountPoint = { - containerPath: "/shared", - readOnly: false, - sourceVolume: volumeName - } - const comparatorEFSArn = `arn:aws:elasticfilesystem:${props.env?.region}:${props.env?.account}:file-system/${volumeId}` - const comparatorEFSMountPolicy = new PolicyStatement( { - effect: Effect.ALLOW, - resources: [comparatorEFSArn], - actions: [ - "elasticfilesystem:ClientMount", - "elasticfilesystem:ClientWrite" - ] - }) - - this.createService({ - serviceName: "traffic-comparator", - dockerFilePath: join(__dirname, "../../../../../", "TrafficCapture/dockerSolution/build/docker/trafficComparator"), - dockerImageCommand: ['/bin/sh', '-c', 'cd containerTC && pip3 install --editable . && nc -v -l -p 9220 | tee /dev/stderr | trafficcomparator -vv stream | trafficcomparator dump-to-sqlite --db /shared/comparisons.db'], - securityGroups: securityGroups, - volumes: [comparatorSQLVolume], - mountPoints: [comparatorSQLMountPoint], - portMappings: [servicePort], - taskRolePolicies: [comparatorEFSMountPolicy], - serviceConnectServices: [serviceConnectService], - taskCpuUnits: 512, - taskMemoryLimitMiB: 2048, - ...props - }); - } - -} \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts index bf676368ac..bec06f6f38 100644 --- a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts +++ b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts @@ -12,8 +12,6 @@ import {MSKUtilityStack} from "./msk-utility-stack"; import {MigrationConsoleStack} from "./service-stacks/migration-console-stack"; import {CaptureProxyESStack} from "./service-stacks/capture-proxy-es-stack"; import {TrafficReplayerStack} from "./service-stacks/traffic-replayer-stack"; -import {TrafficComparatorStack} from "./service-stacks/traffic-comparator-stack"; -import {TrafficComparatorJupyterStack} from "./service-stacks/traffic-comparator-jupyter-stack"; import {CaptureProxyStack} from "./service-stacks/capture-proxy-stack"; import {ElasticsearchStack} from "./service-stacks/elasticsearch-stack"; import {KafkaBrokerStack} from "./service-stacks/kafka-broker-stack"; @@ -86,8 +84,6 @@ export class StackComposer { const trafficReplayerTargetEndpoint = getContextForType('trafficReplayerTargetEndpoint', 'string') const trafficReplayerGroupId = getContextForType('trafficReplayerGroupId', 'string') const trafficReplayerExtraArgs = getContextForType('trafficReplayerExtraArgs', 'string') - const trafficComparatorServiceEnabled = getContextForType('trafficComparatorServiceEnabled', 'boolean') - const trafficComparatorJupyterServiceEnabled = getContextForType('trafficComparatorJupyterServiceEnabled', 'boolean') const captureProxyServiceEnabled = getContextForType('captureProxyServiceEnabled', 'boolean') const captureProxySourceEndpoint = getContextForType('captureProxySourceEndpoint', 'string') const elasticsearchServiceEnabled = getContextForType('elasticsearchServiceEnabled', 'boolean') @@ -216,7 +212,6 @@ export class StackComposer { if (migrationAssistanceEnabled && networkStack && !addOnMigrationDeployId) { migrationStack = new MigrationAssistanceStack(scope, "migrationInfraStack", { vpc: networkStack.vpc, - trafficComparatorEnabled: trafficComparatorServiceEnabled, mskImportARN: mskARN, mskEnablePublicEndpoints: mskEnablePublicEndpoints, mskBrokerNodeCount: mskBrokerNodeCount, @@ -284,7 +279,6 @@ export class StackComposer { customTargetEndpoint: trafficReplayerTargetEndpoint, customKafkaGroupId: trafficReplayerGroupId, extraArgs: trafficReplayerExtraArgs, - enableComparatorLink: trafficComparatorServiceEnabled, stackName: `OSMigrations-${stage}-${region}-${deployId}-TrafficReplayer`, description: "This stack contains resources for the Traffic Replayer ECS service", stage: stage, @@ -302,34 +296,6 @@ export class StackComposer { this.stacks.push(trafficReplayerStack) } - let trafficComparatorStack - if (trafficComparatorServiceEnabled && networkStack && migrationStack) { - trafficComparatorStack = new TrafficComparatorStack(scope, "traffic-comparator", { - vpc: networkStack.vpc, - stackName: `OSMigrations-${stage}-${region}-TrafficComparator`, - description: "This stack contains resources for the Traffic Comparator ECS service", - stage: stage, - defaultDeployId: defaultDeployId, - ...props, - }) - trafficComparatorStack.addDependency(migrationStack) - this.stacks.push(trafficComparatorStack) - } - - let trafficComparatorJupyterStack - if (trafficComparatorJupyterServiceEnabled && networkStack && trafficComparatorStack) { - trafficComparatorJupyterStack = new TrafficComparatorJupyterStack(scope, "traffic-comparator-jupyter", { - vpc: networkStack.vpc, - stackName: `OSMigrations-${stage}-${region}-TrafficComparatorJupyter`, - description: "This stack contains resources for creating a Jupyter Notebook to perform analysis on Traffic Comparator output as an ECS service", - stage: stage, - defaultDeployId: defaultDeployId, - ...props, - }) - trafficComparatorJupyterStack.addDependency(trafficComparatorStack) - this.stacks.push(trafficComparatorJupyterStack) - } - let elasticsearchStack if (elasticsearchServiceEnabled && networkStack && migrationStack) { elasticsearchStack = new ElasticsearchStack(scope, "elasticsearch", { diff --git a/docs/Architecture.md b/docs/Architecture.md index 167107f832..1e15c4e52f 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -82,14 +82,14 @@ A user is using a search/analytics data store to handle some “mission-critical 1. Can be configured to replay at original rate or accelerated rate. 2. The tool can be distributed to meet whatever replay rate is demanded. 3. Allows for real-time and predetermined management of the replay (pause, stop, restart, speed up, etc). - 4. Response logs can be shared for future processing by the Traffic Comparator (see #7). + 4. Response logs can be shared for future processing by comparison tooling (see #7). 5. The replay sequence for mutating requests (PUT, POST, DELETE, etc) is stable across replays. 6. The replayer transforms source requests into appropriate requests for the target cluster. 7. The replay stream can be edited to allow for overrides - newly inserted requests to backfill missing data, reordering specific events, and allowing for new/custom transformations. 8. The replayer can do different degrees of sampling to reduce time/cost for various types of acid tests (only mutations on this index, no queries of the following pattern are to be rerun). 9. The stream of results coming out of the replayer can be monitored by a control plane. That control plane can be triggered by the progress on the streams, or by a UI to perform chaos testing. The replay pace can be accelerated to put undo load on the cluster or by commands could be issued to the target to create stresses on the cluster (node taken offline, node is undergoing a b/g deployment, node is out of disk space, etc). 1. ***This is a potential integration point/ask of the managed service. A number of users complained about a lack of resiliency in their current clusters. Giving users the ability to degrade their user’s performance in test environments in very well-understood ways would prepare users and lower Amazon’s operational burdens.*** -7. As the results from the replayer for all HTTP requests (including queries and control plane commands) are being logged, a **Traffic Comparator** tool compares the responses between the source requests and target requests. +7. As the results from the replayer for all HTTP requests (including queries and control plane commands) are being logged, comparison tooling compares the responses between the source requests and target requests. 1. Compares status codes, response content & performance data (latencies). 2. Contents are normalized and specific types of differences can be ignored. 1. These normalizations and settings are user-configurable. @@ -147,10 +147,6 @@ Right now the plan is to greatly prefer #1 over #2. While appealing, #3 likely ![Replayer Components](diagrams/Replayer.svg) -### Traffic Comparator - -![Traffic Comparator Components](diagrams/TrafficComparator.svg) - ### Cutover ![Cutover](diagrams/SourceTargetCutover.svg) @@ -220,7 +216,7 @@ The following points by component list some of the main concerns that will need * Probably via a multi-tenant control-plane managed by AWS. An OpenSearch-Migrations Service Principal will likely call back into resources on the user’s account. * UI can leverage OpenSearch Dashboards. * Allows OpenSearch to eat our own dog food. - * OpenSearch is good at aggregation reports, making it a good consumer of the final output coming from the traffic comparator. + * OpenSearch is good at aggregation reports, making it a good consumer of the final output. * This might be expensive for the load that we could be sending in. * Will work well for a completely managed solution vs an OpenSource one. * We’ll have much more flexibility in look and feel and to add additional components. @@ -236,7 +232,7 @@ This may have the largest impact to a production workflow. It is also the one t 1. When possible, can we allow the recorder mechanism to induce a performance hit for the source to guarantee that all requests are offloaded to logs? 1. Notice that this doesn’t say, all requests that are processed successfully are offloaded. Doing this, without deep support for a 2-phase commit approach, which is infeasible, means that some requests definitely logged will NOT have been processed correctly. If the fault plane of the coordinating node and the logging are aligned (e.g. same process or machine), any failure in the coordinating node could create a false-positive. 2. Notice that when a response is generated, our downstream systems can be entirely sure of whether the request was committed or not. The ambiguity arises when a response isn’t found, which while relatively rare, still needs to be reconciled. The only real advantage here is that discovery of missing items becomes much simpler. There’s no risk of additional strain on the source cluster to find the missing mutation, which could be very expensive to track down in the case of an update (effectively, running a full-scan on an index’s documents). - 2. The replayer and comparator will need to work to mitigate data gaps that may have occurred in the logs. + 2. The replayer and comparison tooling will need to work to mitigate data gaps that may have occurred in the logs. 3. Should all nodes issue wire logs for all requests to allow redundancy checks on responses to minimize the likelihood of error? This would create a wider performance impact and a much greater cost in offloaded data processing, though the logged items could likely be quickly deduped in stream processing before the replayer. 4. Is it sensible to bootstrap wire logs with additional normative information - like from the translog (or equivalent)? 3. Source traffic that is in TLS creates a host of technical and security challenges. @@ -334,14 +330,6 @@ Other concerns include... * Latencies will likely be measuring different values than the traffic logs did - are there additional calibration requests that should be made so that more representative values can be displayed to the user? * The Replayer may need to run at slower rates, though subsampling will likely be handled upstream in the streaming system. -### Traffic Comparator - in progress - -* How do we run user code? What should “user code” even be? -* We can probably scope a fairly tight environment that is lazily loaded, like here are the mutating messages around n-seconds of this request. -* Can we leverage DataPrepper’s models to parse requests? -* Does a batch/hybrid solution make sense here, especially for rerunning comparisons after the normalization/comparison rules have been updated? MapReduce could be a LOT more efficient than rerunning the stream. -* Being able to explain in what ways things were different could be super valuable. e.g. 33% - ### Production Switchover As mentioned in the description, switching production traffic to the target is the driving factor behind nearly every other design decision. Some considerations across the architecture include... diff --git a/test/README.md b/test/README.md index 797cbf43fd..e7fbe15340 100644 --- a/test/README.md +++ b/test/README.md @@ -25,14 +25,13 @@ The test script, by default, uses the ports assigned to the containers in this [docker-compose file](../TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml), so if the Docker solution in its current setup started with no issues, then the test script will run as is. If for any reason the user changed the ports in that file, they must also either, change the following environment variables: -`PROXY_ENDPOINT`, `SOURCE_ENDPOINT`, `TARGET_ENDPOINT` and `JUPYTER_NOTEBOOK` respectively, or update the default value +`PROXY_ENDPOINT`, `SOURCE_ENDPOINT`, and `TARGET_ENDPOINT` respectively, or update the default value (which can be found below) for them in [tests.py](tests.py). The following are the default values for the only endpoints touched by this script: * `PROXY_ENDPOINT = https://localhost:9200` * `SOURCE_ENDPOINT = http://localhost:19200` * `TARGET_ENDPOINT = https://localhost:29200` -* `JUPYTER_NOTEBOOK = http://localhost:8888/api` #### Clean Up The test script is implemented with a setup and teardown functions that are ran after diff --git a/test/tests.py b/test/tests.py index ae77127899..2e3b399d4c 100644 --- a/test/tests.py +++ b/test/tests.py @@ -53,7 +53,6 @@ def set_common_values(self): self.proxy_endpoint = os.getenv('PROXY_ENDPOINT', 'https://localhost:9200') self.source_endpoint = os.getenv('SOURCE_ENDPOINT', 'https://localhost:19200') self.target_endpoint = os.getenv('TARGET_ENDPOINT', 'https://localhost:29200') - self.jupyter_endpoint = os.getenv('JUPYTER_NOTEBOOK', 'http://localhost:8888/api') self.username = os.getenv('username', 'admin') self.password = os.getenv('password', 'admin') self.auth = (self.username, self.password) @@ -158,11 +157,6 @@ def test_0002_document(self): expected_status_code=HTTPStatus.NOT_FOUND) self.assertEqual(source_response.status_code, HTTPStatus.NOT_FOUND) - def test_0003_jupyterAwake(self): - # Making sure that the Jupyter notebook is up and can be reached. - response = requests.get(self.jupyter_endpoint) - self.assertEqual(response.status_code, HTTPStatus.OK) - def test_0004_negativeAuth_invalidCreds(self): # This test sends negative credentials to the clusters to validate that unauthorized access is prevented. alphabet = string.ascii_letters + string.digits From c64c750b41a613d10593c74b08a81d4e84e15eb3 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Sun, 12 Nov 2023 12:08:16 -0600 Subject: [PATCH 51/55] Remove parenthesis that are no longer needed in gradle.build. Signed-off-by: Brian Presley --- TrafficCapture/dockerSolution/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TrafficCapture/dockerSolution/build.gradle b/TrafficCapture/dockerSolution/build.gradle index e37bbc4445..fdaaccc66c 100644 --- a/TrafficCapture/dockerSolution/build.gradle +++ b/TrafficCapture/dockerSolution/build.gradle @@ -47,7 +47,7 @@ javaContainerServices.each { projectName, dockerImageName -> CommonUtils.createDockerfile(project, projectName, baseImageProjectOverrides, dockerFilesForExternalServices) } -(javaContainerServices).forEach { projectName, dockerImageName -> +javaContainerServices.forEach { projectName, dockerImageName -> def dockerBuildDir = "build/docker/${projectName}" task "buildDockerImage_${projectName}"(type: DockerBuildImage) { dependsOn "createDockerfile_${projectName}" From b9dd32ee0a1da69a2e05701ed0d06282295264b2 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Sun, 12 Nov 2023 12:10:32 -0600 Subject: [PATCH 52/55] Update comment format Signed-off-by: Brian Presley --- .../datahandlers/http/HttpJsonTransformingConsumer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java index f72ca74315..48ed0c3651 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java @@ -58,8 +58,8 @@ public class HttpJsonTransformingConsumer implements IPacketFinalizingConsume private final List> chunkSizes; // This is here for recovery, in case anything goes wrong with a transformation & we want to // just dump it directly. Notice that we're already storing all of the bytes until the response - // comes back so that we can format the output. These should be - // backed by the exact same byte[] arrays, so the memory consumption should already be absorbed. + // comes back so that we can format the output. These should be backed by the exact same + // byte[] arrays, so the memory consumption should already be absorbed. private final List chunks; public HttpJsonTransformingConsumer(IJsonTransformer transformer, From 0de0a3e12c034c694e8e6d2d134f478bb8530040 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Sun, 12 Nov 2023 13:05:13 -0600 Subject: [PATCH 53/55] Remove the dual write option for request/response (tuple) output Signed-off-by: Brian Presley --- .../replay/SourceTargetCaptureTuple.java | 7 +------ .../migrations/replay/TrafficReplayer.java | 15 ++------------- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java index 3625617997..60d7f45e36 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java @@ -12,7 +12,6 @@ import org.opensearch.migrations.replay.datatypes.UniqueSourceRequestKey; import java.io.IOException; -import java.io.OutputStream; import java.io.SequenceInputStream; import java.nio.charset.StandardCharsets; import java.time.Duration; @@ -55,11 +54,9 @@ public void close() { } public static class TupleToFileWriter implements Consumer { - OutputStream outputStream; Logger tupleLogger = LogManager.getLogger("OutputTupleJsonLogger"); - public TupleToFileWriter(OutputStream outputStream){ - this.outputStream = outputStream; + public TupleToFileWriter(){ } private JSONObject jsonFromHttpDataUnsafe(List data) throws IOException { @@ -169,8 +166,6 @@ public void accept(SourceTargetCaptureTuple triple) { JSONObject jsonObject = toJSONObject(triple); tupleLogger.info(jsonObject.toString()); - outputStream.write((jsonObject.toString()+"\n").getBytes(StandardCharsets.UTF_8)); - outputStream.flush(); } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index c4517c4209..0de1d1c4be 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -36,11 +36,8 @@ import software.amazon.awssdk.regions.Region; import javax.net.ssl.SSLException; -import java.io.BufferedOutputStream; import java.io.EOFException; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; import java.lang.ref.WeakReference; import java.net.URI; import java.time.Duration; @@ -224,12 +221,6 @@ static class Parameters { arity=1, description = "input file to read the request/response traces for the source cluster") String inputFilename; - @Parameter(required = false, - names = {"-o", "--output"}, - arity=1, - description = "output file to hold the request/response traces for the source and target cluster") - String outputFilename; - @Parameter(required = false, names = {"-t", PACKET_TIMEOUT_SECONDS_PARAMETER_NAME}, arity = 1, @@ -325,9 +316,7 @@ public static void main(String[] args) return; } - try (OutputStream outputStream = params.outputFilename == null ? System.out : - new FileOutputStream(params.outputFilename, true); - var bufferedOutputStream = new BufferedOutputStream(outputStream); + try ( var blockingTrafficStream = TrafficCaptureSourceFactory.createTrafficCaptureSource(params, Duration.ofSeconds(params.lookaheadTimeSeconds)); var authTransformer = buildAuthTransformerFactory(params)) @@ -335,7 +324,7 @@ public static void main(String[] args) var tr = new TrafficReplayer(uri, params.transformerConfig, authTransformer, params.allowInsecureConnections, params.numClientThreads, params.maxConcurrentRequests); setupShutdownHookForReplayer(tr); - var tupleWriter = new SourceTargetCaptureTuple.TupleToFileWriter(bufferedOutputStream); + var tupleWriter = new SourceTargetCaptureTuple.TupleToFileWriter(); var timeShifter = new TimeShifter(params.speedupFactor); tr.setupRunAndWaitForReplayWithShutdownChecks(Duration.ofSeconds(params.observedPacketConnectionTimeout), blockingTrafficStream, timeShifter, tupleWriter); From 8f268f26bc17cfdcc8bcd0ceaa42d08371ed8e72 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Sun, 12 Nov 2023 21:45:08 -0500 Subject: [PATCH 54/55] Fix a bad merge from earlier that broke the build. The fix was more involved because the way that the test was verifying output changed in the collateral damage. The new test uses Log4J classes directly (just like the SourceTargetCaptureTuple), but is able to confirm that log4j will get the right messages and not throw an error, etc. Signed-off-by: Greg Schohn --- .../replay/SourceTargetCaptureTuple.java | 6 +- .../replay/SourceTargetCaptureTupleTest.java | 70 ++++++++++++++----- 2 files changed, 57 insertions(+), 19 deletions(-) diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java index 1e2bb6d54f..1b28b14ea3 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/SourceTargetCaptureTuple.java @@ -26,6 +26,7 @@ @Slf4j public class SourceTargetCaptureTuple implements AutoCloseable { + public static final String OUTPUT_TUPLE_JSON_LOGGER = "OutputTupleJsonLogger"; final UniqueSourceRequestKey uniqueRequestKey; final RequestResponsePacketPair sourcePair; final TransformedPackets targetRequestData; @@ -56,9 +57,9 @@ public void close() { } public static class TupleToStreamConsumer implements Consumer { - Logger tupleLogger = LogManager.getLogger("OutputTupleJsonLogger"); + Logger tupleLogger = LogManager.getLogger(OUTPUT_TUPLE_JSON_LOGGER); - public TupleToStreamConsumer(OutputStream outputStream){ + public TupleToStreamConsumer() { } private JSONObject jsonFromHttpDataUnsafe(List data) throws IOException { @@ -172,7 +173,6 @@ private JSONObject toJSONObject(SourceTargetCaptureTuple tuple) { @SneakyThrows public void accept(SourceTargetCaptureTuple triple) { JSONObject jsonObject = toJSONObject(triple); - tupleLogger.info(jsonObject.toString()); } } diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SourceTargetCaptureTupleTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SourceTargetCaptureTupleTest.java index 355a12d5ad..e79e12fcbd 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SourceTargetCaptureTupleTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/SourceTargetCaptureTupleTest.java @@ -1,6 +1,12 @@ package org.opensearch.migrations.replay; import lombok.extern.slf4j.Slf4j; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.AbstractAppender; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.opensearch.migrations.replay.datatypes.PojoTrafficStreamKey; @@ -10,6 +16,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -18,11 +26,41 @@ class SourceTargetCaptureTupleTest { private static final String NODE_ID = "n"; public static final String TEST_EXCEPTION_MESSAGE = "TEST_EXCEPTION"; + private static class CloseableLogSetup implements AutoCloseable { + List logEvents = new ArrayList<>(); + AbstractAppender testAppender; + public CloseableLogSetup() { + testAppender = new AbstractAppender(SourceTargetCaptureTuple.OUTPUT_TUPLE_JSON_LOGGER, + null, null, false, null) { + @Override + public void append(LogEvent event) { + logEvents.add(event); + } + }; + var tupleLogger = (Logger) LogManager.getLogger(SourceTargetCaptureTuple.OUTPUT_TUPLE_JSON_LOGGER); + tupleLogger.setLevel(Level.ALL); + testAppender.start(); + tupleLogger.setAdditive(false); + tupleLogger.addAppender(testAppender); + var loggerCtx = ((LoggerContext) LogManager.getContext(false)); + } + + @Override + public void close() { + var tupleLogger = (Logger) LogManager.getLogger(SourceTargetCaptureTuple.OUTPUT_TUPLE_JSON_LOGGER); + tupleLogger.removeAppender(testAppender); + testAppender.stop(); + } + } + @Test public void testTupleNewWithNullKeyThrows() { - Assertions.assertThrows(Exception.class, - ()->new SourceTargetCaptureTuple(null, null, null, - null, null, null, null)); + try (var closeableLogSetup = new CloseableLogSetup()) { + Assertions.assertThrows(Exception.class, + () -> new SourceTargetCaptureTuple(null, null, null, + null, null, null, null)); + Assertions.assertEquals(0, closeableLogSetup.logEvents.size()); + } } @Test @@ -30,14 +68,14 @@ public void testOutputterWithNulls() throws IOException { var emptyTuple = new SourceTargetCaptureTuple( new UniqueReplayerRequestKey(new PojoTrafficStreamKey(NODE_ID,"c",0), 0, 0), null, null, null, null, null, null); - String contents; - try (var byteArrayOutputStream = new ByteArrayOutputStream()) { - var consumer = new SourceTargetCaptureTuple.TupleToStreamConsumer(byteArrayOutputStream); + try (var closeableLogSetup = new CloseableLogSetup()) { + var consumer = new SourceTargetCaptureTuple.TupleToStreamConsumer(); consumer.accept(emptyTuple); - contents = new String(byteArrayOutputStream.toByteArray(), StandardCharsets.UTF_8); + Assertions.assertEquals(1, closeableLogSetup.logEvents.size()); + var contents = closeableLogSetup.logEvents.get(0).getMessage().getFormattedMessage(); + log.info("Output="+contents); + Assertions.assertTrue(contents.contains(NODE_ID)); } - log.info("Output="+contents); - Assertions.assertTrue(contents.contains(NODE_ID)); } @Test @@ -46,14 +84,14 @@ public void testOutputterWithNullsShowsException() throws IOException { var emptyTuple = new SourceTargetCaptureTuple( new UniqueReplayerRequestKey(new PojoTrafficStreamKey(NODE_ID,"c",0), 0, 0), null, null, null, null, exception, null); - String contents; - try (var byteArrayOutputStream = new ByteArrayOutputStream()) { - var consumer = new SourceTargetCaptureTuple.TupleToStreamConsumer(byteArrayOutputStream); + try (var closeableLogSetup = new CloseableLogSetup()) { + var consumer = new SourceTargetCaptureTuple.TupleToStreamConsumer(); consumer.accept(emptyTuple); - contents = new String(byteArrayOutputStream.toByteArray(), StandardCharsets.UTF_8); + Assertions.assertEquals(1, closeableLogSetup.logEvents.size()); + var contents = closeableLogSetup.logEvents.get(0).getMessage().getFormattedMessage(); + log.info("Output="+contents); + Assertions.assertTrue(contents.contains(NODE_ID)); + Assertions.assertTrue(contents.contains(TEST_EXCEPTION_MESSAGE)); } - log.info("Output="+contents); - Assertions.assertTrue(contents.contains(NODE_ID)); - Assertions.assertTrue(contents.contains(TEST_EXCEPTION_MESSAGE)); } } \ No newline at end of file From c33f1f98d8f5f591da15d5668d9405505a4e6ad1 Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Sun, 12 Nov 2023 23:12:38 -0600 Subject: [PATCH 55/55] Remove references to comparator and jupyter from cdk tests Signed-off-by: Brian Presley --- .../service-stacks/traffic-replayer-stack.ts | 5 +-- .../opensearch-service-migration/options.md | 2 -- .../package-lock.json | 1 + .../test/stack-composer-ordering.test.ts | 33 ++----------------- 4 files changed, 4 insertions(+), 37 deletions(-) diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts index 54b3732a5c..e41c7e2904 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts @@ -14,9 +14,7 @@ export interface TrafficReplayerProps extends StackPropsExt { readonly addOnMigrationId?: string, readonly customTargetEndpoint?: string, readonly customKafkaGroupId?: string, - readonly extraArgs?: string, - readonly enableComparatorLink?: boolean - + readonly extraArgs?: string } export class TrafficReplayerStack extends MigrationServiceCore { @@ -101,7 +99,6 @@ export class TrafficReplayerStack extends MigrationServiceCore { replayerCommand = replayerCommand.concat(` --auth-header-user-and-secret ${osUserAndSecret}`) } replayerCommand = props.extraArgs ? replayerCommand.concat(` ${props.extraArgs}`) : replayerCommand - replayerCommand = props.enableComparatorLink ? replayerCommand.concat(" | nc traffic-comparator 9220") : replayerCommand this.createService({ serviceName: `traffic-replayer-${deployId}`, dockerFilePath: join(__dirname, "../../../../../", "TrafficCapture/dockerSolution/build/docker/trafficReplayer"), diff --git a/deployment/cdk/opensearch-service-migration/options.md b/deployment/cdk/opensearch-service-migration/options.md index 5aec5eb5f6..ac524388cf 100644 --- a/deployment/cdk/opensearch-service-migration/options.md +++ b/deployment/cdk/opensearch-service-migration/options.md @@ -10,8 +10,6 @@ These tables list all CDK context configuration values a user can specify for th | captureProxyESServiceEnabled | boolean | true | Enable deploying the given service, via a new CloudFormation stack | | migrationConsoleServiceEnabled | boolean | true | Enable deploying the given service, via a new CloudFormation stack | | trafficReplayerServiceEnabled | boolean | true | Enable deploying the given service, via a new CloudFormation stack | -| trafficComparatorServiceEnabled | boolean | true | Enable deploying the given service, via a new CloudFormation stack **Comparator services are in development mode and should be used with caution** | -| trafficComparatorJupyterServiceEnabled | boolean | true | Enable deploying the given service, via a new CloudFormation stack **Comparator services are in development mode and should be used with caution** | | captureProxyServiceEnabled | boolean | true | Enable deploying the given service, via a new CloudFormation stack | | elasticsearchServiceEnabled | boolean | true | Enable deploying the given service, via a new CloudFormation stack | | kafkaBrokerServiceEnabled | boolean | false | Enable deploying the given service, via a new CloudFormation stack. **This stack is experimental and should only be used for development** | diff --git a/deployment/cdk/opensearch-service-migration/package-lock.json b/deployment/cdk/opensearch-service-migration/package-lock.json index ddaa7f49c8..4e91b26e45 100644 --- a/deployment/cdk/opensearch-service-migration/package-lock.json +++ b/deployment/cdk/opensearch-service-migration/package-lock.json @@ -7,6 +7,7 @@ "": { "name": "opensearch-service-domain-cdk", "version": "0.1.0", + "license": "Apache-2.0", "dependencies": { "@aws-cdk/aws-servicecatalogappregistry-alpha": "2.100.0-alpha.0", "@aws-sdk/client-kafka": "^3.410.0", diff --git a/deployment/cdk/opensearch-service-migration/test/stack-composer-ordering.test.ts b/deployment/cdk/opensearch-service-migration/test/stack-composer-ordering.test.ts index de84e695da..0ca965d8e5 100644 --- a/deployment/cdk/opensearch-service-migration/test/stack-composer-ordering.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/stack-composer-ordering.test.ts @@ -4,8 +4,6 @@ import {CaptureProxyESStack} from "../lib/service-stacks/capture-proxy-es-stack" import {CaptureProxyStack} from "../lib/service-stacks/capture-proxy-stack"; import {ElasticsearchStack} from "../lib/service-stacks/elasticsearch-stack"; import {TrafficReplayerStack} from "../lib/service-stacks/traffic-replayer-stack"; -import {TrafficComparatorStack} from "../lib/service-stacks/traffic-comparator-stack"; -import {TrafficComparatorJupyterStack} from "../lib/service-stacks/traffic-comparator-jupyter-stack"; import {MigrationConsoleStack} from "../lib/service-stacks/migration-console-stack"; import {KafkaBrokerStack} from "../lib/service-stacks/kafka-broker-stack"; import {KafkaZookeeperStack} from "../lib/service-stacks/kafka-zookeeper-stack"; @@ -32,8 +30,6 @@ test('Test all migration services get created when enabled', () => { "trafficReplayerServiceEnabled": true, "captureProxyServiceEnabled": true, "elasticsearchServiceEnabled": true, - "trafficComparatorServiceEnabled": true, - "trafficComparatorJupyterServiceEnabled": true, "kafkaBrokerServiceEnabled": true, "kafkaZookeeperServiceEnabled": true } @@ -41,7 +37,7 @@ test('Test all migration services get created when enabled', () => { const stacks = createStackComposer(contextOptions) const services = [CaptureProxyESStack, CaptureProxyStack, ElasticsearchStack, MigrationConsoleStack, - TrafficReplayerStack, TrafficComparatorStack, TrafficComparatorJupyterStack, KafkaBrokerStack, KafkaZookeeperStack] + TrafficReplayerStack, KafkaBrokerStack, KafkaZookeeperStack] services.forEach( (stackClass) => { const stack = stacks.stacks.filter((s) => s instanceof stackClass)[0] const template = Template.fromStack(stack) @@ -66,8 +62,6 @@ test('Test no migration services get deployed when disabled', () => { "trafficReplayerServiceEnabled": false, "captureProxyServiceEnabled": false, "elasticsearchServiceEnabled": false, - "trafficComparatorServiceEnabled": false, - "trafficComparatorJupyterServiceEnabled": false, "kafkaBrokerServiceEnabled": false, "kafkaZookeeperServiceEnabled": false } @@ -75,33 +69,10 @@ test('Test no migration services get deployed when disabled', () => { const stacks = createStackComposer(contextOptions) const services = [CaptureProxyESStack, CaptureProxyStack, ElasticsearchStack, MigrationConsoleStack, - TrafficReplayerStack, TrafficComparatorStack, TrafficComparatorJupyterStack, KafkaBrokerStack, KafkaZookeeperStack] + TrafficReplayerStack, KafkaBrokerStack, KafkaZookeeperStack] services.forEach( (stackClass) => { const stack = stacks.stacks.filter((s) => s instanceof stackClass)[0] expect(stack).toBeUndefined() }) -}) - -test('Test jupyter service does not get deployed if traffic comparator is not enabled', () => { - - const contextOptions = { - "stage": "test", - "engineVersion": "OS_2.9", - "domainName": "unit-test-opensearch-cluster", - "dataNodeCount": 2, - "availabilityZoneCount": 2, - "openAccessPolicyEnabled": true, - "domainRemovalPolicy": "DESTROY", - "vpcEnabled": true, - "migrationAssistanceEnabled": true, - "trafficComparatorServiceEnabled": false, - "trafficComparatorJupyterServiceEnabled": true - } - - const stacks = createStackComposer(contextOptions) - - const stack = stacks.stacks.filter((s) => s instanceof TrafficComparatorJupyterStack)[0] - expect(stack).toBeUndefined() - }) \ No newline at end of file