diff --git a/FetchMigration/README.md b/FetchMigration/README.md deleted file mode 100644 index 79d5ba0ab..000000000 --- a/FetchMigration/README.md +++ /dev/null @@ -1,108 +0,0 @@ -# Index Configuration Tool - -Python package that automates the creation of indices on a target cluster based on the contents of a source cluster. -Index settings and index mappings are correctly copied over, but no data is transferred. -This tool seeks to eliminate the need to [specify index templates](https://github.com/awslabs/logstash-output-amazon_es#optional-parameters) when migrating data from one cluster to another. -The tool currently supports ElasticSearch or OpenSearch as source and target. - -## Parameters - -The first required input to the tool is a path to a [Data Prepper](https://github.com/opensearch-project/data-prepper) pipeline YAML file, which is parsed to obtain the source and target cluster endpoints. -The second required input is an output path to which a modified version of the pipeline YAML file is written. -This version of the pipeline adds an index inclusion configuration to the sink, specifying only those indices that were created by the index configuration tool. -The tool also supports several optional flags: - -| Flag | Purpose | -| ------------- | ------------- | -| `-h, --help` | Prints help text and exits | -| `--report, -r` | Prints a report of indices indicating which ones will be created, along with indices that are identical or have conflicting settings/mappings. | -| `--dryrun` | Skips the actual creation of indices on the target cluster | - -### Reporting - -If `--report` is specified, the tool prints all processed indices organized into 3 buckets: -* Successfully created on the target cluster -* Skipped due to a conflict in settings/mappings -* Skipped since the index configuration is identical on source and target - -## Current Limitations - -* Only supports ElasticSearch and OpenSearch endpoints for source and target -* Only supports basic auth -* Type mappings for legacy indices are not handled -* Index templates and index aliases are not copied -* Index health is not validated after creation - -## Usage - -### Command-Line - -#### Setup: - -* [Clone](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) this GitHub repo -* Install [Python](https://www.python.org/) -* Ensure that [pip](https://pip.pypa.io/en/stable/installation/#) is installed -* (Optional) Set up and activate a [virtual environment](https://packaging.python.org/en/latest/tutorials/installing-packages/#creating-and-using-virtual-environments) - -Navigate to the cloned GitHub repo. Then, install the required Python dependencies by running: - -```shell -python -m pip install -r index_configuration_tool/requirements.txt -``` - -#### Execution: - -After [setup](#setup), the tool can be executed using: - -```shell -python index_configuration_tool/pre_migration.py -``` - -### Docker - -First build the Docker image from the `Dockerfile`: - -```shell -docker build -t fetch-migration . -``` - -Then run the `fetch-migration` image. -Replace `` in the command below with the path to your Logstash config file: - -```shell -docker run -p 4900:4900 -v :/code/input.yaml ict -``` - -## Development - -The source code for the tool is located under the `index_configuration_tool/` directory. Please refer to the [Setup](#setup) section to ensure that the necessary dependencies are installed prior to development. - -Additionally, you'll also need to install development dependencies by running: - -```shell -python -m pip install -r index_configuration_tool/dev-requirements.txt -``` - -### Unit Tests - -Unit tests are located in a sub-directory named `tests`. Unit tests can be run using: - -```shell -python -m unittest -``` - -### Coverage - -Code coverage metrics can be generated by first running unit tests using _coverage run_: - -```shell -python -m coverage run -m unittest -``` - -Then a report can either be printed on the command line or generated as HTML. -Note that the `--omit` parameter must be specified to avoid tracking code coverage on unit test code: - -```shell -python -m coverage report --omit "*/tests/*" -python -m coverage html --omit "*/tests/*" -``` \ No newline at end of file diff --git a/FetchMigration/python/dev-requirements.txt b/FetchMigration/python/dev-requirements.txt deleted file mode 100644 index ecef90389..000000000 --- a/FetchMigration/python/dev-requirements.txt +++ /dev/null @@ -1 +0,0 @@ -coverage>=7.2.3 \ No newline at end of file diff --git a/FetchMigration/python/endpoint_info.py b/FetchMigration/python/endpoint_info.py deleted file mode 100644 index b77880dc0..000000000 --- a/FetchMigration/python/endpoint_info.py +++ /dev/null @@ -1,8 +0,0 @@ -from dataclasses import dataclass - - -@dataclass -class EndpointInfo: - url: str - auth: tuple = None - verify_ssl: bool = True diff --git a/FetchMigration/python/index_operations.py b/FetchMigration/python/index_operations.py deleted file mode 100644 index c653344f7..000000000 --- a/FetchMigration/python/index_operations.py +++ /dev/null @@ -1,45 +0,0 @@ -import requests - -from endpoint_info import EndpointInfo - -# Constants -SETTINGS_KEY = "settings" -MAPPINGS_KEY = "mappings" -COUNT_KEY = "count" -__INDEX_KEY = "index" -__ALL_INDICES_ENDPOINT = "*" -__COUNT_ENDPOINT = "/_count" -__INTERNAL_SETTINGS_KEYS = ["creation_date", "uuid", "provided_name", "version", "store"] - - -def fetch_all_indices(endpoint: EndpointInfo) -> dict: - actual_endpoint = endpoint.url + __ALL_INDICES_ENDPOINT - resp = requests.get(actual_endpoint, auth=endpoint.auth, verify=endpoint.verify_ssl) - # Remove internal settings - result = dict(resp.json()) - for index in result: - for setting in __INTERNAL_SETTINGS_KEYS: - index_settings = result[index][SETTINGS_KEY] - if __INDEX_KEY in index_settings: - index_settings[__INDEX_KEY].pop(setting, None) - return result - - -def create_indices(indices_data: dict, endpoint: EndpointInfo): - for index in indices_data: - actual_endpoint = endpoint.url + index - data_dict = dict() - data_dict[SETTINGS_KEY] = indices_data[index][SETTINGS_KEY] - data_dict[MAPPINGS_KEY] = indices_data[index][MAPPINGS_KEY] - try: - resp = requests.put(actual_endpoint, auth=endpoint.auth, verify=endpoint.verify_ssl, json=data_dict) - resp.raise_for_status() - except requests.exceptions.RequestException as e: - raise RuntimeError(f"Failed to create index [{index}] - {e!s}") - - -def doc_count(indices: set, endpoint: EndpointInfo) -> int: - actual_endpoint = endpoint.url + ','.join(indices) + __COUNT_ENDPOINT - resp = requests.get(actual_endpoint, auth=endpoint.auth, verify=endpoint.verify_ssl) - result = dict(resp.json()) - return int(result[COUNT_KEY]) diff --git a/FetchMigration/python/pre_migration.py b/FetchMigration/python/pre_migration.py deleted file mode 100644 index eda146f06..000000000 --- a/FetchMigration/python/pre_migration.py +++ /dev/null @@ -1,223 +0,0 @@ -import argparse -import yaml -from typing import Optional - -import index_operations -import utils - -# Constants -from endpoint_info import EndpointInfo -from pre_migration_params import PreMigrationParams -from pre_migration_result import PreMigrationResult - -SUPPORTED_ENDPOINTS = ["opensearch", "elasticsearch"] -SOURCE_KEY = "source" -SINK_KEY = "sink" -HOSTS_KEY = "hosts" -DISABLE_AUTH_KEY = "disable_authentication" -USER_KEY = "username" -PWD_KEY = "password" -INSECURE_KEY = "insecure" -CONNECTION_KEY = "connection" -INDICES_KEY = "indices" -INCLUDE_KEY = "include" -INDEX_NAME_KEY = "index_name_regex" - - -# This config key may be either directly in the main dict (for sink) -# or inside a nested dict (for source). The default value is False. -def is_insecure(config: dict) -> bool: - if INSECURE_KEY in config: - return config[INSECURE_KEY] - elif CONNECTION_KEY in config and INSECURE_KEY in config[CONNECTION_KEY]: - return config[CONNECTION_KEY][INSECURE_KEY] - return False - - -# TODO Only supports basic auth for now -def get_auth(input_data: dict) -> Optional[tuple]: - if not input_data.get(DISABLE_AUTH_KEY, False) and USER_KEY in input_data and PWD_KEY in input_data: - return input_data[USER_KEY], input_data[PWD_KEY] - - -def get_endpoint_info(plugin_config: dict) -> EndpointInfo: - # "hosts" can be a simple string, or an array of hosts for Logstash to hit. - # This tool needs one accessible host, so we pick the first entry in the latter case. - url = plugin_config[HOSTS_KEY][0] if type(plugin_config[HOSTS_KEY]) is list else plugin_config[HOSTS_KEY] - url += "/" - # verify boolean will be the inverse of the insecure SSL key, if present - should_verify = not is_insecure(plugin_config) - return EndpointInfo(url, get_auth(plugin_config), should_verify) - - -def check_supported_endpoint(config: dict) -> Optional[tuple]: - for supported_type in SUPPORTED_ENDPOINTS: - if supported_type in config: - return supported_type, config[supported_type] - - -def get_supported_endpoint(config: dict, key: str) -> tuple: - # The value of each key may be a single plugin (as a dict) - # or a list of plugin configs - supported_tuple = tuple() - if type(config[key]) is dict: - supported_tuple = check_supported_endpoint(config[key]) - elif type(config[key]) is list: - for entry in config[key]: - supported_tuple = check_supported_endpoint(entry) - # Break out of the loop at the first supported type - if supported_tuple: - break - if not supported_tuple: - raise ValueError("Could not find any supported endpoints in section: " + key) - # First tuple value is the name, second value is the config dict - return supported_tuple[0], supported_tuple[1] - - -def validate_plugin_config(config: dict, key: str): - # Raises a ValueError if no supported endpoints are found - supported_endpoint = get_supported_endpoint(config, key) - plugin_config = supported_endpoint[1] - if HOSTS_KEY not in plugin_config: - raise ValueError("No hosts defined for endpoint: " + supported_endpoint[0]) - # Check if auth is disabled. If so, no further validation is required - if plugin_config.get(DISABLE_AUTH_KEY, False): - return - elif USER_KEY not in plugin_config: - raise ValueError("Invalid auth configuration (no username) for endpoint: " + supported_endpoint[0]) - elif PWD_KEY not in plugin_config: - raise ValueError("Invalid auth configuration (no password for username) for endpoint: " + - supported_endpoint[0]) - - -def validate_pipeline_config(config: dict): - if SOURCE_KEY not in config: - raise ValueError("Missing source configuration in Data Prepper pipeline YAML") - if SINK_KEY not in config: - raise ValueError("Missing sink configuration in Data Prepper pipeline YAML") - validate_plugin_config(config, SOURCE_KEY) - validate_plugin_config(config, SINK_KEY) - - -def write_output(yaml_data: dict, new_indices: set, output_path: str): - pipeline_config = next(iter(yaml_data.values())) - # Endpoint is a tuple of (type, config) - source_config = get_supported_endpoint(pipeline_config, SOURCE_KEY)[1] - source_indices = source_config.get(INDICES_KEY, dict()) - included_indices = source_indices.get(INCLUDE_KEY, list()) - for index in new_indices: - included_indices.append({INDEX_NAME_KEY: index}) - source_indices[INCLUDE_KEY] = included_indices - source_config[INDICES_KEY] = source_indices - with open(output_path, 'w') as out_file: - yaml.dump(yaml_data, out_file) - - -# Computes differences in indices between source and target. -# Returns a tuple with 3 elements: -# - The 1st element is the set of indices to create on the target -# - The 2nd element is a set of indices that are identical on source and target -# - The 3rd element is a set of indices that are present on both source and target, -# but differ in their settings or mappings. -def get_index_differences(source: dict, target: dict) -> tuple[set, set, set]: - index_conflicts = set() - indices_in_target = set(source.keys()) & set(target.keys()) - for index in indices_in_target: - # Check settings - if utils.has_differences(index_operations.SETTINGS_KEY, source[index], target[index]): - index_conflicts.add(index) - # Check mappings - if utils.has_differences(index_operations.MAPPINGS_KEY, source[index], target[index]): - index_conflicts.add(index) - identical_indices = set(indices_in_target) - set(index_conflicts) - indices_to_create = set(source.keys()) - set(indices_in_target) - return indices_to_create, identical_indices, index_conflicts - - -# The order of data in the tuple is: -# (indices to create), (identical indices), (indices with conflicts) -def print_report(index_differences: tuple[set, set, set], count: int): # pragma no cover - print("Identical indices in the target cluster (no changes will be made): " + - utils.string_from_set(index_differences[1])) - print("Indices in target cluster with conflicting settings/mappings: " + - utils.string_from_set(index_differences[2])) - print("Indices to create: " + utils.string_from_set(index_differences[0])) - print("Total documents to be moved: " + str(count)) - - -def compute_endpoint_and_fetch_indices(config: dict, key: str) -> tuple[EndpointInfo, dict]: - endpoint = get_supported_endpoint(config, key) - # Endpoint is a tuple of (type, config) - endpoint_info = get_endpoint_info(endpoint[1]) - indices = index_operations.fetch_all_indices(endpoint_info) - return endpoint_info, indices - - -def run(args: PreMigrationParams) -> PreMigrationResult: - # Sanity check - if not args.report and len(args.output_file) == 0: - raise ValueError("No output file specified") - # Parse and validate pipelines YAML file - with open(args.config_file_path, 'r') as pipeline_file: - dp_config = yaml.safe_load(pipeline_file) - # We expect the Data Prepper pipeline to only have a single top-level value - pipeline_config = next(iter(dp_config.values())) - validate_pipeline_config(pipeline_config) - # Fetch EndpointInfo and indices - source_endpoint_info, source_indices = compute_endpoint_and_fetch_indices(pipeline_config, SOURCE_KEY) - target_endpoint_info, target_indices = compute_endpoint_and_fetch_indices(pipeline_config, SINK_KEY) - # Compute index differences and print report - diff = get_index_differences(source_indices, target_indices) - # The first element in the tuple is the set of indices to create - indices_to_create = diff[0] - result = PreMigrationResult() - if indices_to_create: - result.created_indices = indices_to_create - result.target_doc_count = index_operations.doc_count(indices_to_create, source_endpoint_info) - if args.report: - print_report(diff, result.target_doc_count) - if indices_to_create: - # Write output YAML - if len(args.output_file) > 0: - write_output(dp_config, indices_to_create, args.output_file) - if args.report: - print("Wrote output YAML pipeline to: " + args.output_file) - if not args.dryrun: - index_data = dict() - for index_name in indices_to_create: - index_data[index_name] = source_indices[index_name] - index_operations.create_indices(index_data, target_endpoint_info) - return result - - -if __name__ == '__main__': # pragma no cover - # Set up parsing for command line arguments - arg_parser = argparse.ArgumentParser( - prog="python pre_migration.py", - description="This tool creates indices on a target cluster based on the contents of a source cluster.\n" + - "The first input to the tool is a path to a Data Prepper pipeline YAML file, which is parsed to obtain " + - "the source and target cluster endpoints.\nThe second input is an output path to which a modified version " + - "of the pipeline YAML file is written. This version of the pipeline adds an index inclusion configuration " + - "to the sink, specifying only those indices that were created by the index configuration tool.\nThis tool " + - "can also print a report based on the indices in the source cluster, indicating which ones will be created, " + - "along with indices that are identical or have conflicting settings/mappings.", - formatter_class=argparse.RawTextHelpFormatter - ) - # Required positional argument - arg_parser.add_argument( - "config_file_path", - help="Path to the Data Prepper pipeline YAML file to parse for source and target endpoint information" - ) - # Optional positional argument - arg_parser.add_argument( - "output_file", - nargs='?', default="", - help="Output path for the Data Prepper pipeline YAML file that will be generated" - ) - # Flags - arg_parser.add_argument("--report", "-r", action="store_true", - help="Print a report of the index differences") - arg_parser.add_argument("--dryrun", action="store_true", - help="Skips the actual creation of indices on the target cluster") - namespace = arg_parser.parse_args() - run(PreMigrationParams(namespace.config_file_path, namespace.output_file, namespace.report, namespace.dryrun)) diff --git a/FetchMigration/python/tests/resources/expected_parse_output.pickle b/FetchMigration/python/tests/resources/expected_parse_output.pickle deleted file mode 100644 index 568cdae3f..000000000 Binary files a/FetchMigration/python/tests/resources/expected_parse_output.pickle and /dev/null differ diff --git a/FetchMigration/python/tests/resources/test_pipeline_input.yaml b/FetchMigration/python/tests/resources/test_pipeline_input.yaml deleted file mode 100644 index 7c33a7e21..000000000 --- a/FetchMigration/python/tests/resources/test_pipeline_input.yaml +++ /dev/null @@ -1,21 +0,0 @@ -test-pipeline-input: - source: - elasticsearch: - hosts: ["http://host1", "http://host2"] - username: "test_user" - password: "password" - processor: - - plugin1: - str_array: ["abc", "x y z"] - obj_array: - - key: "key1" - value: "val1" - - key: "key2" - value: "val 2" - sink: - - sink1: - num_array: [0] - - opensearch: - hosts: ["https://os_host"] - username: "test_user" - password: "test" diff --git a/FetchMigration/python/tests/test_constants.py b/FetchMigration/python/tests/test_constants.py deleted file mode 100644 index f54206088..000000000 --- a/FetchMigration/python/tests/test_constants.py +++ /dev/null @@ -1,67 +0,0 @@ -# Holds constants for unit tests -from os.path import dirname - -TEST_RESOURCES_SUBPATH = "/resources/" -PIPELINE_CONFIG_RAW_FILE_PATH = dirname(__file__) + TEST_RESOURCES_SUBPATH + "test_pipeline_input.yaml" -PIPELINE_CONFIG_PICKLE_FILE_PATH = dirname(__file__) + TEST_RESOURCES_SUBPATH + "expected_parse_output.pickle" - -INDEX1_NAME = "index1" -INDEX2_NAME = "index2" -INDEX3_NAME = "index3" -SETTINGS_KEY = "settings" -MAPPINGS_KEY = "mappings" -INDEX_KEY = "index" -NUM_SHARDS_SETTING = "number_of_shards" -NUM_REPLICAS_SETTING = "number_of_replicas" -BASE_INDICES_DATA = { - INDEX1_NAME: { - SETTINGS_KEY: { - INDEX_KEY: { - # Internal data - "version": 1, - "uuid": "test", - # Non-internal - "is_filtered": False - }, - "location": "test" - }, - MAPPINGS_KEY: { - "bool_key": {"type": "boolean"}, - "location": {"type": "location"} - } - }, - INDEX2_NAME: { - SETTINGS_KEY: { - INDEX_KEY: { - NUM_SHARDS_SETTING: 2, - NUM_REPLICAS_SETTING: 1 - } - }, - MAPPINGS_KEY: { - "dynamic": "strict" - } - }, - INDEX3_NAME: { - SETTINGS_KEY: { - INDEX_KEY: { - NUM_SHARDS_SETTING: 1, - NUM_REPLICAS_SETTING: 1 - } - }, - MAPPINGS_KEY: { - "id": {"type": "keyword"} - } - } -} -# Based on the contents of test_pipeline_input.yaml -SOURCE_ENDPOINT = "http://host1/" -TARGET_ENDPOINT = "https://os_host/" - -# Utility logic to update the pickle file if/when the input file is updated -# import yaml -# import pickle -# if __name__ == '__main__': -# with open(PIPELINE_CONFIG_RAW_FILE_PATH, 'r') as test_input: -# test_config = yaml.safe_load(test_input) -# with open(PIPELINE_CONFIG_PICKLE_FILE_PATH, 'wb') as out: -# pickle.dump(test_config, out) diff --git a/FetchMigration/python/tests/test_index_operations.py b/FetchMigration/python/tests/test_index_operations.py deleted file mode 100644 index 98ace7cae..000000000 --- a/FetchMigration/python/tests/test_index_operations.py +++ /dev/null @@ -1,62 +0,0 @@ -import copy -import unittest - -import requests -import responses -from responses import matchers - -import index_operations -from endpoint_info import EndpointInfo -from tests import test_constants - - -class TestSearchEndpoint(unittest.TestCase): - @responses.activate - def test_fetch_all_indices(self): - # Set up GET response - responses.get(test_constants.SOURCE_ENDPOINT + "*", json=test_constants.BASE_INDICES_DATA) - # Now send request - index_data = index_operations.fetch_all_indices(EndpointInfo(test_constants.SOURCE_ENDPOINT)) - self.assertEqual(3, len(index_data.keys())) - # Test that internal data has been filtered, but non-internal data is retained - index_settings = index_data[test_constants.INDEX1_NAME][test_constants.SETTINGS_KEY] - self.assertTrue(test_constants.INDEX_KEY in index_settings) - self.assertEqual({"is_filtered": False}, index_settings[test_constants.INDEX_KEY]) - index_mappings = index_data[test_constants.INDEX2_NAME][test_constants.MAPPINGS_KEY] - self.assertEqual("strict", index_mappings["dynamic"]) - - @responses.activate - def test_create_indices(self): - # Set up expected PUT calls with a mock response status - test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) - del test_data[test_constants.INDEX1_NAME] - responses.put(test_constants.TARGET_ENDPOINT + test_constants.INDEX2_NAME, - match=[matchers.json_params_matcher(test_data[test_constants.INDEX2_NAME])]) - responses.put(test_constants.TARGET_ENDPOINT + test_constants.INDEX3_NAME, - match=[matchers.json_params_matcher(test_data[test_constants.INDEX3_NAME])]) - index_operations.create_indices(test_data, EndpointInfo(test_constants.TARGET_ENDPOINT)) - - @responses.activate - def test_create_indices_exception(self): - # Set up expected PUT calls with a mock response status - test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) - del test_data[test_constants.INDEX2_NAME] - del test_data[test_constants.INDEX3_NAME] - responses.put(test_constants.TARGET_ENDPOINT + test_constants.INDEX1_NAME, - body=requests.Timeout()) - self.assertRaises(RuntimeError, index_operations.create_indices, test_data, - EndpointInfo(test_constants.TARGET_ENDPOINT)) - - @responses.activate - def test_doc_count(self): - test_indices = {test_constants.INDEX1_NAME, test_constants.INDEX2_NAME} - expected_count_endpoint = test_constants.SOURCE_ENDPOINT + ",".join(test_indices) + "/_count" - mock_count_response = {"count": "10"} - responses.get(expected_count_endpoint, json=mock_count_response) - # Now send request - count_value = index_operations.doc_count(test_indices, EndpointInfo(test_constants.SOURCE_ENDPOINT)) - self.assertEqual(10, count_value) - - -if __name__ == '__main__': - unittest.main() diff --git a/FetchMigration/python/tests/test_pre_migration.py b/FetchMigration/python/tests/test_pre_migration.py deleted file mode 100644 index 9cfe76311..000000000 --- a/FetchMigration/python/tests/test_pre_migration.py +++ /dev/null @@ -1,318 +0,0 @@ -import copy -import pickle -import random -import unittest -from typing import Optional -from unittest.mock import patch, MagicMock, ANY - -import pre_migration -from pre_migration_params import PreMigrationParams -from tests import test_constants - -# Constants -TEST_KEY = "test_key" -INSECURE_KEY = "insecure" -CONNECTION_KEY = "connection" -BASE_CONFIG_SECTION = { - TEST_KEY: [{"invalid_plugin1": {"key": "val"}}, {"invalid_plugin2": {}}] -} - - -# Utility method to create a test plugin config -def create_plugin_config(host_list: list[str], - user: Optional[str] = None, - password: Optional[str] = None, - disable_auth: Optional[bool] = None) -> dict: - config = dict() - config["hosts"] = host_list - if user: - config["username"] = user - if password: - config["password"] = password - if disable_auth is not None: - config["disable_authentication"] = disable_auth - return config - - -# Utility method to creat a test config section -def create_config_section(plugin_config: dict) -> dict: - valid_plugin = dict() - valid_plugin[random.choice(pre_migration.SUPPORTED_ENDPOINTS)] = plugin_config - config_section = copy.deepcopy(BASE_CONFIG_SECTION) - config_section[TEST_KEY].append(valid_plugin) - return config_section - - -class TestPreMigration(unittest.TestCase): - # Run before each test - def setUp(self) -> None: - with open(test_constants.PIPELINE_CONFIG_PICKLE_FILE_PATH, "rb") as f: - self.loaded_pipeline_config = pickle.load(f) - - def test_is_insecure_default_value(self): - self.assertFalse(pre_migration.is_insecure({})) - - def test_is_insecure_top_level_key(self): - test_input = {"key": 123, INSECURE_KEY: True} - self.assertTrue(pre_migration.is_insecure(test_input)) - - def test_is_insecure_nested_key(self): - test_input = {"key1": 123, CONNECTION_KEY: {"key2": "val", INSECURE_KEY: True}} - self.assertTrue(pre_migration.is_insecure(test_input)) - - def test_is_insecure_missing_nested(self): - test_input = {"key1": 123, CONNECTION_KEY: {"key2": "val"}} - self.assertFalse(pre_migration.is_insecure(test_input)) - - def test_get_auth_returns_none(self): - # The following inputs should not return an auth tuple: - # - Empty input - # - user without password - # - password without user - input_list = [{}, {"username": "test"}, {"password": "test"}] - for test_input in input_list: - self.assertIsNone(pre_migration.get_auth(test_input)) - - def test_get_auth_for_valid_input(self): - # Test valid input - result = pre_migration.get_auth({"username": "user", "password": "pass"}) - self.assertEqual(tuple, type(result)) - self.assertEqual("user", result[0]) - self.assertEqual("pass", result[1]) - - def test_get_endpoint_info(self): - host_input = "test" - expected_endpoint = "test/" - test_user = "user" - test_password = "password" - # Simple base case - test_config = create_plugin_config([host_input]) - result = pre_migration.get_endpoint_info(test_config) - self.assertEqual(expected_endpoint, result.url) - self.assertIsNone(result.auth) - self.assertTrue(result.verify_ssl) - # Invalid auth config - test_config = create_plugin_config([host_input], test_user) - result = pre_migration.get_endpoint_info(test_config) - self.assertEqual(expected_endpoint, result.url) - self.assertIsNone(result.auth) - # Valid auth config - test_config = create_plugin_config([host_input], user=test_user, password=test_password) - result = pre_migration.get_endpoint_info(test_config) - self.assertEqual(expected_endpoint, result.url) - self.assertEqual(test_user, result.auth[0]) - self.assertEqual(test_password, result.auth[1]) - # Array of hosts uses the first entry - test_config = create_plugin_config([host_input, "other_host"], test_user, test_password) - result = pre_migration.get_endpoint_info(test_config) - self.assertEqual(expected_endpoint, result.url) - self.assertEqual(test_user, result.auth[0]) - self.assertEqual(test_password, result.auth[1]) - - def test_get_index_differences_empty(self): - # Base case should return an empty list - result_tuple = pre_migration.get_index_differences(dict(), dict()) - # Invariant - self.assertEqual(3, len(result_tuple)) - # All diffs should be empty - self.assertEqual(set(), result_tuple[0]) - self.assertEqual(set(), result_tuple[1]) - self.assertEqual(set(), result_tuple[2]) - - def test_get_index_differences_empty_target(self): - result_tuple = pre_migration.get_index_differences(test_constants.BASE_INDICES_DATA, dict()) - # Invariant - self.assertEqual(3, len(result_tuple)) - # No conflicts or identical indices - self.assertEqual(set(), result_tuple[1]) - self.assertEqual(set(), result_tuple[2]) - # Indices-to-create - self.assertEqual(3, len(result_tuple[0])) - self.assertTrue(test_constants.INDEX1_NAME in result_tuple[0]) - self.assertTrue(test_constants.INDEX2_NAME in result_tuple[0]) - self.assertTrue(test_constants.INDEX3_NAME in result_tuple[0]) - - def test_get_index_differences_identical_index(self): - test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) - del test_data[test_constants.INDEX2_NAME] - del test_data[test_constants.INDEX3_NAME] - result_tuple = pre_migration.get_index_differences(test_data, test_data) - # Invariant - self.assertEqual(3, len(result_tuple)) - # No indices to move, or conflicts - self.assertEqual(set(), result_tuple[0]) - self.assertEqual(set(), result_tuple[2]) - # Identical indices - self.assertEqual(1, len(result_tuple[1])) - self.assertTrue(test_constants.INDEX1_NAME in result_tuple[1]) - - def test_get_index_differences_settings_conflict(self): - test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) - # Set up conflict in settings - index_settings = test_data[test_constants.INDEX2_NAME][test_constants.SETTINGS_KEY] - index_settings[test_constants.INDEX_KEY][test_constants.NUM_REPLICAS_SETTING] += 1 - result_tuple = pre_migration.get_index_differences(test_constants.BASE_INDICES_DATA, test_data) - # Invariant - self.assertEqual(3, len(result_tuple)) - # No indices to move - self.assertEqual(set(), result_tuple[0]) - # Identical indices - self.assertEqual(2, len(result_tuple[1])) - self.assertTrue(test_constants.INDEX1_NAME in result_tuple[1]) - self.assertTrue(test_constants.INDEX3_NAME in result_tuple[1]) - # Conflicting indices - self.assertEqual(1, len(result_tuple[2])) - self.assertTrue(test_constants.INDEX2_NAME in result_tuple[2]) - - def test_get_index_differences_mappings_conflict(self): - test_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) - # Set up conflict in mappings - test_data[test_constants.INDEX3_NAME][test_constants.MAPPINGS_KEY] = {} - result_tuple = pre_migration.get_index_differences(test_constants.BASE_INDICES_DATA, test_data) - # Invariant - self.assertEqual(3, len(result_tuple)) - # No indices to move - self.assertEqual(set(), result_tuple[0]) - # Identical indices - self.assertEqual(2, len(result_tuple[1])) - self.assertTrue(test_constants.INDEX1_NAME in result_tuple[1]) - self.assertTrue(test_constants.INDEX2_NAME in result_tuple[1]) - # Conflicting indices - self.assertEqual(1, len(result_tuple[2])) - self.assertTrue(test_constants.INDEX3_NAME in result_tuple[2]) - - def test_validate_plugin_config_unsupported_endpoints(self): - # No supported endpoints - self.assertRaises(ValueError, pre_migration.validate_plugin_config, BASE_CONFIG_SECTION, TEST_KEY) - - def test_validate_plugin_config_missing_host(self): - test_data = create_config_section({}) - self.assertRaises(ValueError, pre_migration.validate_plugin_config, test_data, TEST_KEY) - - def test_validate_plugin_config_missing_auth(self): - test_data = create_config_section(create_plugin_config(["host"])) - self.assertRaises(ValueError, pre_migration.validate_plugin_config, test_data, TEST_KEY) - - def test_validate_plugin_config_missing_password(self): - test_data = create_config_section(create_plugin_config(["host"], user="test", disable_auth=False)) - self.assertRaises(ValueError, pre_migration.validate_plugin_config, test_data, TEST_KEY) - - def test_validate_plugin_config_missing_user(self): - test_data = create_config_section(create_plugin_config(["host"], password="test")) - self.assertRaises(ValueError, pre_migration.validate_plugin_config, test_data, TEST_KEY) - - def test_validate_plugin_config_auth_disabled(self): - test_data = create_config_section(create_plugin_config(["host"], user="test", disable_auth=True)) - # Should complete without errors - pre_migration.validate_plugin_config(test_data, TEST_KEY) - - def test_validate_plugin_config_happy_case(self): - plugin_config = create_plugin_config(["host"], "user", "password") - test_data = create_config_section(plugin_config) - # Should complete without errors - pre_migration.validate_plugin_config(test_data, TEST_KEY) - - def test_validate_pipeline_config_missing_required_keys(self): - # Test cases: - # - Empty input - # - missing output - # - missing input - bad_configs = [{}, {"source": {}}, {"sink": {}}] - for config in bad_configs: - self.assertRaises(ValueError, pre_migration.validate_pipeline_config, config) - - def test_validate_pipeline_config_happy_case(self): - # Get top level value - test_config = next(iter(self.loaded_pipeline_config.values())) - pre_migration.validate_pipeline_config(test_config) - - @patch('index_operations.doc_count') - @patch('pre_migration.write_output') - @patch('pre_migration.print_report') - @patch('index_operations.create_indices') - @patch('index_operations.fetch_all_indices') - # Note that mock objects are passed bottom-up from the patch order above - def test_run_report(self, mock_fetch_indices: MagicMock, mock_create_indices: MagicMock, - mock_print_report: MagicMock, mock_write_output: MagicMock, mock_doc_count: MagicMock): - mock_doc_count.return_value = 1 - index_to_create = test_constants.INDEX3_NAME - index_with_conflict = test_constants.INDEX2_NAME - index_exact_match = test_constants.INDEX1_NAME - # Set up expected arguments to mocks so we can verify - expected_create_payload = {index_to_create: test_constants.BASE_INDICES_DATA[index_to_create]} - # Print report accepts a tuple. The elements of the tuple - # are in the order: to-create, exact-match, conflicts - expected_diff = {index_to_create}, {index_exact_match}, {index_with_conflict} - # Create mock data for indices on target - target_indices_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) - del target_indices_data[index_to_create] - # Index with conflict - index_settings = target_indices_data[index_with_conflict][test_constants.SETTINGS_KEY] - index_settings[test_constants.INDEX_KEY][test_constants.NUM_REPLICAS_SETTING] += 1 - # Fetch indices is called first for source, then for target - mock_fetch_indices.side_effect = [test_constants.BASE_INDICES_DATA, target_indices_data] - test_input = PreMigrationParams(test_constants.PIPELINE_CONFIG_RAW_FILE_PATH, report=True) - pre_migration.run(test_input) - mock_create_indices.assert_called_once_with(expected_create_payload, ANY) - mock_doc_count.assert_called() - mock_print_report.assert_called_once_with(expected_diff, 1) - mock_write_output.assert_not_called() - - @patch('index_operations.doc_count') - @patch('pre_migration.print_report') - @patch('pre_migration.write_output') - @patch('index_operations.fetch_all_indices') - # Note that mock objects are passed bottom-up from the patch order above - def test_run_dryrun(self, mock_fetch_indices: MagicMock, mock_write_output: MagicMock, - mock_print_report: MagicMock, mock_doc_count: MagicMock): - index_to_create = test_constants.INDEX1_NAME - mock_doc_count.return_value = 1 - expected_output_path = "dummy" - # Create mock data for indices on target - target_indices_data = copy.deepcopy(test_constants.BASE_INDICES_DATA) - del target_indices_data[index_to_create] - # Fetch indices is called first for source, then for target - mock_fetch_indices.side_effect = [test_constants.BASE_INDICES_DATA, target_indices_data] - test_input = PreMigrationParams(test_constants.PIPELINE_CONFIG_RAW_FILE_PATH, expected_output_path, dryrun=True) - test_result = pre_migration.run(test_input) - self.assertEqual(mock_doc_count.return_value, test_result.target_doc_count) - self.assertEqual({index_to_create}, test_result.created_indices) - mock_write_output.assert_called_once_with(self.loaded_pipeline_config, {index_to_create}, expected_output_path) - mock_doc_count.assert_called() - # Report should not be printed - mock_print_report.assert_not_called() - - @patch('yaml.dump') - def test_write_output(self, mock_dump: MagicMock): - expected_output_path = "dummy" - index_to_create = "good_index" - # Set up expected data that will be dumped - expected_output_data = copy.deepcopy(self.loaded_pipeline_config) - expected_output_data['test-pipeline-input']['source'] = { - 'opensearch': { - 'indices': { - 'exclude': [ - {'index_name_regex': 'bad_index'} - ], - 'include': [ - {'index_name_regex': index_to_create} - ] - } - } - } - # Set up test input - test_input = copy.deepcopy(expected_output_data) - del test_input['test-pipeline-input']['source']['opensearch']['indices']['include'] - # Call method under test - with patch('builtins.open') as mock_open: - pre_migration.write_output(test_input, {index_to_create}, expected_output_path) - mock_open.assert_called_once_with(expected_output_path, 'w') - mock_dump.assert_called_once_with(expected_output_data, ANY) - - def test_missing_output_file_non_report(self): - test_input = PreMigrationParams(test_constants.PIPELINE_CONFIG_RAW_FILE_PATH) - self.assertRaises(ValueError, pre_migration.run, test_input) - - -if __name__ == '__main__': - unittest.main() diff --git a/FetchMigration/python/tests/test_utils.py b/FetchMigration/python/tests/test_utils.py deleted file mode 100644 index 10b531519..000000000 --- a/FetchMigration/python/tests/test_utils.py +++ /dev/null @@ -1,32 +0,0 @@ -import re -import unittest - -import utils - - -class TestUtils(unittest.TestCase): - def test_string_from_set_empty(self): - self.assertEqual("[]", utils.string_from_set(set())) - - def test_string_from_set(self): - test_set = {'a b c', 'xyz', '123'} - # Ordering from a set is not deterministic, but the output should match this regex - pattern = re.compile(r"^\[(.+)(, )*]$") - self.assertTrue(pattern.match(utils.string_from_set(test_set))) - - def test_has_differences(self): - # Base case of empty dicts - self.assertFalse(utils.has_differences("key", dict(), dict())) - # Non empty dicts but differing key - self.assertFalse(utils.has_differences("key", {"test": 0}, {"test": 0})) - # Missing key - self.assertTrue(utils.has_differences("key", dict(), {"key": 0})) - self.assertTrue(utils.has_differences("key", {"key": 0}, {})) - # Differing values - self.assertTrue(utils.has_differences("key", {"key": 0}, {"key": 1})) - # Identical dicts - self.assertFalse(utils.has_differences("key", {"key": 0}, {"key": 0})) - - -if __name__ == '__main__': - unittest.main() diff --git a/FetchMigration/python/utils.py b/FetchMigration/python/utils.py deleted file mode 100644 index 5aafaf148..000000000 --- a/FetchMigration/python/utils.py +++ /dev/null @@ -1,22 +0,0 @@ -from jsondiff import diff - - -# Utility method to make a comma-separated string from a set. -# If the set is empty, "[]" is returned for clarity. -def string_from_set(s: set[str]) -> str: - result = "[" - if s: - result += ", ".join(s) - return result + "]" - - -# Utility method to compare the JSON contents of a key in two dicts. -# This method handles checking if the key exists in either dict. -def has_differences(key: str, dict1: dict, dict2: dict) -> bool: - if key not in dict1 and key not in dict2: - return False - elif key in dict1 and key in dict2: - data_diff = diff(dict1[key], dict2[key]) - return bool(data_diff) - else: - return True diff --git a/TrafficCapture/captureOffloader/src/test/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializerTest.java b/TrafficCapture/captureOffloader/src/test/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializerTest.java index 38a9c0393..477d691fa 100644 --- a/TrafficCapture/captureOffloader/src/test/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializerTest.java +++ b/TrafficCapture/captureOffloader/src/test/java/org/opensearch/migrations/trafficcapture/StreamChannelConnectionCaptureSerializerTest.java @@ -131,7 +131,7 @@ public void testBasicDataConsistencyWhenChunking() throws IOException, Execution var outputBuffersCreated = new ConcurrentLinkedQueue(); // Arbitrarily picking small buffer that can hold the overhead TrafficStream bytes as well as some // data bytes but not all the data bytes and require chunking - var serializer = createSerializerWithTestHandler(outputBuffersCreated, 55); + var serializer = createSerializerWithTestHandler(outputBuffersCreated, 85); var bb = Unpooled.wrappedBuffer(packetBytes); serializer.addWriteEvent(referenceTimestamp, bb); diff --git a/TrafficCapture/dockerSolution/README.md b/TrafficCapture/dockerSolution/README.md index 826669a72..38a2ac191 100644 --- a/TrafficCapture/dockerSolution/README.md +++ b/TrafficCapture/dockerSolution/README.md @@ -8,19 +8,9 @@ down again. Notice that most of the Dockerfiles are dynamically constructed in the build hierarchy. Some efforts have been made to ensure that changes will make it into containers to be launched. -If a user wants to use their own checkout of the traffic-comparator repo, just set the environment variable " -TRAFFIC_COMPARATOR_DIRECTORY" to the directory that contains `setup.py`. Otherwise, if that isn't set, the traffic -comparator repo will be checked out to the build directory and that will be used. Notice that the checkout happens when -the directory wasn't present and there wasn't an environment variable specifying a directory. Once a directory exists, -it will be mounted to the traffic-comparator and jupyter services. - -Netcat is still used to connect several of the components and we're still working on improving the resiliency story -between these containers. The long term approach is to replace fixed streams with message bus approaches directly (i.e. -Kafka). In the short term, we can and are beginning, to leverage things like conditions on dependent services. - ### Running the Docker Solution -While in the TrafficCapture directory, run the following command: +While Docker is running, in the TrafficCapture directory run the following command: `./gradlew :dockerSolution:composeUp` diff --git a/TrafficCapture/dockerSolution/build.gradle b/TrafficCapture/dockerSolution/build.gradle index ecc2197b4..8a99a168d 100644 --- a/TrafficCapture/dockerSolution/build.gradle +++ b/TrafficCapture/dockerSolution/build.gradle @@ -9,19 +9,6 @@ import java.security.MessageDigest import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage import org.apache.tools.ant.taskdefs.condition.Os -def getTrafficComparatorDirectory() { - String overrideTrafficComparatorDirectory = System.getenv(TRAFFIC_COMPARATOR_DIRECTORY_ENV) - String rval = overrideTrafficComparatorDirectory != null ? - overrideTrafficComparatorDirectory : TRAFFIC_COMPARATOR_REPO_DIRECTORY; - return rval -} - -ext { - TRAFFIC_COMPARATOR_REPO_DIRECTORY = "build/traffic-comparator" - TRAFFIC_COMPARATOR_DIRECTORY_ENV = "TRAFFIC_COMPARATOR_DIRECTORY" - REALIZED_TRAFFIC_COMPARATOR_DIRECTORY = project.file(getTrafficComparatorDirectory()) - } - def calculateDockerHash = { projectName -> CommonUtils.calculateDockerHash(projectName, project) } @@ -31,17 +18,6 @@ dependencies { implementation project(':trafficReplayer') } -task cloneComparatorRepoIfNeeded(type: Exec) { - String comparatorDirectory = project.file(REALIZED_TRAFFIC_COMPARATOR_DIRECTORY); - String repo = 'https://github.com/opensearch-project/traffic-comparator.git' - onlyIf { - !(new File(comparatorDirectory).exists()) - } - commandLine = Os.isFamily(Os.FAMILY_WINDOWS) ? - ['git', 'clone', repo, TRAFFIC_COMPARATOR_REPO_DIRECTORY ] : - ['/bin/sh', '-c', "git clone ${repo} ${TRAFFIC_COMPARATOR_REPO_DIRECTORY}"] -} - def dockerFilesForExternalServices = [ "elasticsearchWithSearchGuard": "elasticsearch_searchguard", "migrationConsole": "migration_console" @@ -56,36 +32,6 @@ dockerFilesForExternalServices.each { projectName, dockerImageName -> } } -def trafficComparatorServices = [ - "trafficComparator": "traffic_comparator", - "jupyterNotebook": "jupyter_notebook" -] -trafficComparatorServices.forEach {projectName, dockerImageName -> - def dockerBuildDir = "build/docker/${projectName}" - task("copyArtifact_${projectName}", type: Copy) { - dependsOn(tasks.getByName('cloneComparatorRepoIfNeeded')) - from REALIZED_TRAFFIC_COMPARATOR_DIRECTORY - into dockerBuildDir - include '*.py' - include '/traffic_comparator/*' - if (projectName == 'jupyterNotebook') { - include '*.ipynb' - } - } - - task "createDockerfile_${projectName}"(type: com.bmuschko.gradle.docker.tasks.image.Dockerfile) { - dependsOn "copyArtifact_${projectName}" - destFile = project.file("${dockerBuildDir}/Dockerfile") - from 'python:3.10.10' - runCommand("apt-get update && apt-get install -y netcat lsof") - copyFile("setup.py", "/setup.py") - copyFile(".", "/containerTC/") - runCommand("pip3 install --editable \".[data]\"") - // container stay-alive - defaultCommand('tail', '-f', '/dev/null') - } -} - def javaContainerServices = [ "trafficCaptureProxyServer": "capture_proxy", "trafficReplayer": "traffic_replayer" @@ -101,7 +47,7 @@ javaContainerServices.each { projectName, dockerImageName -> CommonUtils.createDockerfile(project, projectName, baseImageProjectOverrides, dockerFilesForExternalServices) } -(javaContainerServices + trafficComparatorServices).forEach { projectName, dockerImageName -> +(javaContainerServices).forEach { projectName, dockerImageName -> def dockerBuildDir = "build/docker/${projectName}" task "buildDockerImage_${projectName}"(type: DockerBuildImage) { dependsOn "createDockerfile_${projectName}" @@ -112,11 +58,6 @@ javaContainerServices.each { projectName, dockerImageName -> } dockerCompose { - String overrideTrafficComparatorDirectory = System.getenv(TRAFFIC_COMPARATOR_DIRECTORY_ENV) - if (overrideTrafficComparatorDirectory == null) { - environment.put(TRAFFIC_COMPARATOR_DIRECTORY_ENV, REALIZED_TRAFFIC_COMPARATOR_DIRECTORY) - exposeAsEnvironment(this) - } useComposeFiles.add("src/main/docker/docker-compose.yml") } @@ -126,10 +67,7 @@ task buildDockerImages { dependsOn buildDockerImage_trafficCaptureProxyServer dependsOn buildDockerImage_trafficReplayer - dependsOn buildDockerImage_trafficComparator - dependsOn buildDockerImage_jupyterNotebook } tasks.getByName('composeUp') .dependsOn(tasks.getByName('buildDockerImages')) - .dependsOn(tasks.getByName('cloneComparatorRepoIfNeeded')) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 558403335..e666ac330 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -78,9 +78,7 @@ services: condition: service_started opensearchtarget: condition: service_started - trafficcomparator: - condition: service_healthy - command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group | nc trafficcomparator 9220" + command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group >/dev/null" opensearchtarget: image: 'opensearchproject/opensearch:latest' @@ -91,32 +89,33 @@ services: ports: - "29200:9200" - trafficcomparator: - image: 'migrations/traffic_comparator:latest' - networks: - - migrations - ports: - - "9220:9220" - healthcheck: - test: "lsof -i -P -n" - volumes: - - ${TRAFFIC_COMPARATOR_DIRECTORY}:/trafficComparator - - sharedComparatorSqlResults:/shared - command: /bin/sh -c "cd trafficComparator && pip3 install --editable . && nc -v -l -p 9220 | tee /dev/stderr | trafficcomparator -vv stream | trafficcomparator dump-to-sqlite --db /shared/comparisons.db" - - jupyter-notebook: - image: 'migrations/jupyter_notebook:latest' - networks: - - migrations - ports: - - "8888:8888" - volumes: - - ${TRAFFIC_COMPARATOR_DIRECTORY}:/trafficComparator - - sharedComparatorSqlResults:/shared - environment: - # this needs to match the output db that traffic_comparator writes to - - COMPARISONS_DB_LOCATION=/shared/comparisons.db - command: /bin/sh -c 'cd trafficComparator && pip3 install --editable ".[data]" && jupyter notebook --ip=0.0.0.0 --port=8888 --no-browser --allow-root' +# EXPERIMENTAL +# trafficcomparator: +# image: 'migrations/traffic_comparator:latest' +# networks: +# - migrations +# ports: +# - "9220:9220" +# healthcheck: +# test: "lsof -i -P -n" +# volumes: +# - ${TRAFFIC_COMPARATOR_DIRECTORY}:/trafficComparator +# - sharedComparatorSqlResults:/shared +# command: /bin/sh -c "cd trafficComparator && pip3 install --editable . && nc -v -l -p 9220 | tee /dev/stderr | trafficcomparator -vv stream | trafficcomparator dump-to-sqlite --db /shared/comparisons.db" +# +# jupyter-notebook: +# image: 'migrations/jupyter_notebook:latest' +# networks: +# - migrations +# ports: +# - "8888:8888" +# volumes: +# - ${TRAFFIC_COMPARATOR_DIRECTORY}:/trafficComparator +# - sharedComparatorSqlResults:/shared +# environment: +# # this needs to match the output db that traffic_comparator writes to +# - COMPARISONS_DB_LOCATION=/shared/comparisons.db +# command: /bin/sh -c 'cd trafficComparator && pip3 install --editable ".[data]" && jupyter notebook --ip=0.0.0.0 --port=8888 --no-browser --allow-root' migration-console: image: 'migrations/migration_console:latest' @@ -130,11 +129,9 @@ volumes: driver: local kafka_data: driver: local - sharedComparatorSqlResults: - driver: local sharedReplayerOutput: driver: local networks: migrations: - driver: bridge \ No newline at end of file + driver: bridge diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile index 1dd8c5679..7087db04c 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile @@ -14,4 +14,4 @@ RUN chmod ug+x /root/humanReadableLogs.py RUN chmod ug+x /root/catIndices.sh WORKDIR /root -CMD tail -f /dev/null \ No newline at end of file +CMD tail -f /dev/null diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runTestBenchmarks.sh b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runTestBenchmarks.sh index 37581919b..c3792ade1 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runTestBenchmarks.sh +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runTestBenchmarks.sh @@ -54,4 +54,4 @@ opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=$en echo "Running opensearch-benchmark w/ 'nested' workload..." && opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=$endpoint --workload=nested --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1" --client-options=$client_options && echo "Running opensearch-benchmark w/ 'nyc_taxis' workload..." && -opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=$endpoint --workload=nyc_taxis --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1" --client-options=$client_options \ No newline at end of file +opensearch-benchmark execute-test --distribution-version=1.0.0 --target-host=$endpoint --workload=nyc_taxis --pipeline=benchmark-only --test-mode --kill-running-processes --workload-params "target_throughput:0.5,bulk_size:10,bulk_indexing_clients:1,search_clients:1" --client-options=$client_options diff --git a/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java b/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java index 9f75988dd..16d961eb2 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/test/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPoolTest.java @@ -6,6 +6,7 @@ import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Disabled; import java.time.Duration; import java.time.Instant; @@ -41,6 +42,7 @@ class ExpiringSubstitutableItemPoolTest { * in place may further mitigate inconsistencies, though I haven't had any tests fail yet * unless I've stopped threads within the debugger. */ + @Disabled @Test void get() throws Exception { var firstWaveBuildCountdownLatch = new CountDownLatch(NUM_POOLED_ITEMS); @@ -147,4 +149,4 @@ private static Integer getIntegerItem(AtomicInteger builtItemCursor, lastCreation.set(Instant.now()); return Integer.valueOf(builtItemCursor.incrementAndGet()); } -} +} \ No newline at end of file diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java index 1ec1da540..5a9904da5 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumer.java @@ -28,7 +28,7 @@ * transformation and the headers present (such as for gzipped or chunked encodings). * * There will be a number of reasons that we need to reuse the source captured packets - both today - * (for the output to the comparator) and in the future (for retrying transient network errors or + * (for the output and in the future (for retrying transient network errors or * transformation errors). With that in mind, the HttpJsonTransformer now keeps track of all of * the ByteBufs passed into it and can redrive them through the underlying network packet handler. * Cases where that would happen with this edit are where the payload wasn't being modified, nor @@ -58,7 +58,7 @@ public class HttpJsonTransformingConsumer implements IPacketFinalizingConsume private final List> chunkSizes; // This is here for recovery, in case anything goes wrong with a transformation & we want to // just dump it directly. Notice that we're already storing all of the bytes until the response - // comes back so that we can format the output that goes to the comparator. These should be + // comes back so that we can format the output that goes downstream. These should be // backed by the exact same byte[] arrays, so the memory consumption should already be absorbed. private final List chunks; diff --git a/datastash/README.md b/datastash/README.md deleted file mode 100644 index cd2e2ffd8..000000000 --- a/datastash/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Datastash - -## Overview - -The purpose of these Docker configurations is to provide a simplified Logstash container that users can easily customize to migrate all indices from a source cluster (Elasticsearch by default) to a target cluster (OpenSearch by default). - -## Instructions - -### 0. Prerequisites - -* [curl](https://curl.se/) -* [jq](https://stedolan.github.io/jq/) - -### 1. Update the Logstash configuration file - -Edit the `logstash.conf` configuration file to point to your source and target `hosts`. Make sure to also add [any other settings](https://github.com/opensearch-project/logstash-output-opensearch) that are appropriate, such as authentication (`user` and `password`) and `ssl`. Typically, you will not need to change any of the other settings - they are aimed at moving all indices from the source to the target, retaining their names as-is. - -### 2. Configure index templates on the target cluster - -By default, the `logstash.conf` configuration file used by Datastash creates an index on the target cluster with the _same name_ as the source cluster. To ensure that index settings and mappings are correctly configured, perform the following steps _for each index that will be migrated_: - -1. Form the index template by running the following command, replacing `` with the name of the index being migrated: - -``` -indexName=; curl -s http://ec2-18-236-163-55.us-west-2.compute.amazonaws.com:8443/$indexName | jq --arg INDEXNAME "$indexName" '{index_patterns: [$INDEXNAME], priority: 500, template: {settings: .[$INDEXNAME].settings, mappings: .[$INDEXNAME].mappings}} | del(.template.settings.index.creation_date, .template.settings.index.uuid, .template.settings.index.provided_name, .template.settings.index.version)' > /tmp/datastash_template.json -``` - -2. Update the target cluster with the index template, replacing `` with the name of the index being migrated: - -``` -indexName=; curl -XPUT -H 'Content-Type: application/json' ":/_index_template/datastash_template_$indexName?pretty" -d @/tmp/datastash_template.json -``` - -3. Finally, clean up the temporary file we created: - -``` -rm /tmp/datastash_template.json -``` - -### 3. Run the migration - -You're finally ready to kick off the migration! Move your data by running `docker compose up`. Once the process completes, the docker container will shut itself down. - -# Limitations - -This tool has several limitations: - -1. Logstash does not include any support for clustering or load balancing ([[1]](https://discuss.elastic.co/t/how-can-make-a-cluster-with-logstash-is-it-possible/250444/2), [[2]](https://github.com/elastic/logstash/issues/2632), [[3]](https://github.com/elastic/logstash/issues/2633)), so users must set up their own load balancing. This makes it impossible to horizontally scale a Logstash setup that uses a pull-based plugin like the ElasticSearch/OpenSearch input plugin. - -1. Since Logstash cannot make any assumptions about the input or output, it cannot perform any setup steps before sending data to the target cluster, and the validation it performs is minimal. - * Logstash does not validate the configuration of the output plugin. Any mistakes in the OpenSearch output plugin configuration (endpoint, username/password, SSL, etc.) do not fail the Logstash pipeline immediately. Instead, Logstash will continue to process the input (until complete), and then fail each batch at the output. - * Contrary to the output plugin, Logstash does validate connectivity from the input plugin. Any errors therein cause the pipeline to stop processing completely. However, since Logstash cannot make any assumptions about the input, it does not check for a index of the same name on the target cluster. - * Further, these decisions mean that index mappings are not copied to the target cluster, which necessitates the manual steps outlined above. - -1. Logstash also implicitly adds two internal fields to each record in its output - “@version” and “@timestamp”. There is no way to prevent Logstash from adding these fields. The suggested workaround is to apply a mutate filter to remove these fields, but such an approach would remove these fields even if they existed in the original index. - -1. Logstash versions above 7.14 do not support OSS endpoints due to a [change introduced to the ElasticSearch clients](https://discuss.elastic.co/t/logstash-not-start-after-upgrade-to-new-version/291849/2). In order to work around this restriction, this tool uses an older image ([7.13.4](https://hub.docker.com/layers/opensearchproject/logstash-oss-with-opensearch-output-plugin/7.13.4/images/sha256-a0370926a62f5a81078e24f9acb37e6750ee2bc8472a37b071c40b6765319ea5?context=explore)) from Dockerhub. This prevents the client from receiving any further updates, which risks breaking compatibility with future versions of ElasticSearch. diff --git a/datastash/docker-compose.yml b/datastash/docker-compose.yml deleted file mode 100644 index 379c5412b..000000000 --- a/datastash/docker-compose.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: '3' -services: - datastash: - image: opensearchproject/logstash-oss-with-opensearch-output-plugin:7.13.4 - container_name: datastash - environment: - - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping - - "LS_JAVA_OPTS=-Xms1g -Xmx1g" - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 - volumes: - - type: bind - source: ./logstash.conf - target: /usr/share/logstash/pipeline/logstash.conf # Overrides the default conf in the Docker image diff --git a/datastash/logstash.conf b/datastash/logstash.conf deleted file mode 100644 index aa0928fab..000000000 --- a/datastash/logstash.conf +++ /dev/null @@ -1,25 +0,0 @@ -input { - elasticsearch { - hosts => "" - index => "*" - docinfo => true - docinfo_target => "[@metadata][docinfo]" - } -} - -filter { - mutate { - # Remove internal fields added by Logstash - remove_field => ["@version", "@timestamp"] - } -} - -output { - opensearch { - hosts => "" - index => "%{[@metadata][docinfo][_index]}" - document_id => "%{[@metadata][docinfo][_id]}" - action => "update" - doc_as_upsert => true - } -} diff --git a/deployment/cdk/opensearch-service-migration/lib/historical-capture-stack.ts b/deployment/cdk/opensearch-service-migration/lib/historical-capture-stack.ts deleted file mode 100644 index 93f1408a4..000000000 --- a/deployment/cdk/opensearch-service-migration/lib/historical-capture-stack.ts +++ /dev/null @@ -1,62 +0,0 @@ -import {Stack, StackProps} from "aws-cdk-lib"; -import {IVpc} from "aws-cdk-lib/aws-ec2"; -import {Construct} from "constructs"; -import {Cluster, ContainerImage, FargateService, FargateTaskDefinition, LogDrivers} from "aws-cdk-lib/aws-ecs"; -import {DockerImageAsset} from "aws-cdk-lib/aws-ecr-assets"; -import {join} from "path"; -import { readFileSync } from "fs" - -export interface historicalCaptureStackProps extends StackProps { - readonly vpc: IVpc, - readonly logstashConfigFilePath: string, - readonly sourceEndpoint?: string, - readonly targetEndpoint: string -} - -/** - * This stack was a short exploratory task into having a deployable Logstash ECS cluster for historical data migration. - * NOTE: It should only be used for development purposes in its current state - */ -export class HistoricalCaptureStack extends Stack { - - constructor(scope: Construct, id: string, props: historicalCaptureStackProps) { - super(scope, id, props); - - const ecsCluster = new Cluster(this, "ecsHistoricalCaptureCluster", { - vpc: props.vpc - }); - - const historicalCaptureFargateTask = new FargateTaskDefinition(this, "historicalCaptureFargateTask", { - memoryLimitMiB: 2048, - cpu: 512 - }); - - let logstashConfigData: string = readFileSync(props.logstashConfigFilePath, 'utf8'); - if (props.sourceEndpoint) { - logstashConfigData = logstashConfigData.replace("", props.sourceEndpoint) - } - logstashConfigData = logstashConfigData.replace("", props.targetEndpoint + ":80") - // Temporary measure to allow multi-line env variable - logstashConfigData = logstashConfigData.replace(/(\n)/g, "PUT_LINE") - // Create Historical Capture Container - const historicalCaptureImage = new DockerImageAsset(this, "historicalCaptureImage", { - directory: join(__dirname, "../../..", "docker/logstash-setup") - }); - - const historicalCaptureContainer = historicalCaptureFargateTask.addContainer("historicalCaptureContainer", { - image: ContainerImage.fromDockerImageAsset(historicalCaptureImage), - // Add in region and stage - containerName: "logstash", - environment: {"LOGSTASH_CONFIG": '' + logstashConfigData}, - logging: LogDrivers.awsLogs({ streamPrefix: 'logstash-lg', logRetention: 30 }) - }); - - // Create Fargate Service - const historicalCaptureFargateService = new FargateService(this, "historicalCaptureFargateService", { - cluster: ecsCluster, - taskDefinition: historicalCaptureFargateTask, - desiredCount: 1 - }); - - } -} \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts b/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts index 90a1cbdf1..769dfacf7 100644 --- a/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/migration-assistance-stack.ts @@ -103,18 +103,6 @@ export class MigrationAssistanceStack extends Stack { }); this.mskARN = mskCluster.attrArn - const comparatorSQLiteSG = new SecurityGroup(this, 'comparatorSQLiteSG', { - vpc: props.vpc, - allowAllOutbound: false, - }); - comparatorSQLiteSG.addIngressRule(comparatorSQLiteSG, Port.allTraffic()); - - // Create an EFS file system for the traffic-comparator - const comparatorSQLiteEFS = new FileSystem(this, 'comparatorSQLiteEFS', { - vpc: props.vpc, - securityGroup: comparatorSQLiteSG - }); - const replayerOutputSG = new SecurityGroup(this, 'replayerOutputSG', { vpc: props.vpc, allowAllOutbound: false, @@ -132,8 +120,6 @@ export class MigrationAssistanceStack extends Stack { const exports = [ `export MIGRATION_VPC_ID=${props.vpc.vpcId}`, `export MIGRATION_CAPTURE_MSK_SG_ID=${mskSecurityGroup.securityGroupId}`, - `export MIGRATION_COMPARATOR_EFS_ID=${comparatorSQLiteEFS.fileSystemId}`, - `export MIGRATION_COMPARATOR_EFS_SG_ID=${comparatorSQLiteSG.securityGroupId}`, `export MIGRATION_REPLAYER_OUTPUT_EFS_ID=${replayerOutputEFS.fileSystemId}`, `export MIGRATION_REPLAYER_OUTPUT_EFS_SG_ID=${replayerOutputSG.securityGroupId}`] if (publicSubnetString) exports.push(`export MIGRATION_PUBLIC_SUBNETS=${publicSubnetString}`) diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts index e80ca52d9..d416d576f 100644 --- a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts +++ b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts @@ -7,7 +7,6 @@ import {AnyPrincipal, Effect, PolicyStatement} from "aws-cdk-lib/aws-iam"; import * as defaultValuesJson from "../default-values.json" import {NetworkStack} from "./network-stack"; import {MigrationAssistanceStack} from "./migration-assistance-stack"; -import {HistoricalCaptureStack} from "./historical-capture-stack"; import {MSKUtilityStack} from "./msk-utility-stack"; export interface StackPropsExt extends StackProps { @@ -189,25 +188,6 @@ export class StackComposer { this.stacks.push(mskUtilityStack) } - // Currently, placing a requirement on a VPC for a historical capture stack but this can be revisited - // Note: Future work to provide orchestration between historical capture and migration assistance as the current - // state will potentially have both stacks trying to add the same data - if (historicalCaptureEnabled && networkStack) { - const historicalCaptureStack = new HistoricalCaptureStack(scope, "historicalCaptureStack", { - vpc: networkStack.vpc, - logstashConfigFilePath: logstashConfigFilePath, - sourceEndpoint: sourceClusterEndpoint, - targetEndpoint: opensearchStack.domainEndpoint, - stackName: `OSServiceHistoricalCDKStack-${stage}-${region}`, - description: "This stack contains resources to assist migrating historical data to an OpenSearch Service domain", - ...props, - }) - - historicalCaptureStack.addDependency(opensearchStack) - this.stacks.push(historicalCaptureStack) - } - - function getContextForType(optionName: string, expectedType: string): any { const option = scope.node.tryGetContext(optionName) diff --git a/deployment/copilot/README.md b/deployment/copilot/README.md index 7f792eebe..ae7385435 100644 --- a/deployment/copilot/README.md +++ b/deployment/copilot/README.md @@ -12,7 +12,7 @@ Copilot is a tool for deploying containerized applications on AWS ECS. Official ###### Docker Docker is used by Copilot to build container images. If not installed, follow the steps [here](https://docs.docker.com/engine/install/) to set up. Later versions are recommended. ###### Git -Git is used by the opensearch-migrations repo to fetch associated repositories (such as the traffic-comparator repo) for constructing their respective Dockerfiles. Steps to set up can be found [here](https://github.com/git-guides/install-git). +Git is used by the opensearch-migrations repo. Steps to set up can be found [here](https://github.com/git-guides/install-git). ###### Java 11 Java is used by the opensearch-migrations repo and Gradle, its associated build tool. The current required version is Java 11. @@ -87,8 +87,6 @@ export MIGRATION_DOMAIN_ENDPOINT=vpc-aos-domain-123.us-east-1.es.amazonaws.com; export MIGRATION_DOMAIN_USER_AND_SECRET_ARN=admin arn:aws:secretsmanager:us-east-1:123456789123:secret:demo-user-secret-123abc export MIGRATION_VPC_ID=vpc-123; export MIGRATION_CAPTURE_MSK_SG_ID=sg-123; -export MIGRATION_COMPARATOR_EFS_ID=fs-123; -export MIGRATION_COMPARATOR_EFS_SG_ID=sg-123; export MIGRATION_REPLAYER_OUTPUT_EFS_ID=fs-124 export MIGRATION_REPLAYER_OUTPUT_EFS_SG_ID=sg-124 export MIGRATION_PUBLIC_SUBNETS=subnet-123,subnet-124; @@ -97,7 +95,7 @@ export MIGRATION_KAFKA_BROKER_ENDPOINTS=b-1-public.loggingmskcluster.123.45.kafk ``` Additionally, if not using the deploy script, the following export is needed for the Replayer service: ``` -export MIGRATION_REPLAYER_COMMAND=/bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer $MIGRATION_DOMAIN_ENDPOINT --insecure --kafka-traffic-brokers $MIGRATION_KAFKA_BROKER_ENDPOINTS --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --kafka-traffic-enable-msk-auth --auth-header-user-and-secret $MIGRATION_DOMAIN_USER_AND_SECRET_ARN | nc traffic-comparator 9220" +export MIGRATION_REPLAYER_COMMAND=/bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer $MIGRATION_DOMAIN_ENDPOINT --insecure --kafka-traffic-brokers $MIGRATION_KAFKA_BROKER_ENDPOINTS --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --kafka-traffic-enable-msk-auth --auth-header-user-and-secret $MIGRATION_DOMAIN_USER_AND_SECRET_ARN" ``` #### Setting up existing Copilot infrastructure @@ -124,8 +122,6 @@ copilot env init --name dev // Initialize services with their respective required name copilot svc init --name traffic-replayer -copilot svc init --name traffic-comparator -copilot svc init --name traffic-comparator-jupyter copilot svc init --name capture-proxy-es copilot svc init --name migration-console @@ -141,8 +137,6 @@ Currently, it seems that Copilot does not support deploying all services at once copilot env deploy --name dev // Deploy services to a deployed environment -copilot svc deploy --name traffic-comparator-jupyter --env dev -copilot svc deploy --name traffic-comparator --env dev copilot svc deploy --name traffic-replayer --env dev copilot svc deploy --name capture-proxy-es --env dev copilot svc deploy --name migration-console --env dev @@ -182,8 +176,6 @@ curl https://$MIGRATION_DOMAIN_ENDPOINT:443/_cat/indices?v --insecure -u admin:A A command shell can be opened in the service's container if that service has enabled `exec: true` in their `manifest.yml` and the SSM Session Manager plugin is installed when prompted. ``` -copilot svc exec -a migration-copilot -e dev -n traffic-comparator-jupyter -c "bash" -copilot svc exec -a migration-copilot -e dev -n traffic-comparator -c "bash" copilot svc exec -a migration-copilot -e dev -n traffic-replayer -c "bash" copilot svc exec -a migration-copilot -e dev -n elasticsearch -c "bash" copilot svc exec -a migration-copilot -e dev -n capture-proxy -c "bash" @@ -202,7 +194,7 @@ Official documentation on Addons can be found [here](https://aws.github.io/copil `copilot app show` - Provides details on the current app \ `copilot svc show` - Provides details on a particular service -### Removing deloyed resources from AWS +### Removing deployed resources from AWS To remove the resources installed from the steps above, follow these instructions: 1. `./devDeploy.sh --destroy-env` - Destroy all CDK and Copilot CloudFormation stacks deployed, excluding the Copilot app level stack, for the given env/stage and return to a clean state. diff --git a/deployment/copilot/devDeploy.sh b/deployment/copilot/devDeploy.sh index 881dbd2cc..ce906f990 100755 --- a/deployment/copilot/devDeploy.sh +++ b/deployment/copilot/devDeploy.sh @@ -99,8 +99,6 @@ if [ "$DESTROY_ENV" = true ] ; then set +e # Reset AWS_DEFAULT_REGION as the SDK used by Copilot will first check here for region to use to locate the Copilot app (https://github.com/aws/copilot-cli/issues/5138) export AWS_DEFAULT_REGION="" - copilot svc delete -a $COPILOT_APP_NAME --name traffic-comparator-jupyter --env $COPILOT_DEPLOYMENT_STAGE --yes - copilot svc delete -a $COPILOT_APP_NAME --name traffic-comparator --env $COPILOT_DEPLOYMENT_STAGE --yes copilot svc delete -a $COPILOT_APP_NAME --name traffic-replayer --env $COPILOT_DEPLOYMENT_STAGE --yes copilot svc delete -a $COPILOT_APP_NAME --name capture-proxy-es --env $COPILOT_DEPLOYMENT_STAGE --yes copilot svc delete -a $COPILOT_APP_NAME --name migration-console --env $COPILOT_DEPLOYMENT_STAGE --yes @@ -166,8 +164,8 @@ if [ "$SKIP_COPILOT_INIT" = false ] ; then #copilot env init -a $COPILOT_APP_NAME --name $COPILOT_DEPLOYMENT_STAGE --default-config --aws-access-key-id $AWS_ACCESS_KEY_ID --aws-secret-access-key $AWS_SECRET_ACCESS_KEY --aws-session-token $AWS_SESSION_TOKEN --region $REGION # Init services - copilot svc init -a $COPILOT_APP_NAME --name traffic-comparator-jupyter - copilot svc init -a $COPILOT_APP_NAME --name traffic-comparator + # copilot svc init -a $COPILOT_APP_NAME --name traffic-comparator-jupyter # EXPERIMENTAL + # copilot svc init -a $COPILOT_APP_NAME --name traffic-comparator # EXPERIMENTAL copilot svc init -a $COPILOT_APP_NAME --name capture-proxy-es copilot svc init -a $COPILOT_APP_NAME --name migration-console else @@ -179,11 +177,11 @@ fi copilot env deploy -a $COPILOT_APP_NAME --name $COPILOT_DEPLOYMENT_STAGE # Deploy services -copilot svc deploy -a $COPILOT_APP_NAME --name traffic-comparator-jupyter --env $COPILOT_DEPLOYMENT_STAGE --resource-tags $TAGS -copilot svc deploy -a $COPILOT_APP_NAME --name traffic-comparator --env $COPILOT_DEPLOYMENT_STAGE --resource-tags $TAGS +# copilot svc deploy -a $COPILOT_APP_NAME --name traffic-comparator-jupyter --env $COPILOT_DEPLOYMENT_STAGE --resource-tags $TAGS # EXPERIMENTAL +# copilot svc deploy -a $COPILOT_APP_NAME --name traffic-comparator --env $COPILOT_DEPLOYMENT_STAGE --resource-tags $TAGS # EXPERIMENTAL copilot svc deploy -a $COPILOT_APP_NAME --name capture-proxy-es --env $COPILOT_DEPLOYMENT_STAGE --resource-tags $TAGS copilot svc deploy -a $COPILOT_APP_NAME --name migration-console --env $COPILOT_DEPLOYMENT_STAGE --resource-tags $TAGS -./createReplayer.sh --id default --target-uri "https://${MIGRATION_DOMAIN_ENDPOINT}:443" --tags ${TAGS} --extra-args "--auth-header-user-and-secret ${MIGRATION_DOMAIN_USER_AND_SECRET_ARN} | nc traffic-comparator 9220" "${REPLAYER_SKIP_INIT_ARG}" +./createReplayer.sh --id default --target-uri "https://${MIGRATION_DOMAIN_ENDPOINT}:443" --tags ${TAGS} --extra-args "--auth-header-user-and-secret ${MIGRATION_DOMAIN_USER_AND_SECRET_ARN}" "${REPLAYER_SKIP_INIT_ARG}" # Output deployment time diff --git a/deployment/copilot/traffic-comparator-jupyter/manifest.yml b/deployment/copilot/traffic-comparator-jupyter/manifest.yml deleted file mode 100644 index 47e957561..000000000 --- a/deployment/copilot/traffic-comparator-jupyter/manifest.yml +++ /dev/null @@ -1,56 +0,0 @@ -# The manifest for the "traffic-comparator-jupyter" service. -# Read the full specification for the "Load Balanced Web Service" type at: -# https://aws.github.io/copilot-cli/docs/manifest/lb-web-service/ - -# Your service name will be used in naming your resources like log groups, ECS services, etc. -name: traffic-comparator-jupyter -type: Load Balanced Web Service - -# Distribute traffic to your service. -http: - # Requests to this path will be forwarded to your service. - # To match all requests you can use the "/" path. - path: '/' - # Bogus health check to pass startup - healthcheck: - path: '/' - port: 8888 - success_codes: '200-405' - healthy_threshold: 2 - unhealthy_threshold: 2 - interval: 15s - timeout: 10s - grace_period: 60s - -network: - connect: true - vpc: - security_groups: [ "${MIGRATION_COMPARATOR_EFS_SG_ID}" ] - -# Configuration for your containers and service. -image: - # Docker build arguments. For additional overrides: https://aws.github.io/copilot-cli/docs/manifest/backend-service/#image-build - build: - dockerfile: ../TrafficCapture/dockerSolution/build/docker/jupyterNotebook/Dockerfile - port: 8888 - -command: /bin/sh -c 'cd containerTC && pip3 install --editable ".[data]" && jupyter notebook --ip=0.0.0.0 --port=8888 --no-browser --allow-root' - -storage: - volumes: - sharedSQLVolume: # This is a variable key and can be set to an arbitrary string. - path: '/shared' - read_only: false - efs: - id: ${MIGRATION_COMPARATOR_EFS_ID} - -cpu: 512 # Number of CPU units for the task. -memory: 2048 # Amount of memory in MiB used by the task. -count: 1 # Number of tasks that should be running in your service. -exec: true # Enable getting a shell to your container (https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-exec.html). - -environments: - dev: - count: 1 # Number of tasks to run for the "dev" environment. - deployment: # The deployment strategy for the "dev" environment. - rolling: 'recreate' # Stops existing tasks before new ones are started for faster deployments. \ No newline at end of file diff --git a/deployment/copilot/traffic-comparator/manifest.yml b/deployment/copilot/traffic-comparator/manifest.yml deleted file mode 100644 index 10ddf8a80..000000000 --- a/deployment/copilot/traffic-comparator/manifest.yml +++ /dev/null @@ -1,41 +0,0 @@ -# The manifest for the "traffic-comparator" service. -# Read the full specification for the "Backend Service" type at: -# https://aws.github.io/copilot-cli/docs/manifest/backend-service/ - -# Your service name will be used in naming your resources like log groups, ECS services, etc. -name: traffic-comparator -type: Backend Service - -# Allow service-to-service communication with ECS Service Connect -network: - connect: true - vpc: - security_groups: [ "${MIGRATION_COMPARATOR_EFS_SG_ID}" ] - -# Configuration for your containers and service. -image: - # Docker build arguments. For additional overrides: https://aws.github.io/copilot-cli/docs/manifest/backend-service/#image-build - build: - dockerfile: ../TrafficCapture/dockerSolution/build/docker/trafficComparator/Dockerfile - port: 9220 - -command: /bin/sh -c "cd containerTC && pip3 install --editable . && nc -v -l -p 9220 | tee /dev/stderr | trafficcomparator -vv stream | trafficcomparator dump-to-sqlite --db /shared/comparisons.db" - -storage: - volumes: - sharedSQLVolume: # This is a variable key and can be set to an arbitrary string. - path: '/shared' - read_only: false - efs: - id: ${MIGRATION_COMPARATOR_EFS_ID} - -cpu: 512 # Number of CPU units for the task. -memory: 2048 # Amount of memory in MiB used by the task. -count: 1 # Number of tasks that should be running in your service. -exec: true # Enable getting a shell to your container (https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-exec.html). - -environments: - dev: - count: 1 # Number of tasks to run for the "dev" environment. - deployment: # The deployment strategy for the "dev" environment. - rolling: 'recreate' # Stops existing tasks before new ones are started for faster deployments. \ No newline at end of file diff --git a/plugins/elasticsearch/README.md b/plugins/elasticsearch/README.md deleted file mode 100644 index dd36a84ce..000000000 --- a/plugins/elasticsearch/README.md +++ /dev/null @@ -1,6 +0,0 @@ -This part of the code is being removed because the dependencies that were in the elasticsearch logging plugin gradle -file were causing a lot of CVE issues to appear in the mend security check. specifically, due to it using the netty -client for elasticsearch 7.10.2 , which has alot of dependencies that have/cause security issues. - -This is the SHA of the commit that originally added these files: -1107701f9940358afb9bd1c61384aaa8c2093cde \ No newline at end of file diff --git a/plugins/elasticsearch/loggable-transport-netty4/src/main/java/org/elasticsearch/transport/LoggableNetty4ModulePlugin.java b/plugins/elasticsearch/loggable-transport-netty4/src/main/java/org/elasticsearch/transport/LoggableNetty4ModulePlugin.java deleted file mode 100644 index a7ce99782..000000000 --- a/plugins/elasticsearch/loggable-transport-netty4/src/main/java/org/elasticsearch/transport/LoggableNetty4ModulePlugin.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.transport; - -import io.netty.channel.Channel; -import io.netty.channel.ChannelHandler; -import io.netty.handler.logging.LogLevel; -import org.elasticsearch.common.network.NetworkService; -import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.common.util.PageCacheRecycler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.http.HttpHandlingSettings; -import org.elasticsearch.http.HttpServerTransport; -import org.elasticsearch.http.netty4.Netty4HttpServerTransport; -import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.plugins.NetworkPlugin; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.SharedGroupFactory; - -import java.util.Collections; -import java.util.Map; -import java.util.function.Supplier; - -/** - * A plugin that adds returns a subclass of the HttpServerTransport provided by the transport-netty4 module. - * The subclass will return a subclass of an HttpChannelHandler when a serverChannelHandler is configured. - * That innermost HttpChannelHandler will add a handler (an instance of WireLoggingHandler) to the pipeline - * to log HTTP requests and responses. - * - * To use this enhanced trace logging, install this plugin and set the http.type in the - * elasticsearch.yml configuration to "http.type: logging_netty". That will enable logging - * requests through the server's log facilities. - * - * To configure the logging to go to a separate file, a configuration such as the following - * can be added to the log4j.properties file. - * - *
- *     {@code
- * appender.http_trace_rolling.type = RollingFile
- * appender.http_trace_rolling.name = http_trace_rolling
- * appender.http_trace_rolling.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_http_trace.log
- * appender.http_trace_rolling.filePermissions = rw-r-----
- * appender.http_trace_rolling.layout.type = PatternLayout
- * appender.http_trace_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
- *
- * appender.http_trace_rolling.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_http_trace-%i.log.gz
- * appender.http_trace_rolling.policies.type = Policies
- * appender.http_trace_rolling.policies.size.type = SizeBasedTriggeringPolicy
- * appender.http_trace_rolling.policies.size.size = 1MB
- * appender.http_trace_rolling.strategy.type = DefaultRolloverStrategy
- * appender.http_trace_rolling.strategy.max = 4
- *
- * logger.http_trace.name = org.opensearch.http.trace
- * logger.http_trace.level = trace
- * logger.http_trace.appenderRef.http_trace_rolling.ref = http_trace_rolling
- * logger.http_trace.appenderRef.header_warning.ref = header_warning
- * logger.http_trace.additivity = false
- * }
- * 
- */ -public class LoggableNetty4ModulePlugin extends Plugin implements NetworkPlugin { - - public static final String LOG_NAME = "org.opensearch.http.trace.WireLogger"; - - /** - * This is the identifying key that would be referenced by the http.type value in opensearch.yml - */ - public static final String LOGGING_NETTY_TRANSPORT_NAME = "logging_netty"; - - /** - * This c'tor is only present because I needed a javadoc to add for an empty c'tor - */ - public LoggableNetty4ModulePlugin() {} - - public class LoggingHttpServerTransport extends Netty4HttpServerTransport { - - public LoggingHttpServerTransport(Settings settings, - NetworkService networkService, - BigArrays bigArrays, - ThreadPool threadPool, - NamedXContentRegistry xContentRegistry, - Dispatcher dispatcher, - ClusterSettings clusterSettings, - SharedGroupFactory sharedGroupFactory) { - super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher, clusterSettings, sharedGroupFactory); - } - - public class LoggingHttpChannelHandler extends Netty4HttpServerTransport.HttpChannelHandler { - protected LoggingHttpChannelHandler(Netty4HttpServerTransport transport, HttpHandlingSettings handlingSettings) { - super(transport, handlingSettings); - } - - @Override - protected void initChannel(Channel ch) throws Exception { - super.initChannel(ch); - ch.pipeline().addFirst(new WireLoggingHandler(LOG_NAME, LogLevel.TRACE)); - } - } - - public ChannelHandler configureServerChannelHandler() { - return new LoggingHttpChannelHandler(this, handlingSettings); - } - } - - @Override - public Map> getHttpTransports( - Settings settings, - ThreadPool threadPool, - BigArrays bigArrays, - PageCacheRecycler pageCacheRecycler, - CircuitBreakerService circuitBreakerService, - NamedXContentRegistry xContentRegistry, - NetworkService networkService, - HttpServerTransport.Dispatcher dispatcher, - ClusterSettings clusterSettings - ) { - return Collections.singletonMap( - LOGGING_NETTY_TRANSPORT_NAME, - () -> new LoggingHttpServerTransport( - settings, - networkService, - bigArrays, - threadPool, - xContentRegistry, - dispatcher, - clusterSettings, - new SharedGroupFactory(settings) - ) - ); - } -} diff --git a/plugins/opensearch/loggable-transport-netty4/build.gradle b/plugins/opensearch/loggable-transport-netty4/build.gradle deleted file mode 100644 index fd58411fb..000000000 --- a/plugins/opensearch/loggable-transport-netty4/build.gradle +++ /dev/null @@ -1,55 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -apply plugin: 'opensearch.opensearchplugin' -apply plugin: 'opensearch.yaml-rest-test' -apply plugin: 'opensearch.internal-cluster-test' - -plugins { - id 'org.owasp.dependencycheck' version '8.2.1' -} - -opensearchplugin { - description 'Plugin to override the transport-netty4 module surgically to add logging of all HTTP requests' - classname 'org.opensearch.transport.LoggableNetty4ModulePlugin' -} - -dependencies { - api project(':modules:transport-netty4') -} - -restResources { - restApi { - includeCore '_common', 'cluster', 'nodes' - } -} - -artifacts { - restTests(project.file('src/yamlRestTest/resources/rest-api-spec/test')) -} diff --git a/plugins/opensearch/loggable-transport-netty4/src/main/java/org/opensearch/transport/LoggableNetty4ModulePlugin.java b/plugins/opensearch/loggable-transport-netty4/src/main/java/org/opensearch/transport/LoggableNetty4ModulePlugin.java deleted file mode 100644 index 5858adc39..000000000 --- a/plugins/opensearch/loggable-transport-netty4/src/main/java/org/opensearch/transport/LoggableNetty4ModulePlugin.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.transport; - -import io.netty.channel.Channel; -import io.netty.channel.ChannelHandler; -import io.netty.handler.logging.ByteBufFormat; -import io.netty.handler.logging.LogLevel; -import io.netty.handler.logging.LoggingHandler; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.network.NetworkService; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.BigArrays; -import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.common.xcontent.NamedXContentRegistry; -import org.opensearch.http.HttpHandlingSettings; -import org.opensearch.http.HttpServerTransport; -import org.opensearch.http.netty4.Netty4HttpServerTransport; -import org.opensearch.indices.breaker.CircuitBreakerService; -import org.opensearch.plugins.NetworkPlugin; -import org.opensearch.plugins.Plugin; -import org.opensearch.threadpool.ThreadPool; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; - -/** - * A plugin that adds returns a subclass of the HttpServerTransport provided by the transport-netty4 module. - * The subclass will return a subclass of an HttpChannelHandler when a serverChannelHandler is configured. - * That innermost HttpChannelHandler will add a handler (an instance of WireLoggingHandler) to the pipeline - * to log HTTP requests and responses. - * - * To use this enhanced trace logging, install this plugin and set the http.type in the - * opensearch.yml configuration to "http.type: logging_netty". That will enable logging - * requests through the server's log facilities. - * - * To configure the logging to go to a separate file, a configuration such as the following - * can be added to the log4j.properties file. - * - *
- *     {@code
- * appender.http_trace_rolling.type = RollingFile
- * appender.http_trace_rolling.name = http_trace_rolling
- * appender.http_trace_rolling.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_http_trace.log
- * appender.http_trace_rolling.filePermissions = rw-r-----
- * appender.http_trace_rolling.layout.type = PatternLayout
- * appender.http_trace_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
- *
- * appender.http_trace_rolling.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_http_trace-%i.log.gz
- * appender.http_trace_rolling.policies.type = Policies
- * appender.http_trace_rolling.policies.size.type = SizeBasedTriggeringPolicy
- * appender.http_trace_rolling.policies.size.size = 1MB
- * appender.http_trace_rolling.strategy.type = DefaultRolloverStrategy
- * appender.http_trace_rolling.strategy.max = 4
- *
- * logger.http_trace.name = org.opensearch.http.trace
- * logger.http_trace.level = trace
- * logger.http_trace.appenderRef.http_trace_rolling.ref = http_trace_rolling
- * logger.http_trace.appenderRef.header_warning.ref = header_warning
- * logger.http_trace.additivity = false
- * }
- * 
- */ -public class LoggableNetty4ModulePlugin extends Plugin implements NetworkPlugin { - - public static final String LOG_NAME = "org.opensearch.http.trace.WireLogger"; - - /** - * This is the identifying key that would be referenced by the http.type value in opensearch.yml - */ - public static final String LOGGING_NETTY_TRANSPORT_NAME = "logging_netty"; - - /** - * This c'tor is only present because I needed a javadoc to add for an empty c'tor - */ - public LoggableNetty4ModulePlugin() {} - - public class LoggingHttpServerTransport extends Netty4HttpServerTransport { - - public LoggingHttpServerTransport(Settings settings, - NetworkService networkService, - BigArrays bigArrays, - ThreadPool threadPool, - NamedXContentRegistry xContentRegistry, - Dispatcher dispatcher, - ClusterSettings clusterSettings, - SharedGroupFactory sharedGroupFactory) { - super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher, clusterSettings, sharedGroupFactory); - } - - public class LoggingHttpChannelHandler extends Netty4HttpServerTransport.HttpChannelHandler { - protected LoggingHttpChannelHandler(Netty4HttpServerTransport transport, HttpHandlingSettings handlingSettings) { - super(transport, handlingSettings); - } - - @Override - protected void initChannel(Channel ch) throws Exception { - super.initChannel(ch); - ch.pipeline().addFirst(new WireLoggingHandler(LOG_NAME, LogLevel.TRACE)); - } - } - - public ChannelHandler configureServerChannelHandler() { - return new LoggingHttpChannelHandler(this, handlingSettings); - } - } - - @Override - public Map> getHttpTransports( - Settings settings, - ThreadPool threadPool, - BigArrays bigArrays, - PageCacheRecycler pageCacheRecycler, - CircuitBreakerService circuitBreakerService, - NamedXContentRegistry xContentRegistry, - NetworkService networkService, - HttpServerTransport.Dispatcher dispatcher, - ClusterSettings clusterSettings - ) { - return Collections.singletonMap( - LOGGING_NETTY_TRANSPORT_NAME, - () -> new LoggingHttpServerTransport( - settings, - networkService, - bigArrays, - threadPool, - xContentRegistry, - dispatcher, - clusterSettings, - new SharedGroupFactory(settings) - ) - ); - } -} diff --git a/plugins/opensearch/loggable-transport-netty4/src/main/java/org/opensearch/transport/package-info.java b/plugins/opensearch/loggable-transport-netty4/src/main/java/org/opensearch/transport/package-info.java deleted file mode 100644 index cb46c8d9f..000000000 --- a/plugins/opensearch/loggable-transport-netty4/src/main/java/org/opensearch/transport/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * package for traffic logging - */ -package org.opensearch.transport; diff --git a/plugins/opensearch/loggable-transport-netty4/src/main/plugin-metadata/plugin-security.policy b/plugins/opensearch/loggable-transport-netty4/src/main/plugin-metadata/plugin-security.policy deleted file mode 100644 index c8eee6bb5..000000000 --- a/plugins/opensearch/loggable-transport-netty4/src/main/plugin-metadata/plugin-security.policy +++ /dev/null @@ -1,48 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -grant codeBase "${codebase.netty-common}" { - // for reading the system-wide configuration for the backlog of established sockets - permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; - - // netty makes and accepts socket connections - permission java.net.SocketPermission "*", "accept,connect"; - - // Netty sets custom classloader for some of its internal threads - permission java.lang.RuntimePermission "*", "setContextClassLoader"; -}; - -grant codeBase "${codebase.netty-transport}" { - // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854 - // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely! - permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write"; -}; diff --git a/test/README.md b/test/README.md index 797cbf43f..e7fbe1534 100644 --- a/test/README.md +++ b/test/README.md @@ -25,14 +25,13 @@ The test script, by default, uses the ports assigned to the containers in this [docker-compose file](../TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml), so if the Docker solution in its current setup started with no issues, then the test script will run as is. If for any reason the user changed the ports in that file, they must also either, change the following environment variables: -`PROXY_ENDPOINT`, `SOURCE_ENDPOINT`, `TARGET_ENDPOINT` and `JUPYTER_NOTEBOOK` respectively, or update the default value +`PROXY_ENDPOINT`, `SOURCE_ENDPOINT`, and `TARGET_ENDPOINT` respectively, or update the default value (which can be found below) for them in [tests.py](tests.py). The following are the default values for the only endpoints touched by this script: * `PROXY_ENDPOINT = https://localhost:9200` * `SOURCE_ENDPOINT = http://localhost:19200` * `TARGET_ENDPOINT = https://localhost:29200` -* `JUPYTER_NOTEBOOK = http://localhost:8888/api` #### Clean Up The test script is implemented with a setup and teardown functions that are ran after diff --git a/test/tests.py b/test/tests.py index ce27a9604..fc94dc72e 100644 --- a/test/tests.py +++ b/test/tests.py @@ -6,7 +6,6 @@ import os import logging import time -import requests from requests.exceptions import ConnectionError, SSLError logger = logging.getLogger(__name__) @@ -50,7 +49,6 @@ def set_common_values(self): self.proxy_endpoint = os.getenv('PROXY_ENDPOINT', 'https://localhost:9200') self.source_endpoint = os.getenv('SOURCE_ENDPOINT', 'http://localhost:19200') self.target_endpoint = os.getenv('TARGET_ENDPOINT', 'https://localhost:29200') - self.jupyter_endpoint = os.getenv('JUPYTER_NOTEBOOK', 'http://localhost:8888/api') self.username = os.getenv('username', 'admin') self.password = os.getenv('password', 'admin') self.auth = (self.username, self.password) @@ -153,8 +151,3 @@ def test_002_document(self): source_response = retry_request(check_index, args=(self.source_endpoint, self.index, self.auth), expected_status_code=HTTPStatus.NOT_FOUND) self.assertEqual(source_response.status_code, HTTPStatus.NOT_FOUND) - - def test_003_jupyterAwake(self): - # Making sure that the Jupyter notebook is up and can be reached. - response = requests.get(self.jupyter_endpoint) - self.assertEqual(response.status_code, HTTPStatus.OK)