From 5e297ae67e469935c761ccca1022cd31904131a2 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Tue, 5 May 2020 19:08:28 -0500 Subject: [PATCH 01/10] Validate if car allows for using the bundled JDK When the bundled JDK was introduced, it was added to the car config file. This check uses the value from the car config to ensure that the bundled JDK is allowed to be used as specified by the config. If not, it throws an exception. Closes #985 --- esrally/mechanic/java_resolver.py | 5 ++++- esrally/mechanic/launcher.py | 3 ++- esrally/mechanic/provisioner.py | 15 ++++++++------- tests/mechanic/java_resolver_test.py | 18 ++++++++++++++---- tests/mechanic/launcher_test.py | 10 +++++----- tests/mechanic/provisioner_test.py | 9 ++++++--- 6 files changed, 39 insertions(+), 21 deletions(-) diff --git a/esrally/mechanic/java_resolver.py b/esrally/mechanic/java_resolver.py index b61409288..6d76c01bd 100644 --- a/esrally/mechanic/java_resolver.py +++ b/esrally/mechanic/java_resolver.py @@ -21,7 +21,7 @@ from esrally.utils import jvm -def java_home(car_runtime_jdks, cfg): +def java_home(car_runtime_jdks, allow_bundled_jvm, cfg): def determine_runtime_jdks(): override_runtime_jdk = cfg.opts("mechanic", "runtime.jdk") if override_runtime_jdk: @@ -39,6 +39,9 @@ def determine_runtime_jdks(): runtime_jdk_versions = determine_runtime_jdks() if runtime_jdk_versions[0] == "bundled": + if not allow_bundled_jvm: + raise exceptions.SystemSetupError("The bundled JDK is not allowed with the selected car(s): {}" + .format(cfg.opts("mechanic", "car.names"))) logger.info("Using JDK bundled with Elasticsearch.") # assume that the bundled JDK is the highest available; the path is irrelevant return allowed_runtime_jdks[0], None diff --git a/esrally/mechanic/launcher.py b/esrally/mechanic/launcher.py index 7ea41d540..fa7d780a8 100644 --- a/esrally/mechanic/launcher.py +++ b/esrally/mechanic/launcher.py @@ -138,7 +138,8 @@ def _start_node(self, node_configuration, node_count_on_host): data_paths = node_configuration.data_paths node_telemetry_dir = os.path.join(node_configuration.node_root_path, "telemetry") - java_major_version, java_home = java_resolver.java_home(node_configuration.car_runtime_jdks, self.cfg) + java_major_version, java_home = java_resolver.java_home(node_configuration.car_runtime_jdks, + node_configuration.car_allow_bundled_jvm, self.cfg) self.logger.info("Starting node [%s].", node_name) diff --git a/esrally/mechanic/provisioner.py b/esrally/mechanic/provisioner.py index f87606a7e..b30bac3f4 100644 --- a/esrally/mechanic/provisioner.py +++ b/esrally/mechanic/provisioner.py @@ -34,11 +34,8 @@ def local(cfg, car, plugins, cluster_settings, ip, http_port, all_node_ips, all_ node_root_dir = os.path.join(target_root, node_name) runtime_jdk_bundled = convert.to_bool(car.mandatory_var("runtime.jdk.bundled")) - if runtime_jdk_bundled: - java_home = None - else: - runtime_jdk = car.mandatory_var("runtime.jdk") - _, java_home = java_resolver.java_home(runtime_jdk, cfg) + runtime_jdk = car.mandatory_var("runtime.jdk") + _, java_home = java_resolver.java_home(runtime_jdk, runtime_jdk_bundled, cfg) es_installer = ElasticsearchInstaller(car, java_home, node_name, node_root_dir, all_node_ips, all_node_names, ip, http_port) plugin_installers = [PluginInstaller(plugin, java_home) for plugin in plugins] @@ -57,10 +54,12 @@ def docker(cfg, car, cluster_settings, ip, http_port, target_root, node_name): class NodeConfiguration: - def __init__(self, build_type, car_env, car_runtime_jdks, ip, node_name, node_root_path, binary_path, data_paths): + def __init__(self, build_type, car_env, car_runtime_jdks, car_allow_bundled_jvm, ip, node_name, node_root_path, + binary_path, data_paths): self.build_type = build_type self.car_env = car_env self.car_runtime_jdks = car_runtime_jdks + self.car_allow_bundled_jvm = convert.to_bool(car_allow_bundled_jvm) self.ip = ip self.node_name = node_name self.node_root_path = node_root_path @@ -72,6 +71,7 @@ def as_dict(self): "build-type": self.build_type, "car-env": self.car_env, "car-runtime-jdks": self.car_runtime_jdks, + "car-allow-bundled-jvm": self.car_allow_bundled_jvm, "ip": self.ip, "node-name": self.node_name, "node-root-path": self.node_root_path, @@ -81,7 +81,7 @@ def as_dict(self): @staticmethod def from_dict(d): - return NodeConfiguration(d["build-type"], d["car-env"], d["car-runtime-jdks"], d["ip"], d["node-name"], + return NodeConfiguration(d["build-type"], d["car-env"], d["car-runtime-jdks"], d["car-allow-bundled-jvm"], d["ip"], d["node-name"], d["node-root-path"], d["binary-path"], d["data-paths"]) @@ -197,6 +197,7 @@ def prepare(self, binary): installer.invoke_install_hook(team.BootstrapPhase.post_install, provisioner_vars.copy()) return NodeConfiguration("tar", self.es_installer.car.env, self.es_installer.car.mandatory_var("runtime.jdk"), + self.es_installer.car.mandatory_var("runtime.jdk.bundled"), self.es_installer.node_ip, self.es_installer.node_name, self.es_installer.node_root_dir, self.es_installer.es_home_path, self.es_installer.data_paths) diff --git a/tests/mechanic/java_resolver_test.py b/tests/mechanic/java_resolver_test.py index 8bcb07e13..f6c9185fb 100644 --- a/tests/mechanic/java_resolver_test.py +++ b/tests/mechanic/java_resolver_test.py @@ -18,7 +18,7 @@ import unittest.mock as mock from unittest import TestCase -from esrally import config +from esrally import config, exceptions from esrally.mechanic import java_resolver @@ -30,7 +30,7 @@ def test_resolves_java_home_for_default_runtime_jdk(self, resolve_jvm_path): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "runtime.jdk", None) - major, java_home = java_resolver.java_home("12,11,10,9,8", cfg) + major, java_home = java_resolver.java_home("12,11,10,9,8", True, cfg) self.assertEqual(major, 12) self.assertEqual(java_home, "/opt/jdk12") @@ -42,7 +42,7 @@ def test_resolves_java_home_for_specific_runtime_jdk(self, resolve_jvm_path): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "runtime.jdk", 8) - major, java_home = java_resolver.java_home("12,11,10,9,8", cfg) + major, java_home = java_resolver.java_home("12,11,10,9,8", True, cfg) self.assertEqual(major, 8) self.assertEqual(java_home, "/opt/jdk8") @@ -53,9 +53,19 @@ def test_resolves_java_home_for_bundled_jdk(self): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "runtime.jdk", "bundled") - major, java_home = java_resolver.java_home("12,11,10,9,8", cfg) + major, java_home = java_resolver.java_home("12,11,10,9,8", True, cfg) # assumes most recent JDK self.assertEqual(major, 12) # does not set JAVA_HOME for the bundled JDK self.assertEqual(java_home, None) + + def test_disallowed_bundled_jdk(self): + + cfg = config.Config() + cfg.add(config.Scope.application, "mechanic", "runtime.jdk", "bundled") + cfg.add(config.Scope.application, "mechanic", "car.names", ["default"]) + + with self.assertRaises(exceptions.SystemSetupError) as ctx: + java_resolver.java_home("12,11,10,9,8", False, cfg) + self.assertEqual("The bundled JDK is not allowed with the selected car(s): ['default']", ctx.exception.args[0]) diff --git a/tests/mechanic/launcher_test.py b/tests/mechanic/launcher_test.py index 152f6c6b9..b8e1765aa 100644 --- a/tests/mechanic/launcher_test.py +++ b/tests/mechanic/launcher_test.py @@ -177,7 +177,7 @@ def test_daemon_start_stop(self, wait_for_pidfile, chdir, get_size, supports, ja node_configs = [] for node in range(2): - node_configs.append(NodeConfiguration(build_type="tar", car_env={}, car_runtime_jdks="12,11", + node_configs.append(NodeConfiguration(build_type="tar", car_env={}, car_runtime_jdks="12,11", car_allow_bundled_jvm="true", ip="127.0.0.1", node_name="testnode-{}".format(node), node_root_path="/tmp", binary_path="/tmp", data_paths="/tmp")) @@ -326,8 +326,8 @@ def test_starts_container_successfully(self, run_subprocess_with_output, run_sub cfg = config.Config() docker = launcher.DockerLauncher(cfg) - node_config = NodeConfiguration(build_type="docker", car_env={}, car_runtime_jdks="12,11", ip="127.0.0.1", - node_name="testnode", node_root_path="/tmp", binary_path="/bin", + node_config = NodeConfiguration(build_type="docker", car_env={}, car_runtime_jdks="12,11", car_allow_bundled_jvm="true", + ip="127.0.0.1", node_name="testnode", node_root_path="/tmp", binary_path="/bin", data_paths="/tmp") nodes = docker.start([node_config]) @@ -358,8 +358,8 @@ def test_container_not_started(self, run_subprocess_with_output, run_subprocess_ stop_watch = IterationBasedStopWatch(max_iterations=2) docker = launcher.DockerLauncher(cfg, clock=TestClock(stop_watch=stop_watch)) - node_config = NodeConfiguration(build_type="docker", car_env={}, car_runtime_jdks="12,11", ip="127.0.0.1", - node_name="testnode", node_root_path="/tmp", binary_path="/bin", + node_config = NodeConfiguration(build_type="docker", car_env={}, car_runtime_jdks="12,11", car_allow_bundled_jvm="true", + ip="127.0.0.1", node_name="testnode", node_root_path="/tmp", binary_path="/bin", data_paths="/tmp") with self.assertRaisesRegex(exceptions.LaunchError, "No healthy running container after 600 seconds!"): diff --git a/tests/mechanic/provisioner_test.py b/tests/mechanic/provisioner_test.py index ddaaadb6d..1a8165b8b 100644 --- a/tests/mechanic/provisioner_test.py +++ b/tests/mechanic/provisioner_test.py @@ -42,7 +42,7 @@ def null_apply_config(source_root_path, target_root_path, config_vars): names="unit-test-car", root_path=None, config_paths=[HOME_DIR + "/.rally/benchmarks/teams/default/my-car"], - variables={"heap": "4g", "runtime.jdk": "8"}), + variables={"heap": "4g", "runtime.jdk": "8", "runtime.jdk.bundled": "true"}), java_home="/usr/local/javas/java8", node_name="rally-node-0", node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest", @@ -73,6 +73,7 @@ def null_apply_config(source_root_path, target_root_path, config_vars): }, "heap": "4g", "runtime.jdk": "8", + "runtime.jdk.bundled": "true", "cluster_name": "rally-benchmark", "node_name": "rally-node-0", "data_paths": ["/opt/elasticsearch-5.0.0/data"], @@ -150,7 +151,7 @@ def null_apply_config(source_root_path, target_root_path, config_vars): names="unit-test-car", root_path=None, config_paths=[HOME_DIR + "/.rally/benchmarks/teams/default/my-car"], - variables={"heap": "4g", "runtime.jdk": "8"}), + variables={"heap": "4g", "runtime.jdk": "8", "runtime.jdk.bundled": "true"}), java_home="/usr/local/javas/java8", node_name="rally-node-0", node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest", @@ -190,6 +191,7 @@ def null_apply_config(source_root_path, target_root_path, config_vars): }, "heap": "4g", "runtime.jdk": "8", + "runtime.jdk.bundled": "true", "cluster_name": "rally-benchmark", "node_name": "rally-node-0", "data_paths": ["/opt/elasticsearch-5.0.0/data"], @@ -230,7 +232,7 @@ def null_apply_config(source_root_path, target_root_path, config_vars): names="unit-test-car", root_path=None, config_paths=[HOME_DIR + "/.rally/benchmarks/teams/default/my-car"], - variables={"heap": "4g", "runtime.jdk": "8"}), + variables={"heap": "4g", "runtime.jdk": "8", "runtime.jdk.bundled": "true"}), java_home="/usr/local/javas/java8", node_name="rally-node-0", node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest", @@ -270,6 +272,7 @@ def null_apply_config(source_root_path, target_root_path, config_vars): }, "heap": "4g", "runtime.jdk": "8", + "runtime.jdk.bundled": "true", "cluster_name": "rally-benchmark", "node_name": "rally-node-0", "data_paths": ["/opt/elasticsearch-6.3.0/data"], From 76d6528c580abcf78fe7c6b7105fc72f0cfb9d83 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Wed, 6 May 2020 12:33:19 -0500 Subject: [PATCH 02/10] Fix docker --- esrally/mechanic/provisioner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esrally/mechanic/provisioner.py b/esrally/mechanic/provisioner.py index b30bac3f4..f339d6a33 100644 --- a/esrally/mechanic/provisioner.py +++ b/esrally/mechanic/provisioner.py @@ -463,7 +463,8 @@ def prepare(self, binary): with open(os.path.join(self.binary_path, "docker-compose.yml"), mode="wt", encoding="utf-8") as f: f.write(docker_cfg) - return NodeConfiguration("docker", self.car.env, self.car.mandatory_var("runtime.jdk"), self.node_ip, + return NodeConfiguration("docker", self.car.env, self.car.mandatory_var("runtime.jdk"), + self.car.mandatory_var("runtime.jdk.bundled"), self.node_ip, self.node_name, self.node_root_dir, self.binary_path, self.data_paths) def docker_vars(self, mounts): From 7255c9953f33f049eeb1e291277fd2ee96eb9f25 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Thu, 14 May 2020 10:20:21 -0500 Subject: [PATCH 03/10] Fix signature --- esrally/mechanic/java_resolver.py | 13 ++++++------- esrally/mechanic/launcher.py | 3 ++- esrally/mechanic/provisioner.py | 2 +- tests/mechanic/java_resolver_test.py | 10 +++++----- tests/mechanic/launcher_test.py | 1 + 5 files changed, 15 insertions(+), 14 deletions(-) diff --git a/esrally/mechanic/java_resolver.py b/esrally/mechanic/java_resolver.py index 6d76c01bd..8fe0f5308 100644 --- a/esrally/mechanic/java_resolver.py +++ b/esrally/mechanic/java_resolver.py @@ -21,11 +21,10 @@ from esrally.utils import jvm -def java_home(car_runtime_jdks, allow_bundled_jvm, cfg): +def java_home(car_runtime_jdks, specified_runtime_jdk=None, provides_bundled_jdk=False): def determine_runtime_jdks(): - override_runtime_jdk = cfg.opts("mechanic", "runtime.jdk") - if override_runtime_jdk: - return [override_runtime_jdk] + if specified_runtime_jdk: + return [specified_runtime_jdk] else: return allowed_runtime_jdks @@ -39,9 +38,9 @@ def determine_runtime_jdks(): runtime_jdk_versions = determine_runtime_jdks() if runtime_jdk_versions[0] == "bundled": - if not allow_bundled_jvm: - raise exceptions.SystemSetupError("The bundled JDK is not allowed with the selected car(s): {}" - .format(cfg.opts("mechanic", "car.names"))) + if not provides_bundled_jdk: + raise exceptions.SystemSetupError("This Elasticsearch version does not contain a bundled JDK. " + "Please specify a different runtime JDK.") logger.info("Using JDK bundled with Elasticsearch.") # assume that the bundled JDK is the highest available; the path is irrelevant return allowed_runtime_jdks[0], None diff --git a/esrally/mechanic/launcher.py b/esrally/mechanic/launcher.py index 38a1685eb..142a383de 100644 --- a/esrally/mechanic/launcher.py +++ b/esrally/mechanic/launcher.py @@ -140,7 +140,8 @@ def _start_node(self, node_configuration, node_count_on_host): node_telemetry_dir = os.path.join(node_configuration.node_root_path, "telemetry") java_major_version, java_home = java_resolver.java_home(node_configuration.car_runtime_jdks, - node_configuration.car_allow_bundled_jvm, self.cfg) + self.cfg.opts("mechanic", "runtime.jdk"), + node_configuration.car_allow_bundled_jvm) self.logger.info("Starting node [%s].", node_name) diff --git a/esrally/mechanic/provisioner.py b/esrally/mechanic/provisioner.py index f339d6a33..a1a58c862 100644 --- a/esrally/mechanic/provisioner.py +++ b/esrally/mechanic/provisioner.py @@ -35,7 +35,7 @@ def local(cfg, car, plugins, cluster_settings, ip, http_port, all_node_ips, all_ runtime_jdk_bundled = convert.to_bool(car.mandatory_var("runtime.jdk.bundled")) runtime_jdk = car.mandatory_var("runtime.jdk") - _, java_home = java_resolver.java_home(runtime_jdk, runtime_jdk_bundled, cfg) + _, java_home = java_resolver.java_home(runtime_jdk, cfg.opts("mechanic", "runtime.jdk"), runtime_jdk_bundled) es_installer = ElasticsearchInstaller(car, java_home, node_name, node_root_dir, all_node_ips, all_node_names, ip, http_port) plugin_installers = [PluginInstaller(plugin, java_home) for plugin in plugins] diff --git a/tests/mechanic/java_resolver_test.py b/tests/mechanic/java_resolver_test.py index f6c9185fb..2258ad299 100644 --- a/tests/mechanic/java_resolver_test.py +++ b/tests/mechanic/java_resolver_test.py @@ -30,7 +30,7 @@ def test_resolves_java_home_for_default_runtime_jdk(self, resolve_jvm_path): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "runtime.jdk", None) - major, java_home = java_resolver.java_home("12,11,10,9,8", True, cfg) + major, java_home = java_resolver.java_home("12,11,10,9,8", cfg.opts("mechanic", "runtime.jdk"), True) self.assertEqual(major, 12) self.assertEqual(java_home, "/opt/jdk12") @@ -42,7 +42,7 @@ def test_resolves_java_home_for_specific_runtime_jdk(self, resolve_jvm_path): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "runtime.jdk", 8) - major, java_home = java_resolver.java_home("12,11,10,9,8", True, cfg) + major, java_home = java_resolver.java_home("12,11,10,9,8", cfg.opts("mechanic", "runtime.jdk"), True) self.assertEqual(major, 8) self.assertEqual(java_home, "/opt/jdk8") @@ -53,7 +53,7 @@ def test_resolves_java_home_for_bundled_jdk(self): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "runtime.jdk", "bundled") - major, java_home = java_resolver.java_home("12,11,10,9,8", True, cfg) + major, java_home = java_resolver.java_home("12,11,10,9,8", cfg.opts("mechanic", "runtime.jdk"), True) # assumes most recent JDK self.assertEqual(major, 12) @@ -67,5 +67,5 @@ def test_disallowed_bundled_jdk(self): cfg.add(config.Scope.application, "mechanic", "car.names", ["default"]) with self.assertRaises(exceptions.SystemSetupError) as ctx: - java_resolver.java_home("12,11,10,9,8", False, cfg) - self.assertEqual("The bundled JDK is not allowed with the selected car(s): ['default']", ctx.exception.args[0]) + java_resolver.java_home("12,11,10,9,8", cfg.opts("mechanic", "runtime.jdk")) + self.assertEqual("This Elasticsearch version does not contain a bundled JDK. Please specify a different runtime JDK.", ctx.exception.args[0]) diff --git a/tests/mechanic/launcher_test.py b/tests/mechanic/launcher_test.py index d45bfb163..4e2aba8c1 100644 --- a/tests/mechanic/launcher_test.py +++ b/tests/mechanic/launcher_test.py @@ -168,6 +168,7 @@ def test_daemon_start_stop(self, wait_for_pidfile, chdir, get_size, supports, ja cfg = config.Config() cfg.add(config.Scope.application, "node", "root.dir", "test") cfg.add(config.Scope.application, "mechanic", "keep.running", False) + cfg.add(config.Scope.application, "mechanic", "runtime.jdk", None) cfg.add(config.Scope.application, "telemetry", "devices", []) cfg.add(config.Scope.application, "telemetry", "params", None) cfg.add(config.Scope.application, "system", "env.name", "test") From e95be18e3ba49051704d84e4ad5155a9a3e4719b Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Thu, 14 May 2020 10:28:12 -0500 Subject: [PATCH 04/10] named params --- tests/mechanic/java_resolver_test.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/mechanic/java_resolver_test.py b/tests/mechanic/java_resolver_test.py index 2258ad299..dd75c189b 100644 --- a/tests/mechanic/java_resolver_test.py +++ b/tests/mechanic/java_resolver_test.py @@ -30,7 +30,9 @@ def test_resolves_java_home_for_default_runtime_jdk(self, resolve_jvm_path): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "runtime.jdk", None) - major, java_home = java_resolver.java_home("12,11,10,9,8", cfg.opts("mechanic", "runtime.jdk"), True) + major, java_home = java_resolver.java_home("12,11,10,9,8", + specified_runtime_jdk=cfg.opts("mechanic", "runtime.jdk"), + provides_bundled_jdk=True) self.assertEqual(major, 12) self.assertEqual(java_home, "/opt/jdk12") @@ -42,7 +44,9 @@ def test_resolves_java_home_for_specific_runtime_jdk(self, resolve_jvm_path): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "runtime.jdk", 8) - major, java_home = java_resolver.java_home("12,11,10,9,8", cfg.opts("mechanic", "runtime.jdk"), True) + major, java_home = java_resolver.java_home("12,11,10,9,8", + specified_runtime_jdk=cfg.opts("mechanic", "runtime.jdk"), + provides_bundled_jdk=True) self.assertEqual(major, 8) self.assertEqual(java_home, "/opt/jdk8") @@ -53,7 +57,9 @@ def test_resolves_java_home_for_bundled_jdk(self): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "runtime.jdk", "bundled") - major, java_home = java_resolver.java_home("12,11,10,9,8", cfg.opts("mechanic", "runtime.jdk"), True) + major, java_home = java_resolver.java_home("12,11,10,9,8", + specified_runtime_jdk=cfg.opts("mechanic", "runtime.jdk"), + provides_bundled_jdk=True) # assumes most recent JDK self.assertEqual(major, 12) @@ -67,5 +73,5 @@ def test_disallowed_bundled_jdk(self): cfg.add(config.Scope.application, "mechanic", "car.names", ["default"]) with self.assertRaises(exceptions.SystemSetupError) as ctx: - java_resolver.java_home("12,11,10,9,8", cfg.opts("mechanic", "runtime.jdk")) + java_resolver.java_home("12,11,10,9,8", specified_runtime_jdk=cfg.opts("mechanic", "runtime.jdk")) self.assertEqual("This Elasticsearch version does not contain a bundled JDK. Please specify a different runtime JDK.", ctx.exception.args[0]) From c30eef833b745723fc54f4c907c8fc4100627d03 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Thu, 14 May 2020 10:33:12 -0500 Subject: [PATCH 05/10] Rename param --- esrally/mechanic/launcher.py | 2 +- esrally/mechanic/provisioner.py | 10 +++++----- tests/mechanic/launcher_test.py | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/esrally/mechanic/launcher.py b/esrally/mechanic/launcher.py index 142a383de..447898b99 100644 --- a/esrally/mechanic/launcher.py +++ b/esrally/mechanic/launcher.py @@ -141,7 +141,7 @@ def _start_node(self, node_configuration, node_count_on_host): java_major_version, java_home = java_resolver.java_home(node_configuration.car_runtime_jdks, self.cfg.opts("mechanic", "runtime.jdk"), - node_configuration.car_allow_bundled_jvm) + node_configuration.car_provides_bundled_jdk) self.logger.info("Starting node [%s].", node_name) diff --git a/esrally/mechanic/provisioner.py b/esrally/mechanic/provisioner.py index a1a58c862..3e7143c85 100644 --- a/esrally/mechanic/provisioner.py +++ b/esrally/mechanic/provisioner.py @@ -54,12 +54,12 @@ def docker(cfg, car, cluster_settings, ip, http_port, target_root, node_name): class NodeConfiguration: - def __init__(self, build_type, car_env, car_runtime_jdks, car_allow_bundled_jvm, ip, node_name, node_root_path, + def __init__(self, build_type, car_env, car_runtime_jdks, car_provides_bundled_jdk, ip, node_name, node_root_path, binary_path, data_paths): self.build_type = build_type self.car_env = car_env self.car_runtime_jdks = car_runtime_jdks - self.car_allow_bundled_jvm = convert.to_bool(car_allow_bundled_jvm) + self.car_provides_bundled_jdk = convert.to_bool(car_provides_bundled_jdk) self.ip = ip self.node_name = node_name self.node_root_path = node_root_path @@ -71,7 +71,7 @@ def as_dict(self): "build-type": self.build_type, "car-env": self.car_env, "car-runtime-jdks": self.car_runtime_jdks, - "car-allow-bundled-jvm": self.car_allow_bundled_jvm, + "car-provides-bundled-jdk": self.car_provides_bundled_jdk, "ip": self.ip, "node-name": self.node_name, "node-root-path": self.node_root_path, @@ -81,8 +81,8 @@ def as_dict(self): @staticmethod def from_dict(d): - return NodeConfiguration(d["build-type"], d["car-env"], d["car-runtime-jdks"], d["car-allow-bundled-jvm"], d["ip"], d["node-name"], - d["node-root-path"], d["binary-path"], d["data-paths"]) + return NodeConfiguration(d["build-type"], d["car-env"], d["car-runtime-jdks"], d["car-provides-bundled-jdk"], d["ip"], + d["node-name"], d["node-root-path"], d["binary-path"], d["data-paths"]) def save_node_configuration(path, n): diff --git a/tests/mechanic/launcher_test.py b/tests/mechanic/launcher_test.py index 4e2aba8c1..18326ef90 100644 --- a/tests/mechanic/launcher_test.py +++ b/tests/mechanic/launcher_test.py @@ -178,7 +178,7 @@ def test_daemon_start_stop(self, wait_for_pidfile, chdir, get_size, supports, ja node_configs = [] for node in range(2): - node_configs.append(NodeConfiguration(build_type="tar", car_env={}, car_runtime_jdks="12,11", car_allow_bundled_jvm="true", + node_configs.append(NodeConfiguration(build_type="tar", car_env={}, car_runtime_jdks="12,11", car_provides_bundled_jdk="true", ip="127.0.0.1", node_name="testnode-{}".format(node), node_root_path="/tmp", binary_path="/tmp", data_paths="/tmp")) @@ -364,7 +364,7 @@ def test_starts_container_successfully(self, run_subprocess_with_output, run_sub cfg = config.Config() docker = launcher.DockerLauncher(cfg) - node_config = NodeConfiguration(build_type="docker", car_env={}, car_runtime_jdks="12,11", car_allow_bundled_jvm="true", + node_config = NodeConfiguration(build_type="docker", car_env={}, car_runtime_jdks="12,11", car_provides_bundled_jdk="true", ip="127.0.0.1", node_name="testnode", node_root_path="/tmp", binary_path="/bin", data_paths="/tmp") @@ -396,7 +396,7 @@ def test_container_not_started(self, run_subprocess_with_output, run_subprocess_ stop_watch = IterationBasedStopWatch(max_iterations=2) docker = launcher.DockerLauncher(cfg, clock=TestClock(stop_watch=stop_watch)) - node_config = NodeConfiguration(build_type="docker", car_env={}, car_runtime_jdks="12,11", car_allow_bundled_jvm="true", + node_config = NodeConfiguration(build_type="docker", car_env={}, car_runtime_jdks="12,11", car_provides_bundled_jdk="true", ip="127.0.0.1", node_name="testnode", node_root_path="/tmp", binary_path="/bin", data_paths="/tmp") From 20eb79f007982f95d672e1f6b7f4de8fc6d14623 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Thu, 14 May 2020 10:37:57 -0500 Subject: [PATCH 06/10] Fix lint --- tests/mechanic/java_resolver_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/mechanic/java_resolver_test.py b/tests/mechanic/java_resolver_test.py index dd75c189b..5a59fcfae 100644 --- a/tests/mechanic/java_resolver_test.py +++ b/tests/mechanic/java_resolver_test.py @@ -74,4 +74,5 @@ def test_disallowed_bundled_jdk(self): with self.assertRaises(exceptions.SystemSetupError) as ctx: java_resolver.java_home("12,11,10,9,8", specified_runtime_jdk=cfg.opts("mechanic", "runtime.jdk")) - self.assertEqual("This Elasticsearch version does not contain a bundled JDK. Please specify a different runtime JDK.", ctx.exception.args[0]) + self.assertEqual("This Elasticsearch version does not contain a bundled JDK. Please specify a different runtime JDK.", + ctx.exception.args[0]) From d4226484773ceb665ade4b77bd1606431da85262 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Mon, 18 May 2020 12:52:02 -0500 Subject: [PATCH 07/10] Move converter --- esrally/mechanic/provisioner.py | 6 +++--- tests/mechanic/launcher_test.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/esrally/mechanic/provisioner.py b/esrally/mechanic/provisioner.py index 3e7143c85..e73a92238 100644 --- a/esrally/mechanic/provisioner.py +++ b/esrally/mechanic/provisioner.py @@ -59,7 +59,7 @@ def __init__(self, build_type, car_env, car_runtime_jdks, car_provides_bundled_j self.build_type = build_type self.car_env = car_env self.car_runtime_jdks = car_runtime_jdks - self.car_provides_bundled_jdk = convert.to_bool(car_provides_bundled_jdk) + self.car_provides_bundled_jdk = car_provides_bundled_jdk self.ip = ip self.node_name = node_name self.node_root_path = node_root_path @@ -197,7 +197,7 @@ def prepare(self, binary): installer.invoke_install_hook(team.BootstrapPhase.post_install, provisioner_vars.copy()) return NodeConfiguration("tar", self.es_installer.car.env, self.es_installer.car.mandatory_var("runtime.jdk"), - self.es_installer.car.mandatory_var("runtime.jdk.bundled"), + convert.to_bool(self.es_installer.car.mandatory_var("runtime.jdk.bundled")), self.es_installer.node_ip, self.es_installer.node_name, self.es_installer.node_root_dir, self.es_installer.es_home_path, self.es_installer.data_paths) @@ -464,7 +464,7 @@ def prepare(self, binary): f.write(docker_cfg) return NodeConfiguration("docker", self.car.env, self.car.mandatory_var("runtime.jdk"), - self.car.mandatory_var("runtime.jdk.bundled"), self.node_ip, + convert.to_bool(self.car.mandatory_var("runtime.jdk.bundled")), self.node_ip, self.node_name, self.node_root_dir, self.binary_path, self.data_paths) def docker_vars(self, mounts): diff --git a/tests/mechanic/launcher_test.py b/tests/mechanic/launcher_test.py index 18326ef90..297524671 100644 --- a/tests/mechanic/launcher_test.py +++ b/tests/mechanic/launcher_test.py @@ -178,7 +178,7 @@ def test_daemon_start_stop(self, wait_for_pidfile, chdir, get_size, supports, ja node_configs = [] for node in range(2): - node_configs.append(NodeConfiguration(build_type="tar", car_env={}, car_runtime_jdks="12,11", car_provides_bundled_jdk="true", + node_configs.append(NodeConfiguration(build_type="tar", car_env={}, car_runtime_jdks="12,11", car_provides_bundled_jdk=True, ip="127.0.0.1", node_name="testnode-{}".format(node), node_root_path="/tmp", binary_path="/tmp", data_paths="/tmp")) @@ -364,7 +364,7 @@ def test_starts_container_successfully(self, run_subprocess_with_output, run_sub cfg = config.Config() docker = launcher.DockerLauncher(cfg) - node_config = NodeConfiguration(build_type="docker", car_env={}, car_runtime_jdks="12,11", car_provides_bundled_jdk="true", + node_config = NodeConfiguration(build_type="docker", car_env={}, car_runtime_jdks="12,11", car_provides_bundled_jdk=True, ip="127.0.0.1", node_name="testnode", node_root_path="/tmp", binary_path="/bin", data_paths="/tmp") @@ -396,7 +396,7 @@ def test_container_not_started(self, run_subprocess_with_output, run_subprocess_ stop_watch = IterationBasedStopWatch(max_iterations=2) docker = launcher.DockerLauncher(cfg, clock=TestClock(stop_watch=stop_watch)) - node_config = NodeConfiguration(build_type="docker", car_env={}, car_runtime_jdks="12,11", car_provides_bundled_jdk="true", + node_config = NodeConfiguration(build_type="docker", car_env={}, car_runtime_jdks="12,11", car_provides_bundled_jdk=True, ip="127.0.0.1", node_name="testnode", node_root_path="/tmp", binary_path="/bin", data_paths="/tmp") From ac3e578072707e51dc536125edc6c75ccb8e50c4 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Mon, 18 May 2020 12:57:03 -0500 Subject: [PATCH 08/10] Test cleanup --- tests/mechanic/java_resolver_test.py | 33 +++++++--------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/tests/mechanic/java_resolver_test.py b/tests/mechanic/java_resolver_test.py index 5a59fcfae..cc4242b36 100644 --- a/tests/mechanic/java_resolver_test.py +++ b/tests/mechanic/java_resolver_test.py @@ -26,39 +26,27 @@ class JavaResolverTests(TestCase): @mock.patch("esrally.utils.jvm.resolve_path") def test_resolves_java_home_for_default_runtime_jdk(self, resolve_jvm_path): resolve_jvm_path.return_value = (12, "/opt/jdk12") - - cfg = config.Config() - cfg.add(config.Scope.application, "mechanic", "runtime.jdk", None) - major, java_home = java_resolver.java_home("12,11,10,9,8", - specified_runtime_jdk=cfg.opts("mechanic", "runtime.jdk"), + specified_runtime_jdk=None, provides_bundled_jdk=True) - self.assertEqual(major, 12) - self.assertEqual(java_home, "/opt/jdk12") + self.assertEqual(major, resolve_jvm_path.return_value[0]) + self.assertEqual(java_home, resolve_jvm_path.return_value[1]) @mock.patch("esrally.utils.jvm.resolve_path") def test_resolves_java_home_for_specific_runtime_jdk(self, resolve_jvm_path): resolve_jvm_path.return_value = (8, "/opt/jdk8") - - cfg = config.Config() - cfg.add(config.Scope.application, "mechanic", "runtime.jdk", 8) - major, java_home = java_resolver.java_home("12,11,10,9,8", - specified_runtime_jdk=cfg.opts("mechanic", "runtime.jdk"), + specified_runtime_jdk=8, provides_bundled_jdk=True) - self.assertEqual(major, 8) - self.assertEqual(java_home, "/opt/jdk8") + self.assertEqual(major, resolve_jvm_path.return_value[0]) + self.assertEqual(java_home, resolve_jvm_path.return_value[1]) resolve_jvm_path.assert_called_with([8]) def test_resolves_java_home_for_bundled_jdk(self): - - cfg = config.Config() - cfg.add(config.Scope.application, "mechanic", "runtime.jdk", "bundled") - major, java_home = java_resolver.java_home("12,11,10,9,8", - specified_runtime_jdk=cfg.opts("mechanic", "runtime.jdk"), + specified_runtime_jdk="bundled", provides_bundled_jdk=True) # assumes most recent JDK @@ -67,12 +55,7 @@ def test_resolves_java_home_for_bundled_jdk(self): self.assertEqual(java_home, None) def test_disallowed_bundled_jdk(self): - - cfg = config.Config() - cfg.add(config.Scope.application, "mechanic", "runtime.jdk", "bundled") - cfg.add(config.Scope.application, "mechanic", "car.names", ["default"]) - with self.assertRaises(exceptions.SystemSetupError) as ctx: - java_resolver.java_home("12,11,10,9,8", specified_runtime_jdk=cfg.opts("mechanic", "runtime.jdk")) + java_resolver.java_home("12,11,10,9,8", specified_runtime_jdk="bundled") self.assertEqual("This Elasticsearch version does not contain a bundled JDK. Please specify a different runtime JDK.", ctx.exception.args[0]) From 667fa793634b564caf299358f2c2c33f571e72a9 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Mon, 18 May 2020 13:06:19 -0500 Subject: [PATCH 09/10] Fix lint --- tests/mechanic/java_resolver_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mechanic/java_resolver_test.py b/tests/mechanic/java_resolver_test.py index cc4242b36..11fbcba70 100644 --- a/tests/mechanic/java_resolver_test.py +++ b/tests/mechanic/java_resolver_test.py @@ -18,7 +18,7 @@ import unittest.mock as mock from unittest import TestCase -from esrally import config, exceptions +from esrally import exceptions from esrally.mechanic import java_resolver From 451e2cb00876a1b6ec9dd5a362fdefbceefc9467 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Wed, 20 May 2020 13:43:33 -0500 Subject: [PATCH 10/10] Test un-cleanup ;) --- tests/mechanic/java_resolver_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/mechanic/java_resolver_test.py b/tests/mechanic/java_resolver_test.py index 11fbcba70..052651bd4 100644 --- a/tests/mechanic/java_resolver_test.py +++ b/tests/mechanic/java_resolver_test.py @@ -30,8 +30,8 @@ def test_resolves_java_home_for_default_runtime_jdk(self, resolve_jvm_path): specified_runtime_jdk=None, provides_bundled_jdk=True) - self.assertEqual(major, resolve_jvm_path.return_value[0]) - self.assertEqual(java_home, resolve_jvm_path.return_value[1]) + self.assertEqual(major, 12) + self.assertEqual(java_home, "/opt/jdk12") @mock.patch("esrally.utils.jvm.resolve_path") def test_resolves_java_home_for_specific_runtime_jdk(self, resolve_jvm_path): @@ -40,8 +40,8 @@ def test_resolves_java_home_for_specific_runtime_jdk(self, resolve_jvm_path): specified_runtime_jdk=8, provides_bundled_jdk=True) - self.assertEqual(major, resolve_jvm_path.return_value[0]) - self.assertEqual(java_home, resolve_jvm_path.return_value[1]) + self.assertEqual(major, 8) + self.assertEqual(java_home, "/opt/jdk8") resolve_jvm_path.assert_called_with([8]) def test_resolves_java_home_for_bundled_jdk(self):