From 0c6577180f3f5b3b81da763855ae7eeaaf53fce2 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Thu, 14 Jun 2018 08:37:17 +0200 Subject: [PATCH] Derive JDK version at runtime With this commit we derive the appropriate JDK version at benchmark runtime instead of requiring that it is preconfigured. This requires two new variables in the rally-teams repo: `build.jdk` and `runtime.jdk` to resolve the appropriate (major) JDK version depending on the Elasticsearch version. When we know the required JDK version, we check the environment variables `JAVAx_HOME` (where `x` is the JDK version) and `JAVA_HOME` to find the correct JAVA_HOME path. We also allow the user to override the runtime JDK with a new command line parameter `--runtime-jdk` as Rally will default to the highest available and supported JDK version on the target system. Closes #452 Relates #518 --- docs/command_line_reference.rst | 14 ++++++ docs/configuration.rst | 33 ------------ docs/install.rst | 4 +- docs/migrate.rst | 17 +++++++ docs/quickstart.rst | 2 +- esrally/config.py | 89 +++------------------------------ esrally/mechanic/launcher.py | 38 +++++++++----- esrally/mechanic/supplier.py | 28 ++++------- esrally/mechanic/team.py | 6 +++ esrally/rally.py | 13 ++--- esrally/utils/jvm.py | 79 ++++++++++++++++++++++++++++- integration-test.sh | 36 ++++--------- tests/config_test.py | 68 ++----------------------- tests/mechanic/supplier_test.py | 14 +++--- tests/utils/jvm_test.py | 69 ++++++++++++++++++++++++- tox.ini | 7 ++- 16 files changed, 263 insertions(+), 254 deletions(-) diff --git a/docs/command_line_reference.rst b/docs/command_line_reference.rst index 027d9d67a..1d8b857f7 100644 --- a/docs/command_line_reference.rst +++ b/docs/command_line_reference.rst @@ -284,6 +284,20 @@ Example:: This enables the Java flight recorder telemetry device and sets the ``recording-template`` parameter to "profile". +``runtime-jdk`` +~~~~~~~~~~~~~~~ + +By default, Rally will derive the appropriate runtime JDK versions automatically per version of Elasticsearch. For example, it will choose JDK 8 or 7 for Elasticsearch 2.x but only JDK 8 for Elasticsearch 5.0.0. It will choose the highest available version. + +This command line parameter sets the major version of the JDK that Rally should use to run Elasticsearch. It is required that either ``JAVA_HOME`` or ``JAVAx_HOME`` (where ``x`` is the major version, e.g. ``JAVA8_HOME`` for a JDK 8) points to the appropriate JDK. + +Example:: + + # Run a benchmark with defaults (i.e. JDK 8) + esrally --distribution-version=2.4.0 + # Force to run with JDK 7 + esrally --distribution-version=2.4.0 --runtime-jdk=7 + .. _clr_revision: ``revision`` diff --git a/docs/configuration.rst b/docs/configuration.rst index 03d840ec8..7bd36dcbc 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -15,7 +15,6 @@ Rally can build Elasticsearch either from sources or use an `official binary dis Let's go through an example step by step: First run ``esrally``:: dm@io:~ $ esrally - ____ ____ / __ \____ _/ / /_ __ / /_/ / __ `/ / / / / / @@ -27,42 +26,11 @@ Let's go through an example step by step: First run ``esrally``:: esrally configure --advanced-config - * Autodetecting available third-party software - git : [OK] - JDK : [OK] - - * Setting up benchmark data directory in /Users/dm/.rally/benchmarks - Enter the JDK 10 root directory (Press Enter to skip): - -As you can see above, Rally autodetects if git and a JDK are installed. It also searches for a JDK 10; if you don't have it, that's no problem, you are just not able to build Elasticsearch from sources. Let's assume you press Enter and don't specify a path for JDK 10:: - - ******************************************************************************** - You don't have a valid JDK 10 installation and cannot benchmark source builds. - - You can still benchmark binary distributions with e.g.: - - esrally --distribution-version=6.0.0 - ******************************************************************************** - -As you can see, Rally tells you that you cannot build Elasticsearch from sources but you can still benchmark official binary distributions. - -It's also possible that Rally cannot automatically your JDK home directory. In that case, it will ask you later in the configuration process. If you do not provide a JDK home directory, Rally cannot start Elasticsearch on this machine but you can still use it as a load generator to :doc:`benchmark remote clusters `. - -If you specify a valid path for JDK 10, Rally will try to autodetect your Elasticsearch project directory (either in the current directory or in ``../elasticsearch``) or will choose a default directory:: - * Setting up benchmark data directory in /Users/dm/.rally/benchmarks * Setting up benchmark source directory in /Users/dm/.rally/benchmarks/src/elasticsearch -If a valid path for JDK 10 was not found (or entered), it will not ask you for a source directory and just go on. - -Now Rally is done:: - Configuration successfully written to /Users/dm/.rally/rally.ini. Happy benchmarking! - To benchmark Elasticsearch 6.0.0 with the default benchmark, run: - - esrally --distribution-version=6.0.0 - More info about Rally: * Type esrally --help @@ -102,7 +70,6 @@ Rally will ask you a few more things in the advanced setup: * **Benchmark data directory**: Rally stores all benchmark related data in this directory which can take up to several tens of GB. If you want to use a dedicated partition, you can specify a different data directory here. * **Elasticsearch project directory**: This is the directory where the Elasticsearch sources are located. If you don't actively develop on Elasticsearch you can just leave the default but if you want to benchmark local changes you should point Rally to your project directory. Note that Rally will run builds with the Gradle Wrapper in this directory (it runs ``./gradlew clean`` and ``./gradlew :distribution:tar:assemble``). -* **JDK root directory**: Rally will only ask this if it could not autodetect the JDK home by itself. Just enter the root directory of the JDK you want to use. By default, Rally will choose Java 8 if available and fallback to Java 10. * **Metrics store type**: You can choose between ``in-memory`` which requires no additional setup or ``elasticsearch`` which requires that you start a dedicated Elasticsearch instance to store metrics but gives you much more flexibility to analyse results. * **Metrics store settings** (only for metrics store type ``elasticsearch``): Provide the connection details to the Elasticsearch metrics store. This should be an instance that you use just for Rally but it can be a rather small one. A single node cluster with default setting should do it. When using self-signed certificates on the Elasticsearch metrics store, certificate verification can be turned off by setting the ``datastore.ssl.verification_mode`` setting to ``none``. Alternatively you can enter the path to the certificate authority's signing certificate in ``datastore.ssl.certificate_authorities``. Both settings are optional. * **Name for this benchmark environment** (only for metrics store type ``elasticsearch``): You can use the same metrics store for multiple environments (e.g. local, continuous integration etc.) so you can separate metrics from different environments by choosing a different name. diff --git a/docs/install.rst b/docs/install.rst index bc0bfb507..1add07991 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -101,10 +101,12 @@ In all other cases, Rally requires ``git 1.9`` or better. Verify with ``git --ve JDK ~~~ -A JDK is required on all machines where you want to launch Elasticsearch. If you use Rally just as a load generator, no JDK is required. +A JDK is required on all machines where you want to launch Elasticsearch. If you use Rally just as a load generator to :doc:`benchmark remote clusters `, no JDK is required. We recommend to use Oracle JDK but you are free to use OpenJDK as well. For details on how to install a JDK check your operating system's documentation pages. +To find the JDK, Rally expects the environment variable ``JAVA_HOME`` to be set on all targeted machines. To have more specific control, for example when you want to benchmark across a wide range of Elasticsearch releases, you can also set ``JAVAx_HOME`` where ``x`` is the major version of a JDK (e.g. ``JAVA8_HOME`` would point to a JDK 8 installation). Rally will then choose the highest supported JDK per version of Elasticsearch that is available. + .. note:: diff --git a/docs/migrate.rst b/docs/migrate.rst index 673792fdd..b6a0a2801 100644 --- a/docs/migrate.rst +++ b/docs/migrate.rst @@ -1,6 +1,23 @@ Migration Guide =============== +Migrating to Rally 1.0.0 +------------------------ + +Handling of JDK versions +^^^^^^^^^^^^^^^^^^^^^^^^ + +Previously the path to the JDK needed to be configured in Rally's configuration file (``~/.rally/rally.ini``) but this is too inflexible given the increased JDK release cadence. In order to keep up, we define now the allowed runtime JDKs in `rally-teams `_ per Elasticsearch version. + +To resolve the path to the appropriate JDK you need to define the environment variable ``JAVA_HOME`` on each targeted machine. + +You can also set version-specific environment variables, e.g. ``JAVA7_HOME``, ``JAVA8_HOME`` or ``JAVA10_HOME`` which will take precedence over ``JAVA_HOME``. + +.. note:: + + Rally will choose the highest appropriate JDK per Elasticsearch version. You can use ``--runtime-jdk`` to force a specific JDK version but the path will still be resolved according to the logic above. + + Migrating to Rally 0.11.0 ------------------------- diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 2c95d2dcd..d1f4c65b2 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -4,7 +4,7 @@ Quickstart Install ------- -Install Python 3.4+ including ``pip3``, git 1.9+ and at least JDK 8. If you want to benchmark source builds of Elasticsearch you also need JDK 10. Then run the following command, optionally prefixed by ``sudo`` if necessary:: +Install Python 3.4+ including ``pip3``, git 1.9+ and an `appropriate JDK to run Elasticsearch `_ Be sure that ``JAVA_HOME`` points to that JDK. Then run the following command, optionally prefixed by ``sudo`` if necessary:: pip3 install esrally diff --git a/esrally/config.py b/esrally/config.py index aba997e71..1c6e83a2c 100644 --- a/esrally/config.py +++ b/esrally/config.py @@ -106,7 +106,7 @@ def auto_load_local_config(base_config, additional_sections=None, config_file_cl class Config: EARLIEST_SUPPORTED_VERSION = 12 - CURRENT_CONFIG_VERSION = 16 + CURRENT_CONFIG_VERSION = 17 """ Config is the main entry point to retrieve and set benchmark properties. It provides multiple scopes to allow overriding of values on @@ -270,7 +270,7 @@ def __init__(self, i=input, sec_i=getpass.getpass, o=console.println): self.prompter = None self.logger = logging.getLogger(__name__) - def create_config(self, config_file, advanced_config=False, assume_defaults=False, java_home=None, runtime_java_home=None): + def create_config(self, config_file, advanced_config=False, assume_defaults=False): """ Either creates a new configuration file or overwrites an existing one. Will ask the user for input on configurable properties and writes them to the configuration file in ~/.rally/rally.ini. @@ -280,7 +280,6 @@ def create_config(self, config_file, advanced_config=False, assume_defaults=Fals :param assume_defaults: If True, assume the user accepted all values for which defaults are provided. Mainly intended for automatic configuration in CI run. Default: False. """ - benchmark_from_sources = True self.prompter = Prompter(self.i, self.sec_i, self.o, assume_defaults) if advanced_config: @@ -300,48 +299,12 @@ def create_config(self, config_file, advanced_config=False, assume_defaults=Fals else: self.logger.debug("Did not detect a configuration file at [%s]. Running initial configuration routine.", config_file.location) - # Autodetect settings - self.o("* Autodetecting available third-party software") - git_path = io.guess_install_location("git") - - java_8_home = runtime_java_home if runtime_java_home else io.guess_java_home(major_version=8) - java_10_home = java_home if java_home else io.guess_java_home(major_version=10) - from esrally.utils import jvm - if java_8_home: - auto_detected_java_home = java_8_home - # Don't auto-detect an EA release and bring trouble to the user later on. They can still configure it manually if they want to. - elif java_10_home and not jvm.is_early_access_release(java_10_home): - auto_detected_java_home = java_10_home - else: - auto_detected_java_home = None - - self.print_detection_result("git ", git_path) - self.print_detection_result("JDK ", auto_detected_java_home, - warn_if_missing=True, - additional_message="You cannot benchmark Elasticsearch on this machine without a JDK.") - self.o("") - root_dir = io.normalize_path(os.path.abspath(os.path.join(config_file.config_dir, "benchmarks"))) if advanced_config: root_dir = io.normalize_path(self._ask_property("Enter the benchmark data directory", default_value=root_dir)) else: self.o("* Setting up benchmark data directory in %s" % root_dir) - if not java_10_home or jvm.is_early_access_release(java_10_home): - raw_java_10_home = self._ask_property("Enter the JDK 10 root directory", check_path_exists=True, mandatory=False) - if raw_java_10_home and jvm.major_version(raw_java_10_home) == 10 and not jvm.is_early_access_release(raw_java_10_home): - java_10_home = io.normalize_path(raw_java_10_home) if raw_java_10_home else None - else: - benchmark_from_sources = False - self.o("********************************************************************************") - self.o("You don't have a valid JDK 10 installation and cannot benchmark source builds.") - self.o("") - self.o("You can still benchmark binary distributions with e.g.:") - self.o("") - self.o(" %s --distribution-version=6.0.0" % PROGRAM_NAME) - self.o("********************************************************************************") - self.o("") - # We try to autodetect an existing ES source directory guess = self._guess_es_src_dir() if guess: @@ -362,29 +325,6 @@ def create_config(self, config_file, advanced_config=False, assume_defaults=Fals # Not everybody might have SSH access. Play safe with the default. It may be slower but this will work for everybody. repo_url = "https://github.com/elastic/elasticsearch.git" - if auto_detected_java_home: - java_home = auto_detected_java_home - local_benchmarks = True - else: - raw_java_home = self._ask_property("Enter the JDK root directory (version 8 or later)", check_path_exists=True, mandatory=False) - java_home = io.normalize_path(raw_java_home) if raw_java_home else None - if not java_home: - local_benchmarks = False - self.o("") - self.o("********************************************************************************") - self.o("You don't have a JDK installed but Elasticsearch requires one to run. This means") - self.o("that you cannot benchmark Elasticsearch on this machine.") - self.o("") - self.o("You can still benchmark against remote machines e.g.:") - self.o("") - self.o(" %s --pipeline=benchmark-only --target-host=\"NODE_IP:9200\"" % PROGRAM_NAME) - self.o("") - self.o("See %s for further info." % console.format.link("%srecipes.html" % DOC_LINK)) - self.o("********************************************************************************") - self.o("") - else: - local_benchmarks = True - if advanced_config: data_store_choice = self._ask_property("Where should metrics be kept?" "\n\n" @@ -429,12 +369,6 @@ def create_config(self, config_file, advanced_config=False, assume_defaults=Fals # the Elasticsearch directory is just the last path component (relative to the source root directory) config["source"]["elasticsearch.src.subdir"] = io.basename(source_dir) - config["runtime"] = {} - if java_home: - config["runtime"]["java.home"] = java_home - if java_10_home: - config["runtime"]["java10.home"] = java_10_home - config["benchmarks"] = {} config["benchmarks"]["local.dataset.cache"] = "${node:root.dir}/data" @@ -462,20 +396,6 @@ def create_config(self, config_file, advanced_config=False, assume_defaults=Fals self.o("Configuration successfully written to %s. Happy benchmarking!" % config_file.location) self.o("") - if local_benchmarks and benchmark_from_sources: - self.o("To benchmark Elasticsearch with the default benchmark, run:") - self.o("") - self.o(" %s" % PROGRAM_NAME) - self.o("") - elif local_benchmarks: - self.o("To benchmark Elasticsearch 6.0.0 with the default benchmark, run:") - self.o("") - self.o(" %s --distribution-version=6.0.0" % PROGRAM_NAME) - self.o("") - else: - # we've already printed an info for the user. No need to repeat that. - pass - self.o("More info about Rally:") self.o("") self.o("* Type %s --help" % PROGRAM_NAME) @@ -712,6 +632,11 @@ def warn_if_plugin_build_task_is_in_use(config): current_version = 16 config["meta"]["config.version"] = str(current_version) + if current_version == 16 and target_version > current_version: + config.pop("runtime", None) + current_version = 17 + config["meta"]["config.version"] = str(current_version) + # all migrations done config_file.store(config) logger.info("Successfully self-upgraded configuration to version [%s]", target_version) diff --git a/esrally/mechanic/launcher.py b/esrally/mechanic/launcher.py index c91426c5a..55f7b5f93 100644 --- a/esrally/mechanic/launcher.py +++ b/esrally/mechanic/launcher.py @@ -231,8 +231,8 @@ def __init__(self, cfg, metrics_store, races_root_dir, clock=time.Clock): self.metrics_store = metrics_store self._clock = clock self.races_root_dir = races_root_dir - self.java_home = self.cfg.opts("runtime", "java.home") self.keep_running = self.cfg.opts("mechanic", "keep.running") + self.override_runtime_jdk = self.cfg.opts("mechanic", "runtime.jdk") self.logger = logging.getLogger(__name__) def start(self, node_configurations): @@ -241,19 +241,17 @@ def start(self, node_configurations): # # We also do this only once per host otherwise we would kill instances that we've just launched. process.kill_running_es_instances(self.races_root_dir) - java_major_version = jvm.major_version(self.java_home) - self.logger.info("Detected Java major version [%s].", java_major_version) - node_count_on_host = len(node_configurations) - return [self._start_node(node_configuration, node_count_on_host, java_major_version) for node_configuration in node_configurations] + return [self._start_node(node_configuration, node_count_on_host) for node_configuration in node_configurations] - def _start_node(self, node_configuration, node_count_on_host, java_major_version): + def _start_node(self, node_configuration, node_count_on_host): host_name = node_configuration.ip node_name = node_configuration.node_name car = node_configuration.car binary_path = node_configuration.binary_path data_paths = node_configuration.data_paths node_telemetry_dir = "%s/telemetry" % node_configuration.node_root_path + java_major_version, java_home = self._resolve_java_home(car) self.logger.info("Starting node [%s] based on car [%s].", node_name, car) @@ -273,7 +271,7 @@ def _start_node(self, node_configuration, node_count_on_host, java_major_version ] t = telemetry.Telemetry(enabled_devices, devices=node_telemetry) - env = self._prepare_env(car, node_name, t) + env = self._prepare_env(car, node_name, java_home, t) t.on_pre_node_start(node_name) node_process = self._start_process(env, node_name, binary_path) node = cluster.Node(node_process, host_name, node_name, t) @@ -283,21 +281,37 @@ def _start_node(self, node_configuration, node_count_on_host, java_major_version return node - def _prepare_env(self, car, node_name, t): + def _resolve_java_home(self, car): + runtime_jdk_versions = self._determine_runtime_jdks(car) + self.logger.info("Allowed JDK versions are %s.", runtime_jdk_versions) + major, java_home = jvm.resolve_path(runtime_jdk_versions) + self.logger.info("Detected JDK with major version [%s] in [%s].", major, java_home) + return major, java_home + + def _determine_runtime_jdks(self, car): + if self.override_runtime_jdk: + return [self.override_runtime_jdk] + else: + runtime_jdks = car.mandatory_var("runtime.jdk") + try: + return [int(v) for v in runtime_jdks.split(",")] + except ValueError: + raise exceptions.SystemSetupError("Car config key \"runtime.jdk\" is invalid: \"{}\" (must be int)".format(runtime_jdks)) + + def _prepare_env(self, car, node_name, java_home, t): env = {} env.update(os.environ) env.update(car.env) - # Unix specific!: - self._set_env(env, "PATH", "%s/bin" % self.java_home, separator=":") + self._set_env(env, "PATH", os.path.join(java_home, "bin"), separator=os.pathsep) # Don't merge here! - env["JAVA_HOME"] = self.java_home + env["JAVA_HOME"] = java_home # we just blindly trust telemetry here... for k, v in t.instrument_candidate_env(car, node_name).items(): self._set_env(env, k, v) exit_on_oome_flag = "-XX:+ExitOnOutOfMemoryError" - if jvm.supports_option(self.java_home, exit_on_oome_flag): + if jvm.supports_option(java_home, exit_on_oome_flag): self.logger.info("Setting [%s] to detect out of memory errors during the benchmark.", exit_on_oome_flag) self._set_env(env, "ES_JAVA_OPTS", exit_on_oome_flag) else: diff --git a/esrally/mechanic/supplier.py b/esrally/mechanic/supplier.py index 3731d782b..fa001711b 100644 --- a/esrally/mechanic/supplier.py +++ b/esrally/mechanic/supplier.py @@ -5,7 +5,7 @@ import urllib.error from esrally import exceptions, PROGRAM_NAME -from esrally.utils import git, console, io, process, net, versions, convert +from esrally.utils import git, console, io, process, net, jvm, convert from esrally.exceptions import BuildError, SystemSetupError # e.g. my-plugin:current - we cannot simply use String#split(":") as this would not work for timestamp-based revisions @@ -24,9 +24,9 @@ def create(cfg, sources, distribution, build, challenge_root_path, car, plugins= suppliers = [] if build_needed: - java10_home = _java10_home(cfg) + java_home = _java_home(car) es_src_dir = os.path.join(_src_dir(cfg), _config_value(src_config, "elasticsearch.src.subdir")) - builder = Builder(es_src_dir, java10_home, challenge_root_path) + builder = Builder(es_src_dir, java_home, challenge_root_path) else: builder = None @@ -75,13 +75,13 @@ def create(cfg, sources, distribution, build, challenge_root_path, car, plugins= return CompositeSupplier(suppliers) -def _java10_home(cfg): - from esrally import config +def _java_home(car): + build_jdk = car.mandatory_var("build.jdk") try: - return cfg.opts("runtime", "java10.home") - except config.ConfigError: - raise exceptions.SystemSetupError("No JDK 10 is configured. You cannot benchmark source builds of Elasticsearch on this machine. " - "Please install a JDK 10 and reconfigure Rally with %s configure" % PROGRAM_NAME) + _, path = jvm.resolve_path(int(build_jdk)) + return path + except ValueError: + raise exceptions.SystemSetupError("Car config key \"build.jdk\" is invalid: \"{}\" (must be int)".format(build_jdk)) def _required_version(version): @@ -190,23 +190,17 @@ def fetch(self): def prepare(self): if self.builder: - self.builder.build([self.value_of("clean_command"), self.value_of("build_command")]) + self.builder.build([self.car.mandatory_var("clean_command"), self.car.mandatory_var("build_command")]) def add(self, binaries): binaries["elasticsearch"] = self.resolve_binary() def resolve_binary(self): try: - return glob.glob("{}/{}".format(self.src_dir, self.value_of("artifact_path_pattern")))[0] + return glob.glob("{}/{}".format(self.src_dir, self.car.mandatory_var("artifact_path_pattern")))[0] except IndexError: raise SystemSetupError("Couldn't find a tar.gz distribution. Please run Rally with the pipeline 'from-sources-complete'.") - def value_of(self, k): - try: - return self.car.variables[k] - except KeyError: - raise exceptions.SystemSetupError("Car '{}' is missing config variable '{}' to build Elasticsearch.".format(self.car, k)) - class ExternalPluginSourceSupplier: def __init__(self, plugin, revision, src_dir, src_config, builder): diff --git a/esrally/mechanic/team.py b/esrally/mechanic/team.py index 461b30645..4a1db7bfb 100644 --- a/esrally/mechanic/team.py +++ b/esrally/mechanic/team.py @@ -245,6 +245,12 @@ def __init__(self, names, root_path, config_paths, variables=None, env=None): self.variables = variables self.env = env + def mandatory_var(self, name): + try: + return self.variables[name] + except KeyError: + raise exceptions.SystemSetupError("Car \"{}\" requires config key \"{}\"".format(self.name, name)) + @property def name(self): return "+".join(self.names) diff --git a/esrally/rally.py b/esrally/rally.py index 0da53d1bd..2ed82ba04 100644 --- a/esrally/rally.py +++ b/esrally/rally.py @@ -127,9 +127,6 @@ def positive_number(v): help="Automatically accept all options with default values (default: false)", default=False, action="store_true") - # undocumented - only as a workaround for integration tests - p.add_argument("--java-home", default=None) - p.add_argument("--runtime-java-home", default=None) for p in [parser, list_parser, race_parser, generate_parser]: p.add_argument( @@ -137,6 +134,11 @@ def positive_number(v): help="define the version of the Elasticsearch distribution to download. " "Check https://www.elastic.co/downloads/elasticsearch for released versions.", default="") + p.add_argument( + "--runtime-jdk", + type=positive_number, + help="The major version of the runtime JDK to use.", + default=None) track_source_group = p.add_mutually_exclusive_group() track_source_group.add_argument( @@ -331,9 +333,7 @@ def ensure_configuration_present(cfg, args, sub_command): if sub_command == "configure": config.ConfigFactory().create_config(cfg.config_file, advanced_config=args.advanced_config, - assume_defaults=args.assume_defaults, - java_home=args.java_home, - runtime_java_home=args.runtime_java_home) + assume_defaults=args.assume_defaults) exit(0) else: if cfg.config_present(): @@ -536,6 +536,7 @@ def main(): else: cfg.add(config.Scope.applicationOverride, "mechanic", "keep.running", False) cfg.add(config.Scope.applicationOverride, "mechanic", "preserve.install", convert.to_bool(args.preserve_install)) + cfg.add(config.Scope.applicationOverride, "mechanic", "runtime.jdk", args.runtime_jdk) cfg.add(config.Scope.applicationOverride, "mechanic", "telemetry.devices", opts.csv_to_list(args.telemetry)) cfg.add(config.Scope.applicationOverride, "mechanic", "telemetry.params", opts.to_dict(args.telemetry_params)) diff --git a/esrally/utils/jvm.py b/esrally/utils/jvm.py index 2ce17073b..6c38d70e9 100644 --- a/esrally/utils/jvm.py +++ b/esrally/utils/jvm.py @@ -1,5 +1,7 @@ import re +import os +from esrally import exceptions from esrally.utils import process @@ -73,4 +75,79 @@ def is_early_access_release(java_home, sysprop_reader=system_property): :param sysprop_reader: (Optional) only relevant for testing. :return: True iff the JVM available at ``java_home`` is classified as an early access release. """ - return vendor(java_home, sysprop_reader) == "Oracle Corporation" and version(java_home, sysprop_reader).endswith("-ea") \ No newline at end of file + return vendor(java_home, sysprop_reader) == "Oracle Corporation" and version(java_home, sysprop_reader).endswith("-ea") + + +def resolve_path(majors, sysprop_reader=system_property): + """ + Resolves the path to the JDK with the provided major version(s). It checks the versions in the same order specified in ``majors`` + and will return the first match. To achieve this, it first checks the major version x in the environment variable ``JAVAx_HOME`` + and falls back to ``JAVA_HOME``. It also ensures that the environment variable points to the right JDK version. + + If no appropriate version is found, a ``SystemSetupError`` is raised. + + :param majors: Either a list of major versions to check or a single version as an ``int``. + :param sysprop_reader: (Optional) only relevant for testing. + :return: A tuple of (major version, path to Java home directory). + """ + if isinstance(majors, int): + return majors, _resolve_single_path(majors, sysprop_reader=sysprop_reader) + else: + for major in majors: + java_home = _resolve_single_path(major, mandatory=False, sysprop_reader=sysprop_reader) + if java_home: + return major, java_home + raise exceptions.SystemSetupError("Install a JDK with one of the versions {} and point to it with one of {}." + .format(majors, _checked_env_vars(majors))) + + +def _resolve_single_path(major, mandatory=True, sysprop_reader=system_property): + """ + + Resolves the path to a JDK with the provided major version. + + :param major: The major version to check. + :param mandatory: Determines if we expect to find a matching JDK. + :param sysprop_reader: (Optional) only relevant for testing. + :return: The resolved path to the JDK or ``None`` if ``mandatory`` is ``False`` and no appropriate JDK has been found. + """ + def do_resolve(env_var, major): + java_v_home = os.getenv(env_var) + if java_v_home: + actual_major = major_version(java_v_home, sysprop_reader) + if actual_major == major: + return java_v_home + elif mandatory: + raise exceptions.SystemSetupError("{} points to JDK {} but it should point to JDK {}.".format(env_var, actual_major, major)) + else: + return None + else: + return None + + # this has to be consistent with _checked_env_vars() + specific_env_var = "JAVA{}_HOME".format(major) + generic_env_var = "JAVA_HOME" + java_home = do_resolve(specific_env_var, major) + if java_home: + return java_home + else: + java_home = do_resolve(generic_env_var, major) + if java_home: + return java_home + elif mandatory: + raise exceptions.SystemSetupError("Neither {} nor {} point to a JDK {} installation.". + format(specific_env_var, generic_env_var, major)) + else: + return None + + +def _checked_env_vars(majors): + """ + Provides a list of environment variables that are checked for the given list of major versions. + + :param majors: A list of major versions. + :return: A list of checked environment variables. + """ + checked = ["JAVA{}_HOME".format(major) for major in majors] + checked.append("JAVA_HOME") + return checked diff --git a/integration-test.sh b/integration-test.sh index 67f28c2d3..00ff88f63 100755 --- a/integration-test.sh +++ b/integration-test.sh @@ -4,16 +4,10 @@ set -e readonly CONFIGURATIONS=(integration-test es-integration-test) -# we will not test ES 1.x anymore because it does not work with (out of the box) with Java 9 anymore (Java 8 is still fine). On startup it -# fails with: -# -# java.lang.UnsupportedOperationException: Boot class path mechanism is not supported -# -# Temporarily disable testing ES 2.x because it does not start with JDK 10 (sets unrecognized JVM options) -# readonly DISTRIBUTIONS=(2.4.6 5.6.7) -readonly DISTRIBUTIONS=(5.6.7) +readonly DISTRIBUTIONS=(1.7.6 2.4.6 5.6.9) readonly TRACKS=(geonames nyc_taxis http_logs nested) +readonly ES_METRICS_STORE_JAVA_HOME="${JAVA8_HOME}" readonly ES_METRICS_STORE_VERSION="6.2.1" readonly ES_ARTIFACT_PATH="elasticsearch-${ES_METRICS_STORE_VERSION}" readonly ES_ARTIFACT="${ES_ARTIFACT_PATH}.tar.gz" @@ -76,19 +70,10 @@ function set_up { local in_memory_config_file_path="${HOME}/.rally/rally-integration-test.ini" local es_config_file_path="${HOME}/.rally/rally-es-integration-test.ini" - # if the build defines these variables we'll explicitly use them instead of auto-detection - if [ -n "${JAVA_HOME}" ] && [ -n "${RUNTIME_JAVA_HOME}" ]; then - # configure for tests with an in-memory metrics store - esrally configure --java-home="${JAVA_HOME}" --runtime-java-home="${RUNTIME_JAVA_HOME}" --assume-defaults --configuration-name="integration-test" - # configure for tests with an Elasticsearch metrics store - esrally configure --java-home="${JAVA_HOME}" --runtime-java-home="${RUNTIME_JAVA_HOME}" --assume-defaults --configuration-name="es-integration-test" - else - # configure for tests with an in-memory metrics store - esrally configure --assume-defaults --configuration-name="integration-test" - # configure for tests with an Elasticsearch metrics store - esrally configure --assume-defaults --configuration-name="es-integration-test" - - fi + # configure for tests with an in-memory metrics store + esrally configure --assume-defaults --configuration-name="integration-test" + # configure for tests with an Elasticsearch metrics store + esrally configure --assume-defaults --configuration-name="es-integration-test" # configure Elasticsearch instead of in-memory after the fact # this is more portable than using sed's in-place editing which requires "-i" on GNU and "-i ''" elsewhere. @@ -114,6 +99,7 @@ function set_up { # Delete and exit if archive is somehow corrupted, despite getting downloaded correctly. tar -xzf "${ES_ARTIFACT}" || { rm -f "${ES_ARTIFACT}"; exit 1; } cd "${ES_ARTIFACT_PATH}" + export JAVA_HOME=${ES_METRICS_STORE_JAVA_HOME} bin/elasticsearch & # store PID so we can kill ES later ES_PID=$! @@ -164,10 +150,10 @@ function test_sources { # build Elasticsearch and a core plugin info "test sources [--configuration-name=${cfg}], [--revision=latest], [--track=geonames], [--challenge=append-no-conflicts], [--car=4gheap] [--elasticsearch-plugins=analysis-icu]" kill_rally_processes - esrally --configuration-name="${cfg}" --revision=latest --track=geonames --test-mode --challenge=append-no-conflicts --car=4gheap --elasticsearch-plugins=analysis-icu + esrally --configuration-name="${cfg}" --on-error=abort --revision=latest --track=geonames --test-mode --challenge=append-no-conflicts --car=4gheap --elasticsearch-plugins=analysis-icu info "test sources [--configuration-name=${cfg}], [--pipeline=from-sources-skip-build], [--track=geonames], [--challenge=append-no-conflicts-index-only], [--car=4gheap,ea] [--laps=2]" kill_rally_processes - esrally --configuration-name="${cfg}" --pipeline=from-sources-skip-build --track=geonames --test-mode --challenge=append-no-conflicts-index-only --car="4gheap,ea" --laps=2 + esrally --configuration-name="${cfg}" --on-error=abort --pipeline=from-sources-skip-build --track=geonames --test-mode --challenge=append-no-conflicts-index-only --car="4gheap,ea" --laps=2 } function test_distributions { @@ -180,7 +166,7 @@ function test_distributions { random_configuration cfg info "test distributions [--configuration-name=${cfg}], [--distribution-version=${dist}], [--track=${track}], [--car=4gheap]" kill_rally_processes - esrally --configuration-name="${cfg}" --distribution-version="${dist}" --track="${track}" --test-mode --car=4gheap + esrally --configuration-name="${cfg}" --on-error=abort --distribution-version="${dist}" --track="${track}" --test-mode --car=4gheap done done } @@ -193,7 +179,7 @@ function test_benchmark_only { info "test benchmark-only [--configuration-name=${cfg}]" kill_rally_processes - esrally --configuration-name="${cfg}" --pipeline=benchmark-only --track=geonames --test-mode --challenge=append-no-conflicts-index-only --track-params="cluster_health:'yellow'" + esrally --configuration-name="${cfg}" --on-error=abort --pipeline=benchmark-only --track=geonames --test-mode --challenge=append-no-conflicts-index-only --track-params="cluster_health:'yellow'" } function run_test { diff --git a/tests/config_test.py b/tests/config_test.py index fc9e35d6b..9e42da3fe 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -234,13 +234,9 @@ def assert_equals_base_config(self, base_config, local_config, section, key): class ConfigFactoryTests(TestCase): @mock.patch("esrally.utils.git.is_working_copy") - @mock.patch("esrally.utils.jvm.is_early_access_release") - @mock.patch("esrally.utils.io.guess_java_home") @mock.patch("esrally.utils.io.guess_install_location") - def test_create_simple_config(self, guess_install_location, guess_java_home, is_ea_release, working_copy): + def test_create_simple_config(self, guess_install_location, working_copy): guess_install_location.side_effect = ["/tests/usr/bin/git"] - guess_java_home.return_value = "/tests/java10/home" - is_ea_release.return_value = False # Rally checks in the parent and sibling directories whether there is an ES working copy. We don't want this detection logic # to succeed spuriously (e.g. on developer machines). working_copy.return_value = False @@ -270,10 +266,6 @@ def test_create_simple_config(self, guess_install_location, guess_java_home, is_ self.assertEqual("https://github.com/elastic/elasticsearch.git", config_store.config["source"]["remote.repo.url"]) self.assertEqual("elasticsearch", config_store.config["source"]["elasticsearch.src.subdir"]) - self.assertTrue("runtime" in config_store.config) - self.assertEqual("/tests/java10/home", config_store.config["runtime"]["java.home"]) - self.assertEqual("/tests/java10/home", config_store.config["runtime"]["java10.home"]) - self.assertTrue("benchmarks" in config_store.config) self.assertEqual("${node:root.dir}/data", config_store.config["benchmarks"]["local.dataset.cache"]) @@ -297,55 +289,7 @@ def test_create_simple_config(self, guess_install_location, guess_java_home, is_ self.assertTrue("distributions" in config_store.config) self.assertEqual("true", config_store.config["distributions"]["release.cache"]) - @mock.patch("esrally.utils.jvm.is_early_access_release") - @mock.patch("esrally.utils.jvm.major_version") - @mock.patch("esrally.utils.io.guess_java_home") - @mock.patch("esrally.utils.io.guess_install_location") - @mock.patch("esrally.utils.io.normalize_path") - @mock.patch("os.path.exists") - def test_create_simple_config_no_java_detected(self, path_exists, normalize_path, guess_install_location, guess_java_home, - major_jvm_version, jvm_is_early_access_release): - guess_install_location.side_effect = ["/tests/usr/bin/git"] - guess_java_home.return_value = None - normalize_path.side_effect = ["/home/user/.rally/benchmarks", "/tests/java10/home", "/tests/java8/home", - "/home/user/.rally/benchmarks/src"] - major_jvm_version.return_value = 10 - jvm_is_early_access_release.return_value = False - path_exists.return_value = True - - f = config.ConfigFactory(i=MockInput(["/tests/java10/home", "/Projects/elasticsearch/src", "/tests/java8/home"]), o=null_output) - - config_store = InMemoryConfigStore("test") - f.create_config(config_store) - - self.assertIsNotNone(config_store.config) - self.assertTrue("runtime" in config_store.config) - self.assertEqual("/tests/java8/home", config_store.config["runtime"]["java.home"]) - - @mock.patch("esrally.utils.io.guess_java_home") - @mock.patch("esrally.utils.io.guess_install_location") - def test_create_simple_config_no_java_installed(self, guess_install_location, guess_java_home): - guess_install_location.side_effect = ["/tests/usr/bin/git"] - guess_java_home.return_value = None - - # the input is the question for the JDK home and the JDK 10 home directory - the user does not define one - f = config.ConfigFactory(i=MockInput(["", ""]), o=null_output) - - config_store = InMemoryConfigStore("test") - f.create_config(config_store) - - self.assertIsNotNone(config_store.config) - self.assertFalse("java.home" in config_store.config["runtime"]) - self.assertFalse("java10.home" in config_store.config["runtime"]) - - @mock.patch("esrally.utils.jvm.is_early_access_release") - @mock.patch("esrally.utils.io.guess_java_home") - @mock.patch("esrally.utils.io.guess_install_location") - def test_create_advanced_config(self, guess_install_location, guess_java_home, is_ea_release): - guess_install_location.side_effect = ["/tests/usr/bin/git"] - guess_java_home.side_effect = ["/tests/java8/home", "/tests/java10/home"] - is_ea_release.return_value = False - + def test_create_advanced_config(self): f = config.ConfigFactory(i=MockInput([ # benchmark root directory "/var/data/rally", @@ -378,9 +322,6 @@ def test_create_advanced_config(self, guess_install_location, guess_java_home, i self.assertTrue("node" in config_store.config) self.assertEqual("/var/data/rally", config_store.config["node"]["root.dir"]) self.assertTrue("source" in config_store.config) - self.assertTrue("runtime" in config_store.config) - self.assertEqual("/tests/java8/home", config_store.config["runtime"]["java.home"]) - self.assertEqual("/tests/java10/home", config_store.config["runtime"]["java10.home"]) self.assertTrue("benchmarks" in config_store.config) self.assertTrue("reporting" in config_store.config) @@ -434,10 +375,7 @@ def test_does_not_migrate_outdated_config(self): config.migrate(config_file, config.Config.EARLIEST_SUPPORTED_VERSION - 1, config.Config.CURRENT_CONFIG_VERSION, out=null_output) # catch all test, migrations are checked in more detail in the other tests - @mock.patch("esrally.utils.io.get_size") - @mock.patch("esrally.time.sleep") - def test_migrate_from_earliest_supported_to_latest(self, sleep, get_size): - get_size.return_value = 0 + def test_migrate_from_earliest_supported_to_latest(self): config_file = InMemoryConfigStore("test") sample_config = { "meta": { diff --git a/tests/mechanic/supplier_test.py b/tests/mechanic/supplier_test.py index a52c2465a..cbd5dbe64 100644 --- a/tests/mechanic/supplier_test.py +++ b/tests/mechanic/supplier_test.py @@ -180,7 +180,7 @@ def test_raises_error_on_missing_car_variable(self): builder = mock.create_autospec(supplier.Builder) es = supplier.ElasticsearchSourceSupplier(revision="abc", es_src_dir="/src", remote_url="", car=car, builder=builder) with self.assertRaisesRegex(exceptions.SystemSetupError, - "Car 'default' is missing config variable 'build_command' to build Elasticsearch."): + "Car \"default\" requires config key \"build_command\"."): es.prepare() self.assertEqual(0, builder.build.call_count) @@ -371,7 +371,6 @@ def test_create_suppliers_for_es_only_config(self): cfg.add(config.Scope.application, "distributions", "release.url", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz") cfg.add(config.Scope.application, "distributions", "release.cache", True) - cfg.add(config.Scope.application, "runtime", "java10.home", "/usr/local/bin/java10/") cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") car = team.Car("default", root_path=None, config_paths=[]) @@ -390,7 +389,6 @@ def test_create_suppliers_for_es_distribution_plugin_source_skip(self): cfg.add(config.Scope.application, "distributions", "release.url", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz") cfg.add(config.Scope.application, "distributions", "release.cache", True) - cfg.add(config.Scope.application, "runtime", "java10.home", "/usr/local/bin/java10/") cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.dir", "/home/user/Projects/community-plugin") @@ -422,7 +420,6 @@ def test_create_suppliers_for_es_missing_distribution_plugin_source_skip(self): cfg.add(config.Scope.application, "distributions", "release.url", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz") cfg.add(config.Scope.application, "distributions", "release.cache", True) - cfg.add(config.Scope.application, "runtime", "java10.home", "/usr/local/bin/java10/") cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True) @@ -438,6 +435,7 @@ def test_create_suppliers_for_es_missing_distribution_plugin_source_skip(self): ]) self.assertRegex(ctx.exception.args[0], r"Could not determine version..*") + @mock.patch("esrally.utils.jvm.resolve_path", lambda v: (v, "/opt/java/java{}".format(v))) def test_create_suppliers_for_es_distribution_plugin_source_build(self): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "distribution.version", "6.0.0") @@ -447,13 +445,12 @@ def test_create_suppliers_for_es_distribution_plugin_source_build(self): cfg.add(config.Scope.application, "distributions", "release.url", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz") cfg.add(config.Scope.application, "distributions", "release.cache", True) - cfg.add(config.Scope.application, "runtime", "java10.home", "/usr/local/bin/java10/") cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") cfg.add(config.Scope.application, "node", "src.root.dir", "/opt/rally/src") cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir", "elasticsearch") cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.dir", "/home/user/Projects/community-plugin") - car = team.Car("default", root_path=None, config_paths=[]) + car = team.Car("default", root_path=None, config_paths=[], variables={"build.jdk": "10"}) core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True) external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False) @@ -471,6 +468,7 @@ def test_create_suppliers_for_es_distribution_plugin_source_build(self): self.assertEqual(external_plugin, composite_supplier.suppliers[2].plugin) self.assertIsNotNone(composite_supplier.suppliers[2].builder) + @mock.patch("esrally.utils.jvm.resolve_path", lambda v: (v, "/opt/java/java{}".format(v))) def test_create_suppliers_for_es_and_plugin_source_build(self): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "source.revision", "elasticsearch:abc,community-plugin:current") @@ -478,7 +476,6 @@ def test_create_suppliers_for_es_and_plugin_source_build(self): cfg.add(config.Scope.application, "distributions", "release.url", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz") cfg.add(config.Scope.application, "distributions", "release.cache", True) - cfg.add(config.Scope.application, "runtime", "java10.home", "/usr/local/bin/java10/") cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") cfg.add(config.Scope.application, "node", "src.root.dir", "/opt/rally/src") cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir", "elasticsearch") @@ -487,7 +484,8 @@ def test_create_suppliers_for_es_and_plugin_source_build(self): car = team.Car("default", root_path=None, config_paths=[], variables={ "clean_command": "./gradlew clean", - "build_command": "./gradlew assemble" + "build_command": "./gradlew assemble", + "build.jdk": "11" }) core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True) external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False) diff --git a/tests/utils/jvm_test.py b/tests/utils/jvm_test.py index abfc54fd5..95fce0b34 100644 --- a/tests/utils/jvm_test.py +++ b/tests/utils/jvm_test.py @@ -1,5 +1,8 @@ from unittest import TestCase +import unittest.mock as mock + from esrally.utils import jvm +from esrally import exceptions class JvmTests(TestCase): @@ -23,4 +26,68 @@ def test_ga_release(self): def prop_version_reader(self, java_home, prop): props = java_home.split(",") - return props[1] if prop == "java.version" else props[0] \ No newline at end of file + return props[1] if prop == "java.version" else props[0] + + def path_based_prop_version_reader(self, java_home, prop): + props = java_home.split("/") + # assumes a path that contains the major version as last component + return props[-1] if prop == "java.vm.specification.version" else None + + @mock.patch("os.getenv") + def test_resolve_path_for_one_version_via_java_home(self, getenv): + # JAVA8_HOME, JAVA_HOME + getenv.side_effect = [None, "/opt/jdks/jdk/1.8"] + + major, resolved_path = jvm.resolve_path(majors=8, sysprop_reader=self.path_based_prop_version_reader) + self.assertEqual(8, major) + self.assertEqual("/opt/jdks/jdk/1.8", resolved_path) + + @mock.patch("os.getenv") + def test_resolve_path_for_one_version_via_java_x_home(self, getenv): + # JAVA8_HOME, JAVA_HOME + getenv.side_effect = ["/opt/jdks/jdk/1.8", None] + + major, resolved_path = jvm.resolve_path(majors=8, sysprop_reader=self.path_based_prop_version_reader) + self.assertEqual(8, major) + self.assertEqual("/opt/jdks/jdk/1.8", resolved_path) + + @mock.patch("os.getenv") + def test_resolve_path_for_one_version_no_matching_version(self, getenv): + # JAVA8_HOME, JAVA_HOME + getenv.side_effect = [None, "/opt/jdks/jdk/1.7"] + + with self.assertRaisesRegex(expected_exception=exceptions.SystemSetupError, + expected_regex="JAVA_HOME points to JDK 7 but it should point to JDK 8."): + jvm.resolve_path(majors=8, sysprop_reader=self.path_based_prop_version_reader) + + @mock.patch("os.getenv") + def test_resolve_path_for_one_version_no_env_vars_defined(self, getenv): + getenv.return_value = None + + with self.assertRaisesRegex(expected_exception=exceptions.SystemSetupError, + expected_regex="Neither JAVA8_HOME nor JAVA_HOME point to a JDK 8 installation."): + jvm.resolve_path(majors=8, sysprop_reader=self.path_based_prop_version_reader) + + @mock.patch("os.getenv") + def test_resolve_path_for_multiple_versions(self, getenv): + getenv.side_effect = [ + # JAVA_HOME + None, + # JAVA11_HOME + None, + # JAVA_HOME + None, + # JAVA10_HOME, + None, + # JAVA_HOME + None, + # JAVA9_HOME + "/opt/jdks/jdk/9", + # JAVA_HOME + None, + # JAVA8_HOME + "/opt/jdks/jdk/1.8", + ] + major, resolved_path = jvm.resolve_path(majors=[11, 10, 9, 8], sysprop_reader=self.path_based_prop_version_reader) + self.assertEqual(9, major) + self.assertEqual("/opt/jdks/jdk/9", resolved_path) diff --git a/tox.ini b/tox.ini index 51eae32cf..dc1b5b5af 100644 --- a/tox.ini +++ b/tox.ini @@ -33,8 +33,11 @@ deps = pytest-benchmark passenv = HOME - JAVA_HOME - RUNTIME_JAVA_HOME + JAVA7_HOME + JAVA8_HOME + JAVA9_HOME + JAVA10_HOME + JAVA11_HOME SSH_AUTH_SOCK # we do not pass LANG and LC_ALL anymore in order to isolate integration tests # from the test environment. Rally needs to enforce UTF-8 encoding in every