Get distribution download URL from car config

docs/car.rst

@@ -57,6 +57,7 @@ The default car definitions of Rally are stored in ``~/.rally/benchmarks/teams/d
         │           └── jvm.options
         ├── ea.ini
         └── vanilla
+            ├── config.ini
             └── templates
                 └── config
                     ├── elasticsearch.yml
@@ -68,8 +69,12 @@ The top-level directory "v1" denotes the configuration format in version 1. Belo
     [variables]
     clean_command=./gradlew clean
 
-This defines the variable ``clean_command`` for all cars that reference this configuration.
+This defines the variable ``clean_command`` for all cars that reference this configuration. Rally will treat the following variable names specially:
 
+* `clean_command`: The command to clean the Elasticsearch project directory.
+* `build_command`: The command to build an Elasticsearch source distribution.
+* `artifact_path_pattern`: A glob pattern to find a previously built source distribution within the project directory.
+* `release_url`: A download URL for Elasticsearch distributions. The placeholder ``{{VERSION}}`` is replaced by Rally with the actual Elasticsearch version.
 
 Let's have a look at the ``1gheap`` car by inspecting ``1gheap.ini``::
 

docs/migrate.rst

@@ -64,6 +64,41 @@ Starting with Rally 0.11.0, Rally will look for a directory "v1" within ``cars``
                 │           └── elasticsearch.yml
                 └── transport.ini
 
+It is also required that you create a file ``variables.ini`` for all your car config bases (optional for mixins). Therefore, the full directory structure is::
+
+    .
+    ├── cars
+    │   └── v1
+    │       ├── 1gheap.ini
+    │       ├── 2gheap.ini
+    │       ├── defaults.ini
+    │       ├── ea
+    │       │   └── templates
+    │       │       └── config
+    │       │           └── jvm.options
+    │       ├── ea.ini
+    │       └── vanilla
+    │           ├── config.ini
+    │           └── templates
+    │               └── config
+    │                   ├── elasticsearch.yml
+    │                   ├── jvm.options
+    │                   └── log4j2.properties
+    └── plugins
+        └── v1
+            ├── core-plugins.txt
+            └── transport_nio
+                ├── default
+                │   └── templates
+                │       └── config
+                │           └── elasticsearch.yml
+                └── transport.ini
+
+For distribution-based builds, ``config.ini`` file needs to contain a section ``variables`` and a ``release_url`` property::
+
+    [variables]
+    release_url=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{{VERSION}}.tar.gz
+
 
 Migrating to Rally 0.10.0
 -------------------------

esrally/config.py

@@ -108,7 +108,7 @@ def auto_load_local_config(base_config, additional_sections=None, config_file_cl
 class Config:
     EARLIEST_SUPPORTED_VERSION = 12
 
-    CURRENT_CONFIG_VERSION = 15
+    CURRENT_CONFIG_VERSION = 16
 
     """
     Config is the main entry point to retrieve and set benchmark properties. It provides multiple scopes to allow overriding of values on
@@ -457,11 +457,6 @@ def create_config(self, config_file, advanced_config=False, assume_defaults=Fals
         config["defaults"]["preserve_benchmark_candidate"] = str(preserve_install)
 
         config["distributions"] = {}
-        config["distributions"]["release.1.url"] = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-" \
-                                                   "{{VERSION}}.tar.gz"
-        config["distributions"]["release.2.url"] = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/" \
-                                                   "distribution/tar/elasticsearch/{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz"
-        config["distributions"]["release.url"] = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
         config["distributions"]["release.cache"] = "true"
 
         config_file.store(config)
@@ -708,6 +703,15 @@ def warn_if_plugin_build_task_is_in_use(config):
         current_version = 15
         config["meta"]["config.version"] = str(current_version)
 
+    if current_version == 15 and target_version > current_version:
+        if "distributions" in config:
+            # Remove obsolete settings
+            config["distributions"].pop("release.1.url", None)
+            config["distributions"].pop("release.2.url", None)
+            config["distributions"].pop("release.url", None)
+        current_version = 16
+        config["meta"]["config.version"] = str(current_version)
+
     # all migrations done
     config_file.store(config)
     logger.info("Successfully self-upgraded configuration to version [%s]" % target_version)

esrally/mechanic/supplier.py

@@ -40,8 +40,17 @@ def create(cfg, sources, distribution, build, challenge_root_path, car, plugins=
     else:
         es_src_dir = None
         distributions_root = os.path.join(cfg.opts("node", "root.dir"), cfg.opts("source", "distribution.dir"))
+
+        dist_cfg = {}
+        # car / plugin defines defaults...
+        dist_cfg.update(car.variables)
+        for plugin in plugins:
+            for k, v in plugin.variables:
+                dist_cfg["plugin_{}_{}".format(plugin.name, k)] = v
+        # ... but the user can override it in rally.ini
+        dist_cfg.update(cfg.all_opts("distributions"))
         repo = DistributionRepository(name=cfg.opts("mechanic", "distribution.repository"),
-                                      distribution_config=cfg.all_opts("distributions"),
+                                      distribution_config=dist_cfg,
                                       version=es_version)
         suppliers.append(ElasticsearchDistributionSupplier(repo, distributions_root))
 
@@ -333,7 +342,7 @@ def prepare(self):
     def add(self, binaries):
         # if we have multiple plugin configurations for a plugin we will override entries here but as this is always the same
         # key-value pair this is ok.
-        plugin_url = self.repo.plugin_download_url(self.plugin.name)
+        plugin_url = self.repo.plugin_download_url(self.plugin.name, self.plugin.variables)
         if plugin_url:
             binaries[self.plugin.name] = plugin_url
 
@@ -480,34 +489,35 @@ def __init__(self, name, distribution_config, version):
 
     @property
     def download_url(self):
-        major_version = versions.major_version(self.version)
-        version_url_key = "%s.%s.url" % (self.name, str(major_version))
-        default_url_key = "%s.url" % self.name
-        return self._url_for(version_url_key, default_url_key)
+        # team repo
+        default_key = "{}_url".format(self.name)
+        # rally.ini
+        override_key = "{}.url".format(self.name)
+        return self._url_for(override_key, default_key)
 
     def plugin_download_url(self, plugin_name):
-        major_version = versions.major_version(self.version)
-        version_url_key = "plugin.%s.%s.%s.url" % (plugin_name, self.name, str(major_version))
-        default_url_key = "plugin.%s.%s.url" % (plugin_name, self.name)
-        return self._url_for(version_url_key, default_url_key, mandatory=False)
+        # team repo
+        default_key = "plugin_{}_{}_url".format(plugin_name, self.name)
+        # rally.ini
+        override_key = "plugin.{}.{}.url".format(plugin_name, self.name)
+        return self._url_for(override_key, default_key, mandatory=False)
 
-    def _url_for(self, version_url_key, default_url_key, mandatory=True):
+    def _url_for(self, user_defined_key, default_key, mandatory=True):
         try:
-            if version_url_key in self.cfg:
-                url_template = self.cfg[version_url_key]
+            if user_defined_key in self.cfg:
+                url_template = self.cfg[user_defined_key]
             else:
-                url_template = self.cfg[default_url_key]
+                url_template = self.cfg[default_key]
         except KeyError:
             if mandatory:
-                raise exceptions.SystemSetupError("Neither version specific distribution config key [%s] nor a default distribution "
-                                                  "config key [%s] is defined." % (version_url_key, default_url_key))
+                raise exceptions.SystemSetupError("Neither config key [{}] nor [{}] is defined.".format(user_defined_key, default_key))
             else:
                 return None
         return url_template.replace("{{VERSION}}", self.version)
 
     @property
     def cache(self):
-        k = "%s.cache" % self.name
+        k = "{}.cache".format(self.name)
         try:
             raw_value = self.cfg[k]
         except KeyError:

tests/config_test.py

@@ -257,7 +257,7 @@ def test_create_simple_config(self, guess_install_location, guess_java_home, is_
                 print("%s::%s: %s" % (section, k, v))
 
         self.assertTrue("meta" in config_store.config)
-        self.assertEqual("15", config_store.config["meta"]["config.version"])
+        self.assertEqual(str(config.Config.CURRENT_CONFIG_VERSION), config_store.config["meta"]["config.version"])
 
         self.assertTrue("system" in config_store.config)
         self.assertEqual("local", config_store.config["system"]["env.name"])
@@ -295,13 +295,6 @@ def test_create_simple_config(self, guess_install_location, guess_java_home, is_
         self.assertEqual("False", config_store.config["defaults"]["preserve_benchmark_candidate"])
 
         self.assertTrue("distributions" in config_store.config)
-        self.assertEqual("https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
-                         config_store.config["distributions"]["release.1.url"])
-        self.assertEqual("https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/"
-                         "{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz",
-                         config_store.config["distributions"]["release.2.url"])
-        self.assertEqual("https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
-                         config_store.config["distributions"]["release.url"])
         self.assertEqual("true", config_store.config["distributions"]["release.cache"])
 
     @mock.patch("esrally.utils.jvm.is_early_access_release")
@@ -379,7 +372,7 @@ def test_create_advanced_config(self, guess_install_location, guess_java_home, i
 
         self.assertIsNotNone(config_store.config)
         self.assertTrue("meta" in config_store.config)
-        self.assertEqual("15", config_store.config["meta"]["config.version"])
+        self.assertEqual(str(config.Config.CURRENT_CONFIG_VERSION), config_store.config["meta"]["config.version"])
         self.assertTrue("system" in config_store.config)
         self.assertEqual("unittest-env", config_store.config["system"]["env.name"])
         self.assertTrue("node" in config_store.config)
@@ -408,13 +401,6 @@ def test_create_advanced_config(self, guess_install_location, guess_java_home, i
         self.assertEqual("True", config_store.config["defaults"]["preserve_benchmark_candidate"])
 
         self.assertTrue("distributions" in config_store.config)
-        self.assertEqual("https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
-                         config_store.config["distributions"]["release.1.url"])
-        self.assertEqual("https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/"
-                         "{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz",
-                         config_store.config["distributions"]["release.2.url"])
-        self.assertEqual("https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
-                         config_store.config["distributions"]["release.url"])
         self.assertEqual("true", config_store.config["distributions"]["release.cache"])
 
 
@@ -475,6 +461,9 @@ def test_migrate_from_earliest_supported_to_latest(self, sleep, get_size):
             },
             "runtime": {
                 "java8.home": "/opt/jdk/8",
+            },
+            "distributions": {
+                "release.url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
             }
         }
 
@@ -818,3 +807,25 @@ def test_migrate_from_14_to_15_with_source_plugin_definition(self, major_version
                        "plugin.x-pack.build.command",
                        "https://esrally.readthedocs.io/en/latest/elasticsearch_plugins.html#running-a-benchmark-with-plugins"""
             ), string_buffer.read())
+
+    def test_migrate_from_15_to_16(self):
+        config_file = InMemoryConfigStore("test")
+        sample_config = {
+            "meta": {
+                "config.version": 15
+            },
+            "distributions": {
+                "release.url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
+            }
+        }
+        config_file.store(sample_config)
+        config.migrate(config_file, 15, 16, out=null_output)
+
+        self.assertTrue(config_file.backup_created)
+        self.assertEqual("16", config_file.config["meta"]["config.version"])
+        # does not remove section
+        self.assertIn("distributions", config_file.config)
+        self.assertNotIn("release.url", config_file.config["distributions"])
+
+
+

tests/mechanic/supplier_test.py

@@ -498,36 +498,31 @@ def test_create_suppliers_for_es_and_plugin_source_build(self):
 class DistributionRepositoryTests(TestCase):
     def test_release_repo_config_with_default_url(self):
         repo = supplier.DistributionRepository(name="release", distribution_config={
-            "release.2.url": "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/"
-                             "{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz",
-            "release.url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
+            "release_url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
             "release.cache": "true"
         }, version="5.5.0")
         self.assertEqual("https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.5.0.tar.gz", repo.download_url)
         self.assertTrue(repo.cache)
 
-    def test_release_repo_config_with_version_url(self):
+    def test_release_repo_config_with_user_url(self):
         repo = supplier.DistributionRepository(name="release", distribution_config={
-            "release.2.url": "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/"
-                             "{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz",
-            "release.url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
+            "release_url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
+            # user override
+            "release.url": "https://es-mirror.example.org/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
             "release.cache": "false"
         }, version="2.4.3")
-        self.assertEqual("https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.4.3/"
-                         "elasticsearch-2.4.3.tar.gz", repo.download_url)
+        self.assertEqual("https://es-mirror.example.org/downloads/elasticsearch/elasticsearch-2.4.3.tar.gz", repo.download_url)
         self.assertFalse(repo.cache)
 
     def test_missing_url(self):
         repo = supplier.DistributionRepository(name="miss", distribution_config={
-            "release.url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
+            "release_url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
             "release.cache": "true"
         }, version="2.4.3")
         with self.assertRaises(exceptions.SystemSetupError) as ctx:
             # noinspection PyStatementEffect
             repo.download_url
-        self.assertEqual(
-            "Neither version specific distribution config key [miss.2.url] nor a default distribution config key [miss.url] is defined.",
-            ctx.exception.args[0])
+        self.assertEqual("Neither config key [miss.url] nor [miss_url] is defined.", ctx.exception.args[0])
 
     def test_missing_cache(self):
         repo = supplier.DistributionRepository(name="release", distribution_config={
@@ -547,3 +542,21 @@ def test_invalid_cache_value(self):
             # noinspection PyStatementEffect
             repo.cache
         self.assertEqual("Value [Invalid] for config key [release.cache] is not a valid boolean value.", ctx.exception.args[0])
+
+    def test_plugin_config_with_default_url(self):
+        repo = supplier.DistributionRepository(name="release", distribution_config={
+            "plugin_example_release_url": "https://artifacts.example.org/downloads/plugins/example-{{VERSION}}.zip"
+        }, version="5.5.0")
+        self.assertEqual("https://artifacts.example.org/downloads/plugins/example-5.5.0.zip", repo.plugin_download_url("example"))
+
+    def test_plugin_config_with_user_url(self):
+        repo = supplier.DistributionRepository(name="release", distribution_config={
+            "plugin_example_release_url": "https://artifacts.example.org/downloads/plugins/example-{{VERSION}}.zip",
+            # user override
+            "plugin.example.release.url": "https://mirror.example.org/downloads/plugins/example-{{VERSION}}.zip"
+        }, version="5.5.0")
+        self.assertEqual("https://mirror.example.org/downloads/plugins/example-5.5.0.zip", repo.plugin_download_url("example"))
+
+    def test_missing_plugin_config(self):
+        repo = supplier.DistributionRepository(name="release", distribution_config={}, version="5.5.0")
+        self.assertIsNone(repo.plugin_download_url("not existing"))