Use ivy pattern for elasticsearch artifacts download

build.gradle

@@ -221,14 +221,14 @@ allprojects {
     "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
-    "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:archives:zip',
-    "org.elasticsearch.distribution.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip',
-    "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:archives:tar',
-    "org.elasticsearch.distribution.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar',
-    "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
-    "org.elasticsearch.distribution.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm',
-    "org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:packages:deb',
-    "org.elasticsearch.distribution.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb',
+    "downloads.zip:elasticsearch:${version}": ':distribution:archives:zip',
+    "downloads.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip',
+    "downloads.tar:elasticsearch:${version}": ':distribution:archives:tar',
+    "downloads.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar',
+    "downloads.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
+    "downloads.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm',
+    "downloads.deb:elasticsearch:${version}": ':distribution:packages:deb',
+    "downloads.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb',
     "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
     "org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware',
     // for transport client
@@ -244,13 +244,13 @@ allprojects {
   bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion ->
     Version unreleased = unreleasedVersion.version
     String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}"
-    ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${unreleased}"] = snapshotProject
-    ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${unreleased}"] = snapshotProject
-    ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${unreleased}"] = snapshotProject
+    ext.projectSubstitutions["downloads.deb:elasticsearch:${unreleased}"] = snapshotProject
+    ext.projectSubstitutions["downloads.rpm:elasticsearch:${unreleased}"] = snapshotProject
+    ext.projectSubstitutions["downloads.zip:elasticsearch:${unreleased}"] = snapshotProject
     if (unreleased.onOrAfter('6.3.0')) {
-      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch-oss:${unreleased}"] = snapshotProject
-      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject
-      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch-oss:${unreleased}"] = snapshotProject
+      ext.projectSubstitutions["downloads.deb:elasticsearch-oss:${unreleased}"] = snapshotProject
+      ext.projectSubstitutions["downloads.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject
+      ext.projectSubstitutions["downloads.zip:elasticsearch-oss:${unreleased}"] = snapshotProject
     }
   }
 

buildSrc/build.gradle

@@ -178,6 +178,21 @@ if (project != rootProject) {
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
 
+  configurations {
+    distribution
+  }
+
+  dependencies {
+    distribution project(':distribution:archives:zip')
+    distribution project(':distribution:archives:oss-zip')
+  }
+
+  String localDownloads = "${rootProject.buildDir}/local-downloads"
+  task setupLocalDownloads(type:Copy) {
+    from configurations.distribution
+    into localDownloads
+  }
+
   unitTest {
     // The test task is configured to runtimeJava version, but build-tools doesn't support all of them, so test
     // with compiler instead on the ones that are too old.
@@ -192,6 +207,7 @@ if (project != rootProject) {
     dependsOn project.rootProject.allprojects.collect {
       it.tasks.matching { it.name == 'publishNebulaPublicationToLocalTestRepository'}
     }
+    dependsOn setupLocalDownloads
     exclude "**/*Tests.class"
     testClassesDirs = sourceSets.test.output.classesDirs
     classpath = sourceSets.test.runtimeClasspath
@@ -204,6 +220,7 @@ if (project != rootProject) {
             ).asPath,
     )
     systemProperty 'test.local-test-repo-path', "${rootProject.buildDir}/local-test-repo"
+    systemProperty 'test.local-test-downloads-path', localDownloads
     systemProperty 'test.version_under_test', version
     systemProperty 'test.lucene-snapshot-revision', (versions.lucene =~ /\w+-snapshot-([a-z0-9]+)/)[0][1]
   }

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -558,6 +558,12 @@ class BuildPlugin implements Plugin<Project> {
             repos.mavenLocal()
         }
         repos.jcenter()
+        repos.ivy {
+            url "https://artifacts.elastic.co/downloads"
+            patternLayout {
+                artifact "elasticsearch/[module]-[revision](-[classifier]).[ext]"
+            }
+        }
         repos.maven {
             name "elastic"
             url "https://artifacts.elastic.co/maven"

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -18,14 +18,12 @@
  */
 package org.elasticsearch.gradle.test
 
-import java.util.stream.Collectors
 import org.apache.tools.ant.DefaultLogger
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
-
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
 import org.gradle.api.AntBuilder
@@ -45,7 +43,7 @@ import org.gradle.api.tasks.Exec
 import java.nio.charset.StandardCharsets
 import java.nio.file.Paths
 import java.util.concurrent.TimeUnit
-
+import java.util.stream.Collectors
 /**
  * A helper for creating tasks to build a cluster that is used by a task, and tear down the cluster when the task is finished.
  */
@@ -174,24 +172,31 @@ class ClusterFormationTasks {
 
     /** Adds a dependency on the given distribution */
     static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
-        if (Version.fromString(elasticsearchVersion).before('6.3.0') &&
-                distro.startsWith('oss-')
-        ) {
+        Version version = Version.fromString(elasticsearchVersion)
+        if (version.before('6.3.0') && distro.startsWith('oss-')) {
             distro = distro.substring('oss-'.length())
         }
+        String os = "linux"
+        if (Os.FAMILY_WINDOWS) {
+            os = "windows"
+        } else if (Os.FAMILY_MAC) {
+            os = "darwin"
+        }
         String packaging = distro
         if (distro.contains('tar')) {
             packaging = 'tar.gz'\
         } else if (distro.contains('zip')) {
             packaging = 'zip'
         }
-        String subgroup = distro
+        String group = "downloads.${packaging}"
+        if (distro.equals("integ-test-zip")) {
+            group = "org.elasticsearch.distribution.integ-test-zip"
+        }
         String artifactName = 'elasticsearch'
         if (distro.contains('oss')) {
             artifactName += '-oss'
-            subgroup = distro.substring('oss-'.length())
         }
-        project.dependencies.add(configuration.name, "org.elasticsearch.distribution.${subgroup}:${artifactName}:${elasticsearchVersion}@${packaging}")
+        project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@${packaging}")
     }
 
     /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -193,10 +193,10 @@ class VagrantTestPlugin implements Plugin<Project> {
         UPGRADE_FROM_ARCHIVES.each {
             // The version of elasticsearch that we upgrade *from*
             project.dependencies.add(PACKAGING_CONFIGURATION,
-                    "org.elasticsearch.distribution.${it}:elasticsearch:${upgradeFromVersion}@${it}")
+                    "downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
             if (upgradeFromVersion.onOrAfter('6.3.0')) {
                 project.dependencies.add(PACKAGING_CONFIGURATION,
-                        "org.elasticsearch.distribution.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
+                        "downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
             }
         }
 

buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java

@@ -127,6 +127,9 @@ private void adaptBuildScriptForTest() throws IOException {
                 "  maven {\n" +
                 "    url \"" + getLocalTestRepoPath()  + "\"\n" +
                 "  }\n" +
+                "  flatDir {\n" +
+                "    dir '" + getLocalTestDownloadsPath() + "'\n" +
+                "  }\n" +
                 luceneSnapshotRepo +
                 "}\n"
         );

buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java

@@ -144,8 +144,16 @@ protected void assertBuildFileDoesNotExists(BuildResult result, String projectNa
     }
 
     protected String getLocalTestRepoPath() {
-        String property = System.getProperty("test.local-test-repo-path");
-        Objects.requireNonNull(property, "test.local-test-repo-path not passed to tests");
+        return getLocalTestPath("test.local-test-repo-path");
+    }
+
+    protected String getLocalTestDownloadsPath() {
+        return getLocalTestPath("test.local-test-downloads-path");
+    }
+
+    private String getLocalTestPath(String propertyName) {
+        String property = System.getProperty(propertyName);
+        Objects.requireNonNull(property, propertyName + " not passed to tests");
         File file = new File(property);
         assertTrue("Expected " + property + " to exist, but it did not!", file.exists());
         if (File.separator.equals("\\")) {