Use versioned sourceset for REST compat testing (#78418)

This commit changes the sourceset name for REST API compatility testing
from yamlRestCompatTest to yamlRestTestV(N)Compat where (N) is the version
from which the tests are sourced. For example yamlRestTestV7Compat
or yamlRestTestV8Compat.

Two motivating reasons for this change.

    consitent naming - all compatible rest testing will now use a common
    prefix of yamlRestTestV7Compat where the task name to execute is
    yamlRestTestV7CompatTestand the transformation task is yamlRestTestV7CompatTransform
    It makes sense that corresponding source set follows this convention.

    Better support when the major version changes. Gradle uses the source
    set under the covers to execute the compatiblity tests. However, a user
    can add custom REST tests to this source set. By using dynamically named
    source sets, there should little concern of accidentally running the wrong
    version tests. Additionaly it is visualy easy to tell which version the tests are
    from.

This commit also removes the unecessary duplicated v7compat directory
and adds a duplicate checks to ensure that warn if  there are
duplicate test names.

related: #75267

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy

@@ -82,7 +82,8 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         String wrongTest = "wrong_version.yml"
         String additionalTest = "additional_test.yml"
         setupRestResources([wrongApi], [wrongTest]) //setups up resources for current version, which should not be used for this test
-        addRestTestsToProject([additionalTest], "yamlRestCompatTest")
+        String sourceSetName = "yamlRestTestV" + compatibleVersion + "Compat"
+        addRestTestsToProject([additionalTest], sourceSetName)
         //intentionally adding to yamlRestTest source set since the .classes are copied from there
         file("src/yamlRestTest/java/MockIT.java") << "import org.junit.Test;class MockIT { @Test public void doNothing() { }}"
 
@@ -107,17 +108,17 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         file("/build/${testIntermediateDir}/original/rest-api-spec/test/" + test).exists()
         file("/build/${testIntermediateDir}/transformed/rest-api-spec/test/" + test).exists()
         file("/build/${testIntermediateDir}/transformed/rest-api-spec/test/" + test).text.contains("headers") //transformation adds this
-        file("/build/resources/yamlRestCompatTest/rest-api-spec/test/" + additionalTest).exists()
+        file("/build/resources/${sourceSetName}/rest-api-spec/test/" + additionalTest).exists()
 
         //additionalTest is not copied from the prior version, and thus not in the intermediate directory, nor transformed
-        file("/build/resources/yamlRestCompatTest/" + testIntermediateDir + "/rest-api-spec/test/" + additionalTest).exists() == false
-        file("/build/resources/yamlRestCompatTest/rest-api-spec/test/" + additionalTest).text.contains("headers") == false
+        file("/build/resources/${sourceSetName}/" + testIntermediateDir + "/rest-api-spec/test/" + additionalTest).exists() == false
+        file("/build/resources/${sourceSetName}/rest-api-spec/test/" + additionalTest).text.contains("headers") == false
 
         file("/build/classes/java/yamlRestTest/MockIT.class").exists() //The "standard" runner is used to execute the compat test
 
-        file("/build/resources/yamlRestCompatTest/rest-api-spec/api/" + wrongApi).exists() == false
-        file("/build/resources/yamlRestCompatTest/" + testIntermediateDir + "/rest-api-spec/test/" + wrongTest).exists() == false
-        file("/build/resources/yamlRestCompatTest/rest-api-spec/test/" + wrongTest).exists() == false
+        file("/build/resources/${sourceSetName}/rest-api-spec/api/" + wrongApi).exists() == false
+        file("/build/resources/${sourceSetName}/" + testIntermediateDir + "/rest-api-spec/test/" + wrongTest).exists() == false
+        file("/build/resources/${sourceSetName}/rest-api-spec/test/" + wrongTest).exists() == false
 
         result.task(':copyRestApiSpecsTask').outcome == TaskOutcome.NO_SOURCE
         result.task(':copyYamlTestsTask').outcome == TaskOutcome.NO_SOURCE

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/rest/compat/YamlRestCompatTestPlugin.java

@@ -8,17 +8,17 @@
 
 package org.elasticsearch.gradle.internal.rest.compat;
 
-import org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin;
 import org.elasticsearch.gradle.Version;
 import org.elasticsearch.gradle.VersionProperties;
+import org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin;
 import org.elasticsearch.gradle.internal.test.RestIntegTestTask;
 import org.elasticsearch.gradle.internal.test.RestTestBasePlugin;
 import org.elasticsearch.gradle.internal.test.rest.CopyRestApiTask;
 import org.elasticsearch.gradle.internal.test.rest.CopyRestTestsTask;
+import org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin;
 import org.elasticsearch.gradle.internal.test.rest.RestResourcesExtension;
 import org.elasticsearch.gradle.internal.test.rest.RestResourcesPlugin;
 import org.elasticsearch.gradle.internal.test.rest.RestTestUtil;
-import org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin;
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
@@ -46,21 +46,22 @@
  * Apply this plugin to run the YAML based REST tests from a prior major version against this version's cluster.
  */
 public class YamlRestCompatTestPlugin implements Plugin<Project> {
+    public static final String BWC_MINOR_CONFIG_NAME = "bwcMinor";
     private static final String REST_COMPAT_CHECK_TASK_NAME = "checkRestCompat";
     private static final String COMPATIBILITY_APIS_CONFIGURATION = "restCompatSpecs";
     private static final String COMPATIBILITY_TESTS_CONFIGURATION = "restCompatTests";
-    private static final String SOURCE_SET_NAME = "yamlRestCompatTest";
     private static final Path RELATIVE_API_PATH = Path.of("rest-api-spec/api");
     private static final Path RELATIVE_TEST_PATH = Path.of("rest-api-spec/test");
     private static final Path RELATIVE_REST_API_RESOURCES = Path.of("rest-api-spec/src/main/resources");
     private static final Path RELATIVE_REST_XPACK_RESOURCES = Path.of("x-pack/plugin/src/test/resources");
     private static final Path RELATIVE_REST_PROJECT_RESOURCES = Path.of("src/yamlRestTest/resources");
-    public static final String BWC_MINOR_CONFIG_NAME = "bwcMinor";
+    private static final int COMPATIBLE_VERSION = Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() - 1;
+    private static final String SOURCE_SET_NAME = "yamlRestTestV" + COMPATIBLE_VERSION + "Compat";
 
     @Override
     public void apply(Project project) {
-        final int compatibleVersion = Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() - 1;
-        final Path compatRestResourcesDir = Path.of("restResources").resolve("v" + compatibleVersion);
+
+        final Path compatRestResourcesDir = Path.of("restResources").resolve("v" + COMPATIBLE_VERSION);
         final Path compatSpecsDir = compatRestResourcesDir.resolve("yamlSpecs");
         final Path compatTestsDir = compatRestResourcesDir.resolve("yamlTests");
 
@@ -160,7 +161,7 @@ public void apply(Project project) {
 
         // transform the copied tests task
         TaskProvider<RestCompatTestTransformTask> transformCompatTestTask = project.getTasks()
-            .register("yamlRestTestV" + compatibleVersion + "CompatTransform", RestCompatTestTransformTask.class, task -> {
+            .register("yamlRestTestV" + COMPATIBLE_VERSION + "CompatTransform", RestCompatTestTransformTask.class, task -> {
                 task.getSourceDirectory().set(copyCompatYamlTestTask.flatMap(CopyRestTestsTask::getOutputResourceDir));
                 task.getOutputDirectory()
                     .set(project.getLayout().getBuildDirectory().dir(compatTestsDir.resolve("transformed").toString()));
@@ -191,7 +192,7 @@ public void apply(Project project) {
             .named(RestResourcesPlugin.COPY_YAML_TESTS_TASK)
             .flatMap(CopyRestTestsTask::getOutputResourceDir);
 
-        String testTaskName = "yamlRestTestV" + compatibleVersion + "CompatTest";
+        String testTaskName = "yamlRestTestV" + COMPATIBLE_VERSION + "CompatTest";
 
         // setup the test task
         Provider<RestIntegTestTask> yamlRestCompatTestTask = RestTestUtil.registerTestTask(project, yamlCompatTestSourceSet, testTaskName);
@@ -207,11 +208,13 @@ public void apply(Project project) {
                     .minus(project.files(originalYamlSpecsDir))
                     .minus(project.files(originalYamlTestsDir))
             );
+
             // run compatibility tests after "normal" tests
             testTask.mustRunAfter(project.getTasks().named(InternalYamlRestTestPlugin.SOURCE_SET_NAME));
             testTask.onlyIf(t -> isEnabled(project));
         });
 
+
         setupTestDependenciesDefaults(project, yamlCompatTestSourceSet);
 
         // setup IDE

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java

@@ -11,6 +11,8 @@
 import com.carrotsearch.randomizedtesting.RandomizedTest;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 import org.apache.http.HttpHost;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Node;
@@ -42,6 +44,7 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -52,6 +55,7 @@
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
+import java.util.stream.Collectors;
 
 /**
  * Runs a suite of yaml tests shared with all the official Elasticsearch
@@ -270,7 +274,15 @@ private static void addSuite(Path root, Path file, Map<String, Set<Path>> files)
             filesSet = new HashSet<>();
             files.put(groupName, filesSet);
         }
+
         filesSet.add(file);
+        List<String> fileNames = filesSet.stream().map(p -> p.getFileName().toString()).collect(Collectors.toList());
+        if (Collections.frequency(fileNames, file.getFileName().toString()) > 1) {
+            Logger logger = LogManager.getLogger(ESClientYamlSuiteTestCase.class);
+            logger.warn("Found duplicate test name [" + groupName + "/" + file.getFileName() + "] on the class path. " +
+                "This can result in class loader dependent execution commands and reproduction commands " +
+                "(will add #2 to one of the test names dependent on the classloading order)");
+        }
     }
 
     private static String[] resolvePathsProperty(String propertyName, String defaultValue) {

x-pack/plugin/build.gradle

@@ -1,6 +1,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.util.GradleUtils
 import org.elasticsearch.gradle.internal.test.RestIntegTestTask
+import org.elasticsearch.gradle.VersionProperties
 
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.yaml-rest-compat-test'
@@ -18,7 +19,8 @@ dependencies {
 
 // let the yamlRestTests see the classpath of test
 GradleUtils.extendSourceSet(project, "test", "yamlRestTest", tasks.named("yamlRestTest"))
-GradleUtils.extendSourceSet(project, "test", "yamlRestCompatTest")
+int compatVersion = VersionProperties.getElasticsearchVersion().getMajor() - 1;
+GradleUtils.extendSourceSet(project, "test", "yamlRestTestV${compatVersion}Compat")
 
 restResources {
   restApi {