Merge branch 'multiproject/split-custom' into multiproject/split-meta…

…data

.buildkite/scripts/gradle-cache-validation.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+set -euo pipefail
+
+VALIDATION_SCRIPTS_VERSION=2.5.1
+GRADLE_ENTERPRISE_ACCESS_KEY=$(vault kv get -field=value secret/ci/elastic-elasticsearch/gradle-enterprise-api-key)
+export GRADLE_ENTERPRISE_ACCESS_KEY
+
+curl -s -L -O https://github.com/gradle/gradle-enterprise-build-validation-scripts/releases/download/v$VALIDATION_SCRIPTS_VERSION/gradle-enterprise-gradle-build-validation-$VALIDATION_SCRIPTS_VERSION.zip && unzip -q -o gradle-enterprise-gradle-build-validation-$VALIDATION_SCRIPTS_VERSION.zip
+
+# Create a temporary file
+tmpOutputFile=$(mktemp)
+trap "rm $tmpOutputFile" EXIT
+
+gradle-enterprise-gradle-build-validation/03-validate-local-build-caching-different-locations.sh -r https://github.com/elastic/elasticsearch.git -b $BUILDKITE_BRANCH --gradle-enterprise-server https://gradle-enterprise.elastic.co -t precommit --fail-if-not-fully-cacheable | tee $tmpOutputFile
+
+# Capture the return value
+retval=$?
+
+# Now read the content from the temporary file into a variable
+perfOutput=$(cat $tmpOutputFile | sed -n '/Performance Characteristics/,/See https:\/\/gradle.com\/bvs\/main\/Gradle.md#performance-characteristics for details./p' | sed '$d' | sed 's/\x1b\[[0-9;]*m//g')
+investigationOutput=$(cat $tmpOutputFile | sed -n '/Investigation Quick Links/,$p' | sed 's/\x1b\[[0-9;]*m//g')
+
+# Initialize HTML output variable
+summaryHtml="<h4>Performance Characteristics</h4>"
+summaryHtml+="<ul>"
+
+# Process each line of the string
+while IFS=: read -r label value; do
+    if [[ -n "$label" && -n "$value" ]]; then
+        # Trim whitespace from label and value
+        trimmed_label=$(echo "$label" | xargs)
+        trimmed_value=$(echo "$value" | xargs)
+
+        # Append to HTML output variable
+        summaryHtml+="<li><strong>$trimmed_label:</strong> $trimmed_value</li>"
+    fi
+done <<< "$perfOutput"
+
+summaryHtml+="</ul>"
+
+# generate html for links
+summaryHtml+="<h4>Investigation Links</h4>"
+summaryHtml+="<ul>"
+
+# Process each line of the string
+while IFS= read -r line; do
+    if [[ "$line" =~ http.* ]]; then
+        # Extract URL and description using awk
+        url=$(echo "$line" | awk '{print $NF}')
+        description=$(echo "$line" | sed -e "s/:.*//")
+
+        # Append to HTML output variable
+        summaryHtml+="    <li><a href=\"$url\">$description</a></li>"
+    fi
+done <<< "$investigationOutput"
+
+# End of the HTML content
+summaryHtml+="</ul>"
+
+cat << EOF | buildkite-agent annotate --context "ctx-validation-summary" --style "info"
+$summaryHtml
+EOF
+
+# Check if the command was successful
+if [ $retval -eq 0 ]; then
+    echo "Experiment completed successfully"
+elif [ $retval -eq 1 ]; then
+    echo "An invalid input was provided while attempting to run the experiment"
+elif [ $retval -eq 2 ]; then
+    echo "One of the builds that is part of the experiment failed"
+elif [ $retval -eq 3 ]; then
+    echo "The build was not fully cacheable for the given task graph"
+elif [ $retval -eq 3 ]; then
+    echo "An unclassified, fatal error happened while running the experiment"
+fi
+
+exit $retval
+

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java

@@ -23,11 +23,14 @@
 import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.specs.Spec;
+import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputDirectory;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.OutputDirectory;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.TaskAction;
 
 import java.io.File;
@@ -89,6 +92,7 @@
  * for the dependency. This artifact will be redistributed by us with the release to
  * comply with the license terms.
  */
+@CacheableTask
 public abstract class DependencyLicensesTask extends DefaultTask {
 
     private final Pattern regex = Pattern.compile("-v?\\d+.*");
@@ -149,6 +153,7 @@ public DependencyLicensesTask(ObjectFactory objects, ProjectLayout projectLayout
     }
 
     @InputFiles
+    @PathSensitive(PathSensitivity.NAME_ONLY)
     public FileCollection getDependencies() {
         return dependencies;
     }
@@ -159,6 +164,7 @@ public void setDependencies(FileCollection dependencies) {
 
     @Optional
     @InputDirectory
+    @PathSensitive(PathSensitivity.RELATIVE)
     public File getLicensesDir() {
         File asFile = licensesDir.get().getAsFile();
         if (asFile.exists()) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/SplitPackagesAuditTask.java

@@ -20,6 +20,7 @@
 import org.gradle.api.provider.MapProperty;
 import org.gradle.api.provider.Property;
 import org.gradle.api.provider.SetProperty;
+import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.CompileClasspath;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFiles;
@@ -56,6 +57,7 @@
 /**
  * Checks for split packages with dependencies. These are not allowed in a future modularized world.
  */
+@CacheableTask
 public class SplitPackagesAuditTask extends DefaultTask {
 
     private static final Logger LOGGER = Logging.getLogger(SplitPackagesAuditTask.class);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/AbstractVersionsTask.java

@@ -8,19 +8,119 @@
 
 package org.elasticsearch.gradle.internal.release;
 
+import com.github.javaparser.GeneratedJavaParserConstants;
+import com.github.javaparser.ast.CompilationUnit;
+import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration;
+import com.github.javaparser.ast.body.FieldDeclaration;
+import com.github.javaparser.ast.expr.IntegerLiteralExpr;
+import com.github.javaparser.ast.observer.ObservableProperty;
+import com.github.javaparser.printer.ConcreteSyntaxModel;
+import com.github.javaparser.printer.concretesyntaxmodel.CsmElement;
+import com.github.javaparser.printer.lexicalpreservation.LexicalPreservingPrinter;
+
 import org.gradle.api.DefaultTask;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
 import org.gradle.initialization.layout.BuildLayout;
 
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.List;
+import java.util.Map;
+import java.util.OptionalInt;
+import java.util.stream.Collectors;
+
+import static com.github.javaparser.ast.observer.ObservableProperty.TYPE_PARAMETERS;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmConditional.Condition.FLAG;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.block;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.child;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.comma;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.comment;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.conditional;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.list;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.newline;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.none;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.sequence;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.space;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.string;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.token;
 
 public abstract class AbstractVersionsTask extends DefaultTask {
 
+    static {
+        replaceDefaultJavaParserClassCsm();
+    }
+
+    /*
+     * The default JavaParser CSM which it uses to format any new declarations added to a class
+     * inserts two newlines after each declaration. Our version classes only have one newline.
+     * In order to get javaparser lexical printer to use our format, we have to completely replace
+     * the statically declared CSM pattern using hacky reflection
+     * to access the static map where these are stored, and insert a replacement that is identical
+     * apart from only one newline at the end of each member declaration, rather than two.
+     */
+    private static void replaceDefaultJavaParserClassCsm() {
+        try {
+            Field classCsms = ConcreteSyntaxModel.class.getDeclaredField("concreteSyntaxModelByClass");
+            classCsms.setAccessible(true);
+            @SuppressWarnings({ "unchecked", "rawtypes" })
+            Map<Class, CsmElement> csms = (Map) classCsms.get(null);
+
+            // copied from the static initializer in ConcreteSyntaxModel
+            csms.put(
+                ClassOrInterfaceDeclaration.class,
+                sequence(
+                    comment(),
+                    list(ObservableProperty.ANNOTATIONS, newline(), none(), newline()),
+                    list(ObservableProperty.MODIFIERS, space(), none(), space()),
+                    conditional(
+                        ObservableProperty.INTERFACE,
+                        FLAG,
+                        token(GeneratedJavaParserConstants.INTERFACE),
+                        token(GeneratedJavaParserConstants.CLASS)
+                    ),
+                    space(),
+                    child(ObservableProperty.NAME),
+                    list(
+                        TYPE_PARAMETERS,
+                        sequence(comma(), space()),
+                        string(GeneratedJavaParserConstants.LT),
+                        string(GeneratedJavaParserConstants.GT)
+                    ),
+                    list(
+                        ObservableProperty.EXTENDED_TYPES,
+                        sequence(string(GeneratedJavaParserConstants.COMMA), space()),
+                        sequence(space(), token(GeneratedJavaParserConstants.EXTENDS), space()),
+                        none()
+                    ),
+                    list(
+                        ObservableProperty.IMPLEMENTED_TYPES,
+                        sequence(string(GeneratedJavaParserConstants.COMMA), space()),
+                        sequence(space(), token(GeneratedJavaParserConstants.IMPLEMENTS), space()),
+                        none()
+                    ),
+                    space(),
+                    block(sequence(newline(), list(ObservableProperty.MEMBERS, sequence(newline()/*, newline()*/), newline(), newline())))
+                )
+            );
+        } catch (ReflectiveOperationException e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    private static final Logger LOGGER = Logging.getLogger(AbstractVersionsTask.class);
+
     static final String TRANSPORT_VERSION_TYPE = "TransportVersion";
     static final String INDEX_VERSION_TYPE = "IndexVersion";
 
     static final String SERVER_MODULE_PATH = "server/src/main/java/";
-    static final String TRANSPORT_VERSION_FILE_PATH = SERVER_MODULE_PATH + "org/elasticsearch/TransportVersions.java";
-    static final String INDEX_VERSION_FILE_PATH = SERVER_MODULE_PATH + "org/elasticsearch/index/IndexVersions.java";
+
+    static final String VERSION_FILE_PATH = SERVER_MODULE_PATH + "org/elasticsearch/Version.java";
+    static final String TRANSPORT_VERSIONS_FILE_PATH = SERVER_MODULE_PATH + "org/elasticsearch/TransportVersions.java";
+    static final String INDEX_VERSIONS_FILE_PATH = SERVER_MODULE_PATH + "org/elasticsearch/index/IndexVersions.java";
 
     static final String SERVER_RESOURCES_PATH = "server/src/main/resources/";
     static final String TRANSPORT_VERSIONS_RECORD = SERVER_RESOURCES_PATH + "org/elasticsearch/TransportVersions.csv";
@@ -32,4 +132,34 @@ protected AbstractVersionsTask(BuildLayout layout) {
         rootDir = layout.getRootDirectory().toPath();
     }
 
+    static Map<String, Integer> splitVersionIds(List<String> version) {
+        return version.stream().map(l -> {
+            var split = l.split(":");
+            if (split.length != 2) throw new IllegalArgumentException("Invalid tag format [" + l + "]");
+            return split;
+        }).collect(Collectors.toMap(l -> l[0], l -> Integer.parseInt(l[1])));
+    }
+
+    static OptionalInt findSingleIntegerExpr(FieldDeclaration field) {
+        var ints = field.findAll(IntegerLiteralExpr.class);
+        switch (ints.size()) {
+            case 0 -> {
+                return OptionalInt.empty();
+            }
+            case 1 -> {
+                return OptionalInt.of(ints.get(0).asNumber().intValue());
+            }
+            default -> {
+                LOGGER.warn("Multiple integers found in version field declaration [{}]", field); // and ignore it
+                return OptionalInt.empty();
+            }
+        }
+    }
+
+    static void writeOutNewContents(Path file, CompilationUnit unit) throws IOException {
+        if (unit.containsData(LexicalPreservingPrinter.NODE_TEXT_DATA) == false) {
+            throw new IllegalArgumentException("CompilationUnit has no lexical information for output");
+        }
+        Files.writeString(file, LexicalPreservingPrinter.print(unit), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ExtractCurrentVersionsTask.java

@@ -11,7 +11,6 @@
 import com.github.javaparser.StaticJavaParser;
 import com.github.javaparser.ast.CompilationUnit;
 import com.github.javaparser.ast.body.FieldDeclaration;
-import com.github.javaparser.ast.expr.IntegerLiteralExpr;
 
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
@@ -53,11 +52,11 @@ public void executeTask() throws IOException {
         LOGGER.lifecycle("Extracting latest version information");
 
         List<String> output = new ArrayList<>();
-        int transportVersion = readLatestVersion(rootDir.resolve(TRANSPORT_VERSION_FILE_PATH));
+        int transportVersion = readLatestVersion(rootDir.resolve(TRANSPORT_VERSIONS_FILE_PATH));
         LOGGER.lifecycle("Transport version: {}", transportVersion);
         output.add(TRANSPORT_VERSION_TYPE + ":" + transportVersion);
 
-        int indexVersion = readLatestVersion(rootDir.resolve(INDEX_VERSION_FILE_PATH));
+        int indexVersion = readLatestVersion(rootDir.resolve(INDEX_VERSIONS_FILE_PATH));
         LOGGER.lifecycle("Index version: {}", indexVersion);
         output.add(INDEX_VERSION_TYPE + ":" + indexVersion);
 
@@ -74,21 +73,13 @@ Integer highestVersionId() {
 
         @Override
         public void accept(FieldDeclaration fieldDeclaration) {
-            var ints = fieldDeclaration.findAll(IntegerLiteralExpr.class);
-            switch (ints.size()) {
-                case 0 -> {
-                    // No ints in the field declaration, ignore
+            findSingleIntegerExpr(fieldDeclaration).ifPresent(id -> {
+                if (highestVersionId != null && highestVersionId > id) {
+                    LOGGER.warn("Version ids [{}, {}] out of order", highestVersionId, id);
+                } else {
+                    highestVersionId = id;
                 }
-                case 1 -> {
-                    int id = ints.get(0).asNumber().intValue();
-                    if (highestVersionId != null && highestVersionId > id) {
-                        LOGGER.warn("Version ids [{}, {}] out of order", highestVersionId, id);
-                    } else {
-                        highestVersionId = id;
-                    }
-                }
-                default -> LOGGER.warn("Multiple integers found in version field declaration [{}]", fieldDeclaration); // and ignore it
-            }
+            });
         }
     }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java

@@ -52,6 +52,7 @@ public void apply(Project project) {
 
         project.getTasks().register("extractCurrentVersions", ExtractCurrentVersionsTask.class);
         project.getTasks().register("tagVersions", TagVersionsTask.class);
+        project.getTasks().register("setCompatibleVersions", SetCompatibleVersionsTask.class);
 
         final FileTree yamlFiles = projectDirectory.dir("docs/changelog")
             .getAsFileTree()