Skip to content

Commit

Permalink
Merge branch 'main' into optimization/st_extent
Browse files Browse the repository at this point in the history
  • Loading branch information
GalLalouche committed Dec 19, 2024
2 parents 72e3e34 + 6983f9a commit d62e37d
Show file tree
Hide file tree
Showing 299 changed files with 7,327 additions and 1,619 deletions.
101 changes: 2 additions & 99 deletions .ci/init.gradle
Original file line number Diff line number Diff line change
@@ -1,95 +1,3 @@
import com.bettercloud.vault.VaultConfig
import com.bettercloud.vault.Vault

initscript {
repositories {
mavenCentral()
}
dependencies {
classpath 'com.bettercloud:vault-java-driver:4.1.0'
}
}

boolean USE_ARTIFACTORY = false

if (System.getenv('VAULT_ADDR') == null) {
// When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up
if (System.getenv('CI') == null) {
return
}

throw new GradleException("You must set the VAULT_ADDR environment variable to use this init script.")
}

if (System.getenv('VAULT_ROLE_ID') == null && System.getenv('VAULT_SECRET_ID') == null && System.getenv('VAULT_TOKEN') == null) {
// When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up
if (System.getenv('CI') == null) {
return
}

throw new GradleException("You must set either the VAULT_ROLE_ID and VAULT_SECRET_ID environment variables, " +
"or the VAULT_TOKEN environment variable to use this init script.")
}

final String vaultPathPrefix = System.getenv('VAULT_ADDR') ==~ /.+vault-ci.+\.dev.*/ ? "secret/ci/elastic-elasticsearch/migrated" : "secret/elasticsearch-ci"

final String vaultToken = System.getenv('VAULT_TOKEN') ?: new Vault(
new VaultConfig()
.address(System.env.VAULT_ADDR)
.engineVersion(1)
.build()
)
.withRetries(5, 1000)
.auth()
.loginByAppRole("approle", System.env.VAULT_ROLE_ID, System.env.VAULT_SECRET_ID)
.getAuthClientToken()

final Vault vault = new Vault(
new VaultConfig()
.address(System.env.VAULT_ADDR)
.engineVersion(1)
.token(vaultToken)
.build()
)
.withRetries(5, 1000)


if (USE_ARTIFACTORY) {
final Map<String, String> artifactoryCredentials = vault.logical()
.read("${vaultPathPrefix}/artifactory.elstc.co")
.getData()
logger.info("Using elastic artifactory repos")
Closure configCache = {
return {
name "artifactory-gradle-release"
url "https://artifactory.elstc.co/artifactory/gradle-release"
credentials {
username artifactoryCredentials.get("username")
password artifactoryCredentials.get("token")
}
}
}
settingsEvaluated { settings ->
settings.pluginManagement {
repositories {
maven configCache()
}
}
}
projectsLoaded {
allprojects {
buildscript {
repositories {
maven configCache()
}
}
repositories {
maven configCache()
}
}
}
}

gradle.settingsEvaluated { settings ->
settings.pluginManager.withPlugin("com.gradle.develocity") {
settings.develocity {
Expand All @@ -98,14 +6,10 @@ gradle.settingsEvaluated { settings ->
}
}


final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))

if (buildCacheUrl) {
final Map<String, String> buildCacheCredentials = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ? [:] : vault.logical()
.read("${vaultPathPrefix}/gradle-build-cache")
.getData()
gradle.settingsEvaluated { settings ->
settings.buildCache {
local {
Expand All @@ -116,11 +20,10 @@ if (buildCacheUrl) {
url = buildCacheUrl
push = buildCachePush
credentials {
username = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ?: buildCacheCredentials.get("username")
password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD") ?: buildCacheCredentials.get("password")
username = System.getenv("GRADLE_BUILD_CACHE_USERNAME")
password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD")
}
}
}
}
}

Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,15 @@
import org.gradle.api.tasks.bundling.Jar;
import org.gradle.api.tasks.javadoc.Javadoc;
import org.gradle.external.javadoc.CoreJavadocOptions;
import org.gradle.jvm.toolchain.JavaLanguageVersion;
import org.gradle.jvm.toolchain.JavaToolchainService;
import org.gradle.language.base.plugins.LifecycleBasePlugin;

import java.io.File;
import java.util.Map;

import javax.inject.Inject;

import static org.elasticsearch.gradle.internal.conventions.util.Util.toStringable;
import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;

Expand All @@ -44,6 +48,14 @@
* common configuration for production code.
*/
public class ElasticsearchJavaPlugin implements Plugin<Project> {

private final JavaToolchainService javaToolchains;

@Inject
ElasticsearchJavaPlugin(JavaToolchainService javaToolchains) {
this.javaToolchains = javaToolchains;
}

@Override
public void apply(Project project) {
project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class);
Expand All @@ -55,7 +67,7 @@ public void apply(Project project) {
// configureConfigurations(project);
configureJars(project, buildParams.get());
configureJarManifest(project, buildParams.get());
configureJavadoc(project);
configureJavadoc(project, buildParams.get());
testCompileOnlyDeps(project);
}

Expand Down Expand Up @@ -128,14 +140,18 @@ private static void configureJarManifest(Project project, BuildParameterExtensio
project.getPluginManager().apply("nebula.info-jar");
}

private static void configureJavadoc(Project project) {
private void configureJavadoc(Project project, BuildParameterExtension buildParams) {
project.getTasks().withType(Javadoc.class).configureEach(javadoc -> {
/*
* Generate docs using html5 to suppress a warning from `javadoc`
* that the default will change to html5 in the future.
*/
CoreJavadocOptions javadocOptions = (CoreJavadocOptions) javadoc.getOptions();
javadocOptions.addBooleanOption("html5", true);

javadoc.getJavadocTool().set(javaToolchains.javadocToolFor(spec -> {
spec.getLanguageVersion().set(JavaLanguageVersion.of(buildParams.getMinimumRuntimeVersion().getMajorVersion()));
}));
});

TaskProvider<Javadoc> javadoc = project.getTasks().withType(Javadoc.class).named("javadoc");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,12 @@
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.Task;
import org.gradle.api.file.FileSystemOperations;
import org.gradle.api.file.ProjectLayout;
import org.gradle.api.model.ObjectFactory;
import org.gradle.api.plugins.JvmToolchainsPlugin;
import org.gradle.api.provider.Provider;
import org.gradle.api.provider.ProviderFactory;
import org.gradle.api.tasks.Copy;
import org.gradle.api.tasks.PathSensitivity;
import org.gradle.api.tasks.TaskProvider;
import org.gradle.jvm.toolchain.JavaToolchainService;
Expand Down Expand Up @@ -54,11 +54,17 @@ public class InternalDistributionBwcSetupPlugin implements Plugin<Project> {
private final ObjectFactory objectFactory;
private ProviderFactory providerFactory;
private JavaToolchainService toolChainService;
private FileSystemOperations fileSystemOperations;

@Inject
public InternalDistributionBwcSetupPlugin(ObjectFactory objectFactory, ProviderFactory providerFactory) {
public InternalDistributionBwcSetupPlugin(
ObjectFactory objectFactory,
ProviderFactory providerFactory,
FileSystemOperations fileSystemOperations
) {
this.objectFactory = objectFactory;
this.providerFactory = providerFactory;
this.fileSystemOperations = fileSystemOperations;
}

@Override
Expand All @@ -76,7 +82,8 @@ public void apply(Project project) {
providerFactory,
objectFactory,
toolChainService,
isCi
isCi,
fileSystemOperations
);
});
}
Expand All @@ -88,7 +95,8 @@ private static void configureBwcProject(
ProviderFactory providerFactory,
ObjectFactory objectFactory,
JavaToolchainService toolChainService,
Boolean isCi
Boolean isCi,
FileSystemOperations fileSystemOperations
) {
ProjectLayout layout = project.getLayout();
Provider<BwcVersions.UnreleasedVersionInfo> versionInfoProvider = providerFactory.provider(() -> versionInfo);
Expand Down Expand Up @@ -120,11 +128,18 @@ private static void configureBwcProject(
List<DistributionProject> distributionProjects = resolveArchiveProjects(checkoutDir.get(), bwcVersion.get());

// Setup gradle user home directory
project.getTasks().register("setupGradleUserHome", Copy.class, copy -> {
copy.into(project.getGradle().getGradleUserHomeDir().getAbsolutePath() + "-" + project.getName());
copy.from(project.getGradle().getGradleUserHomeDir().getAbsolutePath(), copySpec -> {
copySpec.include("gradle.properties");
copySpec.include("init.d/*");
// We don't use a normal `Copy` task here as snapshotting the entire gradle user home is very expensive. This task is cheap, so
// up-to-date checking doesn't buy us much
project.getTasks().register("setupGradleUserHome", task -> {
task.doLast(t -> {
fileSystemOperations.copy(copy -> {
String gradleUserHome = project.getGradle().getGradleUserHomeDir().getAbsolutePath();
copy.into(gradleUserHome + "-" + project.getName());
copy.from(gradleUserHome, copySpec -> {
copySpec.include("gradle.properties");
copySpec.include("init.d/*");
});
});
});
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,14 +86,14 @@ public void apply(Project project) {
configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.TEST_SOURCE_SET_NAME), 21);
for (int javaVersion : mainVersions) {
String mainSourceSetName = SourceSet.MAIN_SOURCE_SET_NAME + javaVersion;
SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion);
SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion, true);
configureSourceSetInJar(project, mainSourceSet, javaVersion);
addJar(project, mainSourceSet, javaVersion);
mainSourceSets.add(mainSourceSetName);
testSourceSets.add(mainSourceSetName);

String testSourceSetName = SourceSet.TEST_SOURCE_SET_NAME + javaVersion;
SourceSet testSourceSet = addSourceSet(project, javaExtension, testSourceSetName, testSourceSets, javaVersion);
SourceSet testSourceSet = addSourceSet(project, javaExtension, testSourceSetName, testSourceSets, javaVersion, false);
testSourceSets.add(testSourceSetName);
createTestTask(project, buildParams, testSourceSet, javaVersion, mainSourceSets);
}
Expand Down Expand Up @@ -121,7 +121,8 @@ private SourceSet addSourceSet(
JavaPluginExtension javaExtension,
String sourceSetName,
List<String> parentSourceSets,
int javaVersion
int javaVersion,
boolean isMainSourceSet
) {
SourceSet sourceSet = javaExtension.getSourceSets().maybeCreate(sourceSetName);
for (String parentSourceSetName : parentSourceSets) {
Expand All @@ -135,6 +136,13 @@ private SourceSet addSourceSet(
CompileOptions compileOptions = compileTask.getOptions();
compileOptions.getRelease().set(javaVersion);
});
if (isMainSourceSet) {
project.getTasks().create(sourceSet.getJavadocTaskName(), Javadoc.class, javadocTask -> {
javadocTask.getJavadocTool().set(javaToolchains.javadocToolFor(spec -> {
spec.getLanguageVersion().set(JavaLanguageVersion.of(javaVersion));
}));
});
}
configurePreviewFeatures(project, sourceSet, javaVersion);

// Since we configure MRJAR sourcesets to allow preview apis, class signatures for those
Expand Down
2 changes: 1 addition & 1 deletion distribution/src/config/jvm.options
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
## should create one or more files in the jvm.options.d
## directory containing your adjustments.
##
## See https://www.elastic.co/guide/en/elasticsearch/reference/@project.minor.version@/jvm-options.html
## See https://www.elastic.co/guide/en/elasticsearch/reference/@project.minor.version@/advanced-configuration.html#set-jvm-options
## for more information.
##
################################################################
Expand Down
3 changes: 2 additions & 1 deletion docs/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,9 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
setting 'xpack.security.enabled', 'true'
setting 'xpack.security.authc.api_key.enabled', 'true'
setting 'xpack.security.authc.token.enabled', 'true'
// disable the ILM history for doc tests to avoid potential lingering tasks that'd cause test flakiness
// disable the ILM and SLM history for doc tests to avoid potential lingering tasks that'd cause test flakiness
setting 'indices.lifecycle.history_index_enabled', 'false'
setting 'slm.history_index_enabled', 'false'
setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.security.authc.realms.file.file.order', '0'
setting 'xpack.security.authc.realms.native.native.order', '1'
Expand Down
5 changes: 5 additions & 0 deletions docs/changelog/116687.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 116687
summary: Add LogsDB option to route on sort fields
area: Logs
type: enhancement
issues: []
5 changes: 5 additions & 0 deletions docs/changelog/118353.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 118353
summary: Epoch Millis Rounding Down and Not Up 2
area: Infra/Core
type: bug
issues: []
6 changes: 6 additions & 0 deletions docs/changelog/118562.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 118562
summary: Update data stream deprecations warnings to new format and filter searchable
snapshots from response
area: Data streams
type: enhancement
issues: []
7 changes: 7 additions & 0 deletions docs/changelog/118585.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
pr: 118585
summary: Add a generic `rescorer` retriever based on the search request's rescore
functionality
area: Ranking
type: feature
issues:
- 118327
6 changes: 6 additions & 0 deletions docs/changelog/118603.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 118603
summary: Allow DATE_PARSE to read the timezones
area: ES|QL
type: bug
issues:
- 117680
5 changes: 5 additions & 0 deletions docs/changelog/118823.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 118823
summary: Fix attribute set equals
area: ES|QL
type: bug
issues: []
4 changes: 2 additions & 2 deletions docs/changelog/116944.yaml → docs/changelog/118825.yaml
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
pr: 116944
pr: 118825
summary: "Remove support for type, fields, `copy_to` and boost in metadata field definition"
area: Mapping
type: breaking
issues: []
breaking:
title: "Remove support for type, fields, copy_to and boost in metadata field definition"
area: Mapping
details: The type, fields, copy_to and boost parameters are no longer supported in metadata field definition
details: The type, fields, copy_to and boost parameters are no longer supported in metadata field definition starting with version 9.
impact: Users providing type, fields, copy_to or boost as part of metadata field definition should remove them from their mappings.
notable: false
6 changes: 6 additions & 0 deletions docs/changelog/118931.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 118931
summary: Add a `LicenseAware` interface for licensed Nodes
area: ES|QL
type: enhancement
issues:
- 117405
5 changes: 5 additions & 0 deletions docs/changelog/118941.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 118941
summary: Allow archive and searchable snapshots indices in N-2 version
area: Recovery
type: enhancement
issues: []
6 changes: 6 additions & 0 deletions docs/changelog/119007.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 119007
summary: Block-writes cannot be added after read-only
area: Data streams
type: bug
issues:
- 119002
Loading

0 comments on commit d62e37d

Please sign in to comment.