diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 934990a1dbc52..de883187547fd 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -17,6 +17,7 @@ * under the License. */ +import org.gradle.internal.jvm.Jvm import org.gradle.util.GradleVersion plugins { @@ -168,6 +169,9 @@ if (project != rootProject) { forbiddenApisTest.enabled = false jarHell.enabled = false thirdPartyAudit.enabled = false + if (Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))){ + test.enabled = false + } configurations { distribution @@ -223,8 +227,13 @@ if (project != rootProject) { } check.dependsOn(integTest) + // for now we hardcode the tests for our build to use the gradle jvm. + tasks.withType(Test).configureEach { + it.executable = Jvm.current().getJavaExecutable() + } + /* - * We alread configure publication and we don't need or want this one that + * We already configure publication and we don't need or want this one that * comes from the java-gradle-plugin. */ afterEvaluate { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java index 9dba3fbe907ec..07d68ca679f81 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java @@ -41,14 +41,12 @@ public class GenerateGlobalBuildInfoTask extends DefaultTask { private final RegularFileProperty outputFile; private final RegularFileProperty compilerVersionFile; private final RegularFileProperty runtimeVersionFile; - private final RegularFileProperty fipsJvmFile; @Inject public GenerateGlobalBuildInfoTask(ObjectFactory objectFactory) { this.outputFile = objectFactory.fileProperty(); this.compilerVersionFile = objectFactory.fileProperty(); this.runtimeVersionFile = objectFactory.fileProperty(); - this.fipsJvmFile = objectFactory.fileProperty(); } @Input @@ -113,11 +111,6 @@ public RegularFileProperty getRuntimeVersionFile() { return runtimeVersionFile; } - @OutputFile - public RegularFileProperty getFipsJvmFile() { - return fipsJvmFile; - } - @TaskAction public void generate() { String javaVendorVersion = System.getProperty("java.vendor.version", System.getProperty("java.vendor")); @@ -130,7 +123,6 @@ public void generate() { String runtimeJavaVersionDetails = gradleJavaVersionDetails; JavaVersion runtimeJavaVersionEnum = JavaVersion.current(); File gradleJavaHome = Jvm.current().getJavaHome(); - boolean inFipsJvm = false; try { if (Files.isSameFile(compilerJavaHome.toPath(), gradleJavaHome.toPath()) == false) { @@ -146,8 +138,6 @@ public void generate() { if (runtimeJavaHome.exists()) { runtimeJavaVersionDetails = findJavaVersionDetails(runtimeJavaHome); runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(runtimeJavaHome)); - - inFipsJvm = Boolean.parseBoolean(System.getProperty("tests.fips.enabled")); } else { throw new RuntimeException("Runtime Java home path of '" + compilerJavaHome + "' does not exist"); } @@ -213,7 +203,6 @@ public void generate() { writeToFile(compilerVersionFile.getAsFile().get(), compilerJavaVersionEnum.name()); writeToFile(runtimeVersionFile.getAsFile().get(), runtimeJavaVersionEnum.name()); - writeToFile(fipsJvmFile.getAsFile().get(), Boolean.toString(inFipsJvm)); } private void writeToFile(File file, String content) { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java index 4e9e67fb1e8ef..746a06374ace0 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java @@ -76,14 +76,12 @@ public void apply(Project project) { task.getOutputFile().set(new File(project.getBuildDir(), "global-build-info")); task.getCompilerVersionFile().set(new File(project.getBuildDir(), "java-compiler-version")); task.getRuntimeVersionFile().set(new File(project.getBuildDir(), "java-runtime-version")); - task.getFipsJvmFile().set(new File(project.getBuildDir(), "in-fips-jvm")); }); PrintGlobalBuildInfoTask printTask = project.getTasks().create("printGlobalBuildInfo", PrintGlobalBuildInfoTask.class, task -> { task.getBuildInfoFile().set(generateTask.getOutputFile()); task.getCompilerVersionFile().set(generateTask.getCompilerVersionFile()); task.getRuntimeVersionFile().set(generateTask.getRuntimeVersionFile()); - task.getFipsJvmFile().set(generateTask.getFipsJvmFile()); task.setGlobalInfoListeners(extension.listeners); }); @@ -103,6 +101,7 @@ public void apply(Project project) { params.setIsCi(System.getenv("JENKINS_URL") != null); params.setIsInternal(GlobalBuildInfoPlugin.class.getResource("/buildSrc.marker") != null); params.setDefaultParallel(findDefaultParallel(project)); + params.setInFipsJvm(isInFipsJvm()); }); project.allprojects(p -> { @@ -153,6 +152,10 @@ private static String getJavaHomeEnvVarName(String version) { return "JAVA" + version + "_HOME"; } + private static boolean isInFipsJvm() { + return Boolean.parseBoolean(System.getProperty("tests.fips.enabled")); + } + private static String getResourceContents(String resourcePath) { try (BufferedReader reader = new BufferedReader( new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath)) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java index ed0ede14164f2..fdd79e0fc9cb3 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java @@ -16,7 +16,6 @@ public class PrintGlobalBuildInfoTask extends DefaultTask { private final RegularFileProperty buildInfoFile; private final RegularFileProperty compilerVersionFile; private final RegularFileProperty runtimeVersionFile; - private final RegularFileProperty fipsJvmFile; private List globalInfoListeners = new ArrayList<>(); @Inject @@ -24,7 +23,6 @@ public PrintGlobalBuildInfoTask(ObjectFactory objectFactory) { this.buildInfoFile = objectFactory.fileProperty(); this.compilerVersionFile = objectFactory.fileProperty(); this.runtimeVersionFile = objectFactory.fileProperty(); - this.fipsJvmFile = objectFactory.fileProperty(); } @InputFile @@ -42,11 +40,6 @@ public RegularFileProperty getRuntimeVersionFile() { return runtimeVersionFile; } - @InputFile - public RegularFileProperty getFipsJvmFile() { - return fipsJvmFile; - } - public void setGlobalInfoListeners(List globalInfoListeners) { this.globalInfoListeners = globalInfoListeners; } @@ -57,6 +50,7 @@ public void print() { getLogger().quiet("Elasticsearch Build Hamster says Hello!"); getLogger().quiet(getFileText(getBuildInfoFile()).asString()); getLogger().quiet(" Random Testing Seed : " + BuildParams.getTestSeed()); + getLogger().quiet(" In FIPS 140 mode : " + BuildParams.isInFipsJvm()); getLogger().quiet("======================================="); setGlobalProperties(); @@ -76,7 +70,6 @@ private void setGlobalProperties() { BuildParams.init(params -> { params.setCompilerJavaVersion(JavaVersion.valueOf(getFileText(getCompilerVersionFile()).asString())); params.setRuntimeJavaVersion(JavaVersion.valueOf(getFileText(getRuntimeVersionFile()).asString())); - params.setInFipsJvm(Boolean.parseBoolean(getFileText(getFipsJvmFile()).asString())); }); } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 65beedc1860f7..de8e02ed0c7de 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -415,6 +415,11 @@ public synchronized void start() { } catch (IOException e) { throw new UncheckedIOException("Failed to create working directory for " + this, e); } + + copyExtraJars(); + + copyExtraConfigFiles(); + createConfiguration(); if (plugins.isEmpty() == false) { @@ -438,7 +443,7 @@ public synchronized void start() { runElaticsearchBinScript("elasticsearch-keystore", "create"); keystoreSettings.forEach((key, value) -> - runElaticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key) + runElasticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key) ); for (Map.Entry entry : keystoreFiles.entrySet()) { @@ -453,10 +458,6 @@ public synchronized void start() { installModules(); - copyExtraConfigFiles(); - - copyExtraJars(); - if (isSettingTrue("xpack.security.enabled")) { if (credentials.isEmpty()) { user(Collections.emptyMap()); @@ -622,7 +623,7 @@ public void user(Map userSpec) { credentials.add(cred); } - private void runElaticsearchBinScriptWithInput(String input, String tool, String... args) { + private void runElasticsearchBinScriptWithInput(String input, String tool, String... args) { if ( Files.exists(getDistroDir().resolve("bin").resolve(tool)) == false && Files.exists(getDistroDir().resolve("bin").resolve(tool + ".bat")) == false @@ -663,7 +664,7 @@ private void runElaticsearchBinScriptWithInput(String input, String tool, String } private void runElaticsearchBinScript(String tool, String... args) { - runElaticsearchBinScriptWithInput("", tool, args); + runElasticsearchBinScriptWithInput("", tool, args); } private Map getESEnvironment() { @@ -676,6 +677,10 @@ private Map getESEnvironment() { if (systemProperties.isEmpty() == false) { systemPropertiesString = " " + systemProperties.entrySet().stream() .map(entry -> "-D" + entry.getKey() + "=" + entry.getValue()) + // ES_PATH_CONF is also set as an environment variable and for a reference to ${ES_PATH_CONF} + // to work ES_JAVA_OPTS, we need to make sure that ES_PATH_CONF before ES_JAVA_OPTS. Instead, + // we replace the reference with the actual value in other environment variables + .map(p -> p.replace("${ES_PATH_CONF}", configFile.getParent().toString())) .collect(Collectors.joining(" ")); } String jvmArgsString = ""; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 9df33b410a792..2fc23acd13430 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -58,6 +58,7 @@ import org.elasticsearch.client.ml.GetModelSnapshotsRequest; import org.elasticsearch.client.ml.GetOverallBucketsRequest; import org.elasticsearch.client.ml.GetRecordsRequest; +import org.elasticsearch.client.ml.GetTrainedModelsRequest; import org.elasticsearch.client.ml.MlInfoRequest; import org.elasticsearch.client.ml.OpenJobRequest; import org.elasticsearch.client.ml.PostCalendarEventRequest; @@ -709,6 +710,38 @@ static Request estimateMemoryUsage(PutDataFrameAnalyticsRequest estimateRequest) return request; } + static Request getTrainedModels(GetTrainedModelsRequest getTrainedModelsRequest) { + String endpoint = new EndpointBuilder() + .addPathPartAsIs("_ml", "inference") + .addPathPart(Strings.collectionToCommaDelimitedString(getTrainedModelsRequest.getIds())) + .build(); + RequestConverters.Params params = new RequestConverters.Params(); + if (getTrainedModelsRequest.getPageParams() != null) { + PageParams pageParams = getTrainedModelsRequest.getPageParams(); + if (pageParams.getFrom() != null) { + params.putParam(PageParams.FROM.getPreferredName(), pageParams.getFrom().toString()); + } + if (pageParams.getSize() != null) { + params.putParam(PageParams.SIZE.getPreferredName(), pageParams.getSize().toString()); + } + } + if (getTrainedModelsRequest.getAllowNoMatch() != null) { + params.putParam(GetTrainedModelsRequest.ALLOW_NO_MATCH, + Boolean.toString(getTrainedModelsRequest.getAllowNoMatch())); + } + if (getTrainedModelsRequest.getDecompressDefinition() != null) { + params.putParam(GetTrainedModelsRequest.DECOMPRESS_DEFINITION, + Boolean.toString(getTrainedModelsRequest.getDecompressDefinition())); + } + if (getTrainedModelsRequest.getIncludeDefinition() != null) { + params.putParam(GetTrainedModelsRequest.INCLUDE_MODEL_DEFINITION, + Boolean.toString(getTrainedModelsRequest.getIncludeDefinition())); + } + Request request = new Request(HttpGet.METHOD_NAME, endpoint); + request.addParameters(params.asMap()); + return request; + } + static Request putFilter(PutFilterRequest putFilterRequest) throws IOException { String endpoint = new EndpointBuilder() .addPathPartAsIs("_ml") diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java index 62619303685ae..2ddc8839f9648 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java @@ -73,6 +73,8 @@ import org.elasticsearch.client.ml.GetOverallBucketsResponse; import org.elasticsearch.client.ml.GetRecordsRequest; import org.elasticsearch.client.ml.GetRecordsResponse; +import org.elasticsearch.client.ml.GetTrainedModelsRequest; +import org.elasticsearch.client.ml.GetTrainedModelsResponse; import org.elasticsearch.client.ml.MlInfoRequest; import org.elasticsearch.client.ml.MlInfoResponse; import org.elasticsearch.client.ml.OpenJobRequest; @@ -2290,4 +2292,48 @@ public Cancellable estimateMemoryUsageAsync(PutDataFrameAnalyticsRequest request listener, Collections.emptySet()); } + + /** + * Gets trained model configs + *

+ * For additional info + * see + * GET Trained Model Configs documentation + * + * @param request The {@link GetTrainedModelsRequest} + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return {@link GetTrainedModelsResponse} response object + */ + public GetTrainedModelsResponse getTrainedModels(GetTrainedModelsRequest request, + RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, + MLRequestConverters::getTrainedModels, + options, + GetTrainedModelsResponse::fromXContent, + Collections.emptySet()); + } + + /** + * Gets trained model configs asynchronously and notifies listener upon completion + *

+ * For additional info + * see + * GET Trained Model Configs documentation + * + * @param request The {@link GetTrainedModelsRequest} + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener Listener to be notified upon request completion + * @return cancellable that may be used to cancel the request + */ + public Cancellable getTrainedModelsAsync(GetTrainedModelsRequest request, + RequestOptions options, + ActionListener listener) { + return restHighLevelClient.performRequestAsyncAndParseEntity(request, + MLRequestConverters::getTrainedModels, + options, + GetTrainedModelsResponse::fromXContent, + listener, + Collections.emptySet()); + } + } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java new file mode 100644 index 0000000000000..9234770a97a6e --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java @@ -0,0 +1,139 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.ml; + +import org.elasticsearch.client.Validatable; +import org.elasticsearch.client.ValidationException; +import org.elasticsearch.client.core.PageParams; +import org.elasticsearch.common.Nullable; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +public class GetTrainedModelsRequest implements Validatable { + + public static final String ALLOW_NO_MATCH = "allow_no_match"; + public static final String INCLUDE_MODEL_DEFINITION = "include_model_definition"; + public static final String DECOMPRESS_DEFINITION = "decompress_definition"; + + private final List ids; + private Boolean allowNoMatch; + private Boolean includeDefinition; + private Boolean decompressDefinition; + private PageParams pageParams; + + /** + * Helper method to create a request that will get ALL TrainedModelConfigs + * @return new {@link GetTrainedModelsRequest} object for the id "_all" + */ + public static GetTrainedModelsRequest getAllTrainedModelConfigsRequest() { + return new GetTrainedModelsRequest("_all"); + } + + public GetTrainedModelsRequest(String... ids) { + this.ids = Arrays.asList(ids); + } + + public List getIds() { + return ids; + } + + public Boolean getAllowNoMatch() { + return allowNoMatch; + } + + /** + * Whether to ignore if a wildcard expression matches no trained models. + * + * @param allowNoMatch If this is {@code false}, then an error is returned when a wildcard (or {@code _all}) + * does not match any trained models + */ + public GetTrainedModelsRequest setAllowNoMatch(boolean allowNoMatch) { + this.allowNoMatch = allowNoMatch; + return this; + } + + public PageParams getPageParams() { + return pageParams; + } + + public GetTrainedModelsRequest setPageParams(@Nullable PageParams pageParams) { + this.pageParams = pageParams; + return this; + } + + public Boolean getIncludeDefinition() { + return includeDefinition; + } + + /** + * Whether to include the full model definition. + * + * The full model definition can be very large. + * + * @param includeDefinition If {@code true}, the definition is included. + */ + public GetTrainedModelsRequest setIncludeDefinition(Boolean includeDefinition) { + this.includeDefinition = includeDefinition; + return this; + } + + public Boolean getDecompressDefinition() { + return decompressDefinition; + } + + /** + * Whether or not to decompress the trained model, or keep it in its compressed string form + * + * @param decompressDefinition If {@code true}, the definition is decompressed. + */ + public GetTrainedModelsRequest setDecompressDefinition(Boolean decompressDefinition) { + this.decompressDefinition = decompressDefinition; + return this; + } + + @Override + public Optional validate() { + if (ids == null || ids.isEmpty()) { + return Optional.of(ValidationException.withError("trained model id must not be null")); + } + return Optional.empty(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + GetTrainedModelsRequest other = (GetTrainedModelsRequest) o; + return Objects.equals(ids, other.ids) + && Objects.equals(allowNoMatch, other.allowNoMatch) + && Objects.equals(decompressDefinition, other.decompressDefinition) + && Objects.equals(includeDefinition, other.includeDefinition) + && Objects.equals(pageParams, other.pageParams); + } + + @Override + public int hashCode() { + return Objects.hash(ids, allowNoMatch, pageParams, decompressDefinition, includeDefinition); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsResponse.java new file mode 100644 index 0000000000000..c83bcd97f77c1 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsResponse.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.ml; + +import org.elasticsearch.client.ml.inference.TrainedModelConfig; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +public class GetTrainedModelsResponse { + + public static final ParseField TRAINED_MODEL_CONFIGS = new ParseField("trained_model_configs"); + public static final ParseField COUNT = new ParseField("count"); + + @SuppressWarnings("unchecked") + static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>( + "get_trained_model_configs", + true, + args -> new GetTrainedModelsResponse((List) args[0], (Long) args[1])); + + static { + PARSER.declareObjectArray(constructorArg(), (p, c) -> TrainedModelConfig.fromXContent(p), TRAINED_MODEL_CONFIGS); + PARSER.declareLong(constructorArg(), COUNT); + } + + public static GetTrainedModelsResponse fromXContent(final XContentParser parser) { + return PARSER.apply(parser, null); + } + + private final List trainedModels; + private final Long count; + + + public GetTrainedModelsResponse(List trainedModels, Long count) { + this.trainedModels = trainedModels; + this.count = count; + } + + public List getTrainedModels() { + return trainedModels; + } + + /** + * @return The total count of the trained models that matched the ID pattern. + */ + public Long getCount() { + return count; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + GetTrainedModelsResponse other = (GetTrainedModelsResponse) o; + return Objects.equals(this.trainedModels, other.trainedModels) && Objects.equals(this.count, other.count); + } + + @Override + public int hashCode() { + return Objects.hash(trainedModels, count); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelConfig.java index 50775cde8a94e..23eb01fb3b153 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelConfig.java @@ -77,8 +77,8 @@ public class TrainedModelConfig implements ToXContentObject { PARSER.declareString(TrainedModelConfig.Builder::setLicenseLevel, LICENSE_LEVEL); } - public static TrainedModelConfig.Builder fromXContent(XContentParser parser) throws IOException { - return PARSER.parse(parser, null); + public static TrainedModelConfig fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null).build(); } private final String modelId; @@ -293,12 +293,12 @@ public Builder setModelId(String modelId) { return this; } - private Builder setCreatedBy(String createdBy) { + public Builder setCreatedBy(String createdBy) { this.createdBy = createdBy; return this; } - private Builder setVersion(Version version) { + public Builder setVersion(Version version) { this.version = version; return this; } @@ -312,7 +312,7 @@ public Builder setDescription(String description) { return this; } - private Builder setCreateTime(Instant createTime) { + public Builder setCreateTime(Instant createTime) { this.createTime = createTime; return this; } @@ -347,17 +347,17 @@ public Builder setInput(TrainedModelInput input) { return this; } - private Builder setEstimatedHeapMemory(Long estimatedHeapMemory) { + public Builder setEstimatedHeapMemory(Long estimatedHeapMemory) { this.estimatedHeapMemory = estimatedHeapMemory; return this; } - private Builder setEstimatedOperations(Long estimatedOperations) { + public Builder setEstimatedOperations(Long estimatedOperations) { this.estimatedOperations = estimatedOperations; return this; } - private Builder setLicenseLevel(String licenseLevel) { + public Builder setLicenseLevel(String licenseLevel) { this.licenseLevel = licenseLevel; return this; } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java index f68cc6c20cbc2..2ce26cc63c91f 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java @@ -56,6 +56,7 @@ import org.elasticsearch.client.ml.GetModelSnapshotsRequest; import org.elasticsearch.client.ml.GetOverallBucketsRequest; import org.elasticsearch.client.ml.GetRecordsRequest; +import org.elasticsearch.client.ml.GetTrainedModelsRequest; import org.elasticsearch.client.ml.MlInfoRequest; import org.elasticsearch.client.ml.OpenJobRequest; import org.elasticsearch.client.ml.PostCalendarEventRequest; @@ -798,6 +799,31 @@ public void testEstimateMemoryUsage() throws IOException { } } + public void testGetTrainedModels() { + String modelId1 = randomAlphaOfLength(10); + String modelId2 = randomAlphaOfLength(10); + String modelId3 = randomAlphaOfLength(10); + GetTrainedModelsRequest getRequest = new GetTrainedModelsRequest(modelId1, modelId2, modelId3) + .setAllowNoMatch(false) + .setDecompressDefinition(true) + .setIncludeDefinition(false) + .setPageParams(new PageParams(100, 300)); + + Request request = MLRequestConverters.getTrainedModels(getRequest); + assertEquals(HttpGet.METHOD_NAME, request.getMethod()); + assertEquals("/_ml/inference/" + modelId1 + "," + modelId2 + "," + modelId3, request.getEndpoint()); + assertThat(request.getParameters(), allOf(hasEntry("from", "100"), hasEntry("size", "300"), hasEntry("allow_no_match", "false"))); + assertThat(request.getParameters(), + allOf( + hasEntry("from", "100"), + hasEntry("size", "300"), + hasEntry("allow_no_match", "false"), + hasEntry("decompress_definition", "true"), + hasEntry("include_model_definition", "false") + )); + assertNull(request.getEntity()); + } + public void testPutFilter() throws IOException { MlFilter filter = MlFilterTests.createRandomBuilder("foo").build(); PutFilterRequest putFilterRequest = new PutFilterRequest(filter); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java index 1530a324055cf..c6736f09a76fa 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java @@ -73,6 +73,8 @@ import org.elasticsearch.client.ml.GetJobStatsResponse; import org.elasticsearch.client.ml.GetModelSnapshotsRequest; import org.elasticsearch.client.ml.GetModelSnapshotsResponse; +import org.elasticsearch.client.ml.GetTrainedModelsRequest; +import org.elasticsearch.client.ml.GetTrainedModelsResponse; import org.elasticsearch.client.ml.MlInfoRequest; import org.elasticsearch.client.ml.MlInfoResponse; import org.elasticsearch.client.ml.OpenJobRequest; @@ -139,6 +141,9 @@ import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.PrecisionMetric; import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.RecallMetric; import org.elasticsearch.client.ml.filestructurefinder.FileStructure; +import org.elasticsearch.client.ml.inference.TrainedModelConfig; +import org.elasticsearch.client.ml.inference.TrainedModelDefinition; +import org.elasticsearch.client.ml.inference.TrainedModelDefinitionTests; import org.elasticsearch.client.ml.job.config.AnalysisConfig; import org.elasticsearch.client.ml.job.config.DataDescription; import org.elasticsearch.client.ml.job.config.Detector; @@ -148,11 +153,14 @@ import org.elasticsearch.client.ml.job.config.MlFilter; import org.elasticsearch.client.ml.job.process.ModelSnapshot; import org.elasticsearch.client.ml.job.stats.JobStats; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -161,9 +169,11 @@ import org.junit.After; import java.io.IOException; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -171,6 +181,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.zip.GZIPOutputStream; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.anyOf; @@ -186,6 +197,7 @@ import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; public class MachineLearningIT extends ESRestHighLevelClientTestCase { @@ -2002,6 +2014,75 @@ public void testEstimateMemoryUsage() throws IOException { allOf(greaterThanOrEqualTo(response1.getExpectedMemoryWithDisk()), lessThan(upperBound))); } + public void testGetTrainedModels() throws Exception { + MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); + String modelIdPrefix = "get-trained-model-"; + int numberOfModels = 5; + for (int i = 0; i < numberOfModels; ++i) { + String modelId = modelIdPrefix + i; + putTrainedModel(modelId); + } + + { + GetTrainedModelsResponse getTrainedModelsResponse = execute( + new GetTrainedModelsRequest(modelIdPrefix + 0).setDecompressDefinition(true).setIncludeDefinition(true), + machineLearningClient::getTrainedModels, + machineLearningClient::getTrainedModelsAsync); + + assertThat(getTrainedModelsResponse.getCount(), equalTo(1L)); + assertThat(getTrainedModelsResponse.getTrainedModels(), hasSize(1)); + assertThat(getTrainedModelsResponse.getTrainedModels().get(0).getCompressedDefinition(), is(nullValue())); + assertThat(getTrainedModelsResponse.getTrainedModels().get(0).getDefinition(), is(not(nullValue()))); + assertThat(getTrainedModelsResponse.getTrainedModels().get(0).getModelId(), equalTo(modelIdPrefix + 0)); + + getTrainedModelsResponse = execute( + new GetTrainedModelsRequest(modelIdPrefix + 0).setDecompressDefinition(false).setIncludeDefinition(true), + machineLearningClient::getTrainedModels, + machineLearningClient::getTrainedModelsAsync); + + assertThat(getTrainedModelsResponse.getCount(), equalTo(1L)); + assertThat(getTrainedModelsResponse.getTrainedModels(), hasSize(1)); + assertThat(getTrainedModelsResponse.getTrainedModels().get(0).getCompressedDefinition(), is(not(nullValue()))); + assertThat(getTrainedModelsResponse.getTrainedModels().get(0).getDefinition(), is(nullValue())); + assertThat(getTrainedModelsResponse.getTrainedModels().get(0).getModelId(), equalTo(modelIdPrefix + 0)); + + getTrainedModelsResponse = execute( + new GetTrainedModelsRequest(modelIdPrefix + 0).setDecompressDefinition(false).setIncludeDefinition(false), + machineLearningClient::getTrainedModels, + machineLearningClient::getTrainedModelsAsync); + assertThat(getTrainedModelsResponse.getCount(), equalTo(1L)); + assertThat(getTrainedModelsResponse.getTrainedModels(), hasSize(1)); + assertThat(getTrainedModelsResponse.getTrainedModels().get(0).getCompressedDefinition(), is(nullValue())); + assertThat(getTrainedModelsResponse.getTrainedModels().get(0).getDefinition(), is(nullValue())); + assertThat(getTrainedModelsResponse.getTrainedModels().get(0).getModelId(), equalTo(modelIdPrefix + 0)); + + } + { + GetTrainedModelsResponse getTrainedModelsResponse = execute( + GetTrainedModelsRequest.getAllTrainedModelConfigsRequest(), + machineLearningClient::getTrainedModels, machineLearningClient::getTrainedModelsAsync); + assertThat(getTrainedModelsResponse.getTrainedModels(), hasSize(numberOfModels)); + assertThat(getTrainedModelsResponse.getCount(), equalTo(5L)); + } + { + GetTrainedModelsResponse getTrainedModelsResponse = execute( + new GetTrainedModelsRequest(modelIdPrefix + 4, modelIdPrefix + 2, modelIdPrefix + 3), + machineLearningClient::getTrainedModels, machineLearningClient::getTrainedModelsAsync); + assertThat(getTrainedModelsResponse.getTrainedModels(), hasSize(3)); + assertThat(getTrainedModelsResponse.getCount(), equalTo(3L)); + } + { + GetTrainedModelsResponse getTrainedModelsResponse = execute( + new GetTrainedModelsRequest(modelIdPrefix + "*").setPageParams(new PageParams(1, 2)), + machineLearningClient::getTrainedModels, machineLearningClient::getTrainedModelsAsync); + assertThat(getTrainedModelsResponse.getTrainedModels(), hasSize(2)); + assertThat(getTrainedModelsResponse.getCount(), equalTo(5L)); + assertThat( + getTrainedModelsResponse.getTrainedModels().stream().map(TrainedModelConfig::getModelId).collect(Collectors.toList()), + containsInAnyOrder(modelIdPrefix + 1, modelIdPrefix + 2)); + } + } + public void testPutFilter() throws Exception { String filterId = "filter-job-test"; MlFilter mlFilter = MlFilter.builder(filterId) @@ -2179,6 +2260,60 @@ private void openJob(Job job) throws IOException { highLevelClient().machineLearning().openJob(new OpenJobRequest(job.getId()), RequestOptions.DEFAULT); } + private void putTrainedModel(String modelId) throws IOException { + TrainedModelDefinition definition = TrainedModelDefinitionTests.createRandomBuilder().build(); + highLevelClient().index( + new IndexRequest(".ml-inference-000001") + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(modelConfigString(modelId), XContentType.JSON) + .id(modelId), + RequestOptions.DEFAULT); + + highLevelClient().index( + new IndexRequest(".ml-inference-000001") + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(modelDocString(compressDefinition(definition), modelId), XContentType.JSON) + .id("trained_model_definition_doc-" + modelId + "-0"), + RequestOptions.DEFAULT); + } + + private String compressDefinition(TrainedModelDefinition definition) throws IOException { + BytesReference reference = XContentHelper.toXContent(definition, XContentType.JSON, false); + BytesStreamOutput out = new BytesStreamOutput(); + try (OutputStream compressedOutput = new GZIPOutputStream(out, 4096)) { + reference.writeTo(compressedOutput); + } + return new String(Base64.getEncoder().encode(BytesReference.toBytes(out.bytes())), StandardCharsets.UTF_8); + } + + private static String modelConfigString(String modelId) { + return "{\n" + + " \"doc_type\": \"trained_model_config\",\n" + + " \"model_id\": \"" + modelId + "\",\n" + + " \"input\":{\"field_names\":[\"col1\",\"col2\",\"col3\",\"col4\"]}," + + " \"description\": \"test model\",\n" + + " \"version\": \"7.6.0\",\n" + + " \"license_level\": \"platinum\",\n" + + " \"created_by\": \"ml_test\",\n" + + " \"estimated_heap_memory_usage_bytes\": 0," + + " \"estimated_operations\": 0," + + " \"created_time\": 0\n" + + "}"; + } + + private static String modelDocString(String compressedDefinition, String modelId) { + return "" + + "{" + + "\"model_id\": \"" + modelId + "\",\n" + + "\"doc_num\": 0,\n" + + "\"doc_type\": \"trained_model_definition_doc\",\n" + + " \"compression_version\": " + 1 + ",\n" + + " \"total_definition_length\": " + compressedDefinition.length() + ",\n" + + " \"definition_length\": " + compressedDefinition.length() + ",\n" + + "\"definition\": \"" + compressedDefinition + "\"\n" + + "}"; + } + private void waitForJobToClose(String jobId) throws Exception { MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index 6f5c636ccd26d..d844ca622708b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -87,6 +87,8 @@ import org.elasticsearch.client.ml.GetOverallBucketsResponse; import org.elasticsearch.client.ml.GetRecordsRequest; import org.elasticsearch.client.ml.GetRecordsResponse; +import org.elasticsearch.client.ml.GetTrainedModelsRequest; +import org.elasticsearch.client.ml.GetTrainedModelsResponse; import org.elasticsearch.client.ml.MlInfoRequest; import org.elasticsearch.client.ml.MlInfoResponse; import org.elasticsearch.client.ml.OpenJobRequest; @@ -154,6 +156,9 @@ import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.PrecisionMetric; import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.RecallMetric; import org.elasticsearch.client.ml.filestructurefinder.FileStructure; +import org.elasticsearch.client.ml.inference.TrainedModelConfig; +import org.elasticsearch.client.ml.inference.TrainedModelDefinition; +import org.elasticsearch.client.ml.inference.TrainedModelDefinitionTests; import org.elasticsearch.client.ml.job.config.AnalysisConfig; import org.elasticsearch.client.ml.job.config.AnalysisLimits; import org.elasticsearch.client.ml.job.config.DataDescription; @@ -174,10 +179,12 @@ import org.elasticsearch.client.ml.job.results.OverallBucket; import org.elasticsearch.client.ml.job.stats.JobStats; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -188,10 +195,12 @@ import org.junit.After; import java.io.IOException; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -200,6 +209,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.zip.GZIPOutputStream; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.closeTo; @@ -3516,6 +3526,58 @@ public void onFailure(Exception e) { } } + public void testGetTrainedModels() throws Exception { + putTrainedModel("my-trained-model"); + RestHighLevelClient client = highLevelClient(); + { + // tag::get-trained-models-request + GetTrainedModelsRequest request = new GetTrainedModelsRequest("my-trained-model") // <1> + .setPageParams(new PageParams(0, 1)) // <2> + .setIncludeDefinition(false) // <3> + .setDecompressDefinition(false) // <4> + .setAllowNoMatch(true); // <5> + // end::get-trained-models-request + + // tag::get-trained-models-execute + GetTrainedModelsResponse response = client.machineLearning().getTrainedModels(request, RequestOptions.DEFAULT); + // end::get-trained-models-execute + + // tag::get-trained-models-response + List models = response.getTrainedModels(); + // end::get-trained-models-response + + assertThat(models, hasSize(1)); + } + { + GetTrainedModelsRequest request = new GetTrainedModelsRequest("my-trained-model"); + + // tag::get-trained-models-execute-listener + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(GetTrainedModelsResponse response) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::get-trained-models-execute-listener + + // Replace the empty listener by a blocking listener in test + CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::get-trained-models-execute-async + client.machineLearning().getTrainedModelsAsync(request, RequestOptions.DEFAULT, listener); // <1> + // end::get-trained-models-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + } + + public void testCreateFilter() throws Exception { RestHighLevelClient client = highLevelClient(); { @@ -3878,6 +3940,60 @@ private DataFrameAnalyticsState getAnalyticsState(String configId) throws IOExce return stats.getState(); } + private void putTrainedModel(String modelId) throws IOException { + TrainedModelDefinition definition = TrainedModelDefinitionTests.createRandomBuilder().build(); + highLevelClient().index( + new IndexRequest(".ml-inference-000001") + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(modelConfigString(modelId), XContentType.JSON) + .id(modelId), + RequestOptions.DEFAULT); + + highLevelClient().index( + new IndexRequest(".ml-inference-000001") + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(modelDocString(compressDefinition(definition), modelId), XContentType.JSON) + .id("trained_model_definition_doc-" + modelId + "-0"), + RequestOptions.DEFAULT); + } + + private String compressDefinition(TrainedModelDefinition definition) throws IOException { + BytesReference reference = XContentHelper.toXContent(definition, XContentType.JSON, false); + BytesStreamOutput out = new BytesStreamOutput(); + try (OutputStream compressedOutput = new GZIPOutputStream(out, 4096)) { + reference.writeTo(compressedOutput); + } + return new String(Base64.getEncoder().encode(BytesReference.toBytes(out.bytes())), StandardCharsets.UTF_8); + } + + private static String modelConfigString(String modelId) { + return "{\n" + + " \"doc_type\": \"trained_model_config\",\n" + + " \"model_id\": \"" + modelId + "\",\n" + + " \"input\":{\"field_names\":[\"col1\",\"col2\",\"col3\",\"col4\"]}," + + " \"description\": \"test model for\",\n" + + " \"version\": \"7.6.0\",\n" + + " \"license_level\": \"platinum\",\n" + + " \"created_by\": \"ml_test\",\n" + + " \"estimated_heap_memory_usage_bytes\": 0," + + " \"estimated_operations\": 0," + + " \"created_time\": 0\n" + + "}"; + } + + private static String modelDocString(String compressedDefinition, String modelId) { + return "" + + "{" + + "\"model_id\": \"" + modelId + "\",\n" + + "\"doc_num\": 0,\n" + + "\"doc_type\": \"trained_model_definition_doc\",\n" + + " \"compression_version\": " + 1 + ",\n" + + " \"total_definition_length\": " + compressedDefinition.length() + ",\n" + + " \"definition_length\": " + compressedDefinition.length() + ",\n" + + "\"definition\": \"" + compressedDefinition + "\"\n" + + "}"; + } + private static final DataFrameAnalyticsConfig DF_ANALYTICS_CONFIG = DataFrameAnalyticsConfig.builder() .setId("my-analytics-config") diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetTrainedModelsRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetTrainedModelsRequestTests.java new file mode 100644 index 0000000000000..3f26a1ce3b5e4 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetTrainedModelsRequestTests.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.ml; + +import org.elasticsearch.test.ESTestCase; + +import java.util.Optional; + +import static org.hamcrest.Matchers.containsString; + +public class GetTrainedModelsRequestTests extends ESTestCase { + + public void testValidate_Ok() { + assertEquals(Optional.empty(), new GetTrainedModelsRequest("valid-id").validate()); + assertEquals(Optional.empty(), new GetTrainedModelsRequest("").validate()); + } + + public void testValidate_Failure() { + assertThat(new GetTrainedModelsRequest(new String[0]).validate().get().getMessage(), + containsString("trained model id must not be null")); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelConfigTests.java index 7f0ae3ee920a1..95ebbad837d69 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelConfigTests.java @@ -39,7 +39,7 @@ public class TrainedModelConfigTests extends AbstractXContentTestCase> map, fin } values.add(value); } + + public static boolean inFipsJvm() { + return Boolean.parseBoolean(System.getProperty("tests.fips.enabled")); + } } diff --git a/docs/build.gradle b/docs/build.gradle index a99ab46da220a..9cd53e43b8aa9 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -71,6 +71,10 @@ project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { if (subproj.path.startsWith(':plugins:repository-')) { return } + // Do not install ingest-attachment in a FIPS 140 JVM as this is not supported + if (subproj.path.startsWith(':plugins:ingest-attachment') && Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))) { + return + } // FIXME subproj.afterEvaluate { // need to wait until the project has been configured testClusters.integTest { @@ -89,6 +93,12 @@ buildRestTests.docs = fileTree(projectDir) { exclude 'README.asciidoc' // Broken code snippet tests exclude 'reference/graph/explore.asciidoc' + if (Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))) { + // We don't install/support this plugin in FIPS 140 + exclude 'plugins/ingest-attachment.asciidoc' + // We can't conditionally control output, this would be missing the ingest-attachment plugin + exclude 'reference/cat/plugins.asciidoc' + } } listSnippets.docs = buildRestTests.docs diff --git a/docs/java-rest/high-level/ml/get-trained-models.asciidoc b/docs/java-rest/high-level/ml/get-trained-models.asciidoc new file mode 100644 index 0000000000000..4ad9f0091263c --- /dev/null +++ b/docs/java-rest/high-level/ml/get-trained-models.asciidoc @@ -0,0 +1,43 @@ +-- +:api: get-trained-models +:request: GetTrainedModelsRequest +:response: GetTrainedModelsResponse +-- +[role="xpack"] +[id="{upid}-{api}"] +=== Get Trained Models API + +experimental[] + +Retrieves one or more Trained Models. +The API accepts a +{request}+ object and returns a +{response}+. + +[id="{upid}-{api}-request"] +==== Get Trained Models request + +A +{request}+ requires either a Trained Model ID, a comma-separated list of +IDs, or the special wildcard `_all` to get all Trained Models. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request] +-------------------------------------------------- +<1> Constructing a new GET request referencing an existing Trained Model +<2> Set the paging parameters +<3> Indicate if the complete model definition should be included +<4> Should the definition be fully decompressed on GET +<5> Allow empty response if no Trained Models match the provided ID patterns. + If false, an error will be thrown if no Trained Models match the + ID patterns. + +include::../execution.asciidoc[] + +[id="{upid}-{api}-response"] +==== Response + +The returned +{response}+ contains the requested Trained Model. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +-------------------------------------------------- diff --git a/docs/java-rest/high-level/rollup/put_job.asciidoc b/docs/java-rest/high-level/rollup/put_job.asciidoc index 5e76361691919..ff80bca9757a8 100644 --- a/docs/java-rest/high-level/rollup/put_job.asciidoc +++ b/docs/java-rest/high-level/rollup/put_job.asciidoc @@ -21,7 +21,7 @@ include-tagged::{doc-tests}/RollupDocumentationIT.java[x-pack-rollup-put-rollup- ==== Rollup Job Configuration The `RollupJobConfig` object contains all the details about the rollup job -configuration. See {ref}/rollup-job-config.html[Rollup configuration] to learn more +configuration. See {ref}/rollup-put-job.html[create rollup job API] to learn more about the various configuration settings. A `RollupJobConfig` requires the following arguments: @@ -45,7 +45,7 @@ include-tagged::{doc-tests}/RollupDocumentationIT.java[x-pack-rollup-put-rollup- The grouping configuration of the Rollup job is defined in the `RollupJobConfig` using a `GroupConfig` instance. `GroupConfig` reflects all the configuration -settings that can be defined using the REST API. See {ref}/rollup-job-config.html#rollup-groups-config[Grouping Config] +settings that can be defined using the REST API. See {ref}/rollup-put-job.html#rollup-groups-config[Grouping config] to learn more about these settings. Using the REST API, we could define this grouping configuration: @@ -89,7 +89,7 @@ include-tagged::{doc-tests}/RollupDocumentationIT.java[x-pack-rollup-put-rollup- After defining which groups should be generated for the data, you next configure which metrics should be collected. The list of metrics is defined in the `RollupJobConfig` using a `List` instance. `MetricConfig` reflects all the configuration -settings that can be defined using the REST API. See {ref}/rollup-job-config.html#rollup-metrics-config[Metrics Config] +settings that can be defined using the REST API. See {ref}/rollup-put-job.html#rollup-metrics-config[Metrics config] to learn more about these settings. Using the REST API, we could define this metrics configuration: diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 153a0cf577cfd..770866a075522 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -301,6 +301,7 @@ The Java High Level REST Client supports the following Machine Learning APIs: * <<{upid}-stop-data-frame-analytics>> * <<{upid}-evaluate-data-frame>> * <<{upid}-estimate-memory-usage>> +* <<{upid}-get-trained-models>> * <<{upid}-put-filter>> * <<{upid}-get-filters>> * <<{upid}-update-filter>> @@ -353,6 +354,7 @@ include::ml/start-data-frame-analytics.asciidoc[] include::ml/stop-data-frame-analytics.asciidoc[] include::ml/evaluate-data-frame.asciidoc[] include::ml/estimate-memory-usage.asciidoc[] +include::ml/get-trained-models.asciidoc[] include::ml/put-filter.asciidoc[] include::ml/get-filters.asciidoc[] include::ml/update-filter.asciidoc[] diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc index 62cd850a2766d..0e8e0e144b1ce 100644 --- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc @@ -1,5 +1,5 @@ [[search-aggregations-bucket-datehistogram-aggregation]] -=== Date Histogram Aggregation +=== Date histogram aggregation This multi-bucket aggregation is similar to the normal <>, but it can @@ -10,7 +10,8 @@ that here the interval can be specified using date/time expressions. Time-based data requires special support because time-based intervals are not always a fixed length. -==== Calendar and Fixed intervals +[[calendar_and_fixed_intervals]] +==== Calendar and fixed intervals When configuring a date histogram aggregation, the interval can be specified in two manners: calendar-aware time intervals, and fixed time intervals. @@ -42,7 +43,8 @@ are clear to the user immediately and there is no ambiguity. The old `interval` will be removed in the future. ================================== -===== Calendar Intervals +[[calendar_intervals]] +===== Calendar intervals Calendar-aware intervals are configured with the `calendar_interval` parameter. Calendar intervals can only be specified in "singular" quantities of the unit @@ -100,7 +102,8 @@ One year (1y) is the interval between the start day of the month and time of day and the same day of the month and time of day the following year in the specified timezone, so that the date and time are the same at the start and end. + -===== Calendar Interval Examples +[[calendar_interval_examples]] +===== Calendar interval examples As an example, here is an aggregation requesting bucket intervals of a month in calendar time: [source,console] @@ -157,7 +160,8 @@ POST /sales/_search?size=0 -------------------------------------------------- // NOTCONSOLE -===== Fixed Intervals +[[fixed_intervals]] +===== Fixed intervals Fixed intervals are configured with the `fixed_interval` parameter. @@ -192,7 +196,8 @@ All days begin at the earliest possible time, which is usually 00:00:00 Defined as 24 hours (86,400,000 milliseconds) -===== Fixed Interval Examples +[[fixed_interval_examples]] +===== Fixed interval examples If we try to recreate the "month" `calendar_interval` from earlier, we can approximate that with 30 fixed days: diff --git a/docs/reference/analysis/tokenfilters/edgengram-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/edgengram-tokenfilter.asciidoc index e460725523cf6..c41df5095c8f2 100644 --- a/docs/reference/analysis/tokenfilters/edgengram-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/edgengram-tokenfilter.asciidoc @@ -1,16 +1,244 @@ [[analysis-edgengram-tokenfilter]] -=== Edge NGram Token Filter +=== Edge n-gram token filter +++++ +Edge n-gram +++++ -A token filter of type `edge_ngram`. +Forms an https://en.wikipedia.org/wiki/N-gram[n-gram] of a specified length from +the beginning of a token. -The following are settings that can be set for a `edge_ngram` token -filter type: +For example, you can use the `edge_ngram` token filter to change `quick` to +`qu`. -[cols="<,<",options="header",] -|====================================================== -|Setting |Description -|`min_gram` |Defaults to `1`. -|`max_gram` |Defaults to `2`. -|`side` |deprecated. Either `front` or `back`. Defaults to `front`. -|====================================================== +When not customized, the filter creates 1-character edge n-grams by default. +This filter uses Lucene's +https://lucene.apache.org/core/{lucene_version_path}/analyzers-common/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.html[EdgeNGramTokenFilter]. + +[NOTE] +==== +The `edge_ngram` filter is similar to the <>. However, the `edge_ngram` only outputs n-grams that start at the +beginning of a token. These edge n-grams are useful for +<> queries. +==== + +[[analysis-edgengram-tokenfilter-analyze-ex]] +==== Example + +The following <> request uses the `edge_ngram` +filter to convert `the quick brown fox jumps` to 1-character and 2-character +edge n-grams: + +[source,console] +-------------------------------------------------- +GET _analyze +{ + "tokenizer": "standard", + "filter": [ + { "type": "edge_ngram", + "min_gram": 1, + "max_gram": 2 + } + ], + "text": "the quick brown fox jumps" +} +-------------------------------------------------- + +The filter produces the following tokens: + +[source,text] +-------------------------------------------------- +[ t, th, q, ui, b, br, f, fo, j, ju ] +-------------------------------------------------- + +///////////////////// +[source,console-result] +-------------------------------------------------- +{ + "tokens" : [ + { + "token" : "t", + "start_offset" : 0, + "end_offset" : 3, + "type" : "", + "position" : 0 + }, + { + "token" : "th", + "start_offset" : 0, + "end_offset" : 3, + "type" : "", + "position" : 0 + }, + { + "token" : "q", + "start_offset" : 4, + "end_offset" : 9, + "type" : "", + "position" : 1 + }, + { + "token" : "qu", + "start_offset" : 4, + "end_offset" : 9, + "type" : "", + "position" : 1 + }, + { + "token" : "b", + "start_offset" : 10, + "end_offset" : 15, + "type" : "", + "position" : 2 + }, + { + "token" : "br", + "start_offset" : 10, + "end_offset" : 15, + "type" : "", + "position" : 2 + }, + { + "token" : "f", + "start_offset" : 16, + "end_offset" : 19, + "type" : "", + "position" : 3 + }, + { + "token" : "fo", + "start_offset" : 16, + "end_offset" : 19, + "type" : "", + "position" : 3 + }, + { + "token" : "j", + "start_offset" : 20, + "end_offset" : 25, + "type" : "", + "position" : 4 + }, + { + "token" : "ju", + "start_offset" : 20, + "end_offset" : 25, + "type" : "", + "position" : 4 + } + ] +} +-------------------------------------------------- +///////////////////// + +[[analysis-edgengram-tokenfilter-analyzer-ex]] +==== Add to an analyzer + +The following <> request uses the +`edge_ngram` filter to configure a new +<>. + +[source,console] +-------------------------------------------------- +PUT edge_ngram_example +{ + "settings": { + "analysis": { + "analyzer": { + "standard_edge_ngram": { + "tokenizer": "standard", + "filter": [ "edge_ngram" ] + } + } + } + } +} +-------------------------------------------------- + +[[analysis-edgengram-tokenfilter-configure-parms]] +==== Configurable parameters + +`max_gram`:: ++ +-- +(Optional, integer) +Maximum character length of a gram. For custom token filters, defaults to `2`. +For the built-in `edge_ngram` filter, defaults to `1`. + +See <>. +-- + +`min_gram`:: +(Optional, integer) +Minimum character length of a gram. Defaults to `1`. + +`side`:: ++ +-- +(Optional, string) +Deprecated. Indicates whether to truncate tokens from the `front` or `back`. +Defaults to `front`. + +Instead of using the `back` value, you can use the +<> token filter before and after the +`edge_ngram` filter to achieve the same results. +-- + +[[analysis-edgengram-tokenfilter-customize]] +==== Customize + +To customize the `edge_ngram` filter, duplicate it to create the basis +for a new custom token filter. You can modify the filter using its configurable +parameters. + +For example, the following request creates a custom `edge_ngram` +filter that forms n-grams between 3-5 characters. + +[source,console] +-------------------------------------------------- +PUT edge_ngram_custom_example +{ + "settings": { + "analysis": { + "analyzer": { + "default": { + "tokenizer": "whitespace", + "filter": [ "3_5_edgegrams" ] + } + }, + "filter": { + "3_5_edgegrams": { + "type": "edge_ngram", + "min_gram": 3, + "max_gram": 5 + } + } + } + } +} +-------------------------------------------------- + +[[analysis-edgengram-tokenfilter-max-gram-limits]] +==== Limitations of the `max_gram` parameter + +The `edge_ngram` filter's `max_gram` value limits the character length of +tokens. When the `edge_ngram` filter is used with an index analyzer, this +means search terms longer than the `max_gram` length may not match any indexed +terms. + +For example, if the `max_gram` is `3`, searches for `apple` won't match the +indexed term `app`. + +To account for this, you can use the +<> filter with a search analyzer +to shorten search terms to the `max_gram` character length. However, this could +return irrelevant results. + +For example, if the `max_gram` is `3` and search terms are truncated to three +characters, the search term `apple` is shortened to `app`. This means searches +for `apple` return any indexed terms matching `app`, such as `apply`, `snapped`, +and `apple`. + +We recommend testing both approaches to see which best fits your +use case and desired search experience. diff --git a/docs/reference/analysis/tokenfilters/ngram-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/ngram-tokenfilter.asciidoc index 53bda23d12bf9..7bdc913c5793b 100644 --- a/docs/reference/analysis/tokenfilters/ngram-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/ngram-tokenfilter.asciidoc @@ -1,18 +1,228 @@ [[analysis-ngram-tokenfilter]] -=== NGram Token Filter +=== N-gram token filter +++++ +N-gram +++++ -A token filter of type `ngram`. +Forms https://en.wikipedia.org/wiki/N-gram[n-grams] of specified lengths from +a token. -The following are settings that can be set for a `ngram` token filter -type: +For example, you can use the `ngram` token filter to change `fox` to +`[ f, fo, o, ox, x ]`. -[cols="<,<",options="header",] -|============================ -|Setting |Description -|`min_gram` |Defaults to `1`. -|`max_gram` |Defaults to `2`. -|============================ +This filter uses Lucene's +https://lucene.apache.org/core/{lucene_version_path}/analyzers-common/org/apache/lucene/analysis/ngram/NGramTokenFilter.html[NGramTokenFilter]. -The index level setting `index.max_ngram_diff` controls the maximum allowed -difference between `max_gram` and `min_gram`. +[NOTE] +==== +The `ngram` filter is similar to the +<>. However, the +`edge_ngram` only outputs n-grams that start at the beginning of a token. +==== +[[analysis-ngram-tokenfilter-analyze-ex]] +==== Example + +The following <> request uses the `ngram` +filter to convert `Quick fox` to 1-character and 2-character n-grams: + +[source,console] +-------------------------------------------------- +GET _analyze +{ + "tokenizer": "standard", + "filter": [ "ngram" ], + "text": "Quick fox" +} +-------------------------------------------------- + +The filter produces the following tokens: + +[source,text] +-------------------------------------------------- +[ Q, Qu, u, ui, i, ic, c, ck, k, f, fo, o, ox, x ] +-------------------------------------------------- + +///////////////////// +[source,console-result] +-------------------------------------------------- +{ + "tokens" : [ + { + "token" : "Q", + "start_offset" : 0, + "end_offset" : 5, + "type" : "", + "position" : 0 + }, + { + "token" : "Qu", + "start_offset" : 0, + "end_offset" : 5, + "type" : "", + "position" : 0 + }, + { + "token" : "u", + "start_offset" : 0, + "end_offset" : 5, + "type" : "", + "position" : 0 + }, + { + "token" : "ui", + "start_offset" : 0, + "end_offset" : 5, + "type" : "", + "position" : 0 + }, + { + "token" : "i", + "start_offset" : 0, + "end_offset" : 5, + "type" : "", + "position" : 0 + }, + { + "token" : "ic", + "start_offset" : 0, + "end_offset" : 5, + "type" : "", + "position" : 0 + }, + { + "token" : "c", + "start_offset" : 0, + "end_offset" : 5, + "type" : "", + "position" : 0 + }, + { + "token" : "ck", + "start_offset" : 0, + "end_offset" : 5, + "type" : "", + "position" : 0 + }, + { + "token" : "k", + "start_offset" : 0, + "end_offset" : 5, + "type" : "", + "position" : 0 + }, + { + "token" : "f", + "start_offset" : 6, + "end_offset" : 9, + "type" : "", + "position" : 1 + }, + { + "token" : "fo", + "start_offset" : 6, + "end_offset" : 9, + "type" : "", + "position" : 1 + }, + { + "token" : "o", + "start_offset" : 6, + "end_offset" : 9, + "type" : "", + "position" : 1 + }, + { + "token" : "ox", + "start_offset" : 6, + "end_offset" : 9, + "type" : "", + "position" : 1 + }, + { + "token" : "x", + "start_offset" : 6, + "end_offset" : 9, + "type" : "", + "position" : 1 + } + ] +} +-------------------------------------------------- +///////////////////// + +[[analysis-ngram-tokenfilter-analyzer-ex]] +==== Add to an analyzer + +The following <> request uses the `ngram` +filter to configure a new <>. + +[source,console] +-------------------------------------------------- +PUT ngram_example +{ + "settings": { + "analysis": { + "analyzer": { + "standard_ngram": { + "tokenizer": "standard", + "filter": [ "ngram" ] + } + } + } + } +} +-------------------------------------------------- + +[[analysis-ngram-tokenfilter-configure-parms]] +==== Configurable parameters + +`max_gram`:: +(Optional, integer) +Maximum length of characters in a gram. Defaults to `2`. + +`min_gram`:: +(Optional, integer) +Minimum length of characters in a gram. Defaults to `1`. + +You can use the <> index-level +setting to control the maximum allowed difference between the `max_gram` and +`min_gram` values. + +[[analysis-ngram-tokenfilter-customize]] +==== Customize + +To customize the `ngram` filter, duplicate it to create the basis for a new +custom token filter. You can modify the filter using its configurable +parameters. + +For example, the following request creates a custom `ngram` filter that forms +n-grams between 3-5 characters. The request also increases the +`index.max_ngram_diff` setting to `2`. + +[source,console] +-------------------------------------------------- +PUT ngram_custom_example +{ + "settings": { + "index": { + "max_ngram_diff": 2 + }, + "analysis": { + "analyzer": { + "default": { + "tokenizer": "whitespace", + "filter": [ "3_5_grams" ] + } + }, + "filter": { + "3_5_grams": { + "type": "ngram", + "min_gram": 3, + "max_gram": 5 + } + } + } + } +} +-------------------------------------------------- diff --git a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc index b6a67dd8926ed..eb8ac56844e99 100644 --- a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc +++ b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc @@ -1,5 +1,5 @@ [[analysis-edgengram-tokenizer]] -=== Edge NGram Tokenizer +=== Edge n-gram tokenizer The `edge_ngram` tokenizer first breaks text down into words whenever it encounters one of a list of specified characters, then it emits @@ -116,9 +116,10 @@ terms. For example, if the `max_gram` is `3`, searches for `apple` won't match the indexed term `app`. -To account for this, you can use the <> token filter with a search analyzer to shorten search terms to -the `max_gram` character length. However, this could return irrelevant results. +To account for this, you can use the +<> token filter with a search analyzer +to shorten search terms to the `max_gram` character length. However, this could +return irrelevant results. For example, if the `max_gram` is `3` and search terms are truncated to three characters, the search term `apple` is shortened to `app`. This means searches diff --git a/docs/reference/analysis/tokenizers/ngram-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/ngram-tokenizer.asciidoc index a30266d5088ea..3bd69844591f9 100644 --- a/docs/reference/analysis/tokenizers/ngram-tokenizer.asciidoc +++ b/docs/reference/analysis/tokenizers/ngram-tokenizer.asciidoc @@ -1,5 +1,5 @@ [[analysis-ngram-tokenizer]] -=== NGram Tokenizer +=== N-gram tokenizer The `ngram` tokenizer first breaks text down into words whenever it encounters one of a list of specified characters, then it emits diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc index 8ad469682fbf1..d411dbf22b00d 100644 --- a/docs/reference/index-modules.asciidoc +++ b/docs/reference/index-modules.asciidoc @@ -152,6 +152,7 @@ specific index module: The maximum number of `script_fields` that are allowed in a query. Defaults to `32`. +[[index-max-ngram-diff]] `index.max_ngram_diff`:: The maximum allowed difference between min_gram and max_gram for NGramTokenizer and NGramTokenFilter. diff --git a/docs/reference/modules/threadpool.asciidoc b/docs/reference/modules/threadpool.asciidoc index feb244e804670..c69f736feb17c 100644 --- a/docs/reference/modules/threadpool.asciidoc +++ b/docs/reference/modules/threadpool.asciidoc @@ -10,7 +10,7 @@ of discarded. There are several thread pools, but the important ones include: `generic`:: - For generic operations (e.g., background node discovery). + For generic operations (for example, background node discovery). Thread pool type is `scaling`. `search`:: @@ -20,8 +20,9 @@ There are several thread pools, but the important ones include: `1000`. [[search-throttled]]`search_throttled`:: - For count/search/suggest/get operations on `search_throttled indices`. Thread pool type is - `fixed_auto_queue_size` with a size of `1`, and initial queue_size of `100`. + For count/search/suggest/get operations on `search_throttled indices`. + Thread pool type is `fixed_auto_queue_size` with a size of `1`, and initial + queue_size of `100`. `get`:: For get operations. Thread pool type is `fixed` @@ -29,7 +30,8 @@ There are several thread pools, but the important ones include: queue_size of `1000`. `analyze`:: - For analyze requests. Thread pool type is `fixed` with a size of 1, queue size of 16. + For analyze requests. Thread pool type is `fixed` with a size of `1`, queue + size of `16`. `write`:: For single-document index/delete/update and bulk requests. Thread pool type @@ -49,8 +51,9 @@ There are several thread pools, but the important ones include: keep-alive of `5m` and a max of `min(10, (# of available processors)/2)`. `listener`:: - Mainly for java client executing of action when listener threaded is set to true. - Thread pool type is `scaling` with a default max of `min(10, (# of available processors)/2)`. + Mainly for java client executing of action when listener threaded is set to + `true`. Thread pool type is `scaling` with a default max of + `min(10, (# of available processors)/2)`. `fetch_shard_started`:: For listing shard states. @@ -101,8 +104,7 @@ The `fixed` thread pool holds a fixed size of threads to handle the requests with a queue (optionally bounded) for pending requests that have no threads to service them. -The `size` parameter controls the number of threads, and defaults to the -number of cores times 5. +The `size` parameter controls the number of threads. The `queue_size` allows to control the size of the queue of pending requests that have no threads to execute them. By default, it is set to @@ -131,8 +133,7 @@ https://en.wikipedia.org/wiki/Little%27s_law[Little's Law]. These calculations will potentially adjust the `queue_size` up or down by 50 every time `auto_queue_frame_size` operations have been completed. -The `size` parameter controls the number of threads, and defaults to the -number of cores times 5. +The `size` parameter controls the number of threads. The `queue_size` allows to control the initial size of the queue of pending requests that have no threads to execute them. @@ -187,6 +188,7 @@ thread_pool: [float] [[processors]] === Processors setting + The number of processors is automatically detected, and the thread pool settings are automatically set based on it. In some cases it can be useful to override the number of detected processors. This can be done @@ -200,15 +202,14 @@ processors: 2 There are a few use-cases for explicitly overriding the `processors` setting: -. If you are running multiple instances of Elasticsearch on the same -host but want Elasticsearch to size its thread pools as if it only has a -fraction of the CPU, you should override the `processors` setting to the -desired fraction (e.g., if you're running two instances of Elasticsearch -on a 16-core machine, set `processors` to 8). Note that this is an -expert-level use-case and there's a lot more involved than just setting -the `processors` setting as there are other considerations like changing -the number of garbage collector threads, pinning processes to cores, -etc. +. If you are running multiple instances of {es} on the same host but want {es} +to size its thread pools as if it only has a fraction of the CPU, you should +override the `processors` setting to the desired fraction, for example, if +you're running two instances of {es} on a 16-core machine, set `processors` to 8. +Note that this is an expert-level use case and there's a lot more involved +than just setting the `processors` setting as there are other considerations +like changing the number of garbage collector threads, pinning processes to +cores, and so on. . Sometimes the number of processors is wrongly detected and in such cases explicitly setting the `processors` setting will workaround such issues. diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index c985681804ebb..2ea8f6029849d 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -1040,4 +1040,9 @@ See <>. [role="exclude",id="how-security-works"] === How security works -See <>. \ No newline at end of file +See <>. + +[role="exclude",id="rollup-job-config"] +=== Rollup job configuration + +See <>. diff --git a/docs/reference/rollup/apis/put-job.asciidoc b/docs/reference/rollup/apis/put-job.asciidoc index 1816edc8038df..54203dec5bc6b 100644 --- a/docs/reference/rollup/apis/put-job.asciidoc +++ b/docs/reference/rollup/apis/put-job.asciidoc @@ -26,6 +26,14 @@ experimental[] [[rollup-put-job-api-desc]] ==== {api-description-title} +The {rollup-job} configuration contains all the details about how the job should +run, when it indexes documents, and what future queries will be able to execute +against the rollup index. + +There are three main sections to the job configuration: the logistical details +about the job (cron schedule, etc), the fields that are used for grouping, and +what metrics to collect for each group. + Jobs are created in a `STOPPED` state. You can start them with the <>. @@ -33,42 +41,183 @@ Jobs are created in a `STOPPED` state. You can start them with the ==== {api-path-parms-title} ``:: - (Required, string) Identifier for the {rollup-job}. + (Required, string) Identifier for the {rollup-job}. This can be any + alphanumeric string and uniquely identifies the data that is associated with + the {rollup-job}. The ID is persistent; it is stored with the rolled up data. + If you create a job, let it run for a while, then delete the job, the data + that the job rolled up is still be associated with this job ID. You cannot + create a new job with the same ID since that could lead to problems with + mismatched job configurations. [[rollup-put-job-api-request-body]] ==== {api-request-body-title} `cron`:: - (Required, string) A cron string which defines when the {rollup-job} should be executed. + (Required, string) A cron string which defines the intervals when the + {rollup-job} should be executed. When the interval triggers, the indexer + attempts to rollup the data in the index pattern. The cron pattern is + unrelated to the time interval of the data being rolled up. For example, you + may wish to create hourly rollups of your document but to only run the indexer + on a daily basis at midnight, as defined by the cron. The cron pattern is + defined just like a {watcher} cron schedule. +[[rollup-groups-config]] `groups`:: - (Required, object) Defines the grouping fields that are defined for this - {rollup-job}. See <>. + (Required, object) Defines the grouping fields and aggregations that are + defined for this {rollup-job}. These fields will then be available later for + aggregating into buckets. ++ +-- +These aggs and fields can be used in any combination. Think of the `groups` +configuration as defining a set of tools that can later be used in aggregations +to partition the data. Unlike raw data, we have to think ahead to which fields +and aggregations might be used. Rollups provide enough flexibility that you +simply need to determine _which_ fields are needed, not _in what order_ they are +needed. + +There are three types of groupings currently available: +-- + +`date_histogram`::: + (Required, object) A date histogram group aggregates a `date` field into + time-based buckets. This group is *mandatory*; you currently cannot rollup + documents without a timestamp and a `date_histogram` group. The + `date_histogram` group has several parameters: + +`field`:::: + (Required, string) The date field that is to be rolled up. + +`calendar_interval` or `fixed_interval`:::: + (Required, <>) The interval of time buckets to be + generated when rolling up. For example, `60m` produces 60 minute (hourly) + rollups. This follows standard time formatting syntax as used elsewhere in + {es}. The interval defines the _minimum_ interval that can be aggregated only. + If hourly (`60m`) intervals are configured, <> + can execute aggregations with 60m or greater (weekly, monthly, etc) intervals. + So define the interval as the smallest unit that you wish to later query. For + more information about the difference between calendar and fixed time + intervals, see <>. ++ +-- +NOTE: Smaller, more granular intervals take up proportionally more space. + +-- + +`delay`:::: + (Optional,<>) How long to wait before rolling up new + documents. By default, the indexer attempts to roll up all data that is + available. However, it is not uncommon for data to arrive out of order, + sometimes even a few days late. The indexer is unable to deal with data that + arrives after a time-span has been rolled up. That is to say, there is no + provision to update already-existing rollups. ++ +-- +Instead, you should specify a `delay` that matches the longest period of time +you expect out-of-order data to arrive. For example, a `delay` of `1d` +instructs the indexer to roll up documents up to `now - 1d`, which provides +a day of buffer time for out-of-order documents to arrive. +-- + +`time_zone`:::: + (Optional, string) Defines what time_zone the rollup documents are stored as. + Unlike raw data, which can shift timezones on the fly, rolled documents have + to be stored with a specific timezone. By default, rollup documents are stored + in `UTC`. + +`terms`::: + (Optional, object) The terms group can be used on `keyword` or numeric fields + to allow bucketing via the `terms` aggregation at a later point. The indexer + enumerates and stores _all_ values of a field for each time-period. This can + be potentially costly for high-cardinality groups such as IP addresses, + especially if the time-bucket is particularly sparse. ++ +-- +TIP: While it is unlikely that a rollup will ever be larger in size than the raw +data, defining `terms` groups on multiple high-cardinality fields can +effectively reduce the compression of a rollup to a large extent. You should be +judicious which high-cardinality fields are included for that reason. + +The `terms` group has a single parameter: +-- + +`fields`:::: + (Required, string) The set of fields that you wish to collect terms for. This + array can contain fields that are both `keyword` and numerics. Order does not + matter. + +`histogram`::: + (Optional, object) The histogram group aggregates one or more numeric fields + into numeric histogram intervals. ++ +-- +The `histogram` group has a two parameters: +-- + +`fields`:::: + (Required, array) The set of fields that you wish to build histograms for. All fields + specified must be some kind of numeric. Order does not matter. + +`interval`:::: + (Required, integer) The interval of histogram buckets to be generated when + rolling up. For example, a value of `5` creates buckets that are five units + wide (`0-5`, `5-10`, etc). Note that only one interval can be specified in the + `histogram` group, meaning that all fields being grouped via the histogram + must share the same interval. `index_pattern`:: (Required, string) The index or index pattern to roll up. Supports - wildcard-style patterns (`logstash-*`). + wildcard-style patterns (`logstash-*`). The job will + attempt to rollup the entire index or index-pattern. ++ +-- +NOTE: The `index_pattern` cannot be a pattern that would also match the +destination `rollup_index`. For example, the pattern `foo-*` would match the +rollup index `foo-rollup`. This situation would cause problems because the +{rollup-job} would attempt to rollup its own data at runtime. If you attempt to +configure a pattern that matches the `rollup_index`, an exception occurs to +prevent this behavior. + +-- +[[rollup-metrics-config]] `metrics`:: - (Optional, object) Defines the metrics to collect for each grouping tuple. See - <>. + (Optional, object) Defines the metrics to collect for each grouping tuple. + By default, only the doc_counts are collected for each group. To make rollup + useful, you will often add metrics like averages, mins, maxes, etc. Metrics + are defined on a per-field basis and for each field you configure which metric + should be collected. ++ +-- +The `metrics` configuration accepts an array of objects, where each object has +two parameters: +-- + +`field`::: + (Required, string) The field to collect metrics for. This must be a numeric + of some kind. + +`metrics`::: + (Required, array) An array of metrics to collect for the field. At least one + metric must be configured. Acceptable metrics are `min`,`max`,`sum`,`avg`, and + `value_count`. `page_size`:: (Required, integer) The number of bucket results that are processed on each iteration of the rollup indexer. A larger value tends to execute faster, but - requires more memory during processing. + requires more memory during processing. This value has no effect on how the + data is rolled up; it is merely used for tweaking the speed or memory cost of + the indexer. `rollup_index`:: (Required, string) The index that contains the rollup results. The index can - be shared with other {rollup-jobs}. - -For more details about the job configuration, see <>. + be shared with other {rollup-jobs}. The data is stored so that it doesn't + interfere with unrelated jobs. [[rollup-put-job-api-example]] ==== {api-example-title} -The following example creates a {rollup-job} named "sensor", targeting the -"sensor-*" index pattern: +The following example creates a {rollup-job} named `sensor`, targeting the +`sensor-*` index pattern: [source,console] -------------------------------------------------- @@ -78,7 +227,7 @@ PUT _rollup/job/sensor "rollup_index": "sensor_rollup", "cron": "*/30 * * * * ?", "page_size" :1000, - "groups" : { + "groups" : { <1> "date_histogram": { "field": "timestamp", "fixed_interval": "1h", @@ -88,7 +237,7 @@ PUT _rollup/job/sensor "fields": ["node"] } }, - "metrics": [ + "metrics": [ <2> { "field": "temperature", "metrics": ["min", "max", "sum"] @@ -101,6 +250,11 @@ PUT _rollup/job/sensor } -------------------------------------------------- // TEST[setup:sensor_index] +<1> This configuration enables date histograms to be used on the `timestamp` +field and `terms` aggregations to be used on the `node` field. +<2> This configuration defines metrics over two fields: `temperature` and +`voltage`. For the `temperature` field, we are collecting the min, max, and +sum of the temperature. For `voltage`, we are collecting the average. When the job is created, you receive the following results: @@ -109,4 +263,4 @@ When the job is created, you receive the following results: { "acknowledged": true } ----- \ No newline at end of file +---- diff --git a/docs/reference/rollup/apis/rollup-job-config.asciidoc b/docs/reference/rollup/apis/rollup-job-config.asciidoc deleted file mode 100644 index e0ca2b7332092..0000000000000 --- a/docs/reference/rollup/apis/rollup-job-config.asciidoc +++ /dev/null @@ -1,279 +0,0 @@ -[role="xpack"] -[testenv="basic"] -[[rollup-job-config]] -=== Rollup job configuration - -experimental[] - -The Rollup Job Configuration contains all the details about how the rollup job should run, when it indexes documents, -and what future queries will be able to execute against the rollup index. - -There are three main sections to the Job Configuration; the logistical details about the job (cron schedule, etc), what fields -should be grouped on, and what metrics to collect for each group. - -A full job configuration might look like this: - -[source,console] --------------------------------------------------- -PUT _rollup/job/sensor -{ - "index_pattern": "sensor-*", - "rollup_index": "sensor_rollup", - "cron": "*/30 * * * * ?", - "page_size" :1000, - "groups" : { - "date_histogram": { - "field": "timestamp", - "fixed_interval": "60m", - "delay": "7d" - }, - "terms": { - "fields": ["hostname", "datacenter"] - }, - "histogram": { - "fields": ["load", "net_in", "net_out"], - "interval": 5 - } - }, - "metrics": [ - { - "field": "temperature", - "metrics": ["min", "max", "sum"] - }, - { - "field": "voltage", - "metrics": ["avg"] - } - ] -} --------------------------------------------------- -// TEST[setup:sensor_index] - -==== Logistical Details - -In the above example, there are several pieces of logistical configuration for the job itself. - -`{job_id}` (required):: - (string) In the endpoint URL, you specify the name of the job (`sensor` in the above example). This can be any alphanumeric string, - and uniquely identifies the data that is associated with the rollup job. The ID is persistent, in that it is stored with the rolled - up data. So if you create a job, let it run for a while, then delete the job... the data that the job rolled up will still be - associated with this job ID. You will be unable to create a new job with the same ID, as that could lead to problems with mismatched - job configurations - -`index_pattern` (required):: - (string) The index, or index pattern, that you wish to rollup. Supports wildcard-style patterns (`logstash-*`). The job will - attempt to rollup the entire index or index-pattern. Once the "backfill" is finished, it will periodically (as defined by the cron) - look for new data and roll that up too. - -`rollup_index` (required):: - (string) The index that you wish to store rollup results into. All the rollup data that is generated by the job will be - stored in this index. When searching the rollup data, this index will be used in the <> endpoint's URL. - The rollup index can be shared with other rollup jobs. The data is stored so that it doesn't interfere with unrelated jobs. - -`cron` (required):: - (string) A cron string which defines when the rollup job should be executed. The cron string defines an interval of when to run - the job's indexer. When the interval triggers, the indexer will attempt to rollup the data in the index pattern. The cron pattern - is unrelated to the time interval of the data being rolled up. For example, you may wish to create hourly rollups of your document (as - defined in the <>) but to only run the indexer on a daily basis at midnight, as defined by the cron. - The cron pattern is defined just like Watcher's Cron Schedule. - -`page_size` (required):: - (int) The number of bucket results that should be processed on each iteration of the rollup indexer. A larger value - will tend to execute faster, but will require more memory during processing. This has no effect on how the data is rolled up, it is - merely used for tweaking the speed/memory cost of the indexer. - -[NOTE] -The `index_pattern` cannot be a pattern that would also match the destination `rollup_index`. E.g. the pattern -`"foo-*"` would match the rollup index `"foo-rollup"`. This causes problems because the rollup job would attempt -to rollup it's own data at runtime. If you attempt to configure a pattern that matches the `rollup_index`, an exception -will be thrown to prevent this behavior. - -[[rollup-groups-config]] -==== Grouping Config - -The `groups` section of the configuration is where you decide which fields should be grouped on, and with what aggregations. These -fields will then be available later for aggregating into buckets. For example, this configuration: - -[source,js] --------------------------------------------------- -"groups" : { - "date_histogram": { - "field": "timestamp", - "fixed_interval": "60m", - "delay": "7d" - }, - "terms": { - "fields": ["hostname", "datacenter"] - }, - "histogram": { - "fields": ["load", "net_in", "net_out"], - "interval": 5 - } -} --------------------------------------------------- -// NOTCONSOLE - -Allows `date_histogram`'s to be used on the `"timestamp"` field, `terms` aggregations to be used on the `"hostname"` and `"datacenter"` -fields, and `histograms` to be used on any of `"load"`, `"net_in"`, `"net_out"` fields. - -Importantly, these aggs/fields can be used in any combination. Think of the `groups` configuration as defining a set of tools that can -later be used in aggregations to partition the data. Unlike raw data, we have to think ahead to which fields and aggregations might be used. -But Rollups provide enough flexibility that you simply need to determine _which_ fields are needed, not _in what order_ they are needed. - -There are three types of groupings currently available: - -===== Date Histogram - -A `date_histogram` group aggregates a `date` field into time-based buckets. The `date_histogram` group is *mandatory* -- you currently -cannot rollup documents without a timestamp and a `date_histogram` group. - -The `date_histogram` group has several parameters: - -`field` (required):: - The date field that is to be rolled up. - -`interval` (required):: - The interval of time buckets to be generated when rolling up. E.g. `"60m"` will produce 60 minute (hourly) rollups. This follows standard time formatting - syntax as used elsewhere in Elasticsearch. The `interval` defines the _minimum_ interval that can be aggregated only. If hourly (`"60m"`) - intervals are configured, <> can execute aggregations with 60m or greater (weekly, monthly, etc) intervals. - So define the interval as the smallest unit that you wish to later query. - - Note: smaller, more granular intervals take up proportionally more space. - -`delay`:: - How long to wait before rolling up new documents. By default, the indexer attempts to roll up all data that is available. However, it - is not uncommon for data to arrive out of order, sometimes even a few days late. The indexer is unable to deal with data that arrives - after a time-span has been rolled up (e.g. there is no provision to update already-existing rollups). - - Instead, you should specify a `delay` that matches the longest period of time you expect out-of-order data to arrive. E.g. a `delay` of - `"1d"` will instruct the indexer to roll up documents up to `"now - 1d"`, which provides a day of buffer time for out-of-order documents - to arrive. - -`time_zone`:: - Defines what time_zone the rollup documents are stored as. Unlike raw data, which can shift timezones on the fly, rolled documents have - to be stored with a specific timezone. By default, rollup documents are stored in `UTC`, but this can be changed with the `time_zone` - parameter. - -.Calendar vs Fixed time intervals -********************************** -Elasticsearch understands both "calendar" and "fixed" time intervals. Fixed time intervals are fairly easy to understand; -`"60s"` means sixty seconds. But what does `"1M` mean? One month of time depends on which month we are talking about, -some months are longer or shorter than others. This is an example of "calendar" time, and the duration of that unit -depends on context. Calendar units are also affected by leap-seconds, leap-years, etc. - -This is important because the buckets generated by Rollup will be in either calendar or fixed intervals, and will limit -how you can query them later (see <>. - -We recommend sticking with "fixed" time intervals, since they are easier to understand and are more flexible at query -time. It will introduce some drift in your data during leap-events, and you will have to think about months in a fixed -quantity (30 days) instead of the actual calendar length... but it is often easier than dealing with calendar units -at query time. - -Multiples of units are always "fixed" (e.g. `"2h"` is always the fixed quantity `7200` seconds. Single units can be -fixed or calendar depending on the unit: - -[options="header"] -|======= -|Unit |Calendar |Fixed -|millisecond |NA |`1ms`, `10ms`, etc -|second |NA |`1s`, `10s`, etc -|minute |`1m` |`2m`, `10m`, etc -|hour |`1h` |`2h`, `10h`, etc -|day |`1d` |`2d`, `10d`, etc -|week |`1w` |NA -|month |`1M` |NA -|quarter |`1q` |NA -|year |`1y` |NA -|======= - -For some units where there are both fixed and calendar, you may need to express the quantity in terms of the next -smaller unit. For example, if you want a fixed day (not a calendar day), you should specify `24h` instead of `1d`. -Similarly, if you want fixed hours, specify `60m` instead of `1h`. This is because the single quantity entails -calendar time, and limits you to querying by calendar time in the future. - - -********************************** - -===== Terms - -The `terms` group can be used on `keyword` or numeric fields, to allow bucketing via the `terms` aggregation at a later point. The `terms` -group is optional. If defined, the indexer will enumerate and store _all_ values of a field for each time-period. This can be potentially -costly for high-cardinality groups such as IP addresses, especially if the time-bucket is particularly sparse. - -While it is unlikely that a rollup will ever be larger in size than the raw data, defining `terms` groups on multiple high-cardinality fields -can effectively reduce the compression of a rollup to a large extent. You should be judicious which high-cardinality fields are included -for that reason. - -The `terms` group has a single parameter: - -`fields` (required):: - The set of fields that you wish to collect terms for. This array can contain fields that are both `keyword` and numerics. Order - does not matter - - -===== Histogram - -The `histogram` group aggregates one or more numeric fields into numeric histogram intervals. This group is optional - - -The `histogram` group has a two parameters: - -`fields` (required):: - The set of fields that you wish to build histograms for. All fields specified must be some kind of numeric. Order does not matter - -`interval` (required):: - The interval of histogram buckets to be generated when rolling up. E.g. `5` will create buckets that are five units wide - (`0-5`, `5-10`, etc). Note that only one interval can be specified in the `histogram` group, meaning that all fields being grouped via - the histogram must share the same interval. - -[[rollup-metrics-config]] -==== Metrics Config - -After defining which groups should be generated for the data, you next configure which metrics should be collected. By default, only -the doc_counts are collected for each group. To make rollup useful, you will often add metrics like averages, mins, maxes, etc. - -Metrics are defined on a per-field basis, and for each field you configure which metric should be collected. For example: - -[source,js] --------------------------------------------------- -"metrics": [ - { - "field": "temperature", - "metrics": ["min", "max", "sum"] - }, - { - "field": "voltage", - "metrics": ["avg"] - } -] --------------------------------------------------- -// NOTCONSOLE - -This configuration defines metrics over two fields, `"temperature` and `"voltage"`. For the `"temperature"` field, we are collecting -the min, max and sum of the temperature. For `"voltage"`, we are collecting the average. These metrics are collected in a way that makes -them compatible with any combination of defined groups. - -The `metrics` configuration accepts an array of objects, where each object has two parameters: - -`field` (required):: - The field to collect metrics for. This must be a numeric of some kind - -`metrics` (required):: - An array of metrics to collect for the field. At least one metric must be configured. Acceptable metrics are min/max/sum/avg/value_count. - - - -.Averages aren't composable?! -********************************** -If you've worked with rollups before, you may be cautious around averages. If an average is saved for a 10 minute -interval, it usually isn't useful for larger intervals. You cannot average six 10-minute averages to find a -hourly average (average of averages is not equal to the total average). - -For this reason, other systems tend to either omit the ability to average, or store the average at multiple intervals -to support more flexible querying. - -Instead, the Rollup feature saves the `count` and `sum` for the defined time interval. This allows us to reconstruct -the average at any interval greater-than or equal to the defined interval. This gives maximum flexibility for -minimal storage costs... and you don't have to worry about average accuracies (no average of averages here!) -********************************** - diff --git a/docs/reference/rollup/rollup-api.asciidoc b/docs/reference/rollup/rollup-api.asciidoc index c156265c2ffc3..9e56c5f15847b 100644 --- a/docs/reference/rollup/rollup-api.asciidoc +++ b/docs/reference/rollup/rollup-api.asciidoc @@ -10,7 +10,6 @@ * <> or <> * <> or <> * <> -* <> [float] [[rollup-data-endpoint]] @@ -32,6 +31,5 @@ include::apis/get-job.asciidoc[] include::apis/rollup-caps.asciidoc[] include::apis/rollup-index-caps.asciidoc[] include::apis/rollup-search.asciidoc[] -include::apis/rollup-job-config.asciidoc[] include::apis/start-job.asciidoc[] include::apis/stop-job.asciidoc[] \ No newline at end of file diff --git a/docs/reference/rollup/rollup-getting-started.asciidoc b/docs/reference/rollup/rollup-getting-started.asciidoc index 3b57e968a9e5e..740a921713dc4 100644 --- a/docs/reference/rollup/rollup-getting-started.asciidoc +++ b/docs/reference/rollup/rollup-getting-started.asciidoc @@ -72,12 +72,11 @@ seconds worth of data that was indexed into the `sensor-*` indices. If instead the cron was configured to run once a day at midnight, the job would process the last 24 hours worth of data. The choice is largely preference, based on how "realtime" you want the rollups, and if you wish to process continuously or move it to off-peak hours. -Next, we define a set of `groups` and `metrics`. The metrics are fairly straightforward: we want to save the min/max/sum of the `temperature` -field, and the average of the `voltage` field. - -The groups are a little more interesting. Essentially, we are defining the dimensions that we wish to pivot on at a later date when -querying the data. The grouping in this job allows us to use date_histograms aggregations on the `timestamp` field, rolled up at hourly intervals. -It also allows us to run terms aggregations on the `node` field. +Next, we define a set of `groups`. Essentially, we are defining the dimensions +that we wish to pivot on at a later date when querying the data. The grouping in +this job allows us to use `date_histogram` aggregations on the `timestamp` field, +rolled up at hourly intervals. It also allows us to run terms aggregations on +the `node` field. .Date histogram interval vs cron schedule ********************************** @@ -93,8 +92,31 @@ simply go back to sleep. But there's nothing wrong with it either, the job will ********************************** -For more details about the job syntax, see <>. +After defining which groups should be generated for the data, you next configure +which metrics should be collected. By default, only the `doc_counts` are +collected for each group. To make rollup useful, you will often add metrics +like averages, mins, maxes, etc. In this example, the metrics are fairly +straightforward: we want to save the min/max/sum of the `temperature` +field, and the average of the `voltage` field. + +.Averages aren't composable?! +********************************** +If you've worked with rollups before, you may be cautious around averages. If an +average is saved for a 10 minute interval, it usually isn't useful for larger +intervals. You cannot average six 10-minute averages to find a hourly average; +the average of averages is not equal to the total average. + +For this reason, other systems tend to either omit the ability to average or +store the average at multiple intervals to support more flexible querying. + +Instead, the {rollup-features} save the `count` and `sum` for the defined time +interval. This allows us to reconstruct the average at any interval greater-than +or equal to the defined interval. This gives maximum flexibility for minimal +storage costs... and you don't have to worry about average accuracies (no +average of averages here!) +********************************** +For more details about the job syntax, see <>. After you execute the above command and create the job, you'll receive the following response: diff --git a/docs/reference/rollup/understanding-groups.asciidoc b/docs/reference/rollup/understanding-groups.asciidoc index eb1b47e8a16d0..8d0aadb9778d8 100644 --- a/docs/reference/rollup/understanding-groups.asciidoc +++ b/docs/reference/rollup/understanding-groups.asciidoc @@ -119,7 +119,54 @@ Rollup Search to execute: Ultimately, when configuring `groups` for a job, think in terms of how you might wish to partition data in a query at a future date... then include those in the config. Because Rollup Search allows any order or combination of the grouped fields, you just need to decide -if a field is useful for aggregating later, and how you might wish to use it (terms, histogram, etc) +if a field is useful for aggregating later, and how you might wish to use it (terms, histogram, etc). + +[[rollup-understanding-group-intervals]] +==== Calendar vs fixed time intervals + +Each rollup-job must have a date histogram group with a defined interval. {es} +understands both +<>. Fixed time +intervals are fairly easy to understand; `60s` means sixty seconds. But what +does `1M` mean? One month of time depends on which month we are talking about, +some months are longer or shorter than others. This is an example of calendar +time and the duration of that unit depends on context. Calendar units are also +affected by leap-seconds, leap-years, etc. + +This is important because the buckets generated by rollup are in either calendar +or fixed intervals and this limits how you can query them later. See +<>. + +We recommend sticking with fixed time intervals, since they are easier to +understand and are more flexible at query time. It will introduce some drift in +your data during leap-events and you will have to think about months in a fixed +quantity (30 days) instead of the actual calendar length. However, it is often +easier than dealing with calendar units at query time. + +Multiples of units are always "fixed". For example, `2h` is always the fixed +quantity `7200` seconds. Single units can be fixed or calendar depending on the +unit: + +[options="header"] +|======= +|Unit |Calendar |Fixed +|millisecond |NA |`1ms`, `10ms`, etc +|second |NA |`1s`, `10s`, etc +|minute |`1m` |`2m`, `10m`, etc +|hour |`1h` |`2h`, `10h`, etc +|day |`1d` |`2d`, `10d`, etc +|week |`1w` |NA +|month |`1M` |NA +|quarter |`1q` |NA +|year |`1y` |NA +|======= + +For some units where there are both fixed and calendar, you may need to express +the quantity in terms of the next smaller unit. For example, if you want a fixed +day (not a calendar day), you should specify `24h` instead of `1d`. Similarly, +if you want fixed hours, specify `60m` instead of `1h`. This is because the +single quantity entails calendar time, and limits you to querying by calendar +time in the future. ==== Grouping limitations with heterogeneous indices diff --git a/docs/reference/transform/overview.asciidoc b/docs/reference/transform/overview.asciidoc index e3c852d8be941..50930b00c5cb3 100644 --- a/docs/reference/transform/overview.asciidoc +++ b/docs/reference/transform/overview.asciidoc @@ -7,22 +7,19 @@ beta[] -A _{dataframe}_ is a two-dimensional tabular data structure. In the context of -the {stack}, it is a transformation of data that is indexed in {es}. For -example, you can use {dataframes} to _pivot_ your data into a new entity-centric -index. By transforming and summarizing your data, it becomes possible to -visualize and analyze it in alternative and interesting ways. +You can use {transforms} to _pivot_ your data into a new entity-centric index. +By transforming and summarizing your data, it becomes possible to visualize and +analyze it in alternative and interesting ways. A lot of {es} indices are organized as a stream of events: each event is an -individual document, for example a single item purchase. {dataframes-cap} enable +individual document, for example a single item purchase. {transforms-cap} enable you to summarize this data, bringing it into an organized, more analysis-friendly format. For example, you can summarize all the purchases of a single customer. -You can create {dataframes} by using {transforms}. {transforms-cap} enable you to define a pivot, which is a set of features that transform the index into a different, more digestible format. -Pivoting results in a summary of your data, which is the {dataframe}. +Pivoting results in a summary of your data in a new index. To define a pivot, first you select one or more fields that you will use to group your data. You can select categorical fields (terms) and numerical fields @@ -38,34 +35,32 @@ more about the supported aggregations and group-by fields, see As an optional step, you can also add a query to further limit the scope of the aggregation. -The {transform} performs a composite aggregation that -paginates through all the data defined by the source index query. The output of -the aggregation is stored in a destination index. Each time the -{transform} queries the source index, it creates a _checkpoint_. You -can decide whether you want the {transform} to run once (batch -{transform}) or continuously ({transform}). A batch -{transform} is a single operation that has a single checkpoint. -{ctransforms-cap} continually increment and process checkpoints as new -source data is ingested. +The {transform} performs a composite aggregation that paginates through all the +data defined by the source index query. The output of the aggregation is stored +in a destination index. Each time the {transform} queries the source index, it +creates a _checkpoint_. You can decide whether you want the {transform} to run +once (batch {transform}) or continuously ({transform}). A batch {transform} is a +single operation that has a single checkpoint. {ctransforms-cap} continually +increment and process checkpoints as new source data is ingested. .Example -Imagine that you run a webshop that sells clothes. Every order creates a document -that contains a unique order ID, the name and the category of the ordered product, -its price, the ordered quantity, the exact date of the order, and some customer -information (name, gender, location, etc). Your dataset contains all the transactions -from last year. +Imagine that you run a webshop that sells clothes. Every order creates a +document that contains a unique order ID, the name and the category of the +ordered product, its price, the ordered quantity, the exact date of the order, +and some customer information (name, gender, location, etc). Your dataset +contains all the transactions from last year. If you want to check the sales in the different categories in your last fiscal -year, define a {transform} that groups the data by the product -categories (women's shoes, men's clothing, etc.) and the order date. Use the -last year as the interval for the order date. Then add a sum aggregation on the -ordered quantity. The result is a {dataframe} that shows the number of sold +year, define a {transform} that groups the data by the product categories +(women's shoes, men's clothing, etc.) and the order date. Use the last year as +the interval for the order date. Then add a sum aggregation on the ordered +quantity. The result is an entity-centric index that shows the number of sold items in every product category in the last year. [role="screenshot"] image::images/ml-dataframepivot.jpg["Example of a data frame pivot in {kib}"] IMPORTANT: The {transform} leaves your source index intact. It -creates a new index that is dedicated to the {dataframe}. +creates a new index that is dedicated to the transformed data. diff --git a/gradle/runtime-jdk-provision.gradle b/gradle/runtime-jdk-provision.gradle index e67d1d44f507c..e05da2d47c5a6 100644 --- a/gradle/runtime-jdk-provision.gradle +++ b/gradle/runtime-jdk-provision.gradle @@ -12,7 +12,7 @@ jdks { } } -allprojects { +configure(allprojects - project(':build-tools')) { project.tasks.withType(Test).configureEach { Test test -> if (BuildParams.getIsRuntimeJavaHomeSet()) { test.executable = "${BuildParams.runtimeJavaHome}/bin/java" @@ -21,4 +21,4 @@ allprojects { test.executable = rootProject.jdks.provisioned_runtime.getBinJavaPath() } } -} \ No newline at end of file +} diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index 5371107875035..f9d8cb28f6c61 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -174,8 +174,6 @@ public List> getContexts() { public Map>> getAnalyzers() { Map>> analyzers = new TreeMap<>(); analyzers.put("fingerprint", FingerprintAnalyzerProvider::new); - - // TODO remove in 8.0 analyzers.put("pattern", PatternAnalyzerProvider::new); analyzers.put("snowball", SnowballAnalyzerProvider::new); diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index 72d0fc973b79f..b99bb66ee8dba 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -32,7 +32,7 @@ dependencies { compile("com.fasterxml.jackson.core:jackson-databind:2.8.11.3") compile('com.maxmind.db:maxmind-db:1.2.2') - testCompile 'org.elasticsearch:geolite2-databases:20180911' + testCompile 'org.elasticsearch:geolite2-databases:20191119' } task copyDefaultGeoIp2DatabaseFiles(type: Copy) { diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index e3fe54d2f1127..b136fbae0376a 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -55,11 +55,12 @@ public void testCity() throws Exception { assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("8.8.8.8")); @SuppressWarnings("unchecked") Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(5)); + assertThat(geoData.size(), equalTo(6)); assertThat(geoData.get("ip"), equalTo("8.8.8.8")); assertThat(geoData.get("country_iso_code"), equalTo("US")); assertThat(geoData.get("country_name"), equalTo("United States")); assertThat(geoData.get("continent_name"), equalTo("North America")); + assertThat(geoData.get("timezone"), equalTo("America/Chicago")); Map location = new HashMap<>(); location.put("lat", 37.751d); location.put("lon", -97.822d); @@ -129,11 +130,11 @@ public void testCity_withIpV6() throws Exception { assertThat(geoData.get("continent_name"), equalTo("North America")); assertThat(geoData.get("region_iso_code"), equalTo("US-FL")); assertThat(geoData.get("region_name"), equalTo("Florida")); - assertThat(geoData.get("city_name"), equalTo("Hollywood")); + assertThat(geoData.get("city_name"), equalTo("Homestead")); assertThat(geoData.get("timezone"), equalTo("America/New_York")); Map location = new HashMap<>(); - location.put("lat", 25.9825d); - location.put("lon", -80.3434d); + location.put("lat", 25.4573d); + location.put("lon", -80.4572d); assertThat(geoData.get("location"), equalTo(location)); } diff --git a/modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml b/modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml index 95c826eee30de..27ab1f4e8747d 100644 --- a/modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml +++ b/modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml @@ -31,8 +31,8 @@ - length: { _source.geoip: 6 } - match: { _source.geoip.city_name: "Minneapolis" } - match: { _source.geoip.country_iso_code: "US" } - - match: { _source.geoip.location.lon: -93.2323 } - - match: { _source.geoip.location.lat: 44.9733 } + - match: { _source.geoip.location.lon: -93.2548 } + - match: { _source.geoip.location.lat: 44.9399 } - match: { _source.geoip.region_iso_code: "US-MN" } - match: { _source.geoip.region_name: "Minnesota" } - match: { _source.geoip.continent_name: "North America" } @@ -76,8 +76,8 @@ - match: { _source.geoip.city_name: "Minneapolis" } - match: { _source.geoip.country_iso_code: "US" } - match: { _source.geoip.ip: "128.101.101.101" } - - match: { _source.geoip.location.lon: -93.2323 } - - match: { _source.geoip.location.lat: 44.9733 } + - match: { _source.geoip.location.lon: -93.2548 } + - match: { _source.geoip.location.lat: 44.9399 } - match: { _source.geoip.timezone: "America/Chicago" } - match: { _source.geoip.country_name: "United States" } - match: { _source.geoip.region_iso_code: "US-MN" } @@ -181,8 +181,8 @@ - length: { _source.geoip: 6 } - match: { _source.geoip.city_name: "Minneapolis" } - match: { _source.geoip.country_iso_code: "US" } - - match: { _source.geoip.location.lon: -93.2323 } - - match: { _source.geoip.location.lat: 44.9733 } + - match: { _source.geoip.location.lon: -93.2548 } + - match: { _source.geoip.location.lat: 44.9399 } - match: { _source.geoip.region_iso_code: "US-MN" } - match: { _source.geoip.region_name: "Minnesota" } - match: { _source.geoip.continent_name: "North America" } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java index e89a07974a7d4..6dd989ffafdb5 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java @@ -117,6 +117,7 @@ private static SSLContext buildServerSslContext() throws Exception { } public void testClientFailsWithUntrustedCertificate() throws IOException { + assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm()); final List threads = new ArrayList<>(); final Settings settings = Settings.builder() .put("path.home", createTempDir()) diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index 157bcc9714b95..d7b4d56569970 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -112,6 +112,7 @@ thirdPartyAudit { 'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder', 'org.bouncycastle.jce.provider.BouncyCastleProvider', 'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder', + 'org.bouncycastle.asn1.x500.X500Name', // from io.netty.handler.ssl.JettyNpnSslEngine (netty) 'org.eclipse.jetty.npn.NextProtoNego$ClientProvider', @@ -168,7 +169,6 @@ thirdPartyAudit { 'org.eclipse.jetty.alpn.ALPN$ServerProvider', 'org.eclipse.jetty.alpn.ALPN', - 'org.conscrypt.AllocatedBuffer', 'org.conscrypt.BufferAllocator', 'org.conscrypt.Conscrypt', @@ -196,12 +196,10 @@ thirdPartyAudit { ) } -rootProject.globalInfo.ready { - if (BuildParams.inFipsJvm == false) { - // BouncyCastleFIPS provides this class, so the exclusion is invalid when running CI in - // a FIPS JVM with BouncyCastleFIPS Provider - thirdPartyAudit.ignoreMissingClasses( - 'org.bouncycastle.asn1.x500.X500Name' - ) - } +if (BuildParams.inFipsJvm == false) { + // BouncyCastleFIPS provides this class, so the exclusion is invalid when running CI in + // a FIPS JVM with BouncyCastleFIPS Provider + thirdPartyAudit.ignoreMissingClasses( + 'org.bouncycastle.asn1.x500.X500Name' + ) } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoder.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoder.java index 7ef8bfcf59a3b..7d95d44d8d629 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoder.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoder.java @@ -32,6 +32,10 @@ final class Netty4SizeHeaderFrameDecoder extends ByteToMessageDecoder { private static final int HEADER_SIZE = TcpHeader.MARKER_BYTES_SIZE + TcpHeader.MESSAGE_LENGTH_SIZE; + { + setCumulator(COMPOSITE_CUMULATOR); + } + @Override protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception { try { diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle index 771d21765218d..292f8c7595ed2 100644 --- a/plugins/ingest-attachment/build.gradle +++ b/plugins/ingest-attachment/build.gradle @@ -89,7 +89,7 @@ thirdPartyAudit { } thirdPartyAudit.onlyIf { - // FIPS JVM includes manny classes from bouncycastle which count as jar hell for the third party audit, + // FIPS JVM includes many classes from bouncycastle which count as jar hell for the third party audit, // rather than provide a long list of exclusions, disable the check on FIPS. BuildParams.inFipsJvm == false } diff --git a/plugins/transport-nio/build.gradle b/plugins/transport-nio/build.gradle index 368d80e9dad76..5ee0295fc9fed 100644 --- a/plugins/transport-nio/build.gradle +++ b/plugins/transport-nio/build.gradle @@ -64,6 +64,7 @@ thirdPartyAudit { 'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder', 'org.bouncycastle.jce.provider.BouncyCastleProvider', 'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder', + 'org.bouncycastle.asn1.x500.X500Name', // from io.netty.handler.ssl.JettyNpnSslEngine (netty) 'org.eclipse.jetty.npn.NextProtoNego$ClientProvider', @@ -154,12 +155,12 @@ thirdPartyAudit { 'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator' ) } -rootProject.globalInfo.ready { - if (BuildParams.inFipsJvm == false) { - // BouncyCastleFIPS provides this class, so the exclusion is invalid when running CI in - // a FIPS JVM with BouncyCastleFIPS Provider - thirdPartyAudit.ignoreMissingClasses( - 'org.bouncycastle.asn1.x500.X500Name' - ) - } + +if (BuildParams.inFipsJvm == false) { + // BouncyCastleFIPS provides this class, so the exclusion is invalid when running CI in + // a FIPS JVM with BouncyCastleFIPS Provider + thirdPartyAudit.ignoreMissingClasses( + 'org.bouncycastle.asn1.x500.X500Name' + ) } + diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java index 38080740ecc2e..065e77a239bd5 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java @@ -49,6 +49,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class Packages { @@ -301,8 +302,22 @@ public static void startElasticsearchIgnoringFailure(Shell sh) { */ public static void clearJournal(Shell sh) { if (isSystemd()) { - sh.run("rm -rf /run/log/journal/"); - sh.run("systemctl restart systemd-journald"); + sh.run("rm -rf /run/log/journal/*"); + final Result result = sh.runIgnoreExitCode("systemctl restart systemd-journald"); + + // Sometimes the restart fails on Debian 10 with: + // Job for systemd-journald.service failed because the control process exited with error code. + // See "systemctl status systemd-journald.service" and "journalctl -xe" for details.] + // + // ...so run these commands in an attempt to figure out what's going on. + if (result.isSuccess() == false) { + logger.error("Failed to restart systemd-journald: " + result); + + logger.error(sh.runIgnoreExitCode("systemctl status systemd-journald.service")); + logger.error(sh.runIgnoreExitCode("journalctl -xe")); + + fail("Couldn't clear the systemd journal as restarting systemd-journald failed"); + } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json index b2b0741203be0..c95f5ea3b48f3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json @@ -8,7 +8,7 @@ "url":{ "paths":[ { - "path":"{index}/_mapping", + "path":"/{index}/_mapping", "methods":[ "PUT", "POST" diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 9098f280da708..10dcf6943f867 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -218,8 +218,6 @@ import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards; import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction; import org.elasticsearch.index.seqno.RetentionLeaseActions; -import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction; -import org.elasticsearch.index.seqno.RetentionLeaseSyncAction; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData; import org.elasticsearch.persistent.CompletionPersistentTaskAction; @@ -548,8 +546,6 @@ public void reg // internal actions actions.register(GlobalCheckpointSyncAction.TYPE, GlobalCheckpointSyncAction.class); - actions.register(RetentionLeaseBackgroundSyncAction.TYPE, RetentionLeaseBackgroundSyncAction.class); - actions.register(RetentionLeaseSyncAction.TYPE, RetentionLeaseSyncAction.class); actions.register(TransportNodesSnapshotsStatus.TYPE, TransportNodesSnapshotsStatus.class); actions.register(TransportNodesListGatewayMetaState.TYPE, TransportNodesListGatewayMetaState.class); actions.register(TransportVerifyShardBeforeCloseAction.TYPE, TransportVerifyShardBeforeCloseAction.class); diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java index 18051aa99db4f..2d2b9213c63ff 100644 --- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java @@ -113,8 +113,8 @@ abstract class AbstractSearchAsyncAction exten iterators.add(iterator); } } - this.toSkipShardsIts = new GroupShardsIterator<>(toSkipIterators); - this.shardsIts = new GroupShardsIterator<>(iterators); + this.toSkipShardsIts = new GroupShardsIterator<>(toSkipIterators, false); + this.shardsIts = new GroupShardsIterator<>(iterators, false); // we need to add 1 for non active partition, since we count it in the total. This means for each shard in the iterator we sum up // it's number of active shards but use 1 as the default if no replica of a shard is active at this point. // on a per shards level we use shardIt.remaining() to increment the totalOps pointer but add 1 for the current shard result diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java index 66187d5220bbe..aba32d2c850a0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java @@ -23,15 +23,25 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.search.SearchService; +import org.elasticsearch.search.SearchService.CanMatchResponse; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.MinAndMax; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.transport.Transport; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.Executor; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; /** @@ -40,8 +50,12 @@ * from the search. The extra round trip to the search shards is very cheap and is not subject to rejections * which allows to fan out to more shards at the same time without running into rejections even if we are hitting a * large portion of the clusters indices. + * This phase can also be used to pre-sort shards based on min/max values in each shard of the provided primary sort. + * When the query primary sort is perform on a field, this phase extracts the min/max value in each shard and + * sort them according to the provided order. This can be useful for instance to ensure that shards that contain recent + * data are executed first when sorting by descending timestamp. */ -final class CanMatchPreFilterSearchPhase extends AbstractSearchAsyncAction { +final class CanMatchPreFilterSearchPhase extends AbstractSearchAsyncAction { private final Function, SearchPhase> phaseFactory; private final GroupShardsIterator shardsIts; @@ -58,26 +72,26 @@ final class CanMatchPreFilterSearchPhase extends AbstractSearchAsyncAction listener) { + SearchActionListener listener) { getSearchTransport().sendCanMatch(getConnection(shardIt.getClusterAlias(), shard.currentNodeId()), buildShardSearchRequest(shardIt), getTask(), listener); } @Override - protected SearchPhase getNextPhase(SearchPhaseResults results, + protected SearchPhase getNextPhase(SearchPhaseResults results, SearchPhaseContext context) { - return phaseFactory.apply(getIterator((BitSetSearchPhaseResults) results, shardsIts)); + return phaseFactory.apply(getIterator((CanMatchSearchPhaseResults) results, shardsIts)); } - private GroupShardsIterator getIterator(BitSetSearchPhaseResults results, + private GroupShardsIterator getIterator(CanMatchSearchPhaseResults results, GroupShardsIterator shardsIts) { int cardinality = results.getNumPossibleMatches(); FixedBitSet possibleMatches = results.getPossibleMatches(); @@ -86,6 +100,7 @@ private GroupShardsIterator getIterator(BitSetSearchPhaseRe // to produce a valid search result with all the aggs etc. possibleMatches.set(0); } + SearchSourceBuilder source = getRequest().source(); int i = 0; for (SearchShardIterator iter : shardsIts) { if (possibleMatches.get(i++)) { @@ -94,24 +109,48 @@ private GroupShardsIterator getIterator(BitSetSearchPhaseRe iter.resetAndSkip(); } } - return shardsIts; + if (shouldSortShards(results.minAndMaxes) == false) { + return shardsIts; + } + FieldSortBuilder fieldSort = FieldSortBuilder.getPrimaryFieldSortOrNull(source); + return new GroupShardsIterator<>(sortShards(shardsIts, results.minAndMaxes, fieldSort.order()), false); } - private static final class BitSetSearchPhaseResults extends SearchPhaseResults { + private static List sortShards(GroupShardsIterator shardsIts, + MinAndMax[] minAndMaxes, + SortOrder order) { + return IntStream.range(0, shardsIts.size()) + .boxed() + .sorted(shardComparator(shardsIts, minAndMaxes, order)) + .map(ord -> shardsIts.get(ord)) + .collect(Collectors.toList()); + } + private static boolean shouldSortShards(MinAndMax[] minAndMaxes) { + return Arrays.stream(minAndMaxes).anyMatch(Objects::nonNull); + } + + private static Comparator shardComparator(GroupShardsIterator shardsIts, + MinAndMax[] minAndMaxes, + SortOrder order) { + final Comparator comparator = Comparator.comparing(index -> minAndMaxes[index], MinAndMax.getComparator(order)); + return comparator.thenComparing(index -> shardsIts.get(index).shardId()); + } + + private static final class CanMatchSearchPhaseResults extends SearchPhaseResults { private final FixedBitSet possibleMatches; + private final MinAndMax[] minAndMaxes; private int numPossibleMatches; - BitSetSearchPhaseResults(int size) { + CanMatchSearchPhaseResults(int size) { super(size); possibleMatches = new FixedBitSet(size); + minAndMaxes = new MinAndMax[size]; } @Override - void consumeResult(SearchService.CanMatchResponse result) { - if (result.canMatch()) { - consumeShardFailure(result.getShardIndex()); - } + void consumeResult(CanMatchResponse result) { + consumeResult(result.getShardIndex(), result.canMatch(), result.minAndMax()); } @Override @@ -120,12 +159,18 @@ boolean hasResult(int shardIndex) { } @Override - synchronized void consumeShardFailure(int shardIndex) { + void consumeShardFailure(int shardIndex) { // we have to carry over shard failures in order to account for them in the response. - possibleMatches.set(shardIndex); - numPossibleMatches++; + consumeResult(shardIndex, true, null); } + synchronized void consumeResult(int shardIndex, boolean canMatch, MinAndMax minAndMax) { + if (canMatch) { + possibleMatches.set(shardIndex); + numPossibleMatches++; + } + minAndMaxes[shardIndex] = minAndMax; + } synchronized int getNumPossibleMatches() { return numPossibleMatches; @@ -136,7 +181,7 @@ synchronized FixedBitSet getPossibleMatches() { } @Override - Stream getSuccessfulResults() { + Stream getSuccessfulResults() { return Stream.empty(); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchActionListener.java b/server/src/main/java/org/elasticsearch/action/search/SearchActionListener.java index d34c8c61d434a..e9b5598556ff7 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchActionListener.java @@ -28,7 +28,7 @@ */ abstract class SearchActionListener implements ActionListener { - private final int requestIndex; + final int requestIndex; private final SearchShardTarget searchShardTarget; protected SearchActionListener(SearchShardTarget searchShardTarget, diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index d4832fb0d7a10..3e4ae654b2853 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -56,6 +56,7 @@ import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.profile.ProfileShardResult; import org.elasticsearch.search.profile.SearchProfileShardResults; +import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.RemoteClusterAware; @@ -615,9 +616,9 @@ static BiFunction buildConnectionLookup(St private static boolean shouldPreFilterSearchShards(SearchRequest searchRequest, GroupShardsIterator shardIterators) { SearchSourceBuilder source = searchRequest.source(); - return searchRequest.searchType() == QUERY_THEN_FETCH && // we can't do this for DFS it needs to fan out to all shards all the time - SearchService.canRewriteToMatchNone(source) && - searchRequest.getPreFilterShardSize() < shardIterators.size(); + return searchRequest.searchType() == QUERY_THEN_FETCH // we can't do this for DFS it needs to fan out to all shards all the time + && (SearchService.canRewriteToMatchNone(source) || FieldSortBuilder.hasPrimaryFieldSort(source)) + && searchRequest.getPreFilterShardSize() < shardIterators.size(); } static GroupShardsIterator mergeShardsIterators(GroupShardsIterator localShardsIterator, diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/GroupShardsIterator.java b/server/src/main/java/org/elasticsearch/cluster/routing/GroupShardsIterator.java index 21b02043a2249..a9904c96d020f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/GroupShardsIterator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/GroupShardsIterator.java @@ -38,7 +38,16 @@ public final class GroupShardsIterator implements * Constructs a enw GroupShardsIterator from the given list. */ public GroupShardsIterator(List iterators) { - CollectionUtil.timSort(iterators); + this(iterators, true); + } + + /** + * Constructs a new GroupShardsIterator from the given list. + */ + public GroupShardsIterator(List iterators, boolean useSort) { + if (useSort) { + CollectionUtil.timSort(iterators); + } this.iterators = iterators; } diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/AbstractAsyncTask.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/AbstractAsyncTask.java index 1ef9a484a2777..58e48f0215586 100644 --- a/server/src/main/java/org/elasticsearch/common/util/concurrent/AbstractAsyncTask.java +++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/AbstractAsyncTask.java @@ -91,7 +91,7 @@ public synchronized void rescheduleIfNecessary() { if (logger.isTraceEnabled()) { logger.trace("scheduling {} every {}", toString(), interval); } - cancellable = threadPool.schedule(this, interval, getThreadPool()); + cancellable = threadPool.schedule(threadPool.preserveContext(this), interval, getThreadPool()); isScheduledOrRunning = true; } else { logger.trace("scheduled {} disabled", toString()); diff --git a/server/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java b/server/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java index e19cbe38c7e37..03f15d7a85d8e 100644 --- a/server/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java +++ b/server/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java @@ -118,18 +118,6 @@ public void afterIndexShardClosed(ShardId shardId, @Nullable IndexShard indexSha } } - @Override - public void onShardInactive(IndexShard indexShard) { - for (IndexEventListener listener : listeners) { - try { - listener.onShardInactive(indexShard); - } catch (Exception e) { - logger.warn(() -> new ParameterizedMessage("[{}] failed to invoke on shard inactive callback", - indexShard.shardId().getId()), e); - throw e; - } - } - } @Override public void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardState previousState, IndexShardState currentState, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index cfd6fdcec7a25..7a98d9a286ea8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -252,7 +252,7 @@ public static final class DateFieldType extends MappedFieldType { protected DateMathParser dateMathParser; protected Resolution resolution; - DateFieldType() { + public DateFieldType() { super(); setTokenized(false); setHasDocValues(true); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 927bce5d9d6dd..0a473bd189e3b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -816,7 +816,7 @@ public final String typeName() { return name; } /** Get the associated numeric type */ - final NumericType numericType() { + public final NumericType numericType() { return numericType; } public abstract Query termQuery(String field, Object value); @@ -909,6 +909,10 @@ public String typeName() { return type.name; } + public NumericType numericType() { + return type.numericType(); + } + @Override public Query existsQuery(QueryShardContext context) { if (hasDocValues()) { diff --git a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java index 0427d9c152dc8..1c8599f66cf31 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java @@ -827,10 +827,7 @@ private boolean invariant() { assert checkpoints.get(aId) != null : "aId [" + aId + "] is pending in sync but isn't tracked"; } - if (primaryMode - && indexSettings.isSoftDeleteEnabled() - && indexSettings.getIndexMetaData().getState() == IndexMetaData.State.OPEN - && hasAllPeerRecoveryRetentionLeases) { + if (primaryMode && indexSettings.isSoftDeleteEnabled() && hasAllPeerRecoveryRetentionLeases) { // all tracked shard copies have a corresponding peer-recovery retention lease for (final ShardRouting shardRouting : routingTable.assignedShards()) { if (checkpoints.get(shardRouting.allocationId().getId()).tracked) { @@ -898,7 +895,9 @@ public ReplicationTracker( this.pendingInSync = new HashSet<>(); this.routingTable = null; this.replicationGroup = null; - this.hasAllPeerRecoveryRetentionLeases = indexSettings.getIndexVersionCreated().onOrAfter(Version.V_7_4_0); + this.hasAllPeerRecoveryRetentionLeases = indexSettings.getIndexVersionCreated().onOrAfter(Version.V_8_0_0) || + (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_7_4_0) && + indexSettings.getIndexMetaData().getState() == IndexMetaData.State.OPEN); this.fileBasedRecoveryThreshold = IndexSettings.FILE_BASED_RECOVERY_THRESHOLD_SETTING.get(indexSettings.getSettings()); this.safeCommitInfoSupplier = safeCommitInfoSupplier; assert Version.V_EMPTY.equals(indexSettings.getIndexVersionCreated()) == false; @@ -1011,34 +1010,32 @@ private void addPeerRecoveryRetentionLeaseForSolePrimary() { assert primaryMode; assert Thread.holdsLock(this); - if (indexSettings().getIndexMetaData().getState() == IndexMetaData.State.OPEN) { - final ShardRouting primaryShard = routingTable.primaryShard(); - final String leaseId = getPeerRecoveryRetentionLeaseId(primaryShard); - if (retentionLeases.get(leaseId) == null) { - if (replicationGroup.getReplicationTargets().equals(Collections.singletonList(primaryShard))) { - assert primaryShard.allocationId().getId().equals(shardAllocationId) - : routingTable.assignedShards() + " vs " + shardAllocationId; - // Safe to call innerAddRetentionLease() without a subsequent sync since there are no other members of this replication - // group. - logger.trace("addPeerRecoveryRetentionLeaseForSolePrimary: adding lease [{}]", leaseId); - innerAddRetentionLease(leaseId, Math.max(0L, checkpoints.get(shardAllocationId).globalCheckpoint + 1), - PEER_RECOVERY_RETENTION_LEASE_SOURCE); - hasAllPeerRecoveryRetentionLeases = true; - } else { - /* - * We got here here via a rolling upgrade from an older version that doesn't create peer recovery retention - * leases for every shard copy, but in this case we do not expect any leases to exist. - */ - assert hasAllPeerRecoveryRetentionLeases == false : routingTable + " vs " + retentionLeases; - logger.debug("{} becoming primary of {} with missing lease: {}", primaryShard, routingTable, retentionLeases); - } - } else if (hasAllPeerRecoveryRetentionLeases == false && routingTable.assignedShards().stream().allMatch(shardRouting -> - retentionLeases.contains(getPeerRecoveryRetentionLeaseId(shardRouting)) - || checkpoints.get(shardRouting.allocationId().getId()).tracked == false)) { - // Although this index is old enough not to have all the expected peer recovery retention leases, in fact it does, so we - // don't need to do any more work. + final ShardRouting primaryShard = routingTable.primaryShard(); + final String leaseId = getPeerRecoveryRetentionLeaseId(primaryShard); + if (retentionLeases.get(leaseId) == null) { + if (replicationGroup.getReplicationTargets().equals(Collections.singletonList(primaryShard))) { + assert primaryShard.allocationId().getId().equals(shardAllocationId) + : routingTable.assignedShards() + " vs " + shardAllocationId; + // Safe to call innerAddRetentionLease() without a subsequent sync since there are no other members of this replication + // group. + logger.trace("addPeerRecoveryRetentionLeaseForSolePrimary: adding lease [{}]", leaseId); + innerAddRetentionLease(leaseId, Math.max(0L, checkpoints.get(shardAllocationId).globalCheckpoint + 1), + PEER_RECOVERY_RETENTION_LEASE_SOURCE); hasAllPeerRecoveryRetentionLeases = true; + } else { + /* + * We got here here via a rolling upgrade from an older version that doesn't create peer recovery retention + * leases for every shard copy, but in this case we do not expect any leases to exist. + */ + assert hasAllPeerRecoveryRetentionLeases == false : routingTable + " vs " + retentionLeases; + logger.debug("{} becoming primary of {} with missing lease: {}", primaryShard, routingTable, retentionLeases); } + } else if (hasAllPeerRecoveryRetentionLeases == false && routingTable.assignedShards().stream().allMatch(shardRouting -> + retentionLeases.contains(getPeerRecoveryRetentionLeaseId(shardRouting)) + || checkpoints.get(shardRouting.allocationId().getId()).tracked == false)) { + // Although this index is old enough not to have all the expected peer recovery retention leases, in fact it does, so we + // don't need to do any more work. + hasAllPeerRecoveryRetentionLeases = true; } } @@ -1356,10 +1353,7 @@ private synchronized void setHasAllPeerRecoveryRetentionLeases() { * prior to {@link Version#V_7_4_0} that does not create peer-recovery retention leases. */ public synchronized void createMissingPeerRecoveryRetentionLeases(ActionListener listener) { - if (indexSettings().isSoftDeleteEnabled() - && indexSettings().getIndexMetaData().getState() == IndexMetaData.State.OPEN - && hasAllPeerRecoveryRetentionLeases == false) { - + if (indexSettings().isSoftDeleteEnabled() && hasAllPeerRecoveryRetentionLeases == false) { final List shardRoutings = routingTable.assignedShards(); final GroupedActionListener groupedActionListener = new GroupedActionListener<>(ActionListener.wrap(vs -> { setHasAllPeerRecoveryRetentionLeases(); diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java index d93500a5c6a72..e3d3fed4a5107 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java @@ -21,12 +21,15 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.store.AlreadyClosedException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.support.replication.ReplicationResponse; +import org.elasticsearch.action.support.replication.ReplicationTask; import org.elasticsearch.action.support.replication.TransportReplicationAction; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -37,12 +40,19 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.gateway.WriteStateException; import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.IndexShardClosedException; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.node.NodeClosedException; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.Map; import java.util.Objects; /** @@ -56,9 +66,7 @@ public class RetentionLeaseBackgroundSyncAction extends TransportReplicationActi RetentionLeaseBackgroundSyncAction.Request, ReplicationResponse> { - public static String ACTION_NAME = "indices:admin/seq_no/retention_lease_background_sync"; - public static ActionType TYPE = new ActionType<>(ACTION_NAME, ReplicationResponse::new); - + public static final String ACTION_NAME = "indices:admin/seq_no/retention_lease_background_sync"; private static final Logger LOGGER = LogManager.getLogger(RetentionLeaseSyncAction.class); protected Logger getLogger() { @@ -90,6 +98,52 @@ public RetentionLeaseBackgroundSyncAction( ThreadPool.Names.MANAGEMENT); } + @Override + protected void doExecute(Task task, Request request, ActionListener listener) { + assert false : "use RetentionLeaseBackgroundSyncAction#backgroundSync"; + } + + final void backgroundSync(ShardId shardId, String primaryAllocationId, long primaryTerm, RetentionLeases retentionLeases) { + final Request request = new Request(shardId, retentionLeases); + final ReplicationTask task = (ReplicationTask) taskManager.register("transport", "retention_lease_background_sync", request); + transportService.sendChildRequest(clusterService.localNode(), transportPrimaryAction, + new ConcreteShardRequest<>(request, primaryAllocationId, primaryTerm), + task, + transportOptions, + new TransportResponseHandler() { + @Override + public ReplicationResponse read(StreamInput in) throws IOException { + return newResponseInstance(in); + } + + @Override + public String executor() { + return ThreadPool.Names.SAME; + } + + @Override + public void handleResponse(ReplicationResponse response) { + task.setPhase("finished"); + taskManager.unregister(task); + } + + @Override + public void handleException(TransportException e) { + task.setPhase("finished"); + taskManager.unregister(task); + if (ExceptionsHelper.unwrap(e, NodeClosedException.class) != null) { + // node shutting down + return; + } + if (ExceptionsHelper.unwrap(e, AlreadyClosedException.class, IndexShardClosedException.class) != null) { + // the shard is closed + return; + } + getLogger().warn(new ParameterizedMessage("{} retention lease background sync failed", shardId), e); + } + }); + } + @Override protected void shardOperationOnPrimary( final Request request, @@ -137,6 +191,11 @@ public void writeTo(final StreamOutput out) throws IOException { retentionLeases.writeTo(out); } + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new ReplicationTask(id, type, action, "retention_lease_background_sync shardId=" + shardId, parentTaskId, headers); + } + @Override public String toString() { return "RetentionLeaseBackgroundSyncAction.Request{" + diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java index 69de0ed64f9ce..b93deccd51474 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java @@ -21,13 +21,16 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.store.AlreadyClosedException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.WriteResponse; import org.elasticsearch.action.support.replication.ReplicatedWriteRequest; import org.elasticsearch.action.support.replication.ReplicationResponse; +import org.elasticsearch.action.support.replication.ReplicationTask; import org.elasticsearch.action.support.replication.TransportWriteAction; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -39,12 +42,18 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.gateway.WriteStateException; import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.IndexShardClosedException; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.Map; import java.util.Objects; /** @@ -54,9 +63,7 @@ public class RetentionLeaseSyncAction extends TransportWriteAction { - public static String ACTION_NAME = "indices:admin/seq_no/retention_lease_sync"; - public static ActionType TYPE = new ActionType<>(ACTION_NAME, Response::new); - + public static final String ACTION_NAME = "indices:admin/seq_no/retention_lease_sync"; private static final Logger LOGGER = LogManager.getLogger(RetentionLeaseSyncAction.class); protected Logger getLogger() { @@ -88,6 +95,49 @@ public RetentionLeaseSyncAction( ThreadPool.Names.MANAGEMENT, false); } + @Override + protected void doExecute(Task parentTask, Request request, ActionListener listener) { + assert false : "use RetentionLeaseSyncAction#sync"; + } + + final void sync(ShardId shardId, String primaryAllocationId, long primaryTerm, RetentionLeases retentionLeases, + ActionListener listener) { + final Request request = new Request(shardId, retentionLeases); + final ReplicationTask task = (ReplicationTask) taskManager.register("transport", "retention_lease_sync", request); + transportService.sendChildRequest(clusterService.localNode(), transportPrimaryAction, + new ConcreteShardRequest<>(request, primaryAllocationId, primaryTerm), + task, + transportOptions, + new TransportResponseHandler() { + @Override + public ReplicationResponse read(StreamInput in) throws IOException { + return newResponseInstance(in); + } + + @Override + public String executor() { + return ThreadPool.Names.SAME; + } + + @Override + public void handleResponse(ReplicationResponse response) { + task.setPhase("finished"); + taskManager.unregister(task); + listener.onResponse(response); + } + + @Override + public void handleException(TransportException e) { + if (ExceptionsHelper.unwrap(e, AlreadyClosedException.class, IndexShardClosedException.class) == null) { + getLogger().warn(new ParameterizedMessage("{} retention lease sync failed", shardId), e); + } + task.setPhase("finished"); + taskManager.unregister(task); + listener.onFailure(e); + } + }); + } + @Override protected void shardOperationOnPrimary(Request request, IndexShard primary, ActionListener> listener) { @@ -141,6 +191,11 @@ public void writeTo(final StreamOutput out) throws IOException { retentionLeases.writeTo(out); } + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new ReplicationTask(id, type, action, "retention_lease_sync shardId=" + shardId, parentTaskId, headers); + } + @Override public String toString() { return "RetentionLeaseSyncAction.Request{" + diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncer.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncer.java index 7de6bad3f1102..40f80fee2b01d 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncer.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncer.java @@ -21,36 +21,52 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.replication.ReplicationResponse; +import org.elasticsearch.common.inject.Inject; import org.elasticsearch.index.shard.ShardId; -/** - * A functional interface that represents a method for syncing retention leases to replica shards after a new retention lease is added on - * the primary. - */ -public interface RetentionLeaseSyncer { +import java.util.Objects; - /** - * Represents a method that when invoked syncs retention leases to replica shards after a new retention lease is added on the primary. - * The specified listener is invoked when the syncing completes with success or failure. - * - * @param shardId the shard ID - * @param retentionLeases the retention leases to sync - * @param listener the callback when sync completes - */ - void sync(ShardId shardId, RetentionLeases retentionLeases, ActionListener listener); +public class RetentionLeaseSyncer { + private final SyncAction syncAction; + private final BackgroundSyncAction backgroundSyncAction; - void backgroundSync(ShardId shardId, RetentionLeases retentionLeases); + @Inject + public RetentionLeaseSyncer(RetentionLeaseSyncAction syncAction, RetentionLeaseBackgroundSyncAction backgroundSyncAction) { + this(syncAction::sync, backgroundSyncAction::backgroundSync); + } - RetentionLeaseSyncer EMPTY = new RetentionLeaseSyncer() { - @Override - public void sync(final ShardId shardId, final RetentionLeases retentionLeases, final ActionListener listener) { - listener.onResponse(new ReplicationResponse()); - } + public RetentionLeaseSyncer(SyncAction syncAction, BackgroundSyncAction backgroundSyncAction) { + this.syncAction = Objects.requireNonNull(syncAction); + this.backgroundSyncAction = Objects.requireNonNull(backgroundSyncAction); + } - @Override - public void backgroundSync(final ShardId shardId, final RetentionLeases retentionLeases) { + public static final RetentionLeaseSyncer EMPTY = new RetentionLeaseSyncer( + (shardId, primaryAllocationId, primaryTerm, retentionLeases, listener) -> listener.onResponse(new ReplicationResponse()), + (shardId, primaryAllocationId, primaryTerm, retentionLeases) -> { }); - } - }; + public void sync(ShardId shardId, String primaryAllocationId, long primaryTerm, + RetentionLeases retentionLeases, ActionListener listener) { + syncAction.sync(shardId, primaryAllocationId, primaryTerm, retentionLeases, listener); + } + public void backgroundSync(ShardId shardId, String primaryAllocationId, long primaryTerm, RetentionLeases retentionLeases) { + backgroundSyncAction.backgroundSync(shardId, primaryAllocationId, primaryTerm, retentionLeases); + } + + /** + * Represents an action that is invoked to sync retention leases to replica shards after a retention lease is added + * or removed on the primary. The specified listener is invoked when the syncing completes with success or failure. + */ + public interface SyncAction { + void sync(ShardId shardId, String primaryAllocationId, long primaryTerm, + RetentionLeases retentionLeases, ActionListener listener); + } + + /** + * Represents an action that is invoked periodically to sync retention leases to replica shards after some retention + * lease has been renewed or expired. + */ + public interface BackgroundSyncAction { + void backgroundSync(ShardId shardId, String primaryAllocationId, long primaryTerm, RetentionLeases retentionLeases); + } } diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexEventListener.java b/server/src/main/java/org/elasticsearch/index/shard/IndexEventListener.java index 982b42b2c3f66..9cbd9ce4df253 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexEventListener.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexEventListener.java @@ -84,13 +84,6 @@ default void afterIndexShardClosed(ShardId shardId, @Nullable IndexShard indexSh default void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardState previousState, IndexShardState currentState, @Nullable String reason) {} - /** - * Called when a shard is marked as inactive - * - * @param indexShard The shard that was marked inactive - */ - default void onShardInactive(IndexShard indexShard) {} - /** * Called before the index gets created. Note that this is also called * when the index is created on data nodes diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 6de1c0cd052be..8185d8fad5f16 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -340,7 +340,7 @@ public IndexShard( UNASSIGNED_SEQ_NO, globalCheckpointListeners::globalCheckpointUpdated, threadPool::absoluteTimeInMillis, - (retentionLeases, listener) -> retentionLeaseSyncer.sync(shardId, retentionLeases, listener), + (retentionLeases, listener) -> retentionLeaseSyncer.sync(shardId, aId, getPendingPrimaryTerm(), retentionLeases, listener), this::getSafeCommitInfo); // the query cache is a node-level thing, however we want the most popular filters @@ -1578,7 +1578,7 @@ private void innerOpenEngineAndTranslog(LongSupplier globalCheckpointSupplier) t onNewEngine(newEngine); currentEngineReference.set(newEngine); // We set active because we are now writing operations to the engine; this way, - // if we go idle after some time and become inactive, we still give sync'd flush a chance to run. + // we can flush if we go idle after some time and become inactive. active.set(true); } // time elapses after the engine is created above (pulling the config settings) until we set the engine reference, during @@ -1758,19 +1758,28 @@ public void addShardFailureCallback(Consumer onShardFailure) { /** * Called by {@link IndexingMemoryController} to check whether more than {@code inactiveTimeNS} has passed since the last - * indexing operation, and notify listeners that we are now inactive so e.g. sync'd flush can happen. + * indexing operation, so we can flush the index. */ - public void checkIdle(long inactiveTimeNS) { + public void flushOnIdle(long inactiveTimeNS) { Engine engineOrNull = getEngineOrNull(); if (engineOrNull != null && System.nanoTime() - engineOrNull.getLastWriteNanos() >= inactiveTimeNS) { boolean wasActive = active.getAndSet(false); if (wasActive) { - logger.debug("shard is now inactive"); - try { - indexEventListener.onShardInactive(this); - } catch (Exception e) { - logger.warn("failed to notify index event listener", e); - } + logger.debug("flushing shard on inactive"); + threadPool.executor(ThreadPool.Names.FLUSH).execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + if (state != IndexShardState.CLOSED) { + logger.warn("failed to flush shard on inactive", e); + } + } + + @Override + protected void doRun() { + flush(new FlushRequest().waitIfOngoing(false).force(false)); + periodicFlushMetric.inc(); + } + }); } } } @@ -2182,6 +2191,8 @@ public void syncRetentionLeases() { logger.trace("syncing retention leases [{}] after expiration check", retentionLeases.v2()); retentionLeaseSyncer.sync( shardId, + shardRouting.allocationId().getId(), + getPendingPrimaryTerm(), retentionLeases.v2(), ActionListener.wrap( r -> {}, @@ -2191,7 +2202,8 @@ public void syncRetentionLeases() { e))); } else { logger.trace("background syncing retention leases [{}] after expiration check", retentionLeases.v2()); - retentionLeaseSyncer.backgroundSync(shardId, retentionLeases.v2()); + retentionLeaseSyncer.backgroundSync( + shardId, shardRouting.allocationId().getId(), getPendingPrimaryTerm(), retentionLeases.v2()); } } diff --git a/server/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java b/server/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java index e358bc57798b4..8344355a0f372 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java +++ b/server/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java @@ -303,7 +303,7 @@ private void runUnlocked() { long totalBytesWriting = 0; for (IndexShard shard : availableShards()) { - // Give shard a chance to transition to inactive so sync'd flush can happen: + // Give shard a chance to transition to inactive so we can flush checkIdle(shard, inactiveTime.nanos()); // How many bytes this shard is currently (async'd) moving from heap to disk: @@ -400,7 +400,7 @@ private void runUnlocked() { */ protected void checkIdle(IndexShard shard, long inactiveTimeNS) { try { - shard.checkIdle(inactiveTimeNS); + shard.flushOnIdle(inactiveTimeNS); } catch (AlreadyClosedException e) { logger.trace(() -> new ParameterizedMessage("ignore exception while checking if shard {} is inactive", shard.shardId()), e); } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index dae7e1ed4e9b4..7584fda21c329 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -56,6 +56,9 @@ import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TypeFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; +import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction; +import org.elasticsearch.index.seqno.RetentionLeaseSyncAction; +import org.elasticsearch.index.seqno.RetentionLeaseSyncer; import org.elasticsearch.index.shard.PrimaryReplicaSyncer; import org.elasticsearch.indices.cluster.IndicesClusterStateService; import org.elasticsearch.indices.flush.SyncedFlushService; @@ -238,6 +241,9 @@ protected void configure() { bind(SyncedFlushService.class).asEagerSingleton(); bind(TransportResyncReplicationAction.class).asEagerSingleton(); bind(PrimaryReplicaSyncer.class).asEagerSingleton(); + bind(RetentionLeaseSyncAction.class).asEagerSingleton(); + bind(RetentionLeaseBackgroundSyncAction.class).asEagerSingleton(); + bind(RetentionLeaseSyncer.class).asEagerSingleton(); } /** diff --git a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index 9c267e4e97399..b72a56b4a0543 100644 --- a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -26,7 +26,6 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; @@ -58,10 +57,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction; import org.elasticsearch.index.seqno.ReplicationTracker; -import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction; -import org.elasticsearch.index.seqno.RetentionLeaseSyncAction; import org.elasticsearch.index.seqno.RetentionLeaseSyncer; -import org.elasticsearch.index.seqno.RetentionLeases; import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardClosedException; @@ -77,7 +73,6 @@ import org.elasticsearch.indices.recovery.PeerRecoveryTargetService; import org.elasticsearch.indices.recovery.RecoveryFailedException; import org.elasticsearch.indices.recovery.RecoveryState; -import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.search.SearchService; import org.elasticsearch.snapshots.SnapshotShardsService; @@ -91,7 +86,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; @@ -104,7 +98,7 @@ import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED; import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.REOPENED; -public class IndicesClusterStateService extends AbstractLifecycleComponent implements ClusterStateApplier, RetentionLeaseSyncer { +public class IndicesClusterStateService extends AbstractLifecycleComponent implements ClusterStateApplier { private static final Logger logger = LogManager.getLogger(IndicesClusterStateService.class); final AllocatedIndices> indicesService; @@ -127,6 +121,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple private final boolean sendRefreshMapping; private final List buildInIndexListener; private final PrimaryReplicaSyncer primaryReplicaSyncer; + private final RetentionLeaseSyncer retentionLeaseSyncer; private final NodeClient client; @Inject @@ -144,6 +139,7 @@ public IndicesClusterStateService( final PeerRecoverySourceService peerRecoverySourceService, final SnapshotShardsService snapshotShardsService, final PrimaryReplicaSyncer primaryReplicaSyncer, + final RetentionLeaseSyncer retentionLeaseSyncer, final NodeClient client) { this( settings, @@ -159,6 +155,7 @@ public IndicesClusterStateService( peerRecoverySourceService, snapshotShardsService, primaryReplicaSyncer, + retentionLeaseSyncer, client); } @@ -177,15 +174,10 @@ public IndicesClusterStateService( final PeerRecoverySourceService peerRecoverySourceService, final SnapshotShardsService snapshotShardsService, final PrimaryReplicaSyncer primaryReplicaSyncer, + final RetentionLeaseSyncer retentionLeaseSyncer, final NodeClient client) { this.settings = settings; - this.buildInIndexListener = - Arrays.asList( - peerRecoverySourceService, - recoveryTargetService, - searchService, - syncedFlushService, - snapshotShardsService); + this.buildInIndexListener = Arrays.asList(peerRecoverySourceService, recoveryTargetService, searchService, snapshotShardsService); this.indicesService = indicesService; this.clusterService = clusterService; this.threadPool = threadPool; @@ -194,6 +186,7 @@ public IndicesClusterStateService( this.nodeMappingRefreshAction = nodeMappingRefreshAction; this.repositoriesService = repositoriesService; this.primaryReplicaSyncer = primaryReplicaSyncer; + this.retentionLeaseSyncer = retentionLeaseSyncer; this.sendRefreshMapping = settings.getAsBoolean("indices.cluster.send_refresh_mapping", true); this.client = client; } @@ -300,54 +293,6 @@ protected void updateGlobalCheckpointForShard(final ShardId shardId) { } } - @Override - public void sync(ShardId shardId, RetentionLeases retentionLeases, ActionListener listener) { - Objects.requireNonNull(shardId); - Objects.requireNonNull(retentionLeases); - Objects.requireNonNull(listener); - final ThreadContext threadContext = threadPool.getThreadContext(); - try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { - // we have to execute under the system context so that if security is enabled the sync is authorized - threadContext.markAsSystemContext(); - client.executeLocally(RetentionLeaseSyncAction.TYPE, - new RetentionLeaseSyncAction.Request(shardId, retentionLeases), - ActionListener.wrap( - listener::onResponse, - e -> { - if (ExceptionsHelper.unwrap(e, AlreadyClosedException.class, IndexShardClosedException.class) == null) { - getLogger().warn(new ParameterizedMessage("{} retention lease sync failed", shardId), e); - } - listener.onFailure(e); - })); - } - } - - @Override - public void backgroundSync(ShardId shardId, RetentionLeases retentionLeases) { - Objects.requireNonNull(shardId); - Objects.requireNonNull(retentionLeases); - final ThreadContext threadContext = threadPool.getThreadContext(); - try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { - // we have to execute under the system context so that if security is enabled the sync is authorized - threadContext.markAsSystemContext(); - client.executeLocally(RetentionLeaseBackgroundSyncAction.TYPE, - new RetentionLeaseBackgroundSyncAction.Request(shardId, retentionLeases), - ActionListener.wrap( - r -> {}, - e -> { - if (ExceptionsHelper.unwrap(e, NodeClosedException.class) != null) { - // node shutting down - return; - } - if (ExceptionsHelper.unwrap(e, AlreadyClosedException.class, IndexShardClosedException.class) != null) { - // the shard is closed - return; - } - getLogger().warn(new ParameterizedMessage("{} retention lease background sync failed", shardId), e); - })); - } - } - // overrideable by tests Logger getLogger() { return logger; @@ -670,7 +615,7 @@ private void createShard(DiscoveryNodes nodes, RoutingTable routingTable, ShardR repositoriesService, failedShardHandler, this::updateGlobalCheckpointForShard, - this); + retentionLeaseSyncer); } catch (Exception e) { failAndRemoveShard(shardRouting, true, "failed to create shard", e, state); } diff --git a/server/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java b/server/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java index fee11a0a9f54b..c0e0d513b33d7 100644 --- a/server/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java +++ b/server/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java @@ -47,7 +47,6 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.CommitStats; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardNotFoundException; @@ -71,7 +70,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentMap; -public class SyncedFlushService implements IndexEventListener { +public class SyncedFlushService { private static final Logger logger = LogManager.getLogger(SyncedFlushService.class); @@ -101,25 +100,6 @@ public SyncedFlushService(IndicesService indicesService, new InFlightOpCountTransportHandler()); } - @Override - public void onShardInactive(final IndexShard indexShard) { - // we only want to call sync flush once, so only trigger it when we are on a primary - if (indexShard.routingEntry().primary()) { - attemptSyncedFlush(indexShard.shardId(), new ActionListener() { - @Override - public void onResponse(ShardsSyncedFlushResult syncedFlushResult) { - logger.trace("{} sync flush on inactive shard returned successfully for sync_id: {}", - syncedFlushResult.getShardId(), syncedFlushResult.syncId()); - } - - @Override - public void onFailure(Exception e) { - logger.debug(() -> new ParameterizedMessage("{} sync flush on inactive shard failed", indexShard.shardId()), e); - } - }); - } - } - /** * a utility method to perform a synced flush for all shards of multiple indices. * see {@link #attemptSyncedFlush(ShardId, ActionListener)} diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java index 1d45d048c9ba4..c372cc4571a7c 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java @@ -38,7 +38,6 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.ThreadedActionListener; import org.elasticsearch.action.support.replication.ReplicationResponse; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.CheckedSupplier; @@ -154,8 +153,7 @@ public void recoverToTarget(ActionListener listener) { IOUtils.closeWhileHandlingException(releaseResources, () -> wrappedListener.onFailure(e)); }; - final boolean useRetentionLeases = shard.indexSettings().isSoftDeleteEnabled() - && shard.indexSettings().getIndexMetaData().getState() != IndexMetaData.State.CLOSE; + final boolean softDeletesEnabled = shard.indexSettings().isSoftDeleteEnabled(); final SetOnce retentionLeaseRef = new SetOnce<>(); runUnderPrimaryPermit(() -> { @@ -167,7 +165,7 @@ public void recoverToTarget(ActionListener listener) { throw new DelayRecoveryException("source node does not have the shard listed in its state as allocated on the node"); } assert targetShardRouting.initializing() : "expected recovery target to be initializing but was " + targetShardRouting; - retentionLeaseRef.set(useRetentionLeases ? shard.getRetentionLeases().get( + retentionLeaseRef.set(softDeletesEnabled ? shard.getRetentionLeases().get( ReplicationTracker.getPeerRecoveryRetentionLeaseId(targetShardRouting)) : null); }, shardId + " validating recovery target ["+ request.targetAllocationId() + "] registered ", shard, cancellableThreads, logger); @@ -178,7 +176,7 @@ public void recoverToTarget(ActionListener listener) { = request.startingSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO && isTargetSameHistory() && shard.hasCompleteHistoryOperations("peer-recovery", request.startingSeqNo()) - && (useRetentionLeases == false + && (softDeletesEnabled == false || (retentionLeaseRef.get() != null && retentionLeaseRef.get().retainingSequenceNumber() <= request.startingSeqNo())); // NB check hasCompleteHistoryOperations when computing isSequenceNumberBasedRecovery, even if there is a retention lease, // because when doing a rolling upgrade from earlier than 7.4 we may create some leases that are initially unsatisfied. It's @@ -186,7 +184,7 @@ && isTargetSameHistory() // Also it's pretty cheap when soft deletes are enabled, and it'd be a disaster if we tried a sequence-number-based recovery // without having a complete history. - if (isSequenceNumberBasedRecovery && useRetentionLeases) { + if (isSequenceNumberBasedRecovery && softDeletesEnabled) { // all the history we need is retained by an existing retention lease, so we do not need a separate retention lock retentionLock.close(); logger.trace("history is retained by {}", retentionLeaseRef.get()); @@ -225,7 +223,7 @@ && isTargetSameHistory() // advances and not when creating a new safe commit. In any case this is a best-effort thing since future recoveries can // always fall back to file-based ones, and only really presents a problem if this primary fails before things have settled // down. - startingSeqNo = useRetentionLeases + startingSeqNo = softDeletesEnabled ? Long.parseLong(safeCommitRef.getIndexCommit().getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)) + 1L : 0; logger.trace("performing file-based recovery followed by history replay starting at [{}]", startingSeqNo); @@ -243,7 +241,7 @@ && isTargetSameHistory() }); final StepListener deleteRetentionLeaseStep = new StepListener<>(); - if (useRetentionLeases) { + if (softDeletesEnabled) { runUnderPrimaryPermit(() -> { try { // If the target previously had a copy of this shard then a file-based recovery might move its global @@ -266,7 +264,7 @@ && isTargetSameHistory() assert Transports.assertNotTransportThread(RecoverySourceHandler.this + "[phase1]"); final Consumer> createRetentionLeaseAsync; - if (useRetentionLeases) { + if (softDeletesEnabled) { createRetentionLeaseAsync = l -> createRetentionLease(startingSeqNo, l); } else { createRetentionLeaseAsync = l -> l.onResponse(null); @@ -304,7 +302,7 @@ && isTargetSameHistory() final Translog.Snapshot phase2Snapshot = shard.getHistoryOperations("peer-recovery", startingSeqNo); resources.add(phase2Snapshot); - if (useRetentionLeases == false || isSequenceNumberBasedRecovery == false) { + if (softDeletesEnabled == false || isSequenceNumberBasedRecovery == false) { // we can release the retention lock here because the snapshot itself will retain the required operations. retentionLock.close(); } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index dfefef88292b4..6bb4ddc795cef 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.TopDocs; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.OriginalIndices; @@ -92,6 +93,8 @@ import org.elasticsearch.search.query.ScrollQuerySearchResult; import org.elasticsearch.search.rescore.RescorerBuilder; import org.elasticsearch.search.searchafter.SearchAfterBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.MinAndMax; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.suggest.Suggest; @@ -1012,7 +1015,7 @@ public AliasFilter buildAliasFilter(ClusterState state, String index, Setfalse the query won't match any documents on the current * shard. */ - public boolean canMatch(ShardSearchRequest request) throws IOException { + public CanMatchResponse canMatch(ShardSearchRequest request) throws IOException { assert request.searchType() == SearchType.QUERY_THEN_FETCH : "unexpected search type: " + request.searchType(); IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); IndexShard indexShard = indexService.getShard(request.shardId().getId()); @@ -1022,18 +1025,20 @@ public boolean canMatch(ShardSearchRequest request) throws IOException { QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), searcher, request::nowInMillis, request.getClusterAlias()); Rewriteable.rewrite(request.getRewriteable(), context, false); + FieldSortBuilder sortBuilder = FieldSortBuilder.getPrimaryFieldSortOrNull(request.source()); + MinAndMax minMax = sortBuilder != null ? FieldSortBuilder.getMinMaxOrNull(context, sortBuilder) : null; if (canRewriteToMatchNone(request.source())) { QueryBuilder queryBuilder = request.source().query(); - return queryBuilder instanceof MatchNoneQueryBuilder == false; + return new CanMatchResponse(queryBuilder instanceof MatchNoneQueryBuilder == false, minMax); } - return true; // null query means match_all + // null query means match_all + return new CanMatchResponse(true, minMax); } } - public void canMatch(ShardSearchRequest request, ActionListener listener) { try { - listener.onResponse(new CanMatchResponse(canMatch(request))); + listener.onResponse(canMatch(request)); } catch (IOException e) { listener.onFailure(e); } @@ -1052,6 +1057,7 @@ public static boolean canRewriteToMatchNone(SearchSourceBuilder source) { return aggregations == null || aggregations.mustVisitAllDocs() == false; } + /* * Rewrites the search request with a light weight rewrite context in order to fetch resources asynchronously * The action listener is guaranteed to be executed on the search thread-pool @@ -1087,24 +1093,38 @@ public InternalAggregation.ReduceContext createReduceContext(boolean finalReduce public static final class CanMatchResponse extends SearchPhaseResult { private final boolean canMatch; + private final MinAndMax minAndMax; public CanMatchResponse(StreamInput in) throws IOException { super(in); this.canMatch = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_7_6_0)) { + minAndMax = in.readOptionalWriteable(MinAndMax::new); + } else { + minAndMax = null; + } } - public CanMatchResponse(boolean canMatch) { + public CanMatchResponse(boolean canMatch, MinAndMax minAndMax) { this.canMatch = canMatch; + this.minAndMax = minAndMax; } @Override public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(canMatch); + if (out.getVersion().onOrAfter(Version.V_7_6_0)) { + out.writeOptionalWriteable(minAndMax); + } } public boolean canMatch() { return canMatch; } + + public MinAndMax minAndMax() { + return minAndMax; + } } /** diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index ede0616ccfe66..ba03e058a37b9 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -19,21 +19,32 @@ package org.elasticsearch.search.sort; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiTerms; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.Terms; import org.apache.lucene.search.SortField; import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; import org.elasticsearch.index.fielddata.plain.SortedNumericDVIndexFieldData; +import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; +import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; @@ -41,11 +52,16 @@ import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; +import java.util.Collections; import java.util.Locale; import java.util.Objects; +import java.util.function.Function; +import static org.elasticsearch.index.mapper.DateFieldMapper.Resolution.MILLISECONDS; +import static org.elasticsearch.index.mapper.DateFieldMapper.Resolution.NANOSECONDS; import static org.elasticsearch.index.search.NestedHelper.parentObject; import static org.elasticsearch.search.sort.NestedSortBuilder.NESTED_FIELD; @@ -359,6 +375,116 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException { return new SortFieldAndFormat(field, fieldType.docValueFormat(null, null)); } + /** + * Return true if the primary sort in the provided source + * is an instance of {@link FieldSortBuilder}. + */ + public static boolean hasPrimaryFieldSort(SearchSourceBuilder source) { + return getPrimaryFieldSortOrNull(source) != null; + } + + /** + * Return the {@link FieldSortBuilder} if the primary sort in the provided source + * is an instance of this class, null otherwise. + */ + public static FieldSortBuilder getPrimaryFieldSortOrNull(SearchSourceBuilder source) { + if (source == null || source.sorts() == null || source.sorts().isEmpty()) { + return null; + } + return source.sorts().get(0) instanceof FieldSortBuilder ? (FieldSortBuilder) source.sorts().get(0) : null; + } + + /** + * Return a {@link Function} that converts a serialized point into a {@link Number} according to the provided + * {@link SortField}. This is needed for {@link SortField} that converts values from one type to another using + * {@link FieldSortBuilder#setNumericType(String)} )} (e.g.: long to double). + */ + private static Function numericPointConverter(SortField sortField, NumberFieldType numberFieldType) { + switch (IndexSortConfig.getSortFieldType(sortField)) { + case LONG: + return v -> numberFieldType.parsePoint(v).longValue(); + + case INT: + return v -> numberFieldType.parsePoint(v).intValue(); + + case DOUBLE: + return v -> numberFieldType.parsePoint(v).doubleValue(); + + case FLOAT: + return v -> numberFieldType.parsePoint(v).floatValue(); + + default: + return v -> null; + } + } + + /** + * Return a {@link Function} that converts a serialized date point into a {@link Long} according to the provided + * {@link NumericType}. + */ + private static Function datePointConverter(DateFieldType dateFieldType, String numericTypeStr) { + if (numericTypeStr != null) { + NumericType numericType = resolveNumericType(numericTypeStr); + if (dateFieldType.resolution() == MILLISECONDS && numericType == NumericType.DATE_NANOSECONDS) { + return v -> DateUtils.toNanoSeconds(LongPoint.decodeDimension(v, 0)); + } else if (dateFieldType.resolution() == NANOSECONDS && numericType == NumericType.DATE) { + return v -> DateUtils.toMilliSeconds(LongPoint.decodeDimension(v, 0)); + } + } + return v -> LongPoint.decodeDimension(v, 0); + } + + /** + * Return the {@link MinAndMax} indexed value from the provided {@link FieldSortBuilder} or null if unknown. + * The value can be extracted on non-nested indexed mapped fields of type keyword, numeric or date, other fields + * and configurations return null. + */ + public static MinAndMax getMinMaxOrNull(QueryShardContext context, FieldSortBuilder sortBuilder) throws IOException { + SortAndFormats sort = SortBuilder.buildSort(Collections.singletonList(sortBuilder), context).get(); + SortField sortField = sort.sort.getSort()[0]; + if (sortField.getField() == null) { + return null; + } + IndexReader reader = context.getIndexReader(); + MappedFieldType fieldType = context.fieldMapper(sortField.getField()); + if (reader == null || (fieldType == null || fieldType.indexOptions() == IndexOptions.NONE)) { + return null; + } + String fieldName = fieldType.name(); + switch (IndexSortConfig.getSortFieldType(sortField)) { + case LONG: + case INT: + case DOUBLE: + case FLOAT: + final Function converter; + if (fieldType instanceof NumberFieldType) { + converter = numericPointConverter(sortField, (NumberFieldType) fieldType); + } else if (fieldType instanceof DateFieldType) { + converter = datePointConverter((DateFieldType) fieldType, sortBuilder.getNumericType()); + } else { + return null; + } + if (PointValues.size(reader, fieldName) == 0) { + return null; + } + final Comparable min = converter.apply(PointValues.getMinPackedValue(reader, fieldName)); + final Comparable max = converter.apply(PointValues.getMaxPackedValue(reader, fieldName)); + return MinAndMax.newMinMax(min, max); + + case STRING: + case STRING_VAL: + if (fieldType instanceof KeywordFieldMapper.KeywordFieldType) { + Terms terms = MultiTerms.getTerms(reader, fieldName); + if (terms == null) { + return null; + } + return terms.getMin() != null ? MinAndMax.newMinMax(terms.getMin(), terms.getMax()) : null; + } + break; + } + return null; + } + /** * Throws an exception if max children is not located at top level nested sort. */ diff --git a/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java b/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java new file mode 100644 index 0000000000000..28e07c8863b8a --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java @@ -0,0 +1,83 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.sort; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.lucene.Lucene; + +import java.io.IOException; +import java.util.Comparator; +import java.util.Objects; + +/** + * A class that encapsulates a minimum and a maximum {@link Comparable}. + */ +public class MinAndMax> implements Writeable { + private final T minValue; + private final T maxValue; + + private MinAndMax(T minValue, T maxValue) { + this.minValue = Objects.requireNonNull(minValue); + this.maxValue = Objects.requireNonNull(maxValue); + } + + public MinAndMax(StreamInput in) throws IOException { + this.minValue = (T) Lucene.readSortValue(in); + this.maxValue = (T) Lucene.readSortValue(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Lucene.writeSortValue(out, minValue); + Lucene.writeSortValue(out, maxValue); + } + + /** + * Return the minimum value. + */ + public T getMin() { + return minValue; + } + + /** + * Return the maximum value. + */ + public T getMax() { + return maxValue; + } + + public static > MinAndMax newMinMax(T min, T max) { + return new MinAndMax<>(min, max); + } + + /** + * Return a {@link Comparator} for {@link MinAndMax} values according to the provided {@link SortOrder}. + */ + public static Comparator> getComparator(SortOrder order) { + Comparator cmp = order == SortOrder.ASC ? + Comparator.comparing(v -> (Comparable) v.getMin()) : Comparator.comparing(v -> (Comparable) v.getMax()); + if (order == SortOrder.DESC) { + cmp = cmp.reversed(); + } + return Comparator.nullsLast(cmp); + } +} diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 65c4eaeb2609d..0ac0204760ea3 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -1061,20 +1061,21 @@ public void onFailure(final Exception e) { * Removes record of running snapshot from cluster state * @param snapshot snapshot * @param snapshotInfo snapshot info if snapshot was successful - * @param e exception if snapshot failed + * @param e exception if snapshot failed, {@code null} otherwise */ - private void removeSnapshotFromClusterState(final Snapshot snapshot, final SnapshotInfo snapshotInfo, final Exception e) { + private void removeSnapshotFromClusterState(final Snapshot snapshot, final SnapshotInfo snapshotInfo, @Nullable Exception e) { removeSnapshotFromClusterState(snapshot, snapshotInfo, e, null); } /** * Removes record of running snapshot from cluster state and notifies the listener when this action is complete * @param snapshot snapshot - * @param failure exception if snapshot failed + * @param failure exception if snapshot failed, {@code null} otherwise * @param listener listener to notify when snapshot information is removed from the cluster state */ - private void removeSnapshotFromClusterState(final Snapshot snapshot, @Nullable SnapshotInfo snapshotInfo, final Exception failure, + private void removeSnapshotFromClusterState(final Snapshot snapshot, @Nullable SnapshotInfo snapshotInfo, @Nullable Exception failure, @Nullable CleanupAfterErrorListener listener) { + assert snapshotInfo != null || failure != null : "Either snapshotInfo or failure must be supplied"; clusterService.submitStateUpdateTask("remove snapshot metadata", new ClusterStateUpdateTask() { @Override diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java index f24bbf5b999b2..88a090757d5bc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java @@ -533,7 +533,7 @@ public void testShrinkCommitsMergeOnIdle() throws Exception { IndexService indexShards = service.indexService(target.getIndex()); IndexShard shard = indexShards.getShard(0); assertTrue(shard.isActive()); - shard.checkIdle(0); + shard.flushOnIdle(0); assertFalse(shard.isActive()); } } diff --git a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java index f23f40cde7809..06e63f7b32936 100644 --- a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java @@ -26,21 +26,32 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchService; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.ShardSearchRequest; +import org.elasticsearch.search.sort.MinAndMax; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.Transport; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.IntStream; public class CanMatchPreFilterSearchPhaseTests extends ESTestCase { @@ -62,7 +73,7 @@ public void testFilterShards() throws InterruptedException { public void sendCanMatch(Transport.Connection connection, ShardSearchRequest request, SearchTask task, ActionListener listener) { new Thread(() -> listener.onResponse(new SearchService.CanMatchResponse(request.shardId().id() == 0 ? shard1 : - shard2))).start(); + shard2, null))).start(); } }; @@ -124,7 +135,7 @@ public void sendCanMatch(Transport.Connection connection, ShardSearchRequest req } else { new Thread(() -> { if (throwException == false) { - listener.onResponse(new SearchService.CanMatchResponse(shard1)); + listener.onResponse(new SearchService.CanMatchResponse(shard1, null)); } else { listener.onFailure(new NullPointerException()); } @@ -187,7 +198,7 @@ public void sendCanMatch( ShardSearchRequest request, SearchTask task, ActionListener listener) { - listener.onResponse(new SearchService.CanMatchResponse(randomBoolean())); + listener.onResponse(new SearchService.CanMatchResponse(randomBoolean(), null)); } }; @@ -265,4 +276,77 @@ protected void executePhaseOnShard( latch.await(); executor.shutdown(); } + + public void testSortShards() throws InterruptedException { + final TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(0, System.nanoTime(), + System::nanoTime); + + Map lookup = new ConcurrentHashMap<>(); + DiscoveryNode primaryNode = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNode replicaNode = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT); + lookup.put("node1", new SearchAsyncActionTests.MockConnection(primaryNode)); + lookup.put("node2", new SearchAsyncActionTests.MockConnection(replicaNode)); + + for (SortOrder order : SortOrder.values()) { + List shardIds = new ArrayList<>(); + List> minAndMaxes = new ArrayList<>(); + Set shardToSkip = new HashSet<>(); + + SearchTransportService searchTransportService = new SearchTransportService(null, null) { + @Override + public void sendCanMatch(Transport.Connection connection, ShardSearchRequest request, SearchTask task, + ActionListener listener) { + Long min = rarely() ? null : randomLong(); + Long max = min == null ? null : randomLongBetween(min, Long.MAX_VALUE); + MinAndMax minMax = min == null ? null : MinAndMax.newMinMax(min, max); + boolean canMatch = frequently(); + synchronized (shardIds) { + shardIds.add(request.shardId()); + minAndMaxes.add(minMax); + if (canMatch == false) { + shardToSkip.add(request.shardId()); + } + } + new Thread(() -> listener.onResponse(new SearchService.CanMatchResponse(canMatch, minMax))).start(); + } + }; + + AtomicReference> result = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + GroupShardsIterator shardsIter = SearchAsyncActionTests.getShardsIter("logs", + new OriginalIndices(new String[]{"logs"}, SearchRequest.DEFAULT_INDICES_OPTIONS), + randomIntBetween(2, 20), randomBoolean(), primaryNode, replicaNode); + final SearchRequest searchRequest = new SearchRequest(); + searchRequest.source(new SearchSourceBuilder().sort(SortBuilders.fieldSort("timestamp").order(order))); + searchRequest.allowPartialSearchResults(true); + + CanMatchPreFilterSearchPhase canMatchPhase = new CanMatchPreFilterSearchPhase(logger, + searchTransportService, + (clusterAlias, node) -> lookup.get(node), + Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)), + Collections.emptyMap(), Collections.emptyMap(), EsExecutors.newDirectExecutorService(), + searchRequest, null, shardsIter, timeProvider, 0, null, + (iter) -> new SearchPhase("test") { + @Override + public void run() { + result.set(iter); + latch.countDown(); + } + }, SearchResponse.Clusters.EMPTY); + + canMatchPhase.start(); + latch.await(); + ShardId[] expected = IntStream.range(0, shardIds.size()) + .boxed() + .sorted(Comparator.comparing(minAndMaxes::get, MinAndMax.getComparator(order)).thenComparing(shardIds::get)) + .map(shardIds::get) + .toArray(ShardId[]::new); + + int pos = 0; + for (SearchShardIterator i : result.get()) { + assertEquals(shardToSkip.contains(i.shardId()), i.skip()); + assertEquals(expected[pos++], i.shardId()); + } + } + } } diff --git a/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java b/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java index 554b75a3dc539..35064b0063676 100644 --- a/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java @@ -323,6 +323,40 @@ public void testDoNotCancelRecoveryForBrokenNode() throws Exception { transportService.clearAllRules(); } + public void testPeerRecoveryForClosedIndices() throws Exception { + String indexName = "peer_recovery_closed_indices"; + internalCluster().ensureAtLeastNumDataNodes(1); + createIndex(indexName, Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) + .put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "100ms") + .put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms") + .build()); + indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), IntStream.range(0, randomIntBetween(1, 100)) + .mapToObj(n -> client().prepareIndex(indexName).setSource("num", n)).collect(Collectors.toList())); + ensureActivePeerRecoveryRetentionLeasesAdvanced(indexName); + assertAcked(client().admin().indices().prepareClose(indexName)); + int numberOfReplicas = randomIntBetween(1, 2); + internalCluster().ensureAtLeastNumDataNodes(2 + numberOfReplicas); + assertAcked(client().admin().indices().prepareUpdateSettings(indexName) + .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numberOfReplicas))); + ensureGreen(indexName); + ensureActivePeerRecoveryRetentionLeasesAdvanced(indexName); + assertAcked(client().admin().cluster().prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put("cluster.routing.allocation.enable", "primaries").build())); + internalCluster().fullRestart(); + ensureYellow(indexName); + if (randomBoolean()) { + assertAcked(client().admin().indices().prepareOpen(indexName)); + client().admin().indices().prepareForceMerge(indexName).get(); + } + assertAcked(client().admin().cluster().prepareUpdateSettings() + .setPersistentSettings(Settings.builder().putNull("cluster.routing.allocation.enable").build())); + ensureGreen(indexName); + assertNoOpRecoveries(indexName); + } + private void ensureActivePeerRecoveryRetentionLeasesAdvanced(String indexName) throws Exception { assertBusy(() -> { Index index = resolveIndex(indexName); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index e524e7bad5d49..86fa8c613b628 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -167,23 +166,6 @@ public void testLockTryingToDelete() throws Exception { } } - public void testMarkAsInactiveTriggersSyncedFlush() throws Exception { - assertAcked(client().admin().indices().prepareCreate("test") - .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0))); - client().prepareIndex("test").setSource("{}", XContentType.JSON).get(); - ensureGreen("test"); - IndicesService indicesService = getInstanceFromNode(IndicesService.class); - indicesService.indexService(resolveIndex("test")).getShardOrNull(0).checkIdle(0); - assertBusy(() -> { - IndexStats indexStats = client().admin().indices().prepareStats("test").clear().get().getIndex("test"); - assertNotNull(indexStats.getShards()[0].getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID)); - indicesService.indexService(resolveIndex("test")).getShardOrNull(0).checkIdle(0); - } - ); - IndexStats indexStats = client().admin().indices().prepareStats("test").get().getIndex("test"); - assertNotNull(indexStats.getShards()[0].getCommitStats().getUserData().get(Engine.SYNC_COMMIT_ID)); - } - public void testDurableFlagHasEffect() throws Exception { createIndex("test"); ensureGreen(); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 59e437da4f397..93867f4aa4c77 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -3419,7 +3419,7 @@ public void testScheduledRefresh() throws Exception { indexDoc(primary, "_doc", "2", "{\"foo\" : \"bar\"}"); assertFalse(primary.scheduledRefresh()); assertTrue(primary.isSearchIdle()); - primary.checkIdle(0); + primary.flushOnIdle(0); assertTrue(primary.scheduledRefresh()); // make sure we refresh once the shard is inactive try (Engine.Searcher searcher = primary.acquireSearcher("test")) { assertEquals(3, searcher.getIndexReader().numDocs()); @@ -3629,38 +3629,13 @@ public void testSegmentMemoryTrackedWithRandomSearchers() throws Exception { assertThat(breaker.getUsed(), equalTo(0L)); } - public void testFlushOnInactive() throws Exception { - Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .build(); - IndexMetaData metaData = IndexMetaData.builder("test") - .putMapping("{ \"properties\": { \"foo\": { \"type\": \"text\"}}}") - .settings(settings) - .primaryTerm(0, 1).build(); - ShardRouting shardRouting = - TestShardRouting.newShardRouting(new ShardId(metaData.getIndex(), 0), "n1", true, - ShardRoutingState.INITIALIZING, RecoverySource.EmptyStoreRecoverySource.INSTANCE); - final ShardId shardId = shardRouting.shardId(); - final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(createTempDir()); - ShardPath shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId); - AtomicBoolean markedInactive = new AtomicBoolean(); - AtomicReference primaryRef = new AtomicReference<>(); - IndexShard primary = newShard(shardRouting, shardPath, metaData, null, null, new InternalEngineFactory(), () -> { }, - RetentionLeaseSyncer.EMPTY, new IndexEventListener() { - @Override - public void onShardInactive(IndexShard indexShard) { - markedInactive.set(true); - primaryRef.get().flush(new FlushRequest()); - } - }); - primaryRef.set(primary); - recoverShardFromStore(primary); + public void testFlushOnIdle() throws Exception { + IndexShard shard = newStartedShard(); for (int i = 0; i < 3; i++) { - indexDoc(primary, "_doc", "" + i, "{\"foo\" : \"" + randomAlphaOfLength(10) + "\"}"); - primary.refresh("test"); // produce segments + indexDoc(shard, "_doc", Integer.toString(i)); + shard.refresh("test"); // produce segments } - List segments = primary.segments(false); + List segments = shard.segments(false); Set names = new HashSet<>(); for (Segment segment : segments) { assertFalse(segment.committed); @@ -3668,10 +3643,10 @@ public void onShardInactive(IndexShard indexShard) { names.add(segment.getName()); } assertEquals(3, segments.size()); - primary.flush(new FlushRequest()); - primary.forceMerge(new ForceMergeRequest().maxNumSegments(1).flush(false)); - primary.refresh("test"); - segments = primary.segments(false); + shard.flush(new FlushRequest()); + shard.forceMerge(new ForceMergeRequest().maxNumSegments(1).flush(false)); + shard.refresh("test"); + segments = shard.segments(false); for (Segment segment : segments) { if (names.contains(segment.getName())) { assertTrue(segment.committed); @@ -3683,20 +3658,18 @@ public void onShardInactive(IndexShard indexShard) { } assertEquals(4, segments.size()); - assertFalse(markedInactive.get()); - assertBusy(() -> { - primary.checkIdle(0); - assertFalse(primary.isActive()); - }); + shard.flushOnIdle(0); + assertFalse(shard.isActive()); - assertTrue(markedInactive.get()); - segments = primary.segments(false); - assertEquals(1, segments.size()); - for (Segment segment : segments) { - assertTrue(segment.committed); - assertTrue(segment.search); - } - closeShards(primary); + assertBusy(() -> { // flush happens in the background using the flush threadpool + List segmentsAfterFlush = shard.segments(false); + assertEquals(1, segmentsAfterFlush.size()); + for (Segment segment : segmentsAfterFlush) { + assertTrue(segment.committed); + assertTrue(segment.search); + } + }); + closeShards(shard); } public void testOnCloseStats() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java index 6c28c7bc2dc2c..3590c52e505bf 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java @@ -21,7 +21,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.Version; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; @@ -30,7 +29,6 @@ import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; -import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; @@ -52,7 +50,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.Index; -import org.elasticsearch.index.seqno.RetentionLeases; +import org.elasticsearch.index.seqno.RetentionLeaseSyncer; import org.elasticsearch.index.shard.PrimaryReplicaSyncer; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.recovery.PeerRecoveryTargetService; @@ -484,12 +482,9 @@ private IndicesClusterStateService createIndicesClusterStateService(DiscoveryNod null, null, primaryReplicaSyncer, + RetentionLeaseSyncer.EMPTY, client) { @Override - public void sync(ShardId shardId, RetentionLeases retentionLeases, ActionListener listener) {} - @Override - public void backgroundSync(ShardId shardId, RetentionLeases retentionLeases) {} - @Override protected void updateGlobalCheckpointForShard(final ShardId shardId) {} }; } diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceTests.java deleted file mode 100644 index 6a35f268c987e..0000000000000 --- a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceTests.java +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.indices.cluster; - -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.lucene.store.AlreadyClosedException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.TransportAction; -import org.elasticsearch.action.support.replication.ReplicationResponse; -import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction; -import org.elasticsearch.index.seqno.RetentionLeaseSyncAction; -import org.elasticsearch.index.seqno.RetentionLeases; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.shard.IndexShardClosedException; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.node.NodeClosedException; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.tasks.TaskManager; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.SendRequestTransportException; -import org.elasticsearch.transport.TransportException; -import org.elasticsearch.transport.TransportService; -import org.junit.Before; -import org.mockito.ArgumentCaptor; - -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.elasticsearch.mock.orig.Mockito.verifyNoMoreInteractions; -import static org.elasticsearch.mock.orig.Mockito.when; -import static org.hamcrest.Matchers.arrayContaining; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.sameInstance; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.same; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; - -public class IndicesClusterStateServiceTests extends ESTestCase { - - ThreadPool threadPool; - - @Before - public void createServices() { - threadPool = mock(ThreadPool.class); - when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); - } - - public void testRetentionLeaseBackgroundSyncExecution() { - final IndicesService indicesService = mock(IndicesService.class); - - final Index index = new Index("index", "uuid"); - final IndexService indexService = mock(IndexService.class); - when(indicesService.indexServiceSafe(index)).thenReturn(indexService); - - final int id = randomIntBetween(0, 4); - final IndexShard indexShard = mock(IndexShard.class); - when(indexService.getShard(id)).thenReturn(indexShard); - - final ShardId shardId = new ShardId(index, id); - when(indexShard.shardId()).thenReturn(shardId); - - final Logger mockLogger = mock(Logger.class); - final TaskManager taskManager = mock(TaskManager.class); - when(taskManager.registerAndExecute(any(), any(), any(), any(), any())).thenCallRealMethod(); - final TransportService transportService = mock(TransportService.class); - when(transportService.getTaskManager()).thenReturn(taskManager); - - final RetentionLeases retentionLeases = mock(RetentionLeases.class); - final AtomicBoolean invoked = new AtomicBoolean(); - final RetentionLeaseBackgroundSyncAction action = new RetentionLeaseBackgroundSyncAction( - Settings.EMPTY, - transportService, - null, - indicesService, - threadPool, - null, - new ActionFilters(Collections.emptySet()), - new IndexNameExpressionResolver()) { - - @Override - protected void doExecute(Task task, RetentionLeaseBackgroundSyncAction.Request request, - ActionListener listener) { - assertTrue(threadPool.getThreadContext().isSystemContext()); - assertThat(request.shardId(), sameInstance(indexShard.shardId())); - assertThat(request.getRetentionLeases(), sameInstance(retentionLeases)); - if (randomBoolean()) { - listener.onResponse(new ReplicationResponse()); - } else { - final Exception e = randomFrom( - new AlreadyClosedException("closed"), - new IndexShardClosedException(indexShard.shardId()), - new TransportException("failed"), - new SendRequestTransportException(null, randomFrom( - "some-action", - "indices:admin/seq_no/retention_lease_background_sync[p]" - ), new NodeClosedException((DiscoveryNode) null)), - new RuntimeException("failed")); - listener.onFailure(e); - if (e.getMessage().equals("failed")) { - final ArgumentCaptor captor = ArgumentCaptor.forClass(ParameterizedMessage.class); - verify(mockLogger).warn(captor.capture(), same(e)); - final ParameterizedMessage message = captor.getValue(); - assertThat(message.getFormat(), equalTo("{} retention lease background sync failed")); - assertThat(message.getParameters(), arrayContaining(indexShard.shardId())); - } - verifyNoMoreInteractions(mockLogger); - } - invoked.set(true); - } - }; - NodeClient client = new NodeClient(Settings.EMPTY, null); - Map actions = Collections.singletonMap(RetentionLeaseBackgroundSyncAction.TYPE, action); - client.initialize(actions, taskManager, null, null); - IndicesClusterStateService service = new IndicesClusterStateService(Settings.EMPTY, null, null, threadPool, null, null, null, - null, null, null, null, null, null, client) { - @Override - protected Logger getLogger() { - return mockLogger; - } - }; - - service.backgroundSync(indexShard.shardId(), retentionLeases); - assertTrue(invoked.get()); - } - - - public void testRetentionLeaseSyncExecution() { - final IndicesService indicesService = mock(IndicesService.class); - - final Index index = new Index("index", "uuid"); - final IndexService indexService = mock(IndexService.class); - when(indicesService.indexServiceSafe(index)).thenReturn(indexService); - final int id = randomIntBetween(0, 4); - final IndexShard indexShard = mock(IndexShard.class); - when(indexService.getShard(id)).thenReturn(indexShard); - - final ShardId shardId = new ShardId(index, id); - when(indexShard.shardId()).thenReturn(shardId); - - final Logger mockLogger = mock(Logger.class); - final TaskManager taskManager = mock(TaskManager.class); - when(taskManager.registerAndExecute(any(), any(), any(), any(), any())).thenCallRealMethod(); - final TransportService transportService = mock(TransportService.class); - when(transportService.getTaskManager()).thenReturn(taskManager); - - final RetentionLeases retentionLeases = mock(RetentionLeases.class); - final AtomicBoolean invoked = new AtomicBoolean(); - final RetentionLeaseSyncAction action = new RetentionLeaseSyncAction( - Settings.EMPTY, - transportService, - null, - indicesService, - threadPool, - null, - new ActionFilters(Collections.emptySet()), - new IndexNameExpressionResolver()) { - - @Override - protected void doExecute(Task task, Request request, ActionListener listener) { - assertTrue(threadPool.getThreadContext().isSystemContext()); - assertThat(request.shardId(), sameInstance(indexShard.shardId())); - assertThat(request.getRetentionLeases(), sameInstance(retentionLeases)); - if (randomBoolean()) { - listener.onResponse(new Response()); - } else { - final Exception e = randomFrom( - new AlreadyClosedException("closed"), - new IndexShardClosedException(indexShard.shardId()), - new RuntimeException("failed")); - listener.onFailure(e); - if (e instanceof AlreadyClosedException == false && e instanceof IndexShardClosedException == false) { - final ArgumentCaptor captor = ArgumentCaptor.forClass(ParameterizedMessage.class); - verify(mockLogger).warn(captor.capture(), same(e)); - final ParameterizedMessage message = captor.getValue(); - assertThat(message.getFormat(), equalTo("{} retention lease sync failed")); - assertThat(message.getParameters(), arrayContaining(indexShard.shardId())); - } - verifyNoMoreInteractions(mockLogger); - } - invoked.set(true); - } - }; - NodeClient client = new NodeClient(Settings.EMPTY, null); - Map actions = Collections.singletonMap(RetentionLeaseSyncAction.TYPE, action); - client.initialize(actions, taskManager, null, null); - IndicesClusterStateService service = new IndicesClusterStateService(Settings.EMPTY, null, null, threadPool, null, null, null, - null, null, null, null, null, null, client) { - @Override - protected Logger getLogger() { - return mockLogger; - } - }; - - // execution happens on the test thread, so no need to register an actual listener to callback - service.sync(indexShard.shardId(), retentionLeases, ActionListener.wrap(() -> {})); - assertTrue(invoked.get()); - } - -} diff --git a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java index 79211202a14bd..5b4b80aab8c57 100644 --- a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java +++ b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.InternalEngine; @@ -47,15 +48,22 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndexingMemoryController; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.test.InternalTestCluster; import java.io.IOException; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -71,6 +79,11 @@ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class FlushIT extends ESIntegTestCase { + @Override + protected Collection> nodePlugins() { + return Collections.singletonList(InternalSettingsPlugin.class); + } + public void testWaitIfOngoing() throws InterruptedException { createIndex("test"); ensureGreen("test"); @@ -369,4 +382,29 @@ public void testDoNotRenewSyncedFlushWhenAllSealed() throws Exception { assertThat(forthSeal.successfulShards(), equalTo(numberOfReplicas + 1)); assertThat(forthSeal.syncId(), not(equalTo(thirdSeal.syncId()))); } + + public void testFlushOnInactive() throws Exception { + final String indexName = "flush_on_inactive"; + List dataNodes = internalCluster().startDataOnlyNodes(2, Settings.builder() + .put(IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.getKey(), randomTimeValue(10, 1000, "ms")).build()); + assertAcked(client().admin().indices().prepareCreate(indexName).setSettings(Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), randomTimeValue(50, 200, "ms")) + .put("index.routing.allocation.include._name", String.join(",", dataNodes)) + .build())); + ensureGreen(indexName); + int numDocs = randomIntBetween(1, 10); + for (int i = 0; i < numDocs; i++) { + client().prepareIndex(indexName).setSource("f", "v").get(); + } + if (randomBoolean()) { + internalCluster().restartNode(randomFrom(dataNodes), new InternalTestCluster.RestartCallback()); + ensureGreen(indexName); + } + assertBusy(() -> { + for (ShardStats shardStats : client().admin().indices().prepareStats(indexName).get().getShards()) { + assertThat(shardStats.getStats().getTranslog().getUncommittedOperations(), equalTo(0)); + } + }, 30, TimeUnit.SECONDS); + } } diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java index 00c6a0cc8b15f..16632094f5aa7 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java @@ -601,28 +601,28 @@ public void testCanMatch() throws IOException { SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true); int numWrapReader = numWrapInvocations.get(); assertTrue(service.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); searchRequest.source(new SearchSourceBuilder()); assertTrue(service.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); searchRequest.source(new SearchSourceBuilder().query(new MatchAllQueryBuilder())); assertTrue(service.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); searchRequest.source(new SearchSourceBuilder().query(new MatchNoneQueryBuilder()) .aggregation(new TermsAggregationBuilder("test", ValueType.STRING).minDocCount(0))); assertTrue(service.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); searchRequest.source(new SearchSourceBuilder().query(new MatchNoneQueryBuilder()) .aggregation(new GlobalAggregationBuilder("test"))); assertTrue(service.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); searchRequest.source(new SearchSourceBuilder().query(new MatchNoneQueryBuilder())); assertFalse(service.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); assertEquals(numWrapReader, numWrapInvocations.get()); // make sure that the wrapper is called when the context is actually created diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 7256a46715a05..9cda381df243b 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.sort; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.SortField; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -182,7 +183,11 @@ public void testEqualsAndHashcode() { } } - protected QueryShardContext createMockShardContext() { + protected final QueryShardContext createMockShardContext() { + return createMockShardContext(null); + } + + protected final QueryShardContext createMockShardContext(IndexSearcher searcher) { Index index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_"); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()); @@ -192,7 +197,8 @@ protected QueryShardContext createMockShardContext() { return builder.build(idxSettings, fieldType, new IndexFieldDataCache.None(), null, null); }; return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, - null, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, null, () -> randomNonNegativeLong(), null, null) { + null, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, searcher, + () -> randomNonNegativeLong(), null, null) { @Override public MappedFieldType fieldMapper(String name) { diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java index 1623bc2bb378d..750ec4f34dfa5 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java @@ -19,20 +19,37 @@ package org.elasticsearch.search.sort; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.HalfFloatPoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.search.AssertingIndexSearcher; import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedNumericSelector; import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.SortedSetSelector; import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.xcontent.XContentParseException; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.TypeFieldMapper; import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -43,11 +60,15 @@ import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Locale; +import static org.elasticsearch.search.sort.FieldSortBuilder.getMinMaxOrNull; +import static org.elasticsearch.search.sort.FieldSortBuilder.getPrimaryFieldSortOrNull; import static org.elasticsearch.search.sort.NestedSortBuilderTests.createRandomNestedSort; import static org.hamcrest.Matchers.instanceOf; @@ -306,6 +327,32 @@ protected MappedFieldType provideMappedFieldType(String name) { fieldType.setName(name); fieldType.setHasDocValues(true); return fieldType; + } else if (name.startsWith("custom-")) { + final MappedFieldType fieldType; + if (name.startsWith("custom-keyword")) { + fieldType = new KeywordFieldMapper.KeywordFieldType(); + } else if (name.startsWith("custom-date")) { + fieldType = new DateFieldMapper.DateFieldType(); + } else { + String type = name.split("-")[1]; + if (type.equals("INT")) { + type = "integer"; + } + NumberFieldMapper.NumberType numberType = NumberFieldMapper.NumberType.valueOf(type.toUpperCase(Locale.ENGLISH)); + if (numberType != null) { + fieldType = new NumberFieldMapper.NumberFieldType(numberType); + } else { + fieldType = new KeywordFieldMapper.KeywordFieldType(); + } + } + fieldType.setName(name); + fieldType.setHasDocValues(true); + if (name.endsWith("-ni")) { + fieldType.setIndexOptions(IndexOptions.NONE); + } else { + fieldType.setIndexOptions(IndexOptions.DOCS); + } + return fieldType; } else { return super.provideMappedFieldType(name); } @@ -377,6 +424,147 @@ public QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOEx assertNotSame(rangeQuery, rewritten.getNestedSort().getFilter()); } + public void testGetPrimaryFieldSort() { + assertNull(getPrimaryFieldSortOrNull(null)); + assertNull(getPrimaryFieldSortOrNull(new SearchSourceBuilder())); + assertNull(getPrimaryFieldSortOrNull(new SearchSourceBuilder().sort(SortBuilders.scoreSort()))); + FieldSortBuilder sortBuilder = new FieldSortBuilder(MAPPED_STRING_FIELDNAME); + assertEquals(sortBuilder, getPrimaryFieldSortOrNull(new SearchSourceBuilder().sort(sortBuilder))); + assertNull(getPrimaryFieldSortOrNull(new SearchSourceBuilder() + .sort(SortBuilders.scoreSort()).sort(sortBuilder))); + assertNull(getPrimaryFieldSortOrNull(new SearchSourceBuilder() + .sort(SortBuilders.geoDistanceSort("field", 0d, 0d)).sort(sortBuilder))); + } + + public void testGetMaxNumericSortValue() throws IOException { + QueryShardContext context = createMockShardContext(); + for (NumberFieldMapper.NumberType numberType : NumberFieldMapper.NumberType.values()) { + String fieldName = "custom-" + numberType.numericType(); + assertNull(getMinMaxOrNull(context, SortBuilders.fieldSort(fieldName))); + assertNull(getMinMaxOrNull(context, SortBuilders.fieldSort(fieldName + "-ni"))); + + try (Directory dir = newDirectory()) { + int numDocs = randomIntBetween(10, 30); + final Comparable[] values = new Comparable[numDocs]; + try (RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + switch (numberType) { + case LONG: + long v1 = randomLong(); + values[i] = v1; + doc.add(new LongPoint(fieldName, v1)); + break; + + case INTEGER: + int v2 = randomInt(); + values[i] = (long) v2; + doc.add(new IntPoint(fieldName, v2)); + break; + + case DOUBLE: + double v3 = randomDouble(); + values[i] = v3; + doc.add(new DoublePoint(fieldName, v3)); + break; + + case FLOAT: + float v4 = randomFloat(); + values[i] = v4; + doc.add(new FloatPoint(fieldName, v4)); + break; + + case HALF_FLOAT: + float v5 = randomFloat(); + values[i] = (double) v5; + doc.add(new HalfFloatPoint(fieldName, v5)); + break; + + case BYTE: + byte v6 = randomByte(); + values[i] = (long) v6; + doc.add(new IntPoint(fieldName, v6)); + break; + + case SHORT: + short v7 = randomShort(); + values[i] = (long) v7; + doc.add(new IntPoint(fieldName, v7)); + break; + + default: + throw new AssertionError("unknown type " + numberType); + } + writer.addDocument(doc); + } + Arrays.sort(values); + try (DirectoryReader reader = writer.getReader()) { + QueryShardContext newContext = createMockShardContext(new AssertingIndexSearcher(random(), reader)); + if (numberType == NumberFieldMapper.NumberType.HALF_FLOAT) { + assertNull(getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName + "-ni"))); + assertNull(getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName))); + } else { + assertNull(getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName + "-ni"))); + assertEquals(values[numDocs - 1], + getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName)).getMax()); + assertEquals(values[0], getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName)).getMin()); + } + } + } + } + } + } + + public void testGetMaxNumericDateValue() throws IOException { + QueryShardContext context = createMockShardContext(); + String fieldName = "custom-date"; + assertNull(getMinMaxOrNull(context, SortBuilders.fieldSort(fieldName))); + assertNull(getMinMaxOrNull(context, SortBuilders.fieldSort(fieldName + "-ni"))); + try (Directory dir = newDirectory()) { + int numDocs = randomIntBetween(10, 30); + final long[] values = new long[numDocs]; + try (RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + values[i] = randomNonNegativeLong(); + doc.add(new LongPoint(fieldName, values[i])); + writer.addDocument(doc); + } + Arrays.sort(values); + try (DirectoryReader reader = writer.getReader()) { + QueryShardContext newContext = createMockShardContext(new AssertingIndexSearcher(random(), reader)); + assertEquals(values[numDocs - 1], getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName)).getMax()); + assertEquals(values[0], getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName)).getMin()); + } + } + } + } + + public void testGetMaxKeywordValue() throws IOException { + QueryShardContext context = createMockShardContext(); + String fieldName = "custom-keyword"; + assertNull(getMinMaxOrNull(context, SortBuilders.fieldSort(fieldName))); + assertNull(getMinMaxOrNull(context, SortBuilders.fieldSort(fieldName + "-ni"))); + try (Directory dir = newDirectory()) { + int numDocs = randomIntBetween(10, 30); + final BytesRef[] values = new BytesRef[numDocs]; + try (RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new KeywordAnalyzer())) { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + values[i] = new BytesRef(randomAlphaOfLengthBetween(5, 10)); + doc.add(new TextField(fieldName, values[i].utf8ToString(), Field.Store.NO)); + writer.addDocument(doc); + } + Arrays.sort(values); + try (DirectoryReader reader = writer.getReader()) { + QueryShardContext newContext = createMockShardContext(new AssertingIndexSearcher(random(), reader)); + assertEquals(values[numDocs - 1], getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName)).getMax()); + assertEquals(values[0], getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName)).getMin()); + } + } + } + } + @Override protected FieldSortBuilder fromXContent(XContentParser parser, String fieldName) throws IOException { return FieldSortBuilder.fromXContent(parser, fieldName); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 454a407103382..b4ba7fbc9269b 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -139,8 +139,7 @@ import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction; -import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction; -import org.elasticsearch.index.seqno.RetentionLeaseSyncAction; +import org.elasticsearch.index.seqno.RetentionLeaseSyncer; import org.elasticsearch.index.shard.PrimaryReplicaSyncer; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; @@ -1081,12 +1080,6 @@ public void onFailure(final Exception e) { actions.put(GlobalCheckpointSyncAction.TYPE, new GlobalCheckpointSyncAction(settings, transportService, clusterService, indicesService, threadPool, shardStateAction, actionFilters, indexNameExpressionResolver)); - actions.put(RetentionLeaseBackgroundSyncAction.TYPE, - new RetentionLeaseBackgroundSyncAction(settings, transportService, clusterService, indicesService, threadPool, - shardStateAction, actionFilters, indexNameExpressionResolver)); - actions.put(RetentionLeaseSyncAction.TYPE, - new RetentionLeaseSyncAction(settings, transportService, clusterService, indicesService, threadPool, - shardStateAction, actionFilters, indexNameExpressionResolver)); final MetaDataMappingService metaDataMappingService = new MetaDataMappingService(clusterService, indicesService); indicesClusterStateService = new IndicesClusterStateService( settings, @@ -1112,6 +1105,7 @@ public void onFailure(final Exception e) { shardStateAction, actionFilters, indexNameExpressionResolver)), + RetentionLeaseSyncer.EMPTY, client); final MetaDataCreateIndexService metaDataCreateIndexService = new MetaDataCreateIndexService(settings, clusterService, indicesService, diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/DeterministicTaskQueue.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/DeterministicTaskQueue.java index 0837f431fff9c..db3818832f6e4 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/DeterministicTaskQueue.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/DeterministicTaskQueue.java @@ -383,7 +383,7 @@ public Cancellable scheduleWithFixedDelay(Runnable command, TimeValue interval, @Override public Runnable preserveContext(Runnable command) { - throw new UnsupportedOperationException(); + return command; } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java index 413af96615638..62f624335c864 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java @@ -178,21 +178,15 @@ protected class ReplicationGroup implements AutoCloseable, Iterable } }); - private final RetentionLeaseSyncer retentionLeaseSyncer = new RetentionLeaseSyncer() { - @Override - public void sync(ShardId shardId, RetentionLeases retentionLeases, ActionListener listener) { - syncRetentionLeases(shardId, retentionLeases, listener); - } - - @Override - public void backgroundSync(ShardId shardId, RetentionLeases retentionLeases) { - sync(shardId, retentionLeases, ActionListener.wrap( + private final RetentionLeaseSyncer retentionLeaseSyncer = new RetentionLeaseSyncer( + (shardId, primaryAllocationId, primaryTerm, retentionLeases, listener) -> + syncRetentionLeases(shardId, retentionLeases, listener), + (shardId, primaryAllocationId, primaryTerm, retentionLeases) -> syncRetentionLeases(shardId, retentionLeases, + ActionListener.wrap( r -> { }, e -> { throw new AssertionError("failed to background sync retention lease", e); - })); - } - }; + }))); protected ReplicationGroup(final IndexMetaData indexMetaData) throws IOException { final ShardRouting primaryRouting = this.createShardRouting("s0", true); diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index dc119f940de21..ad88d88d4941f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -1230,7 +1230,7 @@ private void assertNoSnapshottedIndexCommit() throws Exception { } } } - }); + }, 60, TimeUnit.SECONDS); } /** @@ -1983,9 +1983,13 @@ public List startMasterOnlyNodes(int numNodes, Settings settings) { } public List startDataOnlyNodes(int numNodes) { + return startDataOnlyNodes(numNodes, Settings.EMPTY); + } + + public List startDataOnlyNodes(int numNodes, Settings settings) { return startNodes( numNodes, - Settings.builder().put(Settings.EMPTY).put(Node.NODE_MASTER_SETTING.getKey(), false) + Settings.builder().put(settings).put(Node.NODE_MASTER_SETTING.getKey(), false) .put(Node.NODE_DATA_SETTING.getKey(), true).build()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java b/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java index 77d1c315b6970..81178c9019845 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java +++ b/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java @@ -126,11 +126,6 @@ public void indexShardStateChanged(IndexShard indexShard, @Nullable IndexShardSt delegate.indexShardStateChanged(indexShard, previousState, currentState, reason); } - @Override - public void onShardInactive(IndexShard indexShard) { - delegate.onShardInactive(indexShard); - } - @Override public void beforeIndexCreated(Index index, Settings indexSettings) { delegate.beforeIndexCreated(index, indexSettings); diff --git a/x-pack/plugin/ccr/qa/security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java b/x-pack/plugin/ccr/qa/security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java index c9c74e658f4fd..9f41ae758bf94 100644 --- a/x-pack/plugin/ccr/qa/security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java +++ b/x-pack/plugin/ccr/qa/security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java @@ -202,7 +202,7 @@ public void testForgetFollower() throws IOException { assertOK(client().performRequest(new Request("POST", "/" + forgetFollower + "/_ccr/pause_follow"))); - try (RestClient leaderClient = buildLeaderClient(restClientSettings())) { + try (RestClient leaderClient = buildLeaderClient(restAdminSettings())) { final Request request = new Request("POST", "/" + forgetLeader + "/_ccr/forget_follower"); final String requestBody = "{" + "\"follower_cluster\":\"follow-cluster\"," + diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/FrequencyEncoding.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/FrequencyEncoding.java index cea99d3edc8f6..ed693460edcc7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/FrequencyEncoding.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/FrequencyEncoding.java @@ -103,11 +103,11 @@ public String getName() { @Override public void process(Map fields) { - String value = (String)fields.get(field); + Object value = fields.get(field); if (value == null) { return; } - fields.put(featureName, frequencyMap.getOrDefault(value, 0.0)); + fields.put(featureName, frequencyMap.getOrDefault(value.toString(), 0.0)); } @Override @@ -152,7 +152,8 @@ public long ramBytesUsed() { long size = SHALLOW_SIZE; size += RamUsageEstimator.sizeOf(field); size += RamUsageEstimator.sizeOf(featureName); - size += RamUsageEstimator.sizeOfMap(frequencyMap); + // defSize:0 indicates that there is not a defined size. Finding the shallowSize of Double gives the best estimate + size += RamUsageEstimator.sizeOfMap(frequencyMap, 0); return size; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/OneHotEncoding.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/OneHotEncoding.java index 9784ed8cbe7aa..a4924a277c0ad 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/OneHotEncoding.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/OneHotEncoding.java @@ -86,12 +86,12 @@ public String getName() { @Override public void process(Map fields) { - String value = (String)fields.get(field); + Object value = fields.get(field); if (value == null) { return; } hotMap.forEach((val, col) -> { - int encoding = value.equals(val) ? 1 : 0; + int encoding = value.toString().equals(val) ? 1 : 0; fields.put(col, encoding); }); } @@ -134,7 +134,8 @@ public int hashCode() { public long ramBytesUsed() { long size = SHALLOW_SIZE; size += RamUsageEstimator.sizeOf(field); - size += RamUsageEstimator.sizeOfMap(hotMap); + // defSize:0 does not do much in this case as sizeOf(String) is a known quantity + size += RamUsageEstimator.sizeOfMap(hotMap, 0); return size; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/TargetMeanEncoding.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/TargetMeanEncoding.java index 914b43f98e967..8276fc2c8fefb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/TargetMeanEncoding.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/TargetMeanEncoding.java @@ -114,11 +114,11 @@ public String getName() { @Override public void process(Map fields) { - String value = (String)fields.get(field); + Object value = fields.get(field); if (value == null) { return; } - fields.put(featureName, meanMap.getOrDefault(value, defaultValue)); + fields.put(featureName, meanMap.getOrDefault(value.toString(), defaultValue)); } @Override @@ -166,7 +166,8 @@ public long ramBytesUsed() { long size = SHALLOW_SIZE; size += RamUsageEstimator.sizeOf(field); size += RamUsageEstimator.sizeOf(featureName); - size += RamUsageEstimator.sizeOfMap(meanMap); + // defSize:0 indicates that there is not a defined size. Finding the shallowSize of Double gives the best estimate + size += RamUsageEstimator.sizeOfMap(meanMap, 0); return size; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/FrequencyEncodingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/FrequencyEncodingTests.java index 72047178e9f54..4c0497fa409f9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/FrequencyEncodingTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/FrequencyEncodingTests.java @@ -15,7 +15,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; @@ -48,13 +47,14 @@ protected Writeable.Reader instanceReader() { public void testProcessWithFieldPresent() { String field = "categorical"; - List values = Arrays.asList("foo", "bar", "foobar", "baz", "farequote"); - Map valueMap = values.stream().collect(Collectors.toMap(Function.identity(), + List values = Arrays.asList("foo", "bar", "foobar", "baz", "farequote", 1.5); + Map valueMap = values.stream().collect(Collectors.toMap(Object::toString, v -> randomDoubleBetween(0.0, 1.0, false))); String encodedFeatureName = "encoded"; FrequencyEncoding encoding = new FrequencyEncoding(field, encodedFeatureName, valueMap); - String fieldValue = randomFrom(values); - Map> matchers = Collections.singletonMap(encodedFeatureName, equalTo(valueMap.get(fieldValue))); + Object fieldValue = randomFrom(values); + Map> matchers = Collections.singletonMap(encodedFeatureName, + equalTo(valueMap.get(fieldValue.toString()))); Map fieldValues = randomFieldValues(field, fieldValue); testProcess(encoding, fieldValues, matchers); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/OneHotEncodingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/OneHotEncodingTests.java index f0627719ec47c..8b35b77b5a69c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/OneHotEncodingTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/OneHotEncodingTests.java @@ -47,10 +47,10 @@ protected Writeable.Reader instanceReader() { public void testProcessWithFieldPresent() { String field = "categorical"; - List values = Arrays.asList("foo", "bar", "foobar", "baz", "farequote"); - Map valueMap = values.stream().collect(Collectors.toMap(Function.identity(), v -> "Column_" + v)); + List values = Arrays.asList("foo", "bar", "foobar", "baz", "farequote", 1.0); + Map valueMap = values.stream().collect(Collectors.toMap(Object::toString, v -> "Column_" + v.toString())); OneHotEncoding encoding = new OneHotEncoding(field, valueMap); - String fieldValue = randomFrom(values); + Object fieldValue = randomFrom(values); Map fieldValues = randomFieldValues(field, fieldValue); Map> matchers = values.stream().map(v -> "Column_" + v) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/PreProcessingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/PreProcessingTests.java index 4301b09c5ece7..c4e8b879bcdd2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/PreProcessingTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/PreProcessingTests.java @@ -58,9 +58,9 @@ Map randomFieldValues() { return fieldValues; } - Map randomFieldValues(String categoricalField, String catigoricalValue) { + Map randomFieldValues(String categoricalField, Object categoricalValue) { Map fieldValues = randomFieldValues(); - fieldValues.put(categoricalField, catigoricalValue); + fieldValues.put(categoricalField, categoricalValue); return fieldValues; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/TargetMeanEncodingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/TargetMeanEncodingTests.java index d86d9e09f0238..e2aaf1e1256c6 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/TargetMeanEncodingTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/TargetMeanEncodingTests.java @@ -15,7 +15,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; @@ -51,14 +50,15 @@ protected Writeable.Reader instanceReader() { public void testProcessWithFieldPresent() { String field = "categorical"; - List values = Arrays.asList("foo", "bar", "foobar", "baz", "farequote"); - Map valueMap = values.stream().collect(Collectors.toMap(Function.identity(), + List values = Arrays.asList("foo", "bar", "foobar", "baz", "farequote", 1.0); + Map valueMap = values.stream().collect(Collectors.toMap(Object::toString, v -> randomDoubleBetween(0.0, 1.0, false))); String encodedFeatureName = "encoded"; Double defaultvalue = randomDouble(); TargetMeanEncoding encoding = new TargetMeanEncoding(field, encodedFeatureName, valueMap, defaultvalue); - String fieldValue = randomFrom(values); - Map> matchers = Collections.singletonMap(encodedFeatureName, equalTo(valueMap.get(fieldValue))); + Object fieldValue = randomFrom(values); + Map> matchers = Collections.singletonMap(encodedFeatureName, + equalTo(valueMap.get(fieldValue.toString()))); Map fieldValues = randomFieldValues(field, fieldValue); testProcess(encoding, fieldValues, matchers); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLServiceTests.java index 67bb99e508af3..b2bf6974e319d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLServiceTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLServiceTests.java @@ -578,11 +578,12 @@ public void testReadCertificateInformation() throws Exception { final SSLService sslService = new SSLService(settings, env); final List certificates = new ArrayList<>(sslService.getLoadedCertificates()); - assertThat(certificates, iterableWithSize(10)); + assertThat(certificates, iterableWithSize(13)); Collections.sort(certificates, Comparator.comparing((CertificateInfo c) -> c.alias() == null ? "" : c.alias()).thenComparing(CertificateInfo::path)); final Iterator iterator = certificates.iterator(); + CertificateInfo cert = iterator.next(); assertThat(cert.alias(), nullValue()); assertThat(cert.path(), equalTo(pemPath.toString())); @@ -646,6 +647,15 @@ public void testReadCertificateInformation() throws Exception { assertThat(cert.expiry(), equalTo(ZonedDateTime.parse("2045-10-02T09:43:18.000Z"))); assertThat(cert.hasPrivateKey(), equalTo(true)); + cert = iterator.next(); + assertThat(cert.alias(), equalTo("testnode_dsa")); + assertThat(cert.path(), equalTo(p12Path.toString())); + assertThat(cert.format(), equalTo("PKCS12")); + assertThat(cert.serialNumber(), equalTo("223c736a")); + assertThat(cert.subjectDn(), equalTo("CN=Elasticsearch Test Node")); + assertThat(cert.expiry(), equalTo(ZonedDateTime.parse("2045-10-02T09:43:18.000Z"))); + assertThat(cert.hasPrivateKey(), equalTo(true)); + cert = iterator.next(); assertThat(cert.alias(), equalTo("testnode_ec")); assertThat(cert.path(), equalTo(jksPath.toString())); @@ -655,6 +665,15 @@ public void testReadCertificateInformation() throws Exception { assertThat(cert.expiry(), equalTo(ZonedDateTime.parse("2045-10-02T09:36:10.000Z"))); assertThat(cert.hasPrivateKey(), equalTo(true)); + cert = iterator.next(); + assertThat(cert.alias(), equalTo("testnode_ec")); + assertThat(cert.path(), equalTo(p12Path.toString())); + assertThat(cert.format(), equalTo("PKCS12")); + assertThat(cert.serialNumber(), equalTo("7268203b")); + assertThat(cert.subjectDn(), equalTo("CN=Elasticsearch Test Node")); + assertThat(cert.expiry(), equalTo(ZonedDateTime.parse("2045-10-02T09:36:10.000Z"))); + assertThat(cert.hasPrivateKey(), equalTo(true)); + cert = iterator.next(); assertThat(cert.alias(), equalTo("testnode_rsa")); assertThat(cert.path(), equalTo(jksPath.toString())); @@ -670,9 +689,18 @@ public void testReadCertificateInformation() throws Exception { assertThat(cert.format(), equalTo("PKCS12")); assertThat(cert.serialNumber(), equalTo("b8b96c37e332cccb")); assertThat(cert.subjectDn(), equalTo("CN=Elasticsearch Test Node, OU=elasticsearch, O=org")); - assertThat(cert.expiry(), equalTo(ZonedDateTime.parse("2019-09-22T18:52:57Z"))); + assertThat(cert.expiry(), equalTo(ZonedDateTime.parse("2019-09-22T18:52:57.000Z"))); assertThat(cert.hasPrivateKey(), equalTo(true)); + cert = iterator.next(); + assertThat(cert.alias(), equalTo("trusted_testnode_ec")); + assertThat(cert.path(), equalTo(jksPath.toString())); + assertThat(cert.format(), equalTo("jks")); + assertThat(cert.serialNumber(), equalTo("7268203b")); + assertThat(cert.subjectDn(), equalTo("CN=Elasticsearch Test Node")); + assertThat(cert.expiry(), equalTo(ZonedDateTime.parse("2045-10-02T09:36:10.000Z"))); + assertThat(cert.hasPrivateKey(), equalTo(false)); + assertFalse(iterator.hasNext()); } diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks index ebe6146124e8f..7c0c4f1aae7fb 100644 Binary files a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks and b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks differ diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.p12 b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.p12 index 0e6bcaa4f8b76..c61c5d99574fa 100644 Binary files a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.p12 and b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.p12 differ diff --git a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java index 120f03bddec57..191285239a945 100644 --- a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java +++ b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java @@ -252,17 +252,17 @@ public void testCanMatch() throws IOException { SearchService searchService = getInstanceFromNode(SearchService.class); SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true); assertTrue(searchService.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, shard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); searchRequest.source(sourceBuilder); sourceBuilder.query(QueryBuilders.rangeQuery("field").gte("2010-01-03||+2d").lte("2010-01-04||+2d/d")); assertTrue(searchService.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, shard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); sourceBuilder.query(QueryBuilders.rangeQuery("field").gt("2010-01-06T02:00").lt("2010-01-07T02:00")); assertFalse(searchService.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, shard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); } assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index")).actionGet()); @@ -276,17 +276,17 @@ public void testCanMatch() throws IOException { SearchService searchService = getInstanceFromNode(SearchService.class); SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true); assertTrue(searchService.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, shard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.query(QueryBuilders.rangeQuery("field").gte("2010-01-03||+2d").lte("2010-01-04||+2d/d")); searchRequest.source(sourceBuilder); assertTrue(searchService.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, shard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); sourceBuilder.query(QueryBuilders.rangeQuery("field").gt("2010-01-06T02:00").lt("2010-01-07T02:00")); assertFalse(searchService.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, shard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); IndicesStatsResponse response = client().admin().indices().prepareStats("index").clear().setRefresh(true).get(); assertEquals(0, response.getTotal().refresh.getTotal()); // never opened a reader diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java index dbc34d0e0f886..c71efadf99eca 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.junit.annotations.TestIssueLogging; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ilm.OperationMode; @@ -99,9 +98,9 @@ public void testNothingScheduledWhenNotRunning() throws InterruptedException { try (ClusterService clusterService = ClusterServiceUtils.createClusterService(initialState, threadPool); SnapshotLifecycleService sls = new SnapshotLifecycleService(Settings.EMPTY, () -> new FakeSnapshotTask(e -> logger.info("triggered")), clusterService, clock)) { - + sls.offMaster(); - + SnapshotLifecyclePolicyMetadata newPolicy = SnapshotLifecyclePolicyMetadata.builder() .setPolicy(createPolicy("foo", "*/1 * * * * ?")) .setHeaders(Collections.emptyMap()) @@ -114,27 +113,27 @@ public void testNothingScheduledWhenNotRunning() throws InterruptedException { createState(new SnapshotLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING, new SnapshotLifecycleStats())); ClusterState state = createState(new SnapshotLifecycleMetadata(policies, OperationMode.RUNNING, new SnapshotLifecycleStats())); - + sls.clusterChanged(new ClusterChangedEvent("1", state, emptyState)); - + // Since the service does not think it is master, it should not be triggered or scheduled assertThat(sls.getScheduler().scheduledJobIds(), equalTo(Collections.emptySet())); - + sls.onMaster(); assertThat(sls.getScheduler().scheduledJobIds(), equalTo(Collections.singleton("initial-1"))); - + state = createState(new SnapshotLifecycleMetadata(policies, OperationMode.STOPPING, new SnapshotLifecycleStats())); sls.clusterChanged(new ClusterChangedEvent("2", state, emptyState)); - + // Since the service is stopping, jobs should have been cancelled assertThat(sls.getScheduler().scheduledJobIds(), equalTo(Collections.emptySet())); - + state = createState(new SnapshotLifecycleMetadata(policies, OperationMode.STOPPED, new SnapshotLifecycleStats())); sls.clusterChanged(new ClusterChangedEvent("3", state, emptyState)); - + // Since the service is stopped, jobs should have been cancelled assertThat(sls.getScheduler().scheduledJobIds(), equalTo(Collections.emptySet())); - + // No jobs should be scheduled when service is closed state = createState(new SnapshotLifecycleMetadata(policies, OperationMode.RUNNING, new SnapshotLifecycleStats())); sls.close(); @@ -151,7 +150,7 @@ public void testNothingScheduledWhenNotRunning() throws InterruptedException { * Test new policies getting scheduled correctly, updated policies also being scheduled, * and deleted policies having their schedules cancelled. */ - @TestIssueLogging(value = "org.elasticsearch.xpack.slm:TRACE", issueUrl = "https://github.com/elastic/elasticsearch/issues/44997") + @AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/44997") public void testPolicyCRUD() throws Exception { ClockMock clock = new ClockMock(); final AtomicInteger triggerCount = new AtomicInteger(0); diff --git a/x-pack/plugin/security/cli/build.gradle b/x-pack/plugin/security/cli/build.gradle index 8957e98b1e341..82dda5ffa3998 100644 --- a/x-pack/plugin/security/cli/build.gradle +++ b/x-pack/plugin/security/cli/build.gradle @@ -19,18 +19,16 @@ dependencyLicenses { mapping from: /bc.*/, to: 'bouncycastle' } -rootProject.globalInfo.ready { - if (BuildParams.inFipsJvm) { - test.enabled = false - testingConventions.enabled = false - // Forbiden APIs non-portable checks fail because bouncy castle classes being used from the FIPS JDK since those are - // not part of the Java specification - all of this is as designed, so we have to relax this check for FIPS. - tasks.withType(CheckForbiddenApis) { - bundledSignatures -= "jdk-non-portable" - } - // FIPS JVM includes many classes from bouncycastle which count as jar hell for the third party audit, - // rather than provide a long list of exclusions, disable the check on FIPS. - thirdPartyAudit.enabled = false - +if (BuildParams.inFipsJvm) { + test.enabled = false + testingConventions.enabled = false + // Forbiden APIs non-portable checks fail because bouncy castle classes being used from the FIPS JDK since those are + // not part of the Java specification - all of this is as designed, so we have to relax this check for FIPS. + tasks.withType(CheckForbiddenApis) { + bundledSignatures -= "jdk-non-portable" } + // FIPS JVM includes many classes from bouncycastle which count as jar hell for the third party audit, + // rather than provide a long list of exclusions, disable the check on FIPS. + thirdPartyAudit.enabled = false + } diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.p12 b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.p12 index 0e6bcaa4f8b76..c61c5d99574fa 100644 Binary files a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.p12 and b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.p12 differ diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/Case.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/Case.java index 7536612a67dd7..1354bb27034ac 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/Case.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/Case.java @@ -116,12 +116,21 @@ protected TypeResolution resolveType() { /** * All foldable conditions that fold to FALSE should have - * been removed by the {@link Optimizer}. + * been removed by the {@link Optimizer}#SimplifyCase. */ @Override public boolean foldable() { - return (conditions.isEmpty() && elseResult.foldable()) || - (conditions.size() == 1 && conditions.get(0).condition().foldable() && conditions.get(0).result().foldable()); + if (conditions.isEmpty() && elseResult.foldable()) { + return true; + } + if (conditions.size() == 1 && conditions.get(0).condition().foldable()) { + if (conditions.get(0).condition().fold() == Boolean.TRUE) { + return conditions().get(0).result().foldable(); + } else { + return elseResult().foldable(); + } + } + return false; } @Override diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java index d2eeb98a25bda..8efb687428945 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java @@ -671,6 +671,8 @@ public void testSimplifyCaseConditionsFoldWhenFalse() { new IfConditional(EMPTY, new Equals(EMPTY, TWO, ONE), Literal.of(EMPTY, "bar2")), new IfConditional(EMPTY, new GreaterThan(EMPTY, getFieldAttribute(), ONE), Literal.of(EMPTY, "foo2")), Literal.of(EMPTY, "default"))); + assertFalse(c.foldable()); + Expression e = new SimplifyCase().rule(c); assertEquals(Case.class, e.getClass()); c = (Case) e; @@ -696,13 +698,15 @@ public void testSimplifyCaseConditionsFoldWhenTrue() { // ELSE 'default' // END - SimplifyCase rule = new SimplifyCase(); Case c = new Case(EMPTY, Arrays.asList( new IfConditional(EMPTY, new Equals(EMPTY, getFieldAttribute(), ONE), Literal.of(EMPTY, "foo1")), new IfConditional(EMPTY, new Equals(EMPTY, ONE, ONE), Literal.of(EMPTY, "bar1")), new IfConditional(EMPTY, new Equals(EMPTY, TWO, ONE), Literal.of(EMPTY, "bar2")), new IfConditional(EMPTY, new GreaterThan(EMPTY, getFieldAttribute(), ONE), Literal.of(EMPTY, "foo2")), Literal.of(EMPTY, "default"))); + assertFalse(c.foldable()); + + SimplifyCase rule = new SimplifyCase(); Expression e = rule.rule(c); assertEquals(Case.class, e.getClass()); c = (Case) e; @@ -713,7 +717,7 @@ public void testSimplifyCaseConditionsFoldWhenTrue() { assertEquals(TypeResolution.TYPE_RESOLVED, c.typeResolved()); } - public void testSimplifyCaseConditionsFoldCompletely() { + public void testSimplifyCaseConditionsFoldCompletely_FoldableElse() { // CASE WHEN 1 = 2 THEN 'foo1' // WHEN 1 = 1 THEN 'foo2' // ELSE 'default' @@ -723,11 +727,13 @@ public void testSimplifyCaseConditionsFoldCompletely() { // // 'foo2' - SimplifyCase rule = new SimplifyCase(); Case c = new Case(EMPTY, Arrays.asList( new IfConditional(EMPTY, new Equals(EMPTY, ONE, TWO), Literal.of(EMPTY, "foo1")), new IfConditional(EMPTY, new Equals(EMPTY, ONE, ONE), Literal.of(EMPTY, "foo2")), Literal.of(EMPTY, "default"))); + assertFalse(c.foldable()); + + SimplifyCase rule = new SimplifyCase(); Expression e = rule.rule(c); assertEquals(Case.class, e.getClass()); c = (Case) e; @@ -738,9 +744,34 @@ public void testSimplifyCaseConditionsFoldCompletely() { assertEquals(TypeResolution.TYPE_RESOLVED, c.typeResolved()); } - public void testSimplifyIif_ConditionTrue() { + public void testSimplifyCaseConditionsFoldCompletely_NonFoldableElse() { + // CASE WHEN 1 = 2 THEN 'foo1' + // ELSE myField + // END + // + // ==> + // + // myField (non-foldable) + + Case c = new Case(EMPTY, Arrays.asList( + new IfConditional(EMPTY, new Equals(EMPTY, ONE, TWO), Literal.of(EMPTY, "foo1")), + getFieldAttribute("myField"))); + assertFalse(c.foldable()); + + SimplifyCase rule = new SimplifyCase(); + Expression e = rule.rule(c); + assertEquals(Case.class, e.getClass()); + c = (Case) e; + assertEquals(0, c.conditions().size()); + assertFalse(c.foldable()); + assertEquals("myField", Expressions.name(c.elseResult())); + } + + public void testSimplifyIif_ConditionTrue_FoldableResult() { SimplifyCase rule = new SimplifyCase(); Iif iif = new Iif(EMPTY, new Equals(EMPTY, ONE, ONE), Literal.of(EMPTY, "foo"), Literal.of(EMPTY, "bar")); + assertTrue(iif.foldable()); + Expression e = rule.rule(iif); assertEquals(Iif.class, e.getClass()); iif = (Iif) e; @@ -750,9 +781,26 @@ public void testSimplifyIif_ConditionTrue() { assertEquals(TypeResolution.TYPE_RESOLVED, iif.typeResolved()); } - public void testSimplifyIif_ConditionFalse() { + public void testSimplifyIif_ConditionTrue_NonFoldableResult() { + SimplifyCase rule = new SimplifyCase(); + Iif iif = new Iif(EMPTY, new Equals(EMPTY, ONE, ONE), getFieldAttribute("myField"), Literal.of(EMPTY, "bar")); + assertFalse(iif.foldable()); + + Expression e = rule.rule(iif); + assertEquals(Iif.class, e.getClass()); + iif = (Iif) e; + assertEquals(1, iif.conditions().size()); + assertFalse(iif.foldable()); + assertTrue(iif.conditions().get(0).condition().foldable()); + assertEquals(Boolean.TRUE, iif.conditions().get(0).condition().fold()); + assertEquals("myField", Expressions.name(iif.conditions().get(0).result())); + } + + public void testSimplifyIif_ConditionFalse_FoldableResult() { SimplifyCase rule = new SimplifyCase(); Iif iif = new Iif(EMPTY, new Equals(EMPTY, ONE, TWO), Literal.of(EMPTY, "foo"), Literal.of(EMPTY, "bar")); + assertTrue(iif.foldable()); + Expression e = rule.rule(iif); assertEquals(Iif.class, e.getClass()); iif = (Iif) e; @@ -762,6 +810,19 @@ public void testSimplifyIif_ConditionFalse() { assertEquals(TypeResolution.TYPE_RESOLVED, iif.typeResolved()); } + public void testSimplifyIif_ConditionFalse_NonFoldableResult() { + SimplifyCase rule = new SimplifyCase(); + Iif iif = new Iif(EMPTY, new Equals(EMPTY, ONE, TWO), Literal.of(EMPTY, "foo"), getFieldAttribute("myField")); + assertFalse(iif.foldable()); + + Expression e = rule.rule(iif); + assertEquals(Iif.class, e.getClass()); + iif = (Iif) e; + assertEquals(0, iif.conditions().size()); + assertFalse(iif.foldable()); + assertEquals("myField", Expressions.name(iif.elseResult())); + } + // // Logical simplifications // diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java index 613c98a9337d4..018f4b09b5593 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java @@ -86,16 +86,6 @@ protected NamedXContentRegistry xContentRegistry() { return namedXContentRegistry; } - - /* - Had to disable warnings because tests get random date histo configs, and changing to - new interval format was non-trivial. Best for ML team to fix - */ - @Override - protected boolean enableWarningsCheck() { - return false; - } - public void testValidateExistingIndex() throws Exception { SourceConfig source = new SourceConfig(new String[]{"existing_source_index"}, QueryConfig.matchAll()); Pivot pivot = new Pivot(getValidPivotConfig());