Merge remote-tracking branch 'elastic/master' into packaging-script-o…

…ne-liners

* elastic/master:
  [TEST] Enable DEBUG logging on testAutoQueueSizingWithMax
  [ML] Don't install empty ML metadata on startup (elastic#30751)
  Add assertion on removing copy_settings (elastic#30748)
  bump lucene version for 6_3_0
  [DOCS] Mark painless execute api as experimental (elastic#30710)
  disable annotation processor for docs (elastic#30610)
  Add more script contexts (elastic#30721)
  Fix default shards count in create index docs (elastic#30747)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy

@@ -24,6 +24,7 @@ import org.elasticsearch.gradle.BuildPlugin
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.plugins.JavaBasePlugin
+import org.gradle.api.tasks.compile.JavaCompile
 
 /**
  * Configures the build to compile against Elasticsearch's test framework and
@@ -49,5 +50,12 @@ public class StandaloneTestPlugin implements Plugin<Project> {
         test.testClassesDir project.sourceSets.test.output.classesDir
         test.mustRunAfter(project.precommit)
         project.check.dependsOn(test)
+
+        project.tasks.withType(JavaCompile) {
+            // This will be the default in Gradle 5.0
+            if (options.compilerArgs.contains("-processor") == false) {
+                options.compilerArgs << '-proc:none'
+            }
+        }
     }
 }

docs/painless/painless-execute-script.asciidoc

@@ -1,6 +1,8 @@
 [[painless-execute-api]]
 === Painless execute API
 
+experimental[The painless execute api is new and the request / response format may change in a breaking way in the future]
+
 The Painless execute API allows an arbitrary script to be executed and a result to be returned.
 
 [[painless-execute-api-parameters]]

docs/reference/indices/create-index.asciidoc

@@ -25,7 +25,7 @@ PUT twitter
 }
 --------------------------------------------------
 // CONSOLE
-<1> Default for `number_of_shards` is 5
+<1> Default for `number_of_shards` is 1
 <2> Default for `number_of_replicas` is 1 (ie one replica for each primary shard)
 
 The above second curl example shows how an index called `twitter` can be

plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java

@@ -54,7 +54,7 @@ public String getType() {
 
         @Override
         public <T> T compile(String scriptName, String scriptSource, ScriptContext<T> context, Map<String, String> params) {
-            if (context.equals(SearchScript.CONTEXT) == false) {
+            if (context.equals(SearchScript.SCRIPT_SCORE_CONTEXT) == false) {
                 throw new IllegalArgumentException(getType() + " scripts cannot be used for context [" + context.name + "]");
             }
             // we use the script "source" as the script identifier

server/src/main/java/org/elasticsearch/Version.java

@@ -169,7 +169,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final int V_6_2_5_ID = 6020599;
     public static final Version V_6_2_5 = new Version(V_6_2_5_ID, LUCENE_7_2_1);
     public static final int V_6_3_0_ID = 6030099;
-    public static final Version V_6_3_0 = new Version(V_6_3_0_ID, org.apache.lucene.util.Version.LUCENE_7_3_0);
+    public static final Version V_6_3_0 = new Version(V_6_3_0_ID, org.apache.lucene.util.Version.LUCENE_7_3_1);
     public static final int V_6_4_0_ID = 6040099;
     public static final Version V_6_4_0 = new Version(V_6_4_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
     public static final int V_7_0_0_alpha1_ID = 7000001;

server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java

@@ -249,7 +249,8 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
             IndexNumericFieldData fieldData = context.getForField(msmFieldType);
             longValuesSource = new FieldValuesSource(fieldData);
         } else if (minimumShouldMatchScript != null) {
-            SearchScript.Factory factory = context.getScriptService().compile(minimumShouldMatchScript, SearchScript.CONTEXT);
+            SearchScript.Factory factory = context.getScriptService().compile(minimumShouldMatchScript,
+                SearchScript.TERMS_SET_QUERY_CONTEXT);
             Map<String, Object> params = new HashMap<>();
             params.putAll(minimumShouldMatchScript.getParams());
             params.put("num_terms", queries.size());

server/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java

@@ -92,7 +92,7 @@ protected int doHashCode() {
     @Override
     protected ScoreFunction doToFunction(QueryShardContext context) {
         try {
-            SearchScript.Factory factory = context.getScriptService().compile(script, SearchScript.CONTEXT);
+            SearchScript.Factory factory = context.getScriptService().compile(script, SearchScript.SCRIPT_SCORE_CONTEXT);
             SearchScript.LeafFactory searchScript = factory.newFactory(script.getParams(), context.lookup());
             return new ScriptScoreFunction(script, searchScript);
         } catch (Exception e) {

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.rest.action.admin.indices;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
 import org.elasticsearch.action.support.ActiveShardCount;
@@ -47,6 +48,8 @@ public abstract class RestResizeHandler extends BaseRestHandler {
     public final RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final ResizeRequest resizeRequest = new ResizeRequest(request.param("target"), request.param("index"));
         resizeRequest.setResizeType(getResizeType());
+        // copy_settings should be removed in Elasticsearch 8.0.0; cf. https://github.com/elastic/elasticsearch/issues/28347
+        assert Version.CURRENT.major < 8;
         final String rawCopySettings = request.param("copy_settings");
         final Boolean copySettings;
         if (rawCopySettings == null) {

server/src/main/java/org/elasticsearch/script/ScriptModule.java

@@ -42,6 +42,9 @@ public class ScriptModule {
         CORE_CONTEXTS = Stream.of(
             SearchScript.CONTEXT,
             SearchScript.AGGS_CONTEXT,
+            SearchScript.SCRIPT_SCORE_CONTEXT,
+            SearchScript.SCRIPT_SORT_CONTEXT,
+            SearchScript.TERMS_SET_QUERY_CONTEXT,
             ExecutableScript.CONTEXT,
             ExecutableScript.AGGS_CONTEXT,
             ExecutableScript.UPDATE_CONTEXT,

server/src/main/java/org/elasticsearch/script/SearchScript.java

@@ -158,6 +158,12 @@ public interface Factory {
 
     /** The context used to compile {@link SearchScript} factories. */
     public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("search", Factory.class);
-    // TODO: remove aggs context when it has its own interface
+    // TODO: remove these contexts when it has its own interface
     public static final ScriptContext<Factory> AGGS_CONTEXT = new ScriptContext<>("aggs", Factory.class);
-}
+    // Can return a double. (For ScriptSortType#NUMBER only, for ScriptSortType#STRING normal CONTEXT should be used)
+    public static final ScriptContext<Factory> SCRIPT_SORT_CONTEXT = new ScriptContext<>("sort", Factory.class);
+    // Can return a float
+    public static final ScriptContext<Factory> SCRIPT_SCORE_CONTEXT = new ScriptContext<>("score", Factory.class);
+    // Can return a long
+    public static final ScriptContext<Factory> TERMS_SET_QUERY_CONTEXT = new ScriptContext<>("terms_set", Factory.class);
+}

server/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java

@@ -305,7 +305,7 @@ public static ScriptSortBuilder fromXContent(XContentParser parser, String eleme
 
     @Override
     public SortFieldAndFormat build(QueryShardContext context) throws IOException {
-        final SearchScript.Factory factory = context.getScriptService().compile(script, SearchScript.CONTEXT);
+        final SearchScript.Factory factory = context.getScriptService().compile(script, SearchScript.SCRIPT_SORT_CONTEXT);
         final SearchScript.LeafFactory searchScript = factory.newFactory(script.getParams(), context.lookup());
 
         MultiValueMode valueMode = null;

server/src/test/java/org/elasticsearch/common/util/concurrent/QueueResizingEsThreadPoolExecutorTests.java

@@ -154,6 +154,7 @@ public void testAutoQueueSizingWithMin() throws Exception {
         context.close();
     }
 
+    @TestLogging("org.elasticsearch.common.util.concurrent:DEBUG")
     public void testAutoQueueSizingWithMax() throws Exception {
         ThreadContext context = new ThreadContext(Settings.EMPTY);
         ResizableBlockingQueue<Runnable> queue =

server/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java

@@ -76,7 +76,7 @@ public String getType() {
                 @Override
                 public <T> T compile(String scriptName, String scriptSource, ScriptContext<T> context, Map<String, String> params) {
                     assert scriptSource.equals("explainable_script");
-                    assert context == SearchScript.CONTEXT;
+                    assert context == SearchScript.SCRIPT_SCORE_CONTEXT;
                     SearchScript.Factory factory = (p, lookup) -> new SearchScript.LeafFactory() {
                         @Override
                         public SearchScript newInstance(LeafReaderContext context) throws IOException {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java

@@ -9,6 +9,7 @@
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.cluster.NamedDiff;
@@ -467,6 +468,14 @@ public static DatafeedState getDatafeedState(String datafeedId, @Nullable Persis
         }
     }
 
+    public static MlMetadata getMlMetadata(ClusterState state) {
+        MlMetadata mlMetadata = (state == null) ? null : state.getMetaData().custom(MLMetadataField.TYPE);
+        if (mlMetadata == null) {
+            return EMPTY_METADATA;
+        }
+        return mlMetadata;
+    }
+
     public static class JobAlreadyMarkedAsDeletedException extends RuntimeException {
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java

@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.core.ml.job.persistence;
 
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 
 /**
@@ -47,8 +46,7 @@ public static String resultsWriteAlias(String jobId) {
      * @return The index name
      */
     public static String getPhysicalIndexFromState(ClusterState state, String jobId) {
-        MlMetadata meta = state.getMetaData().custom(MLMetadataField.TYPE);
-        return meta.getJobs().get(jobId).getResultsIndexName();
+        return MlMetadata.getMlMetadata(state).getJobs().get(jobId).getResultsIndexName();
     }
 
     /**

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java

@@ -23,7 +23,6 @@
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.XPackField;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MachineLearningFeatureSetUsage;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction;
@@ -132,15 +131,7 @@ public Map<String, Object> nativeCodeInfo() {
     @Override
     public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
         ClusterState state = clusterService.state();
-        MlMetadata mlMetadata = state.getMetaData().custom(MLMetadataField.TYPE);
-
-        // Handle case when usage is called but MlMetadata has not been installed yet
-        if (mlMetadata == null) {
-            listener.onResponse(new MachineLearningFeatureSetUsage(available(), enabled,
-                    Collections.emptyMap(), Collections.emptyMap()));
-        } else {
-            new Retriever(client, mlMetadata, available(), enabled()).execute(listener);
-        }
+        new Retriever(client, MlMetadata.getMlMetadata(state), available(), enabled()).execute(listener);
     }
 
     public static class Retriever {

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java

@@ -13,7 +13,6 @@
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
 import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
@@ -90,8 +89,7 @@ public void clusterChanged(ClusterChangedEvent event) {
                 }
             } else if (StartDatafeedAction.TASK_NAME.equals(currentTask.getTaskName())) {
                 String datafeedId = ((StartDatafeedAction.DatafeedParams) currentTask.getParams()).getDatafeedId();
-                MlMetadata mlMetadata = event.state().getMetaData().custom(MLMetadataField.TYPE);
-                DatafeedConfig datafeedConfig = mlMetadata.getDatafeed(datafeedId);
+                DatafeedConfig datafeedConfig = MlMetadata.getMlMetadata(event.state()).getDatafeed(datafeedId);
                 if (currentAssignment.getExecutorNode() == null) {
                     String msg = "No node found to start datafeed [" + datafeedId +"]. Reasons [" +
                             currentAssignment.getExplanation() + "]";

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java

@@ -7,29 +7,20 @@
 
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
-import org.elasticsearch.xpack.core.ml.MlMetadata;
-
-import java.util.concurrent.atomic.AtomicBoolean;
 
 class MlInitializationService extends AbstractComponent implements ClusterStateListener {
 
     private final ThreadPool threadPool;
     private final ClusterService clusterService;
     private final Client client;
 
-    private final AtomicBoolean installMlMetadataCheck = new AtomicBoolean(false);
-
     private volatile MlDailyMaintenanceService mlDailyMaintenanceService;
 
     MlInitializationService(Settings settings, ThreadPool threadPool, ClusterService clusterService, Client client) {
@@ -48,45 +39,12 @@ public void clusterChanged(ClusterChangedEvent event) {
         }
 
         if (event.localNodeMaster()) {
-            MetaData metaData = event.state().metaData();
-            installMlMetadata(metaData);
             installDailyMaintenanceService();
         } else {
             uninstallDailyMaintenanceService();
         }
     }
 
-    private void installMlMetadata(MetaData metaData) {
-        if (metaData.custom(MLMetadataField.TYPE) == null) {
-            if (installMlMetadataCheck.compareAndSet(false, true)) {
-                threadPool.executor(ThreadPool.Names.GENERIC).execute(() ->
-                    clusterService.submitStateUpdateTask("install-ml-metadata", new ClusterStateUpdateTask() {
-                        @Override
-                        public ClusterState execute(ClusterState currentState) throws Exception {
-                            // If the metadata has been added already don't try to update
-                            if (currentState.metaData().custom(MLMetadataField.TYPE) != null) {
-                                return currentState;
-                            }
-                            ClusterState.Builder builder = new ClusterState.Builder(currentState);
-                            MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData());
-                            metadataBuilder.putCustom(MLMetadataField.TYPE, MlMetadata.EMPTY_METADATA);
-                            builder.metaData(metadataBuilder.build());
-                            return builder.build();
-                        }
-
-                        @Override
-                        public void onFailure(String source, Exception e) {
-                            installMlMetadataCheck.set(false);
-                            logger.error("unable to install ml metadata", e);
-                        }
-                    })
-                );
-            }
-        } else {
-            installMlMetadataCheck.set(false);
-        }
-    }
-
     private void installDailyMaintenanceService() {
         if (mlDailyMaintenanceService == null) {
             mlDailyMaintenanceService = new MlDailyMaintenanceService(clusterService.getClusterName(), threadPool, client);

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java

@@ -27,7 +27,6 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
 import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
@@ -92,8 +91,7 @@ public TransportCloseJobAction(Settings settings, TransportService transportServ
     static void resolveAndValidateJobId(CloseJobAction.Request request, ClusterState state, List<String> openJobIds,
                                         List<String> closingJobIds) {
         PersistentTasksCustomMetaData tasksMetaData = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
-        MlMetadata maybeNull = state.metaData().custom(MLMetadataField.TYPE);
-        final MlMetadata mlMetadata = (maybeNull == null) ? MlMetadata.EMPTY_METADATA : maybeNull;
+        final MlMetadata mlMetadata = MlMetadata.getMlMetadata(state);
 
         List<String> failedJobs = new ArrayList<>();
 
@@ -107,7 +105,7 @@ static void resolveAndValidateJobId(CloseJobAction.Request request, ClusterState
         };
 
         Set<String> expandedJobIds = mlMetadata.expandJobIds(request.getJobId(), request.allowNoJobs());
-        expandedJobIds.stream().forEach(jobIdProcessor::accept);
+        expandedJobIds.forEach(jobIdProcessor::accept);
         if (request.isForce() == false && failedJobs.size() > 0) {
             if (expandedJobIds.size() == 1) {
                 throw ExceptionsHelper.conflictStatusException("cannot close job [{}] because it failed, use force close",

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java

@@ -119,8 +119,8 @@ protected DeleteDatafeedAction.Response newResponse(boolean acknowledged) {
                     }
 
                     @Override
-                    public ClusterState execute(ClusterState currentState) throws Exception {
-                        MlMetadata currentMetadata = currentState.getMetaData().custom(MLMetadataField.TYPE);
+                    public ClusterState execute(ClusterState currentState) {
+                        MlMetadata currentMetadata = MlMetadata.getMlMetadata(currentState);
                         PersistentTasksCustomMetaData persistentTasks =
                                 currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
                         MlMetadata newMetadata = new MlMetadata.Builder(currentMetadata)

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteFilterAction.java

@@ -24,7 +24,6 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.DeleteFilterAction;
-import org.elasticsearch.xpack.core.ml.MLMetadataField;
 import org.elasticsearch.xpack.core.ml.MlMetaIndex;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.job.config.Detector;
@@ -60,8 +59,7 @@ protected void doExecute(DeleteFilterAction.Request request, ActionListener<Dele
 
         final String filterId = request.getFilterId();
         ClusterState state = clusterService.state();
-        MlMetadata currentMlMetadata = state.metaData().custom(MLMetadataField.TYPE);
-        Map<String, Job> jobs = currentMlMetadata.getJobs();
+        Map<String, Job> jobs = MlMetadata.getMlMetadata(state).getJobs();
         List<String> currentlyUsedBy = new ArrayList<>();
         for (Job job : jobs.values()) {
             List<Detector> detectors = job.getAnalysisConfig().getDetectors();