Merge branch 'master' into feature/data_stream_support_routing

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-nojdk.yml

@@ -0,0 +1,60 @@
+---
+- job:
+    name: "elastic+elasticsearch+pull-request+packaging-tests-windows-nojdk"
+    display-name: "elastic / elasticsearch - pull request packaging-tests-windows-nojdk"
+    description: "Testing of Elasticsearch pull requests - packaging-tests-windows-nojdk"
+    # We use a hard-coded workspace directory here to avoid hitting windows path length limits
+    child-workspace: "C:\\Users\\jenkins\\workspace\\pr-packaging-windows\\${BUILD_NUMBER}"
+    project-type: matrix
+    node: master
+    scm:
+      - git:
+          refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+          branches:
+            - "${ghprbActualCommit}"
+    triggers:
+      - github-pull-request:
+          org-list:
+            - elastic
+          allow-whitelist-orgs-as-admins: true
+          trigger-phrase: '.*run\W+elasticsearch-ci/packaging-tests-windows.*'
+          github-hooks: true
+          status-context: elasticsearch-ci/packaging-tests-windows
+          cancel-builds-on-update: true
+          # We've removed the no-jdk distribution on master as well
+          black-list-target-branches:
+            - master
+            - 6.8
+          excluded-regions:
+            - ^docs/.*
+          white-list-labels:
+            - ':Delivery/Packaging'
+          black-list-labels:
+            - '>test-mute'
+    axes:
+      - axis:
+          type: label-expression
+          name: os
+          values:
+            - "windows-2012-r2"
+            - "windows-2016"
+            - "windows-2019"
+      - axis:
+          type: user-defined
+          name: PACKAGING_TASK
+          values:
+            - 'default-windows-archive'
+            - 'default-windows-archive-no-jdk'
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA
+      - batch: |
+          del /f /s /q %USERPROFILE%\.gradle\init.d\*.*
+          mkdir %USERPROFILE%\.gradle\init.d
+          copy .ci\init.gradle %USERPROFILE%\.gradle\init.d\
+          (
+             echo powershell.exe .\.ci\scripts\packaging-test.ps1 -GradleTasks destructiveDistroTest.%PACKAGING_TASK%   ^|^| exit /b 1
+          ) | java -jar "C:\Program Files\infra\bin\runbld" --redirect-stderr -

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample-nojdk.yml

@@ -0,0 +1,57 @@
+---
+- job:
+    name: "elastic+elasticsearch+pull-request+packaging-tests-windows-sample-nojdk"
+    display-name: "elastic / elasticsearch - pull request packaging-tests-windows-sample-nojdk"
+    description: "Testing of Elasticsearch pull requests - packaging-tests-windows-sample-nojdk"
+    # We use a hard-coded workspace directory here to avoid hitting windows path length limits
+    child-workspace: "C:\\Users\\jenkins\\workspace\\pr-packaging-windows\\${BUILD_NUMBER}"
+    project-type: matrix
+    node: master
+    scm:
+      - git:
+          refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+          branches:
+            - "${ghprbActualCommit}"
+    triggers:
+      - github-pull-request:
+          org-list:
+            - elastic
+          allow-whitelist-orgs-as-admins: true
+          trigger-phrase: '.*run\W+elasticsearch-ci/packaging-tests-windows-sample.*'
+          github-hooks: true
+          status-context: elasticsearch-ci/packaging-tests-windows-sample
+          cancel-builds-on-update: true
+          # We've removed the no-jdk distribution on master as well
+          black-list-target-branches:
+            - master
+            - 6.8
+          excluded-regions:
+            - ^docs/.*
+          black-list-labels:
+            - '>test-mute'
+            - ':Delivery/Packaging'
+    axes:
+      - axis:
+          type: label-expression
+          name: os
+          values:
+            - "windows-2019"
+      - axis:
+          type: user-defined
+          name: PACKAGING_TASK
+          values:
+            - 'default-windows-archive'
+            - 'default-windows-archive-no-jdk'
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA
+      - batch: |
+          del /f /s /q %USERPROFILE%\.gradle\init.d\*.*
+          mkdir %USERPROFILE%\.gradle\init.d
+          copy .ci\init.gradle %USERPROFILE%\.gradle\init.d\
+          (
+             echo powershell.exe .\.ci\scripts\packaging-test.ps1 -GradleTasks destructiveDistroTest.%PACKAGING_TASK%   ^|^| exit /b 1
+          ) | java -jar "C:\Program Files\infra\bin\runbld" --redirect-stderr -

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample.yml

@@ -21,8 +21,8 @@
           github-hooks: true
           status-context: elasticsearch-ci/packaging-tests-windows-sample
           cancel-builds-on-update: true
-          black-list-target-branches:
-            - 6.8
+          white-list-target-branches:
+            - master
           excluded-regions:
             - ^docs/.*
           black-list-labels:
@@ -39,7 +39,6 @@
           name: PACKAGING_TASK
           values:
             - 'default-windows-archive'
-            - 'default-windows-archive-no-jdk'
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows.yml

@@ -21,8 +21,8 @@
           github-hooks: true
           status-context: elasticsearch-ci/packaging-tests-windows
           cancel-builds-on-update: true
-          black-list-target-branches:
-            - 6.8
+          white-list-target-branches:
+            - master
           excluded-regions:
             - ^docs/.*
           white-list-labels:
@@ -42,7 +42,6 @@
           name: PACKAGING_TASK
           values:
             - 'default-windows-archive'
-            - 'default-windows-archive-no-jdk'
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

.github/workflows/sync-main-branch.yml

@@ -0,0 +1,26 @@
+# Synchronize all pushes to 'master' branch with 'main' branch to facilitate migration
+name: "Sync main branch"
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  sync_latest_from_upstream:
+    runs-on: ubuntu-latest
+    name: Sync latest commits from master branch
+
+    steps:
+      - name: Checkout target repo
+        uses: actions/checkout@v2
+        with:
+          ref: main
+
+      - name: Sync upstream changes
+        id: sync
+        uses: aormsby/[email protected]
+        with:
+          target_sync_branch: main
+          target_repo_token: ${{ secrets.GITHUB_TOKEN }}
+          upstream_sync_branch: master
+          upstream_sync_repo: elastic/elasticsearch

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -129,7 +129,8 @@ if (providers.systemProperty('idea.active').forUseAtConfigurationTime().getOrNul
               '--add-opens=java.base/javax.net.ssl=ALL-UNNAMED',
               '--add-opens=java.base/java.nio.file=ALL-UNNAMED',
               '--add-opens=java.base/java.time=ALL-UNNAMED',
-              '--add-opens=java.base/java.lang=ALL-UNNAMED'
+              '--add-opens=java.base/java.lang=ALL-UNNAMED',
+              '--add-opens=java.management/java.lang.management=ALL-UNNAMED'
             ].join(' ')
           }
         }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -103,6 +103,7 @@ public void execute(Task t) {
                 "--add-opens=java.base/javax.net.ssl=ALL-UNNAMED",
                 "--add-opens=java.base/java.nio.file=ALL-UNNAMED",
                 "--add-opens=java.base/java.time=ALL-UNNAMED",
+                "--add-opens=java.management/java.lang.management=ALL-UNNAMED",
                 "-XX:+HeapDumpOnOutOfMemoryError"
             );
 

client/rest-high-level/src/main/java/org/elasticsearch/client/TransformRequestConverters.java

@@ -137,11 +137,16 @@ static Request stopTransform(StopTransformRequest stopRequest) {
     }
 
     static Request previewTransform(PreviewTransformRequest previewRequest) throws IOException {
-        String endpoint = new RequestConverters.EndpointBuilder()
-                .addPathPartAsIs("_transform", "_preview")
-                .build();
+        RequestConverters.EndpointBuilder endpointBuilder = new RequestConverters.EndpointBuilder().addPathPartAsIs("_transform");
+        if (previewRequest.getTransformId() != null) {
+            endpointBuilder.addPathPart(previewRequest.getTransformId());
+        }
+        endpointBuilder.addPathPartAsIs("_preview");
+        String endpoint = endpointBuilder.build();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        request.setEntity(createEntity(previewRequest, REQUEST_BODY_CONTENT_TYPE));
+        if (previewRequest.getTransformId() == null) {
+            request.setEntity(createEntity(previewRequest, REQUEST_BODY_CONTENT_TYPE));
+        }
         return request;
     }
 

client/rest-high-level/src/main/java/org/elasticsearch/client/transform/PreviewTransformRequest.java

@@ -21,10 +21,21 @@
 
 public class PreviewTransformRequest implements ToXContentObject, Validatable {
 
+    private final String transformId;
     private final TransformConfig config;
 
+    public PreviewTransformRequest(String transformId) {
+        this.transformId = Objects.requireNonNull(transformId);
+        this.config = null;
+    }
+
     public PreviewTransformRequest(TransformConfig config) {
-        this.config = config;
+        this.transformId = null;
+        this.config = Objects.requireNonNull(config);
+    }
+
+    public String getTransformId() {
+        return transformId;
     }
 
     public TransformConfig getConfig() {
@@ -33,16 +44,20 @@ public TransformConfig getConfig() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
-        return config.toXContent(builder, params);
+        if (this.config != null) {
+            return this.config.toXContent(builder, params);
+        } else {
+            return builder
+                .startObject()
+                .field(TransformConfig.ID.getPreferredName(), this.transformId)
+                .endObject();
+        }
     }
 
     @Override
     public Optional<ValidationException> validate() {
         ValidationException validationException = new ValidationException();
-        if (config == null) {
-            validationException.addValidationError("preview requires a non-null transform config");
-            return Optional.of(validationException);
-        } else {
+        if (config != null) {
             if (config.getSource() == null) {
                 validationException.addValidationError("transform source cannot be null");
             }
@@ -57,7 +72,7 @@ public Optional<ValidationException> validate() {
 
     @Override
     public int hashCode() {
-        return Objects.hash(config);
+        return Objects.hash(transformId, config);
     }
 
     @Override
@@ -69,6 +84,7 @@ public boolean equals(Object obj) {
             return false;
         }
         PreviewTransformRequest other = (PreviewTransformRequest) obj;
-        return Objects.equals(config, other.config);
+        return Objects.equals(transformId, other.transformId)
+            && Objects.equals(config, other.config);
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java

@@ -336,7 +336,6 @@ public void testStartStop() throws IOException {
         assertThat(taskState, is(TransformStats.State.STOPPED));
     }
 
-    @SuppressWarnings("unchecked")
     public void testPreview() throws IOException {
         String sourceIndex = "transform-source";
         createIndex(sourceIndex);
@@ -345,12 +344,28 @@ public void testPreview() throws IOException {
         TransformConfig transform = validDataFrameTransformConfig("test-preview", sourceIndex, null);
 
         TransformClient client = highLevelClient().transform();
-        PreviewTransformResponse preview = execute(
-            new PreviewTransformRequest(transform),
-            client::previewTransform,
-            client::previewTransformAsync
-        );
+        PreviewTransformResponse preview =
+            execute(new PreviewTransformRequest(transform), client::previewTransform, client::previewTransformAsync);
+        assertExpectedPreview(preview);
+    }
+
+    public void testPreviewById() throws IOException {
+        String sourceIndex = "transform-source";
+        createIndex(sourceIndex);
+        indexData(sourceIndex);
 
+        String transformId = "test-preview-by-id";
+        TransformConfig transform = validDataFrameTransformConfig(transformId, sourceIndex, "pivot-dest");
+        putTransform(transform);
+
+        TransformClient client = highLevelClient().transform();
+        PreviewTransformResponse preview =
+            execute(new PreviewTransformRequest(transformId), client::previewTransform, client::previewTransformAsync);
+        assertExpectedPreview(preview);
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void assertExpectedPreview(PreviewTransformResponse preview) {
         List<Map<String, Object>> docs = preview.getDocs();
         assertThat(docs, hasSize(2));
         Optional<Map<String, Object>> theresa = docs.stream().filter(doc -> "theresa".equals(doc.get("reviewer"))).findFirst();

client/rest-high-level/src/test/java/org/elasticsearch/client/TransformRequestConvertersTests.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.client.transform.GetTransformRequest;
 import org.elasticsearch.client.transform.GetTransformStatsRequest;
 import org.elasticsearch.client.transform.PreviewTransformRequest;
+import org.elasticsearch.client.transform.PreviewTransformRequestTests;
 import org.elasticsearch.client.transform.PutTransformRequest;
 import org.elasticsearch.client.transform.StartTransformRequest;
 import org.elasticsearch.client.transform.StopTransformRequest;
@@ -44,6 +45,7 @@
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
 
 public class TransformRequestConvertersTests extends ESTestCase {
 
@@ -176,19 +178,28 @@ public void testStopDataFrameTransform() {
     }
 
     public void testPreviewDataFrameTransform() throws IOException {
-        PreviewTransformRequest previewRequest = new PreviewTransformRequest(
-                TransformConfigTests.randomTransformConfig());
+        PreviewTransformRequest previewRequest = new PreviewTransformRequest(TransformConfigTests.randomTransformConfig());
         Request request = TransformRequestConverters.previewTransform(previewRequest);
 
         assertEquals(HttpPost.METHOD_NAME, request.getMethod());
         assertThat(request.getEndpoint(), equalTo("/_transform/_preview"));
 
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, request.getEntity().getContent())) {
-            TransformConfig parsedConfig = TransformConfig.PARSER.apply(parser, null);
-            assertThat(parsedConfig, equalTo(previewRequest.getConfig()));
+            PreviewTransformRequest parsedRequest = PreviewTransformRequestTests.fromXContent(parser);
+            assertThat(parsedRequest, equalTo(previewRequest));
         }
     }
 
+    public void testPreviewDataFrameTransformById() throws IOException {
+        String transformId = randomAlphaOfLengthBetween(1, 10);
+        PreviewTransformRequest previewRequest = new PreviewTransformRequest(transformId);
+        Request request = TransformRequestConverters.previewTransform(previewRequest);
+
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
+        assertThat(request.getEndpoint(), equalTo("/_transform/" + transformId + "/_preview"));
+        assertThat(request.getEntity(), nullValue());
+    }
+
     public void testGetDataFrameTransformStats() {
         GetTransformStatsRequest getStatsRequest = new GetTransformStatsRequest("foo");
         Request request = TransformRequestConverters.getTransformStats(getStatsRequest);