Merge branch 'master' into ccr

* master: (31 commits)
  [TEST] Fix `GeoShapeQueryTests#testPointsOnly` failure
  Transition transport apis to use void listeners (#27440)
  AwaitsFix GeoShapeQueryTests#testPointsOnly #27454
  Bump test version after backport
  Ensure nested documents have consistent version and seq_ids (#27455)
  Tests: Add Fedora-27 to packaging tests
  Delete some seemingly unused exceptions (#27439)
  #26800: Fix docs rendering
  Remove config prompting for secrets and text (#27216)
  Move the CLI into its own subproject (#27114)
  Correct usage of "an" to "a" in getting started docs
  Avoid NPE when getting build information
  Removes BWC snapshot status handler used in 6.x (#27443)
  Remove manual tracking of registered channels (#27445)
  Remove parameters on HandshakeResponseHandler (#27444)
  [GEO] fix pointsOnly bug for MULTIPOINT
  Standardize underscore requirements in parameters (#27414)
  peanut butter hamburgers
  Log primary-replica resync failures
  Uses TransportMasterNodeAction to update shard snapshot status (#27165)
  ...

README.textile

@@ -208,8 +208,7 @@ In order to create a distribution, simply run the @gradle assemble@ command in t
 
 The distribution for each project will be created under the @build/distributions@ directory in that project.
 
-See the "TESTING":TESTING.asciidoc file for more information about
-running the Elasticsearch test suite.
+See the "TESTING":TESTING.asciidoc file for more information about running the Elasticsearch test suite.
 
 h3. Upgrading from Elasticsearch 1.x?
 

TESTING.asciidoc

@@ -351,8 +351,8 @@ These are the linux flavors the Vagrantfile currently supports:
 * debian-9 aka stretch, the current debian stable distribution
 * centos-6
 * centos-7
-* fedora-25
 * fedora-26
+* fedora-27
 * oel-6 aka Oracle Enterprise Linux 6
 * oel-7 aka Oracle Enterprise Linux 7
 * sles-12
@@ -428,23 +428,23 @@ sudo -E bats $BATS_TESTS/*.bats
 You can also use Gradle to prepare the test environment and then starts a single VM:
 
 -------------------------------------------------
-gradle vagrantFedora25#up
+gradle vagrantFedora27#up
 -------------------------------------------------
 
 Or any of vagrantCentos6#up, vagrantCentos7#up, vagrantDebian8#up,
-vagrantFedora25#up, vagrantOel6#up, vagrantOel7#up, vagrantOpensuse13#up,
-vagrantSles12#up, vagrantUbuntu1404#up, vagrantUbuntu1604#up.
+vagrantDebian9#up, vagrantFedora26#up, vagrantFedora27#up, vagrantOel6#up, vagrantOel7#up,
+vagrantOpensuse42#up,vagrantSles12#up, vagrantUbuntu1404#up, vagrantUbuntu1604#up.
 
 Once up, you can then connect to the VM using SSH from the elasticsearch directory:
 
 -------------------------------------------------
-vagrant ssh fedora-25
+vagrant ssh fedora-27
 -------------------------------------------------
 
 Or from another directory:
 
 -------------------------------------------------
-VAGRANT_CWD=/path/to/elasticsearch vagrant ssh fedora-25
+VAGRANT_CWD=/path/to/elasticsearch vagrant ssh fedora-27
 -------------------------------------------------
 
 Note: Starting vagrant VM outside of the elasticsearch folder requires to

Vagrantfile

@@ -60,14 +60,14 @@ Vagrant.configure(2) do |config|
     config.vm.box = "elastic/oraclelinux-7-x86_64"
     rpm_common config
   end
-  config.vm.define "fedora-25" do |config|
-    config.vm.box = "elastic/fedora-25-x86_64"
-    dnf_common config
-  end
   config.vm.define "fedora-26" do |config|
     config.vm.box = "elastic/fedora-26-x86_64"
     dnf_common config
   end
+  config.vm.define "fedora-27" do |config|
+    config.vm.box = "elastic/fedora-27-x86_64"
+    dnf_common config
+  end
   config.vm.define "opensuse-42" do |config|
     config.vm.box = "elastic/opensuse-42-x86_64"
     opensuse_common config

build.gradle

@@ -81,6 +81,7 @@ List<Version> versions = []
 // keep track of the previous major version's last minor, so we know where wire compat begins
 int prevMinorIndex = -1 // index in the versions list of the last minor from the prev major
 int lastPrevMinor = -1 // the minor version number from the prev major we most recently seen
+int prevBugfixIndex = -1 // index in the versions list of the last bugfix release from the prev major
 for (String line : versionLines) {
   /* Note that this skips alphas and betas which is fine because they aren't
    * compatible with anything. */
@@ -108,12 +109,19 @@ for (String line : versionLines) {
         lastPrevMinor = minor
       }
     }
+    if (major == prevMajor) {
+      prevBugfixIndex = versions.size() - 1
+    }
   }
 }
 if (versions.toSorted { it.id } != versions) {
   println "Versions: ${versions}"
   throw new GradleException("Versions.java contains out of order version constants")
 }
+if (prevBugfixIndex != -1) {
+  versions[prevBugfixIndex] = new Version(versions[prevBugfixIndex].major, versions[prevBugfixIndex].minor,
+                                          versions[prevBugfixIndex].bugfix, versions[prevBugfixIndex].suffix, true)
+}
 if (currentVersion.bugfix == 0) {
   // If on a release branch, after the initial release of that branch, the bugfix version will
   // be bumped, and will be != 0. On master and N.x branches, we want to test against the
@@ -223,6 +231,7 @@ subprojects {
     "org.elasticsearch.gradle:build-tools:${version}": ':build-tools',
     "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
     "org.elasticsearch:elasticsearch:${version}": ':core',
+    "org.elasticsearch:elasticsearch-cli:${version}": ':core:cli',
     "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
     "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer',
     "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',
@@ -262,6 +271,11 @@ subprojects {
       ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
       ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
     }
+  } else if (indexCompatVersions[-2].snapshot) {
+    /* This is a terrible hack for the bump to 6.0.1 which will be fixed by #27397 */
+    ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
+    ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
+    ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
   }
   project.afterEvaluate {
     configurations.all {

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -19,8 +19,8 @@ class VagrantTestPlugin implements Plugin<Project> {
             'centos-7',
             'debian-8',
             'debian-9',
-            'fedora-25',
             'fedora-26',
+            'fedora-27',
             'oel-6',
             'oel-7',
             'opensuse-42',

client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java

@@ -191,23 +191,23 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
                         metadata.field("_id", request.id());
                     }
                     if (Strings.hasLength(request.routing())) {
-                        metadata.field("_routing", request.routing());
+                        metadata.field("routing", request.routing());
                     }
                     if (Strings.hasLength(request.parent())) {
-                        metadata.field("_parent", request.parent());
+                        metadata.field("parent", request.parent());
                     }
                     if (request.version() != Versions.MATCH_ANY) {
-                        metadata.field("_version", request.version());
+                        metadata.field("version", request.version());
                     }
 
                     VersionType versionType = request.versionType();
                     if (versionType != VersionType.INTERNAL) {
                         if (versionType == VersionType.EXTERNAL) {
-                            metadata.field("_version_type", "external");
+                            metadata.field("version_type", "external");
                         } else if (versionType == VersionType.EXTERNAL_GTE) {
-                            metadata.field("_version_type", "external_gte");
+                            metadata.field("version_type", "external_gte");
                         } else if (versionType == VersionType.FORCE) {
-                            metadata.field("_version_type", "force");
+                            metadata.field("version_type", "force");
                         }
                     }
 
@@ -219,7 +219,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
                     } else if (opType == DocWriteRequest.OpType.UPDATE) {
                         UpdateRequest updateRequest = (UpdateRequest) request;
                         if (updateRequest.retryOnConflict() > 0) {
-                            metadata.field("_retry_on_conflict", updateRequest.retryOnConflict());
+                            metadata.field("retry_on_conflict", updateRequest.retryOnConflict());
                         }
                         if (updateRequest.fetchSource() != null) {
                             metadata.field("_source", updateRequest.fetchSource());

core/build.gradle

@@ -58,7 +58,7 @@ dependencies {
   compile 'org.elasticsearch:securesm:1.1'
 
   // utilities
-  compile 'net.sf.jopt-simple:jopt-simple:5.0.2'
+  compile "org.elasticsearch:elasticsearch-cli:${version}"
   compile 'com.carrotsearch:hppc:0.7.1'
 
   // time handling, remove with java 8 time
@@ -265,6 +265,12 @@ if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
 dependencyLicenses {
   mapping from: /lucene-.*/, to: 'lucene'
   mapping from: /jackson-.*/, to: 'jackson'
+  dependencies = project.configurations.runtime.fileCollection {
+    it.group.startsWith('org.elasticsearch') == false ||
+            // keep the following org.elasticsearch jars in
+            (it.name == 'jna' ||
+             it.name == 'securesm')
+  }
 }
 
 if (isEclipse == false || project.path == ":core-tests") {

core/cli/build.gradle

@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.precommit.PrecommitTasks
+
+apply plugin: 'elasticsearch.build'
+
+archivesBaseName = 'elasticsearch-cli'
+
+dependencies {
+    compile 'net.sf.jopt-simple:jopt-simple:5.0.2'
+}
+
+test.enabled = false
+// Since CLI does not depend on :core, it cannot run the jarHell task
+jarHell.enabled = false
+
+forbiddenApisMain {
+    signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+}

core/src/main/java/org/elasticsearch/cli/Command.java → core/cli/src/main/java/org/elasticsearch/cli/Command.java

@@ -23,11 +23,6 @@
 import joptsimple.OptionParser;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
-import org.apache.logging.log4j.Level;
-import org.apache.lucene.util.SetOnce;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.logging.LogConfigurator;
-import org.elasticsearch.common.settings.Settings;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -55,12 +50,13 @@ public Command(String description) {
         this.description = description;
     }
 
-    final SetOnce<Thread> shutdownHookThread = new SetOnce<>();
+    private Thread shutdownHookThread;
 
     /** Parses options for this command from args and executes it. */
     public final int main(String[] args, Terminal terminal) throws Exception {
         if (addShutdownHook()) {
-            shutdownHookThread.set(new Thread(() -> {
+
+            shutdownHookThread = new Thread(() -> {
                 try {
                     this.close();
                 } catch (final IOException e) {
@@ -75,16 +71,11 @@ public final int main(String[] args, Terminal terminal) throws Exception {
                         throw new AssertionError(impossible);
                     }
                 }
-            }));
-            Runtime.getRuntime().addShutdownHook(shutdownHookThread.get());
+            });
+            Runtime.getRuntime().addShutdownHook(shutdownHookThread);
         }
 
-        if (shouldConfigureLoggingWithoutConfig()) {
-            // initialize default for es.logger.level because we will not read the log4j2.properties
-            final String loggerLevel = System.getProperty("es.logger.level", Level.INFO.name());
-            final Settings settings = Settings.builder().put("logger.level", loggerLevel).build();
-            LogConfigurator.configureWithoutConfig(settings);
-        }
+        beforeExecute();
 
         try {
             mainWithoutErrorHandling(args, terminal);
@@ -103,14 +94,10 @@ public final int main(String[] args, Terminal terminal) throws Exception {
     }
 
     /**
-     * Indicate whether or not logging should be configured without reading a log4j2.properties. Most commands should do this because we do
-     * not configure logging for CLI tools. Only commands that configure logging on their own should not do this.
-     *
-     * @return true if logging should be configured without reading a log4j2.properties file
+     * Setup method to be executed before parsing or execution of the command being run. Any exceptions thrown by the
+     * method will not be cleanly caught by the parser.
      */
-    protected boolean shouldConfigureLoggingWithoutConfig() {
-        return true;
-    }
+    protected void beforeExecute() {}
 
     /**
      * Executes the command, but all errors are thrown.
@@ -166,6 +153,11 @@ protected boolean addShutdownHook() {
         return true;
     }
 
+    /** Gets the shutdown hook thread if it exists **/
+    Thread getShutdownHookThread() {
+        return shutdownHookThread;
+    }
+
     @Override
     public void close() throws IOException {
 

core/cli/src/main/java/org/elasticsearch/cli/SuppressForbidden.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cli;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation to suppress forbidden-apis errors inside a whole class, a method, or a field.
+ */
+@Retention(RetentionPolicy.CLASS)
+@Target({ ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE })
+public @interface SuppressForbidden {
+    String reason();
+}
+

core/src/main/java/org/elasticsearch/cli/Terminal.java → core/cli/src/main/java/org/elasticsearch/cli/Terminal.java

@@ -19,8 +19,6 @@
 
 package org.elasticsearch.cli;
 
-import org.elasticsearch.common.SuppressForbidden;
-
 import java.io.BufferedReader;
 import java.io.Console;
 import java.io.IOException;

core/src/main/java/org/elasticsearch/Build.java

@@ -26,6 +26,7 @@
 
 import java.io.IOException;
 import java.net.URL;
+import java.security.CodeSource;
 import java.util.jar.JarInputStream;
 import java.util.jar.Manifest;
 
@@ -45,8 +46,8 @@ public class Build {
         final boolean isSnapshot;
 
         final String esPrefix = "elasticsearch-" + Version.CURRENT;
-        final URL url = getElasticsearchCodebase();
-        final String urlStr = url.toString();
+        final URL url = getElasticsearchCodeSourceLocation();
+        final String urlStr = url == null ? "" : url.toString();
         if (urlStr.startsWith("file:/") && (urlStr.endsWith(esPrefix + ".jar") || urlStr.endsWith(esPrefix + "-SNAPSHOT.jar"))) {
             try (JarInputStream jar = new JarInputStream(FileSystemUtils.openFileURLStream(url))) {
                 Manifest manifest = jar.getManifest();
@@ -88,10 +89,13 @@ public class Build {
     private final boolean isSnapshot;
 
     /**
-     * Returns path to elasticsearch codebase path
+     * The location of the code source for Elasticsearch
+     *
+     * @return the location of the code source for Elasticsearch which may be null
      */
-    static URL getElasticsearchCodebase() {
-        return Build.class.getProtectionDomain().getCodeSource().getLocation();
+    static URL getElasticsearchCodeSourceLocation() {
+        final CodeSource codeSource = Build.class.getProtectionDomain().getCodeSource();
+        return codeSource == null ? null : codeSource.getLocation();
     }
 
     private final String shortHash;