Merge branch 'master' into composite_sorted_index_optim

README.textile

@@ -124,7 +124,7 @@ curl -XGET 'http://localhost:9200/twitter/_search?pretty=true' -H 'Content-Type:
 
 There are many more options to perform search, after all, it's a search product no? All the familiar Lucene queries are available through the JSON query language, or through the query parser.
 
-h3. Multi Tenant - Indices and Types
+h3. Multi Tenant and Indices
 
 Man, that twitter index might get big (in this case, index size == valuation). Let's see if we can structure our twitter system a bit differently in order to support such large amounts of data.
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -22,7 +22,6 @@ import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin
 import com.github.jengelman.gradle.plugins.shadow.ShadowExtension
 import com.github.jengelman.gradle.plugins.shadow.ShadowJavaPlugin
 import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
-import groovy.transform.CompileDynamic
 import groovy.transform.CompileStatic
 import org.apache.commons.io.IOUtils
 import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
@@ -41,13 +40,10 @@ import org.gradle.api.NamedDomainObjectContainer
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.Task
-import org.gradle.api.XmlProvider
 import org.gradle.api.artifacts.Configuration
 import org.gradle.api.artifacts.Dependency
 import org.gradle.api.artifacts.ModuleDependency
-import org.gradle.api.artifacts.ModuleVersionIdentifier
 import org.gradle.api.artifacts.ProjectDependency
-import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.artifacts.dsl.RepositoryHandler
 import org.gradle.api.artifacts.repositories.IvyArtifactRepository
 import org.gradle.api.artifacts.repositories.IvyPatternRepositoryLayout
@@ -330,11 +326,6 @@ class BuildPlugin implements Plugin<Project> {
         return javaVersions.find { it.version == version }.javaHome.absolutePath
     }
 
-    /** Return the configuration name used for finding transitive deps of the given dependency. */
-    private static String transitiveDepConfigName(String groupId, String artifactId, String version) {
-        return "_transitive_${groupId}_${artifactId}_${version}"
-    }
-
     /**
      * Makes dependencies non-transitive.
      *
@@ -361,11 +352,6 @@ class BuildPlugin implements Plugin<Project> {
         }
         // fail on any conflicting dependency versions
         project.configurations.all({ Configuration configuration ->
-            if (configuration.name.startsWith('_transitive_')) {
-                // don't force transitive configurations to not conflict with themselves, since
-                // we just have them to find *what* transitive deps exist
-                return
-            }
             if (configuration.name.endsWith('Fixture')) {
                 // just a self contained test-fixture configuration, likely transitive and hellacious
                 return
@@ -380,14 +366,6 @@ class BuildPlugin implements Plugin<Project> {
             if (dep instanceof ModuleDependency && !(dep instanceof ProjectDependency)
                     && dep.group.startsWith('org.elasticsearch') == false) {
                 dep.transitive = false
-
-                // also create a configuration just for this dependency version, so that later
-                // we can determine which transitive dependencies it has
-                String depConfig = transitiveDepConfigName(dep.group, dep.name, dep.version)
-                if (project.configurations.findByName(depConfig) == null) {
-                    project.configurations.create(depConfig)
-                    project.dependencies.add(depConfig, "${dep.group}:${dep.name}:${dep.version}")
-                }
             }
         }
 
@@ -457,77 +435,6 @@ class BuildPlugin implements Plugin<Project> {
         }
     }
 
-    /**
-     * Returns a closure which can be used with a MavenPom for fixing problems with gradle generated poms.
-     *
-     * <ul>
-     *     <li>Remove transitive dependencies. We currently exclude all artifacts explicitly instead of using wildcards
-     *         as Ivy incorrectly translates POMs with * excludes to Ivy XML with * excludes which results in the main artifact
-     *         being excluded as well (see https://issues.apache.org/jira/browse/IVY-1531). Note that Gradle 2.14+ automatically
-     *         translates non-transitive dependencies to * excludes. We should revisit this when upgrading Gradle.</li>
-     *     <li>Set compile time deps back to compile from runtime (known issue with maven-publish plugin)</li>
-     * </ul>
-     */
-    @CompileDynamic
-    private static Closure fixupDependencies(Project project) {
-        return { XmlProvider xml ->
-            // first find if we have dependencies at all, and grab the node
-            NodeList depsNodes = xml.asNode().get('dependencies')
-            if (depsNodes.isEmpty()) {
-                return
-            }
-
-            // check each dependency for any transitive deps
-            for (Node depNode : depsNodes.get(0).children()) {
-                String groupId = depNode.get('groupId').get(0).text()
-                String artifactId = depNode.get('artifactId').get(0).text()
-                String version = depNode.get('version').get(0).text()
-
-                // fix deps incorrectly marked as runtime back to compile time deps
-                // see https://discuss.gradle.org/t/maven-publish-plugin-generated-pom-making-dependency-scope-runtime/7494/4
-                boolean isCompileDep = project.configurations.compile.allDependencies.find { dep ->
-                    dep.name == depNode.artifactId.text()
-                }
-                if (depNode.scope.text() == 'runtime' && isCompileDep) {
-                    depNode.scope*.value = 'compile'
-                }
-
-                // remove any exclusions added by gradle, they contain wildcards and systems like ivy have bugs with wildcards
-                // see https://github.com/elastic/elasticsearch/issues/24490
-                NodeList exclusionsNode = depNode.get('exclusions')
-                if (exclusionsNode.size() > 0) {
-                    depNode.remove(exclusionsNode.get(0))
-                }
-
-                // collect the transitive deps now that we know what this dependency is
-                String depConfig = transitiveDepConfigName(groupId, artifactId, version)
-                Configuration configuration = project.configurations.findByName(depConfig)
-                if (configuration == null) {
-                    continue // we did not make this dep non-transitive
-                }
-                Set<ResolvedArtifact> artifacts = configuration.resolvedConfiguration.resolvedArtifacts
-                if (artifacts.size() <= 1) {
-                    // this dep has no transitive deps (or the only artifact is itself)
-                    continue
-                }
-
-                // we now know we have something to exclude, so add exclusions for all artifacts except the main one
-                Node exclusions = depNode.appendNode('exclusions')
-                for (ResolvedArtifact artifact : artifacts) {
-                    ModuleVersionIdentifier moduleVersionIdentifier = artifact.moduleVersion.id;
-                    String depGroupId = moduleVersionIdentifier.group
-                    String depArtifactId = moduleVersionIdentifier.name
-                    // add exclusions for all artifacts except the main one
-                    if (depGroupId != groupId || depArtifactId != artifactId) {
-                        Node exclusion = exclusions.appendNode('exclusion')
-                        exclusion.appendNode('groupId', depGroupId)
-                        exclusion.appendNode('artifactId', depArtifactId)
-                    }
-                }
-            }
-        }
-    }
-
     /**Configuration generation of maven poms. */
     static void configurePomGeneration(Project project) {
         project.plugins.withType(MavenPublishPlugin).whenPluginAdded {
@@ -545,11 +452,6 @@ class BuildPlugin implements Plugin<Project> {
 
             PublishingExtension publishing = project.extensions.getByType(PublishingExtension)
 
-            project.extensions.getByType(PublishingExtension).publications.all { MavenPublication publication -> // we only deal with maven
-                // add exclusions to the pom directly, for each of the transitive deps of this project's deps
-                publication.pom.withXml(fixupDependencies(project))
-            }
-
             project.pluginManager.withPlugin('com.github.johnrengelman.shadow') {
                 MavenPublication publication = publishing.publications.maybeCreate('shadow', MavenPublication)
                 ShadowExtension shadow = project.extensions.getByType(ShadowExtension)

buildSrc/version.properties

@@ -23,7 +23,7 @@ slf4j             = 1.6.2
 jna               = 4.5.1
 
 netty             = 4.1.43.Final
-joda              = 2.10.3
+joda              = 2.10.4
 
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)

distribution/archives/build.gradle

@@ -59,7 +59,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
         with binFiles(distributionType, oss, jdk)
       }
       if (jdk) {
-        into('jdk') {
+        into("darwin".equals(platform) ? 'jdk.app' : 'jdk') {
           with jdkFiles(project, platform)
         }
       }

distribution/src/bin/elasticsearch-env

@@ -41,8 +41,8 @@ if [ ! -z "$JAVA_HOME" ]; then
   JAVA_TYPE="JAVA_HOME"
 else
   if [ "$(uname -s)" = "Darwin" ]; then
-    # OSX has a different structure
-    JAVA="$ES_HOME/jdk/Contents/Home/bin/java"
+    # macOS has a different structure
+    JAVA="$ES_HOME/jdk.app/Contents/Home/bin/java"
   else
     JAVA="$ES_HOME/jdk/bin/java"
   fi

docs/reference/analysis/tokenfilters/common-grams-tokenfilter.asciidoc

@@ -30,7 +30,7 @@ GET /_analyze
 {
   "tokenizer" : "whitespace",
   "filter" : [
-    "common_grams", {
+    {
       "type": "common_grams",
       "common_words": ["is", "the"]
     }

docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc

@@ -34,7 +34,7 @@ The above configures a `synonym` filter, with a path of
 `analysis/synonym.txt` (relative to the `config` location). The
 `synonym` analyzer is then configured with the filter.
 
-This filter tokenize synonyms with whatever tokenizer and token filters
+This filter tokenizes synonyms with whatever tokenizer and token filters
 appear before it in the chain.
 
 Additional settings are:

docs/reference/ilm/apis/explain.asciidoc

@@ -239,8 +239,11 @@ information for the step that's being performed on the index.
 
 If the index is in the ERROR step, something went wrong while executing a
 step in the policy and you will need to take action for the index to proceed
-to the next step. To help you diagnose the problem, the explain response shows
-the step that failed and the step info provides information about the error.
+to the next step. Some steps are safe to automatically be retried in certain
+circumstances. To help you diagnose the problem, the explain response shows
+the step that failed, the step info which provides information about the error,
+and information about the retry attempts executed for the failed step if it's
+the case.
 
 [source,console-result]
 --------------------------------------------------
@@ -262,10 +265,12 @@ the step that failed and the step info provides information about the error.
       "step": "ERROR",
       "step_time_millis": 1538475653317,
       "step_time": "2018-10-15T13:45:22.577Z",
-      "failed_step": "attempt-rollover", <1>
-      "step_info": { <2>
-        "type": "resource_already_exists_exception",
-        "reason": "index [test-000057/H7lF9n36Rzqa-KfKcnGQMg] already exists",
+      "failed_step": "check-rollover-ready", <1>
+      "is_auto_retryable_error": true, <2>
+      "failed_step_retry_count": 1, <3>
+      "step_info": { <4>
+        "type": "cluster_block_exception",
+        "reason": "index [test-000057/H7lF9n36Rzqa-KfKcnGQMg] blocked by: [FORBIDDEN/5/index read-only (api)",
         "index_uuid": "H7lF9n36Rzqa-KfKcnGQMg",
         "index": "test-000057"
       },
@@ -290,4 +295,8 @@ the step that failed and the step info provides information about the error.
 // TESTRESPONSE[skip:not possible to get the cluster into this state in a docs test]
 
 <1> The step that caused the error
-<2> What went wrong
+<2> Indicates if retrying the failed step can overcome the error. If this
+is true, ILM will retry the failed step automatically.
+<3> Shows the number of attempted automatic retries to execute the failed
+step.
+<4> What went wrong

docs/reference/ilm/policy-definitions.asciidoc

@@ -89,7 +89,7 @@ executing.
 The below list shows the actions which are available in each phase.
 
 NOTE: The order that configured actions are performed in within each phase is
-determined by automatically by {ilm-init}, and cannot be changed by changing the
+determined automatically by {ilm-init}, and cannot be changed by changing the
 policy definition.
 
 * Hot

docs/reference/migration/migrate_8_0/search.asciidoc

@@ -34,4 +34,12 @@ The `nested_filter` and `nested_path` options, deprecated in 6.x, have been remo
 
 {es} will no longer prefer using shards in the same location (with the same awareness attribute values) to process
 `_search` and `_get` requests. Adaptive replica selection (activated by default in this version) will route requests
-more efficiently using the service time of prior inter-node communications.
+more efficiently using the service time of prior inter-node communications.
+
+[float]
+==== Update to vector function signatures
+The vector functions of the form `function(query, doc['field'])` were
+deprecated in 7.6, and are now removed in 8.x. The form
+`function(query, 'field')` should be used instead. For example,
+`cosineSimilarity(query, doc['field'])` is replaced by
+`cosineSimilarity(query, 'field')`.