Merge branch 'main' into ingest-geoip-boundary-refactor-wip

docs/changelog/114234.yaml

@@ -0,0 +1,5 @@
+pr: 114234
+summary: Prevent flattening of ordered and unordered interval sources
+area: Search
+type: bug
+issues: []

docs/changelog/114321.yaml

@@ -0,0 +1,5 @@
+pr: 114321
+summary: Stream Anthropic Completion
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/114368.yaml

@@ -0,0 +1,5 @@
+pr: 114368
+summary: "ESQL: Delay construction of warnings"
+area: EQL
+type: enhancement
+issues: []

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java

@@ -165,7 +165,7 @@ public PrivacyDetectionResult(
             // @MaxMindDbParameter(name = "network") String network, // for now we're not exposing this
             @MaxMindDbParameter(name = "proxy") String proxy,
             @MaxMindDbParameter(name = "relay") String relay,
-            @MaxMindDbParameter(name = "service") String service, // n.b. this remains a string, the rest are parsed booleans
+            @MaxMindDbParameter(name = "service") String service, // n.b. this remains a string, the rest are parsed as booleans
             @MaxMindDbParameter(name = "tor") String tor,
             @MaxMindDbParameter(name = "vpn") String vpn
         ) {

modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java

@@ -313,6 +313,7 @@ public void testPrivacyDetection() throws IOException {
         ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache);
         configDatabases.initialize(resourceWatcherService);
 
+        // testing the first row in the sample database
         try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("privacy_detection_sample.mmdb")) {
             IpDataLookup lookup = new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties());
             Map<String, Object> data = lookup.getData(loader, "1.53.59.33");
@@ -331,6 +332,7 @@ public void testPrivacyDetection() throws IOException {
             );
         }
 
+        // testing a row with a non-empty service in the sample database
         try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("privacy_detection_sample.mmdb")) {
             IpDataLookup lookup = new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties());
             Map<String, Object> data = lookup.getData(loader, "216.131.74.65");

muted-tests.yml

@@ -361,6 +361,9 @@ tests:
 - class: org.elasticsearch.xpack.inference.InferenceRestIT
   method: test {p0=inference/30_semantic_text_inference/Calculates embeddings using the default ELSER 2 endpoint}
   issue: https://github.com/elastic/elasticsearch/issues/114412
+- class: org.elasticsearch.xpack.inference.InferenceRestIT
+  method: test {p0=inference/40_semantic_text_query/Query a field that uses the default ELSER 2 endpoint}
+  issue: https://github.com/elastic/elasticsearch/issues/114376
 
 # Examples:
 #

server/src/internalClusterTest/java/org/elasticsearch/search/query/IntervalQueriesIT.java

@@ -31,6 +31,7 @@
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
 
 public class IntervalQueriesIT extends ESIntegTestCase {
 
@@ -64,6 +65,58 @@ public void testEmptyIntervalsWithNestedMappings() throws InterruptedException {
         );
     }
 
+    public void testPreserveInnerGap() {
+        assertAcked(prepareCreate("index").setMapping("""
+            {
+                "_doc" : {
+                    "properties" : {
+                        "text" : { "type" : "text" }
+                    }
+                }
+            }
+            """));
+
+        indexRandom(true, prepareIndex("index").setId("1").setSource("text", "w1 w2 w3 w4 w5"));
+
+        // ordered
+        {
+            var res = prepareSearch("index").setQuery(
+                new IntervalQueryBuilder(
+                    "text",
+                    new IntervalsSourceProvider.Combine(
+                        Arrays.asList(
+                            new IntervalsSourceProvider.Match("w1 w4", -1, true, null, null, null),
+                            new IntervalsSourceProvider.Match("w5", -1, true, null, null, null)
+                        ),
+                        true,
+                        1,
+                        null
+                    )
+                )
+            );
+            assertSearchHits(res, "1");
+        }
+
+        // unordered
+        {
+            var res = prepareSearch("index").setQuery(
+                new IntervalQueryBuilder(
+                    "text",
+                    new IntervalsSourceProvider.Combine(
+                        Arrays.asList(
+                            new IntervalsSourceProvider.Match("w3", 0, false, null, null, null),
+                            new IntervalsSourceProvider.Match("w4 w1", -1, false, null, null, null)
+                        ),
+                        false,
+                        0,
+                        null
+                    )
+                )
+            );
+            assertSearchHits(res, "1");
+        }
+    }
+
     private static class EmptyAnalyzer extends Analyzer {
 
         @Override

server/src/main/java/org/elasticsearch/index/query/IntervalBuilder.java

@@ -126,7 +126,7 @@ protected static IntervalsSource combineSources(List<IntervalsSource> sources, i
         if (maxGaps == 0 && ordered) {
             return Intervals.phrase(sourcesArray);
         }
-        IntervalsSource inner = ordered ? Intervals.ordered(sourcesArray) : Intervals.unordered(sourcesArray);
+        IntervalsSource inner = ordered ? XIntervals.ordered(sourcesArray) : XIntervals.unordered(sourcesArray);
         if (maxGaps == -1) {
             return inner;
         }

server/src/main/java/org/elasticsearch/index/query/XIntervals.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.query;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.queries.intervals.IntervalIterator;
+import org.apache.lucene.queries.intervals.IntervalMatchesIterator;
+import org.apache.lucene.queries.intervals.Intervals;
+import org.apache.lucene.queries.intervals.IntervalsSource;
+import org.apache.lucene.search.QueryVisitor;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Objects;
+
+/**
+ * Copy of {@link Intervals} that exposes versions of {@link Intervals#ordered} and {@link Intervals#unordered}
+ * that preserve their inner gaps.
+ * NOTE: Remove this hack when a version of Lucene with https://github.com/apache/lucene/pull/13819 is used (10.1.0).
+ */
+public final class XIntervals {
+
+    /**
+     * Create an ordered {@link IntervalsSource}
+     *
+     * <p>Returns intervals in which the subsources all appear in the given order
+     *
+     * @param subSources an ordered set of {@link IntervalsSource} objects
+     */
+    public static IntervalsSource ordered(IntervalsSource... subSources) {
+        return new DelegateIntervalsSource(Intervals.ordered(subSources));
+    }
+
+    /**
+     * Create an ordered {@link IntervalsSource}
+     *
+     * <p>Returns intervals in which the subsources all appear in the given order
+     *
+     * @param subSources an ordered set of {@link IntervalsSource} objects
+     */
+    public static IntervalsSource unordered(IntervalsSource... subSources) {
+        return new DelegateIntervalsSource(Intervals.unordered(subSources));
+    }
+
+    /**
+     * Wraps a source to avoid aggressive flattening of the ordered and unordered sources.
+     * The flattening modifies the final gap and is removed in the latest unreleased version of Lucene (10.1).
+     */
+    private static class DelegateIntervalsSource extends IntervalsSource {
+        private final IntervalsSource delegate;
+
+        private DelegateIntervalsSource(IntervalsSource delegate) {
+            this.delegate = delegate;
+        }
+
+        @Override
+        public IntervalIterator intervals(String field, LeafReaderContext ctx) throws IOException {
+            return delegate.intervals(field, ctx);
+        }
+
+        @Override
+        public IntervalMatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException {
+            return delegate.matches(field, ctx, doc);
+        }
+
+        @Override
+        public void visit(String field, QueryVisitor visitor) {
+            delegate.visit(field, visitor);
+        }
+
+        @Override
+        public int minExtent() {
+            return delegate.minExtent();
+        }
+
+        @Override
+        public Collection<IntervalsSource> pullUpDisjunctions() {
+            return delegate.pullUpDisjunctions();
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            DelegateIntervalsSource that = (DelegateIntervalsSource) o;
+            return Objects.equals(delegate, that.delegate);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(delegate);
+        }
+
+        @Override
+        public String toString() {
+            return delegate.toString();
+        }
+    }
+}

server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java

@@ -11,8 +11,10 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.common.document.DocumentField;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.IgnoredFieldMapper;
+import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.index.mapper.LegacyTypeFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
@@ -61,6 +63,21 @@ public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) {
             : fetchFieldsContext.fields().isEmpty() ? null
             : FieldFetcher.create(searchExecutionContext, fetchFieldsContext.fields());
 
+        // NOTE: Collect stored metadata fields requested via `fields` (in FetchFieldsContext`) like for instance the _ignored source field
+        final Set<FieldAndFormat> fetchContextMetadataFields = new HashSet<>();
+        if (fetchFieldsContext != null && fetchFieldsContext.fields() != null && fetchFieldsContext.fields().isEmpty() == false) {
+            for (final FieldAndFormat fieldAndFormat : fetchFieldsContext.fields()) {
+                // NOTE: _id and _source are always retrieved anyway, no need to do it explicitly. See FieldsVisitor.
+                if (SourceFieldMapper.NAME.equals(fieldAndFormat.field) || IdFieldMapper.NAME.equals(fieldAndFormat.field)) {
+                    continue;
+                }
+                if (searchExecutionContext.isMetadataField(fieldAndFormat.field)
+                    && searchExecutionContext.getFieldType(fieldAndFormat.field).isStored()) {
+                    fetchContextMetadataFields.add(fieldAndFormat);
+                }
+            }
+        }
+
         final FieldFetcher metadataFieldFetcher;
         if (storedFieldsContext != null
             && storedFieldsContext.fieldNames() != null
@@ -73,6 +90,10 @@ public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) {
                         continue;
                     }
                     final MappedFieldType fieldType = searchExecutionContext.getFieldType(matchingFieldName);
+                    // NOTE: Exclude _ignored_source when requested via wildcard '*'
+                    if (matchingFieldName.equals(IgnoredSourceFieldMapper.NAME) && Regex.isSimpleMatchPattern(storedField)) {
+                        continue;
+                    }
                     // NOTE: checking if the field is stored is required for backward compatibility reasons and to make
                     // sure we also handle here stored fields requested via `stored_fields`, which was previously a
                     // responsibility of StoredFieldsPhase.
@@ -81,9 +102,14 @@ public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) {
                     }
                 }
             }
+            // NOTE: Include also metadata stored fields requested via `fields`
+            metadataFields.addAll(fetchContextMetadataFields);
             metadataFieldFetcher = FieldFetcher.create(searchExecutionContext, metadataFields);
         } else {
-            metadataFieldFetcher = FieldFetcher.create(searchExecutionContext, DEFAULT_METADATA_FIELDS);
+            // NOTE: Include also metadata stored fields requested via `fields`
+            final Set<FieldAndFormat> allMetadataFields = new HashSet<>(DEFAULT_METADATA_FIELDS);
+            allMetadataFields.addAll(fetchContextMetadataFields);
+            metadataFieldFetcher = FieldFetcher.create(searchExecutionContext, allMetadataFields);
         }
         return new FetchSubPhaseProcessor() {
             @Override