Merge branch 'main' into ml-token-bucket

jonathan-buttner · Mar 19, 2024 · 9ba1a0d · 9ba1a0d
2 parents 4a65523 + bceb38d
commit 9ba1a0d
Show file tree

Hide file tree

Showing 80 changed files with 4,890 additions and 751 deletions.
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.benchmark.compute.operator;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.Block;
@@ -26,11 +27,13 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.esql.planner.Layout;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.expression.Literal;
+import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataTypes;
 import org.elasticsearch.xpack.ql.type.EsField;
@@ -58,7 +61,6 @@
 @State(Scope.Thread)
 @Fork(1)
 public class EvalBenchmark {
-    private static final BigArrays BIG_ARRAYS = BigArrays.NON_RECYCLING_INSTANCE;  // TODO real big arrays?
     private static final BlockFactory blockFactory = BlockFactory.getInstance(
         new NoopCircuitBreaker("noop"),
         BigArrays.NON_RECYCLING_INSTANCE
@@ -82,7 +84,9 @@ public class EvalBenchmark {
         }
     }
 
-    @Param({ "abs", "add", "date_trunc", "equal_to_const", "long_equal_to_long", "long_equal_to_int", "mv_min", "mv_min_ascending" })
+    @Param(
+        { "abs", "add", "date_trunc", "equal_to_const", "long_equal_to_long", "long_equal_to_int", "mv_min", "mv_min_ascending", "rlike" }
+    )
     public String operation;
 
     private static Operator operator(String operation) {
@@ -134,6 +138,11 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                 FieldAttribute longField = longField();
                 yield EvalMapper.toEvaluator(new MvMin(Source.EMPTY, longField), layout(longField)).get(driverContext);
             }
+            case "rlike" -> {
+                FieldAttribute keywordField = keywordField();
+                RLike rlike = new RLike(Source.EMPTY, keywordField, new RLikePattern(".ar"));
+                yield EvalMapper.toEvaluator(rlike, layout(keywordField)).get(driverContext);
+            }
             default -> throw new UnsupportedOperationException();
         };
     }
@@ -146,6 +155,10 @@ private static FieldAttribute intField() {
         return new FieldAttribute(Source.EMPTY, "int", new EsField("int", DataTypes.INTEGER, Map.of(), true));
     }
 
+    private static FieldAttribute keywordField() {
+        return new FieldAttribute(Source.EMPTY, "keyword", new EsField("keyword", DataTypes.KEYWORD, Map.of(), true));
+    }
+
     private static Layout layout(FieldAttribute... fields) {
         Layout.Builder layout = new Layout.Builder();
         layout.append(Arrays.asList(fields));
@@ -205,6 +218,15 @@ private static void checkExpected(String operation, Page actual) {
                     }
                 }
             }
+            case "rlike" -> {
+                BooleanVector v = actual.<BooleanBlock>getBlock(1).asVector();
+                for (int i = 0; i < BLOCK_LENGTH; i++) {
+                    boolean expected = i % 2 == 1;
+                    if (v.getBoolean(i) != expected) {
+                        throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + v.getBoolean(i) + "]");
+                    }
+                }
+            }
             default -> throw new UnsupportedOperationException();
         }
     }
@@ -250,6 +272,14 @@ private static Page page(String operation) {
                 }
                 yield new Page(builder.build());
             }
+            case "rlike" -> {
+                var builder = blockFactory.newBytesRefVectorBuilder(BLOCK_LENGTH);
+                BytesRef[] values = new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") };
+                for (int i = 0; i < BLOCK_LENGTH; i++) {
+                    builder.appendBytesRef(values[i % 2]);
+                }
+                yield new Page(builder.build().asBlock());
+            }
             default -> throw new UnsupportedOperationException();
         };
     }

diff --git a/docs/changelog/104907.yaml b/docs/changelog/104907.yaml
@@ -0,0 +1,6 @@
+pr: 104907
+summary: Support ST_INTERSECTS between geometry column and other geometry or string
+area: "ES|QL"
+type: enhancement
+issues:
+- 104874
diff --git a/docs/changelog/106381.yaml b/docs/changelog/106381.yaml
@@ -0,0 +1,5 @@
+pr: 106381
+summary: Dedupe terms in terms queries
+area: Mapping
+type: bug
+issues: []
diff --git a/docs/changelog/106429.yaml b/docs/changelog/106429.yaml
@@ -0,0 +1,5 @@
+pr: 106429
+summary: "ESQL: Regex improvements"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/106435.yaml b/docs/changelog/106435.yaml
@@ -0,0 +1,6 @@
+pr: 106435
+summary: "ENRICH support for TEXT fields"
+area: ES|QL
+type: enhancement
+issues:
+ - 105384
diff --git a/docs/reference/esql/functions/signature/st_intersects.svg b/docs/reference/esql/functions/signature/st_intersects.svg
diff --git a/docs/reference/esql/functions/spatial-functions.asciidoc b/docs/reference/esql/functions/spatial-functions.asciidoc
@@ -8,9 +8,11 @@
 {esql} supports these spatial functions:
 
 // tag::spatial_list[]
+* <<esql-st_intersects>>
 * <<esql-st_x>>
 * <<esql-st_y>>
 // end::spatial_list[]
 
+include::st_intersects.asciidoc[]
 include::st_x.asciidoc[]
 include::st_y.asciidoc[]
diff --git a/docs/reference/esql/functions/st_intersects.asciidoc b/docs/reference/esql/functions/st_intersects.asciidoc
@@ -0,0 +1,40 @@
+[discrete]
+[[esql-st_intersects]]
+=== `ST_INTERSECTS`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/st_intersects.svg[Embedded,opts=inline]
+
+*Parameters*
+
+`geomA`::
+Expression of type `geo_point`, `cartesian_point`, `geo_shape` or `cartesian_shape`. If `null`, the function returns `null`.
+
+`geomB`::
+Expression of type `geo_point`, `cartesian_point`, `geo_shape` or `cartesian_shape`. If `null`, the function returns `null`.
+The second parameter must also have the same coordinate system as the first.
+This means it is not possible to combine `geo_*` and `cartesian_*` parameters.
+
+*Description*
+
+Returns true if two geometries intersect.
+They intersect if they have any point in common, including their interior points
+(points along lines or within polygons).
+In mathematical terms: ST_Intersects(A, B) ⇔ A ⋂ B ≠ ∅
+
+*Supported types*
+
+include::types/st_intersects.asciidoc[]
+
+*Example*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/spatial.csv-spec[tag=st_intersects-airports]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/spatial.csv-spec[tag=st_intersects-airports-results]
+|===
diff --git a/docs/reference/esql/functions/types/st_intersects.asciidoc b/docs/reference/esql/functions/types/st_intersects.asciidoc
@@ -0,0 +1,12 @@
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+geomA | geomB | result
+cartesian_point | cartesian_point | boolean
+cartesian_point | cartesian_shape | boolean
+cartesian_shape | cartesian_point | boolean
+cartesian_shape | cartesian_shape | boolean
+geo_point | geo_point | boolean
+geo_point | geo_shape | boolean
+geo_shape | geo_point | boolean
+geo_shape | geo_shape | boolean
+|===
diff --git a/docs/reference/watcher/input/search.asciidoc b/docs/reference/watcher/input/search.asciidoc
@@ -194,7 +194,7 @@ accurately.
                                                                                     When a search generates a large response, you can use `extract` to select the
                                                                                     relevant fields instead of loading the entire response.
 
-| `timeout`                                     | no        | 30s                 | The timeout for waiting for the search api call to return. If no response is
+| `timeout`                                     | no        | 1m                  | The timeout for waiting for the search api call to return. If no response is
                                                                                     returned within this time, the search input times out and fails. This setting
                                                                                     overrides the default search operations timeouts.
 |======

diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkSystemdLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkSystemdLibrary.java
@@ -58,7 +58,7 @@ static String findLibSystemd() {
     @Override
     public int sd_notify(int unset_environment, String state) {
         try (Arena arena = Arena.ofConfined()) {
-            MemorySegment nativeState = arena.allocateUtf8String(state);
+            MemorySegment nativeState = MemorySegmentUtil.allocateString(arena, state);
             return (int) sd_notify$mh.invokeExact(unset_environment, nativeState);
         } catch (Throwable t) {
             throw new AssertionError(t);

diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.nativeaccess.jdk;
 
+import java.lang.foreign.Arena;
 import java.lang.foreign.MemorySegment;
 
 /**
@@ -19,5 +20,9 @@ static String getString(MemorySegment segment, long offset) {
         return segment.getUtf8String(offset);
     }
 
+    static MemorySegment allocateString(Arena arena, String s) {
+        return arena.allocateUtf8String(s);
+    }
+
     private MemorySegmentUtil() {}
 }
diff --git a/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java b/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.nativeaccess.jdk;
 
+import java.lang.foreign.Arena;
 import java.lang.foreign.MemorySegment;
 
 public class MemorySegmentUtil {
@@ -16,5 +17,9 @@ static String getString(MemorySegment segment, long offset) {
         return segment.getString(offset);
     }
 
+    static MemorySegment allocateString(Arena arena, String s) {
+        return arena.allocateFrom(s);
+    }
+
     private MemorySegmentUtil() {}
 }
diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java
@@ -289,15 +289,15 @@ protected void dumpDebug() {
     protected void assertWhileRunning(Platforms.PlatformAction assertions) throws Exception {
         try {
             awaitElasticsearchStartup(runElasticsearchStartCommand(null, true, false));
-        } catch (Exception e) {
+        } catch (AssertionError | Exception e) {
             dumpDebug();
             throw e;
         }
 
         try {
             assertions.run();
-        } catch (Exception e) {
-            logger.warn("Elasticsearch log:\n" + FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"));
+        } catch (AssertionError | Exception e) {
+            dumpDebug();
             throw e;
         }
         stopElasticsearch();
@@ -392,15 +392,8 @@ public Shell.Result awaitElasticsearchStartupWithResult(Shell.Result result) thr
     public void startElasticsearch() throws Exception {
         try {
             awaitElasticsearchStartup(runElasticsearchStartCommand(null, true, false));
-        } catch (Exception e) {
-            if (Files.exists(installation.home.resolve("elasticsearch.pid"))) {
-                String pid = FileUtils.slurp(installation.home.resolve("elasticsearch.pid")).trim();
-                logger.info("elasticsearch process ({}) failed to start", pid);
-                if (sh.run("jps").stdout().contains(pid)) {
-                    logger.info("Dumping jstack of elasticsearch process ({}) ", pid);
-                    sh.runIgnoreExitCode("jstack " + pid);
-                }
-            }
+        } catch (AssertionError | Exception e) {
+            dumpDebug();
             throw e;
         }
     }