EQL: add support for partial shard results

docs/changelog/116388.yaml

@@ -0,0 +1,5 @@
+pr: 116388
+summary: Add support for partial shard results
+area: EQL
+type: enhancement
+issues: []

docs/reference/eql/eql-search-api.asciidoc

@@ -88,6 +88,53 @@ request that targets only `bar*` still returns an error.
 +
 Defaults to `true`.
 
+`allow_partial_search_results`::
+(Optional, Boolean)
+
+If `false`, the request returns an error if one or more shards involved in the query are unavailable.
++
+If `true`, the query is executed only on the available shards, ignoring shard request timeouts and
+<<shard-failures,shard failures>>.
++
+Defaults to `false`.
++
+To override the default for this field, set the
+`xpack.eql.default_allow_partial_results` cluster setting to `true`.
+
+
+[IMPORTANT]
+====
+You can also specify this value using the `allow_partial_search_results` request body parameter.
+If both parameters are specified, only the query parameter is used.
+====
+
+
+`allow_partial_sequence_results`::
+(Optional, Boolean)
+
+
+Used together with `allow_partial_search_results=true`, controls the behavior of sequence queries specifically
+(if `allow_partial_search_results=false`, this setting has no effect).
+If `true` and if some shards are unavailable, the sequences are calculated on available shards only.
++
+If `false` and if some shards are unavailable, the query only returns information about the shard failures,
+but no further results.
++
+Defaults to `false`.
++
+Consider that sequences calculated with `allow_partial_search_results=true` can return incorrect results
+(eg. if a <<eql-missing-events, missing event>> clause matches records in unavailable shards)
++
+To override the default for this field, set the
+`xpack.eql.default_allow_partial_sequence_results` cluster setting to `true`.
+
+
+[IMPORTANT]
+====
+You can also specify this value using the `allow_partial_sequence_results` request body parameter.
+If both parameters are specified, only the query parameter is used.
+====
+
 `ccs_minimize_roundtrips`::
 (Optional, Boolean) If `true`, network round-trips between the local and the
 remote cluster are minimized when running cross-cluster search (CCS) requests.

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -209,6 +209,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion REINDEX_DATA_STREAMS = def(8_799_00_0);
     public static final TransportVersion ESQL_REMOVE_NODE_LEVEL_PLAN = def(8_800_00_0);
     public static final TransportVersion LOGSDB_TELEMETRY_CUSTOM_CUTOFF_DATE = def(8_801_00_0);
+    public static final TransportVersion EQL_ALLOW_PARTIAL_SEARCH_RESULTS = def(8_802_00_0);
+
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -2453,7 +2453,7 @@ public static void afterClass() throws Exception {
     /**
      *  After the cluster is stopped, there are a few netty threads that can linger, so we make sure we don't leak any tasks on them.
      */
-    static void awaitGlobalNettyThreadsFinish() throws Exception {
+    public static void awaitGlobalNettyThreadsFinish() throws Exception {
         // Don't use GlobalEventExecutor#awaitInactivity. It will waste up to 1s for every call and we expect no tasks queued for it
         // except for the odd scheduled shutdown task.
         assertBusy(() -> assertEquals(0, GlobalEventExecutor.INSTANCE.pendingTasks()));

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java

@@ -163,13 +163,19 @@ protected ObjectPath runQuery(String index, String query) throws Exception {
         if (maxSamplesPerKey > 0) {
             builder.field("max_samples_per_key", maxSamplesPerKey);
         }
+        if (randomBoolean()) {
+            builder.field("allow_partial_search_results", randomBoolean());
+        }
         builder.endObject();
 
         Request request = new Request("POST", "/" + index + "/_eql/search");
         Boolean ccsMinimizeRoundtrips = ccsMinimizeRoundtrips();
         if (ccsMinimizeRoundtrips != null) {
             request.addParameter("ccs_minimize_roundtrips", ccsMinimizeRoundtrips.toString());
         }
+        if (randomBoolean()) {
+            request.addParameter("allow_partial_search_results", String.valueOf(randomBoolean()));
+        }
         int timeout = Math.toIntExact(timeout().millis());
         RequestConfig config = RequestConfig.copy(RequestConfig.DEFAULT)
             .setConnectionRequestTimeout(timeout)

x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml

@@ -478,3 +478,34 @@ setup:
           query: 'sequence with maxspan=10d [network where user == "ADMIN"] ![network where used == "SYSTEM"]'
   - match: { error.root_cause.0.type: "verification_exception" }
   - match: { error.root_cause.0.reason: "Found 1 problem\nline 1:75: Unknown column [used], did you mean [user]?" }
+
+
+---
+"Execute query with allow_partial_search_results":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'process where user == "SYSTEM"'
+          fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
+          allow_partial_search_results: true
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 3}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.events.0._source.user: "SYSTEM"}
+  - match: {hits.events.0._id: "1"}
+  - match: {hits.events.0.fields.@timestamp: ["1580733296000"]}
+  - match: {hits.events.0.fields.id: [123]}
+  - match: {hits.events.0.fields.valid: [false]}
+  - match: {hits.events.0.fields.day_of_week: ["Monday"]}
+  - match: {hits.events.1._id: "2"}
+  - match: {hits.events.1.fields.@timestamp: ["1580819696000"]}
+  - match: {hits.events.1.fields.id: [123]}
+  - match: {hits.events.1.fields.valid: [true]}
+  - match: {hits.events.1.fields.day_of_week: ["Tuesday"]}
+  - match: {hits.events.2._id: "3"}
+  - match: {hits.events.2.fields.@timestamp: ["1580906096000"]}
+  - match: {hits.events.2.fields.id: [123]}
+  - match: {hits.events.2.fields.valid: [true]}
+  - match: {hits.events.2.fields.day_of_week: ["Wednesday"]}