From 4aa4a97703a670cae34c079e84f47f05ece5bd48 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 7 Nov 2024 10:35:30 +0100 Subject: [PATCH 01/23] EQL: add support for partial shard failures --- .../org/elasticsearch/TransportVersions.java | 1 + .../eql/action/PartialSearchResultsIT.java | 245 ++++++++++++++++++ .../xpack/eql/action/EqlSearchRequest.java | 26 +- .../execution/assembler/ExecutionManager.java | 6 +- .../eql/execution/sample/SampleIterator.java | 8 +- .../execution/search/BasicQueryClient.java | 18 +- .../execution/search/PITAwareQueryClient.java | 25 +- .../eql/execution/search/RuntimeUtils.java | 25 +- .../execution/sequence/TumblingWindow.java | 14 +- .../eql/plugin/TransportEqlSearchAction.java | 1 + .../xpack/eql/session/EqlConfiguration.java | 7 + .../elasticsearch/xpack/eql/EqlTestUtils.java | 2 + .../eql/action/EqlSearchRequestTests.java | 4 + .../eql/action/LocalStateEQLXPackPlugin.java | 21 +- .../assembler/ImplicitTiebreakerTests.java | 2 +- .../assembler/SequenceSpecTests.java | 2 +- .../execution/sample/CircuitBreakerTests.java | 5 +- .../search/PITAwareQueryClientTests.java | 3 +- .../sequence/CircuitBreakerTests.java | 7 +- .../execution/sequence/PITFailureTests.java | 3 +- 20 files changed, 379 insertions(+), 46 deletions(-) create mode 100644 x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index f7f13c6266540..6f3aefdd0a16e 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -188,6 +188,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_CCS_EXEC_INFO_WITH_FAILURES = def(8_783_00_0); public static final TransportVersion LOGSDB_TELEMETRY = def(8_784_00_0); public static final TransportVersion LOGSDB_TELEMETRY_STATS = def(8_785_00_0); + public static final TransportVersion EQL_ALLOW_PARTIAL_SEARCH_RESULTS = def(8_786_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java new file mode 100644 index 0000000000000..60e7c897f45ee --- /dev/null +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java @@ -0,0 +1,245 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.eql.action; + +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchService; +import org.elasticsearch.test.transport.MockTransportService; + +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class PartialSearchResultsIT extends AbstractEqlIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return CollectionUtils.appendToCopy(super.nodePlugins(), MockTransportService.TestPlugin.class); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal, otherSettings)) + .put(SearchService.KEEPALIVE_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(randomIntBetween(100, 500))) + .build(); + } + + public void testPartialResults() throws Exception { + internalCluster().ensureAtLeastNumDataNodes(2); + final List dataNodes = internalCluster().clusterService() + .state() + .nodes() + .getDataNodes() + .values() + .stream() + .map(DiscoveryNode::getName) + .toList(); + final String assignedNodeForIndex1 = randomFrom(dataNodes); + + assertAcked( + indicesAdmin().prepareCreate("test-1") + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.routing.allocation.include._name", assignedNodeForIndex1) + .build() + ) + .setMapping("@timestamp", "type=date") + ); + assertAcked( + indicesAdmin().prepareCreate("test-2") + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.routing.allocation.exclude._name", assignedNodeForIndex1) + .build() + ) + .setMapping("@timestamp", "type=date") + ); + + for (int i = 0; i < 5; i++) { + int val = i * 2; + prepareIndex("test-1").setId(Integer.toString(i)) + .setSource("@timestamp", 100000 + val, "event.category", "process", "key", "same", "value", val) + .get(); + } + for (int i = 0; i < 5; i++) { + int val = i * 2 + 1; + prepareIndex("test-2").setId(Integer.toString(i)) + .setSource("@timestamp", 100000 + val, "event.category", "process", "key", "same", "value", val) + .get(); + } + refresh(); + + // ------------------------------------------------------------------------ + // queries with full cluster (no missing shards) + // ------------------------------------------------------------------------ + + // event query + var request = new EqlSearchRequest().indices("test-*").query("process where true").allowPartialSearchResults(true); + EqlSearchResponse response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().events().size(), equalTo(10)); + for (int i = 0; i < 10; i++) { + assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + i)); + } + + // sequence query on both shards + request = new EqlSearchRequest().indices("test-*") + .query("sequence [process where value == 1] [process where value == 2]") + .allowPartialSearchResults(true); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + EqlSearchResponse.Sequence sequence = response.hits().sequences().get(0); + assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); + assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 2")); + + // sequence query on the available shard only + request = new EqlSearchRequest().indices("test-*") + .query("sequence [process where value == 1] [process where value == 3]") + .allowPartialSearchResults(true); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sequence = response.hits().sequences().get(0); + assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); + assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 3")); + + // sequence query on the unavailable shard only + request = new EqlSearchRequest().indices("test-*") + .query("sequence [process where value == 0] [process where value == 2]") + .allowPartialSearchResults(true); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sequence = response.hits().sequences().get(0); + assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 0")); + assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 2")); + + // sequence query with missing event on unavailable shard + request = new EqlSearchRequest().indices("test-*") + .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]") + .allowPartialSearchResults(true); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + + // sample query on both shards + request = new EqlSearchRequest().indices("test-*") + .query("sample by key [process where value == 2] [process where value == 1]") + .allowPartialSearchResults(true); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + EqlSearchResponse.Sequence sample = response.hits().sequences().get(0); + assertThat(sample.events().get(0).toString(), containsString("\"value\" : 2")); + assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); + + // sample query on the available shard only + request = new EqlSearchRequest().indices("test-*") + .query("sample by key [process where value == 3] [process where value == 1]") + .allowPartialSearchResults(true); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sample = response.hits().sequences().get(0); + assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); + assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); + + // sample query on the unavailable shard only + request = new EqlSearchRequest().indices("test-*") + .query("sample by key [process where value == 2] [process where value == 0]") + .allowPartialSearchResults(true); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sample = response.hits().sequences().get(0); + assertThat(sample.events().get(0).toString(), containsString("\"value\" : 2")); + assertThat(sample.events().get(1).toString(), containsString("\"value\" : 0")); + + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + internalCluster().stopNode(assignedNodeForIndex1); + + // ------------------------------------------------------------------------ + // same queries, with missing shards + // ------------------------------------------------------------------------ + + // event query + request = new EqlSearchRequest().indices("test-*").query("process where true").allowPartialSearchResults(true); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().events().size(), equalTo(5)); + for (int i = 0; i < 5; i++) { + assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1))); + } + + // sequence query on both shards + request = new EqlSearchRequest().indices("test-*") + .query("sequence [process where value == 1] [process where value == 2]") + .allowPartialSearchResults(true); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + + // sequence query on the available shard only + request = new EqlSearchRequest().indices("test-*") + .query("sequence [process where value == 1] [process where value == 3]") + .allowPartialSearchResults(true); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sequence = response.hits().sequences().get(0); + assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); + assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 3")); + + // sequence query on the unavailable shard only + request = new EqlSearchRequest().indices("test-*") + .query("sequence [process where value == 0] [process where value == 2]") + .allowPartialSearchResults(true); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + + // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE + request = new EqlSearchRequest().indices("test-*") + .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]") + .allowPartialSearchResults(true); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sequence = response.hits().sequences().get(0); + assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); + assertThat(sequence.events().get(2).toString(), containsString("\"value\" : 3")); + + // sample query on both shards + request = new EqlSearchRequest().indices("test-*") + .query("sample by key [process where value == 2] [process where value == 1]") + .allowPartialSearchResults(true); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + + // sample query on the available shard only + request = new EqlSearchRequest().indices("test-*") + .query("sample by key [process where value == 3] [process where value == 1]") + .allowPartialSearchResults(true); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sample = response.hits().sequences().get(0); + assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); + assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); + + // sample query on the unavailable shard only + request = new EqlSearchRequest().indices("test-*") + .query("sample by key [process where value == 2] [process where value == 0]") + .allowPartialSearchResults(true); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + + } + +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java index 0aeddd525e317..68d325ed6ee6c 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java @@ -63,6 +63,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re private List fetchFields; private Map runtimeMappings = emptyMap(); private int maxSamplesPerKey = RequestDefaults.MAX_SAMPLES_PER_KEY; + private boolean allowPartialSearchResults; // Async settings private TimeValue waitForCompletionTimeout = null; @@ -83,6 +84,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re static final String KEY_FETCH_FIELDS = "fields"; static final String KEY_RUNTIME_MAPPINGS = "runtime_mappings"; static final String KEY_MAX_SAMPLES_PER_KEY = "max_samples_per_key"; + static final String KEY_ALLOW_PARTIAL_SEARCH_RESULTS = "allow_partial_search_results"; static final ParseField FILTER = new ParseField(KEY_FILTER); static final ParseField TIMESTAMP_FIELD = new ParseField(KEY_TIMESTAMP_FIELD); @@ -97,6 +99,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re static final ParseField RESULT_POSITION = new ParseField(KEY_RESULT_POSITION); static final ParseField FETCH_FIELDS_FIELD = SearchSourceBuilder.FETCH_FIELDS_FIELD; static final ParseField MAX_SAMPLES_PER_KEY = new ParseField(KEY_MAX_SAMPLES_PER_KEY); + static final ParseField ALLOW_PARTIAL_SEARCH_RESULTS = new ParseField(KEY_ALLOW_PARTIAL_SEARCH_RESULTS); private static final ObjectParser PARSER = objectParser(EqlSearchRequest::new); @@ -135,6 +138,9 @@ public EqlSearchRequest(StreamInput in) throws IOException { if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_7_0)) { maxSamplesPerKey = in.readInt(); } + if (in.getTransportVersion().onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS)) { + allowPartialSearchResults = in.readBoolean(); + } } @Override @@ -245,6 +251,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(KEY_RUNTIME_MAPPINGS, runtimeMappings); } builder.field(KEY_MAX_SAMPLES_PER_KEY, maxSamplesPerKey); + builder.field(KEY_ALLOW_PARTIAL_SEARCH_RESULTS, allowPartialSearchResults); return builder; } @@ -279,6 +286,7 @@ protected static ObjectParser objectParser parser.declareField(EqlSearchRequest::fetchFields, EqlSearchRequest::parseFetchFields, FETCH_FIELDS_FIELD, ValueType.VALUE_ARRAY); parser.declareObject(EqlSearchRequest::runtimeMappings, (p, c) -> p.map(), SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD); parser.declareInt(EqlSearchRequest::maxSamplesPerKey, MAX_SAMPLES_PER_KEY); + parser.declareBoolean(EqlSearchRequest::allowPartialSearchResults, ALLOW_PARTIAL_SEARCH_RESULTS); return parser; } @@ -427,6 +435,15 @@ public EqlSearchRequest maxSamplesPerKey(int maxSamplesPerKey) { return this; } + public boolean allowPartialSearchResults() { + return allowPartialSearchResults; + } + + public EqlSearchRequest allowPartialSearchResults(Boolean val) { + this.allowPartialSearchResults = Boolean.TRUE.equals(val); + return this; + } + private static List parseFetchFields(XContentParser parser) throws IOException { List result = new ArrayList<>(); Token token = parser.currentToken(); @@ -470,6 +487,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_7_0)) { out.writeInt(maxSamplesPerKey); } + if (out.getTransportVersion().onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS)) { + out.writeBoolean(allowPartialSearchResults); + } } @Override @@ -496,7 +516,8 @@ public boolean equals(Object o) { && Objects.equals(resultPosition, that.resultPosition) && Objects.equals(fetchFields, that.fetchFields) && Objects.equals(runtimeMappings, that.runtimeMappings) - && Objects.equals(maxSamplesPerKey, that.maxSamplesPerKey); + && Objects.equals(maxSamplesPerKey, that.maxSamplesPerKey) + && Objects.equals(allowPartialSearchResults, that.allowPartialSearchResults); } @Override @@ -517,7 +538,8 @@ public int hashCode() { resultPosition, fetchFields, runtimeMappings, - maxSamplesPerKey + maxSamplesPerKey, + allowPartialSearchResults ); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java index b26c815c1a2b5..951a169d94ee3 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java @@ -167,7 +167,8 @@ public Executable assemble( criteria.subList(0, completionStage), criteria.get(completionStage), matcher, - listOfKeys + listOfKeys, + cfg.allowPartialSearchResults() ); return w; @@ -235,7 +236,8 @@ public Executable assemble(List> listOfKeys, List cfg.fetchSize(), limit, session.circuitBreaker(), - cfg.maxSamplesPerKey() + cfg.maxSamplesPerKey(), + cfg.allowPartialSearchResults() ); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java index 89f1c4d1eb041..5127edcdb4b7d 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java @@ -84,13 +84,16 @@ public class SampleIterator implements Executable { */ private long previousTotalPageSize = 0; + private boolean allowPartialSearchResults; + public SampleIterator( QueryClient client, List criteria, int fetchSize, Limit limit, CircuitBreaker circuitBreaker, - int maxSamplesPerKey + int maxSamplesPerKey, + boolean allowPartialSearchResults ) { this.client = client; this.criteria = criteria; @@ -100,6 +103,7 @@ public SampleIterator( this.limit = limit; this.circuitBreaker = circuitBreaker; this.maxSamplesPerKey = maxSamplesPerKey; + this.allowPartialSearchResults = allowPartialSearchResults; } @Override @@ -209,7 +213,7 @@ private void finalStep(ActionListener listener) { for (SampleCriterion criterion : criteria) { SampleQueryRequest r = criterion.finalQuery(); r.singleKeyPair(compositeKeyValues, maxCriteria, maxSamplesPerKey); - searches.add(prepareRequest(r.searchSource(), false, EMPTY_ARRAY)); + searches.add(prepareRequest(r.searchSource(), false, allowPartialSearchResults, EMPTY_ARRAY)); } sampleKeys.add(new SequenceKey(compositeKeyValues.toArray())); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java index 6cbe5298b5950..38ae983900206 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java @@ -46,12 +46,14 @@ public class BasicQueryClient implements QueryClient { final Client client; final String[] indices; final List fetchFields; + private final boolean allowPartialSearchResults; public BasicQueryClient(EqlSession eqlSession) { this.cfg = eqlSession.configuration(); this.client = eqlSession.client(); this.indices = cfg.indices(); this.fetchFields = cfg.fetchFields(); + this.allowPartialSearchResults = eqlSession.configuration().allowPartialSearchResults(); } @Override @@ -60,11 +62,11 @@ public void query(QueryRequest request, ActionListener listener) // set query timeout searchSource.timeout(cfg.requestTimeout()); - SearchRequest search = prepareRequest(searchSource, false, indices); - search(search, searchLogListener(listener, log)); + SearchRequest search = prepareRequest(searchSource, false, allowPartialSearchResults, indices); + search(search, allowPartialSearchResults, searchLogListener(listener, log, allowPartialSearchResults)); } - protected void search(SearchRequest search, ActionListener listener) { + protected void search(SearchRequest search, boolean allowPartialSearchResults, ActionListener listener) { if (cfg.isCancelled()) { listener.onFailure(new TaskCancelledException("cancelled")); return; @@ -77,7 +79,7 @@ protected void search(SearchRequest search, ActionListener liste client.search(search, listener); } - protected void search(MultiSearchRequest search, ActionListener listener) { + protected void search(MultiSearchRequest search, boolean allowPartialSearchResults, ActionListener listener) { if (cfg.isCancelled()) { listener.onFailure(new TaskCancelledException("cancelled")); return; @@ -91,7 +93,7 @@ protected void search(MultiSearchRequest search, ActionListener> refs, ActionListener { + search(multiSearchBuilder.request(), allowPartialSearchResults, listener.delegateFailureAndWrap((delegate, r) -> { for (MultiSearchResponse.Item item : r.getResponses()) { // check for failures if (item.isFailure()) { @@ -187,6 +189,6 @@ public void multiQuery(List searches, ActionListener listener) { + protected void search(SearchRequest search, boolean allowPartialSearchResults, ActionListener listener) { // no pitId, ask for one if (pitId == null) { - openPIT(listener, () -> searchWithPIT(search, listener)); + openPIT(listener, () -> searchWithPIT(search, listener, allowPartialSearchResults), allowPartialSearchResults); } else { - searchWithPIT(search, listener); + searchWithPIT(search, listener, allowPartialSearchResults); } } - private void searchWithPIT(SearchRequest request, ActionListener listener) { + private void searchWithPIT(SearchRequest request, ActionListener listener, boolean allowPartialSearchResults) { makeRequestPITCompatible(request); // get the pid on each response - super.search(request, pitListener(SearchResponse::pointInTimeId, listener)); + super.search(request, allowPartialSearchResults, pitListener(SearchResponse::pointInTimeId, listener)); } @Override - protected void search(MultiSearchRequest search, ActionListener listener) { + protected void search(MultiSearchRequest search, boolean allowPartialSearchResults, ActionListener listener) { // no pitId, ask for one if (pitId == null) { - openPIT(listener, () -> searchWithPIT(search, listener)); + openPIT(listener, () -> searchWithPIT(search, allowPartialSearchResults, listener), allowPartialSearchResults); } else { - searchWithPIT(search, listener); + searchWithPIT(search, allowPartialSearchResults, listener); } } - private void searchWithPIT(MultiSearchRequest search, ActionListener listener) { + private void searchWithPIT(MultiSearchRequest search, boolean allowPartialSearchResults, ActionListener listener) { for (SearchRequest request : search.requests()) { makeRequestPITCompatible(request); } // get the pid on each request - super.search(search, pitListener(r -> { + super.search(search, allowPartialSearchResults, pitListener(r -> { // get pid for (MultiSearchResponse.Item item : r.getResponses()) { // pick the first non-failing response @@ -135,9 +135,10 @@ private ActionListener pitListener( ); } - private void openPIT(ActionListener listener, Runnable runnable) { + private void openPIT(ActionListener listener, Runnable runnable, boolean allowPartialSearchResults) { OpenPointInTimeRequest request = new OpenPointInTimeRequest(indices).indicesOptions(IndexResolver.FIELD_CAPS_INDICES_OPTIONS) - .keepAlive(keepAlive); + .keepAlive(keepAlive) + .allowPartialSearchResults(allowPartialSearchResults); request.indexFilter(filter); client.execute(TransportOpenPointInTimeAction.TYPE, request, listener.delegateFailureAndWrap((l, r) -> { pitId = r.getPointInTimeId(); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java index 40f7f7139efa1..94504cad02633 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java @@ -56,10 +56,14 @@ public final class RuntimeUtils { private RuntimeUtils() {} - public static ActionListener searchLogListener(ActionListener listener, Logger log) { + public static ActionListener searchLogListener( + ActionListener listener, + Logger log, + boolean allowPartialResults + ) { return listener.delegateFailureAndWrap((delegate, response) -> { ShardSearchFailure[] failures = response.getShardFailures(); - if (CollectionUtils.isEmpty(failures) == false) { + if (CollectionUtils.isEmpty(failures) == false && allowPartialResults == false) { delegate.onFailure(new EqlIllegalArgumentException(failures[0].reason(), failures[0].getCause())); return; } @@ -70,7 +74,11 @@ public static ActionListener searchLogListener(ActionListener multiSearchLogListener(ActionListener listener, Logger log) { + public static ActionListener multiSearchLogListener( + ActionListener listener, + boolean allowPartialSearchResults, + Logger log + ) { return listener.delegateFailureAndWrap((delegate, items) -> { for (MultiSearchResponse.Item item : items) { Exception failure = item.getFailure(); @@ -78,7 +86,7 @@ public static ActionListener multiSearchLogListener(ActionL if (failure == null) { ShardSearchFailure[] failures = response.getShardFailures(); - if (CollectionUtils.isEmpty(failures) == false) { + if (CollectionUtils.isEmpty(failures) == false && allowPartialSearchResults == false) { failure = new EqlIllegalArgumentException(failures[0].reason(), failures[0].getCause()); } } @@ -170,11 +178,16 @@ public static HitExtractor createExtractor(FieldExtraction ref, EqlConfiguration throw new EqlIllegalArgumentException("Unexpected value reference {}", ref.getClass()); } - public static SearchRequest prepareRequest(SearchSourceBuilder source, boolean includeFrozen, String... indices) { + public static SearchRequest prepareRequest( + SearchSourceBuilder source, + boolean includeFrozen, + boolean allowPartialSearchResults, + String... indices + ) { SearchRequest searchRequest = new SearchRequest(); searchRequest.indices(indices); searchRequest.source(source); - searchRequest.allowPartialSearchResults(false); + searchRequest.allowPartialSearchResults(allowPartialSearchResults); searchRequest.indicesOptions( includeFrozen ? IndexResolver.FIELD_CAPS_FROZEN_INDICES_OPTIONS : IndexResolver.FIELD_CAPS_INDICES_OPTIONS ); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java index eabf6df518ad4..271b0fdb3d40b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java @@ -103,6 +103,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) { private final boolean hasKeys; private final List> listOfKeys; + private final boolean allowPartialSearchResults; // flag used for DESC sequences to indicate whether // the window needs to restart (since the DESC query still has results) @@ -127,7 +128,8 @@ public TumblingWindow( List criteria, SequenceCriterion until, SequenceMatcher matcher, - List> listOfKeys + List> listOfKeys, + boolean allowPartialSearchResults ) { this.client = client; @@ -141,6 +143,7 @@ public TumblingWindow( this.hasKeys = baseRequest.keySize() > 0; this.restartWindowFromTailQuery = baseRequest.descending(); this.listOfKeys = listOfKeys; + this.allowPartialSearchResults = allowPartialSearchResults; } @Override @@ -316,7 +319,14 @@ private List prepareQueryForMissingEvents(List toCheck) } addKeyFilter(i, sequence, builder); RuntimeUtils.combineFilters(builder, range); - result.add(RuntimeUtils.prepareRequest(builder.size(1).trackTotalHits(false), false, Strings.EMPTY_ARRAY)); + result.add( + RuntimeUtils.prepareRequest( + builder.size(1).trackTotalHits(false), + false, + allowPartialSearchResults, + Strings.EMPTY_ARRAY + ) + ); } else { leading = false; } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java index c0141da2432ce..ca55e6f7a7e54 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java @@ -231,6 +231,7 @@ public static void operation( request.indicesOptions(), request.fetchSize(), request.maxSamplesPerKey(), + request.allowPartialSearchResults(), clientId, new TaskId(nodeId, task.getId()), task diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java index 8dd8220fb63bc..590b484399436 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java @@ -30,6 +30,7 @@ public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu private final EqlSearchTask task; private final int fetchSize; private final int maxSamplesPerKey; + private final boolean allowPartialSearchResults; @Nullable private final QueryBuilder filter; @@ -50,6 +51,7 @@ public EqlConfiguration( IndicesOptions indicesOptions, int fetchSize, int maxSamplesPerKey, + boolean allowPartialSearchResults, String clientId, TaskId taskId, EqlSearchTask task @@ -67,6 +69,7 @@ public EqlConfiguration( this.task = task; this.fetchSize = fetchSize; this.maxSamplesPerKey = maxSamplesPerKey; + this.allowPartialSearchResults = allowPartialSearchResults; } public String[] indices() { @@ -89,6 +92,10 @@ public int maxSamplesPerKey() { return maxSamplesPerKey; } + public boolean allowPartialSearchResults() { + return allowPartialSearchResults; + } + public QueryBuilder filter() { return filter; } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java index a1aa8e4bd98d7..99906935bcd59 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java @@ -51,6 +51,7 @@ private EqlTestUtils() {} null, 123, 1, + false, "", new TaskId("test", 123), null @@ -69,6 +70,7 @@ public static EqlConfiguration randomConfiguration() { randomIndicesOptions(), randomIntBetween(1, 1000), randomIntBetween(1, 1000), + randomBoolean(), randomAlphaOfLength(16), new TaskId(randomAlphaOfLength(10), randomNonNegativeLong()), randomTask() diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java index 0ff9fa9131b27..eae32902b941b 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java @@ -80,6 +80,7 @@ protected EqlSearchRequest createTestInstance() { .waitForCompletionTimeout(randomTimeValue()) .keepAlive(randomTimeValue()) .keepOnCompletion(randomBoolean()) + .allowPartialSearchResults(randomBoolean()) .fetchFields(randomFetchFields) .runtimeMappings(randomRuntimeMappings()) .resultPosition(randomFrom("tail", "head")) @@ -136,6 +137,9 @@ protected EqlSearchRequest mutateInstanceForVersion(EqlSearchRequest instance, T mutatedInstance.runtimeMappings(version.onOrAfter(TransportVersions.V_7_13_0) ? instance.runtimeMappings() : emptyMap()); mutatedInstance.resultPosition(version.onOrAfter(TransportVersions.V_7_17_8) ? instance.resultPosition() : "tail"); mutatedInstance.maxSamplesPerKey(version.onOrAfter(TransportVersions.V_8_7_0) ? instance.maxSamplesPerKey() : 1); + mutatedInstance.allowPartialSearchResults( + version.onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS) ? instance.allowPartialSearchResults() : false + ); return mutatedInstance; } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/LocalStateEQLXPackPlugin.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/LocalStateEQLXPackPlugin.java index 4d5201f544d72..33573b99546fb 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/LocalStateEQLXPackPlugin.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/LocalStateEQLXPackPlugin.java @@ -7,26 +7,41 @@ package org.elasticsearch.xpack.eql.action; +import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.indices.breaker.BreakerSettings; import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.plugins.CircuitBreakerPlugin; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.eql.plugin.EqlPlugin; import org.elasticsearch.xpack.ql.plugin.QlPlugin; import java.nio.file.Path; -public class LocalStateEQLXPackPlugin extends LocalStateCompositeXPackPlugin { +public class LocalStateEQLXPackPlugin extends LocalStateCompositeXPackPlugin implements CircuitBreakerPlugin { + + private final EqlPlugin eqlPlugin; public LocalStateEQLXPackPlugin(final Settings settings, final Path configPath) { super(settings, configPath); LocalStateEQLXPackPlugin thisVar = this; - plugins.add(new EqlPlugin() { + this.eqlPlugin = new EqlPlugin() { @Override protected XPackLicenseState getLicenseState() { return thisVar.getLicenseState(); } - }); + }; + plugins.add(eqlPlugin); plugins.add(new QlPlugin()); } + @Override + public BreakerSettings getCircuitBreaker(Settings settings) { + return eqlPlugin.getCircuitBreaker(settings); + } + + @Override + public void setCircuitBreaker(CircuitBreaker circuitBreaker) { + eqlPlugin.setCircuitBreaker(circuitBreaker); + } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java index 7bb6a228f6e48..28f5d042a8eef 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java @@ -141,7 +141,7 @@ public void testImplicitTiebreakerBeingSet() { booleanArrayOf(stages, false), NOOP_CIRCUIT_BREAKER ); - TumblingWindow window = new TumblingWindow(client, criteria, null, matcher, Collections.emptyList()); + TumblingWindow window = new TumblingWindow(client, criteria, null, matcher, Collections.emptyList(), randomBoolean()); window.execute(wrap(p -> {}, ex -> { throw ExceptionsHelper.convertToRuntime(ex); })); } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java index a8ed842e94c44..40c88eeac3214 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java @@ -277,7 +277,7 @@ public void test() throws Exception { ); QueryClient testClient = new TestQueryClient(); - TumblingWindow window = new TumblingWindow(testClient, criteria, null, matcher, Collections.emptyList()); + TumblingWindow window = new TumblingWindow(testClient, criteria, null, matcher, Collections.emptyList(), randomBoolean()); // finally make the assertion at the end of the listener window.execute(ActionTestUtils.assertNoFailureListener(this::checkResults)); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java index 9cd6549b4be2c..a21f891964f4c 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java @@ -89,7 +89,7 @@ public void query(QueryRequest r, ActionListener l) {} @Override public void fetchHits(Iterable> refs, ActionListener>> listener) {} - }, mockCriteria(), randomIntBetween(10, 500), new Limit(1000, 0), CIRCUIT_BREAKER, 1); + }, mockCriteria(), randomIntBetween(10, 500), new Limit(1000, 0), CIRCUIT_BREAKER, 1, randomBoolean()); CIRCUIT_BREAKER.startBreaking(); iterator.pushToStack(new SampleIterator.Page(CB_STACK_SIZE_PRECISION - 1)); @@ -144,7 +144,8 @@ public void fetchHits(Iterable> refs, ActionListener> refs, ActionListener { // do nothing, we don't care about the query results }, ex -> { fail("Shouldn't have failed"); })); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java index ecf5ef61ac49a..b6eed31893a77 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java @@ -146,7 +146,7 @@ public void testCircuitBreakerTumblingWindow() { booleanArrayOf(stages, false), CIRCUIT_BREAKER ); - TumblingWindow window = new TumblingWindow(client, criteria, null, matcher, Collections.emptyList()); + TumblingWindow window = new TumblingWindow(client, criteria, null, matcher, Collections.emptyList(), randomBoolean()); window.execute(ActionTestUtils.assertNoFailureListener(p -> {})); CIRCUIT_BREAKER.startBreaking(); @@ -230,7 +230,7 @@ private void assertMemoryCleared( booleanArrayOf(sequenceFiltersCount, false), eqlCircuitBreaker ); - TumblingWindow window = new TumblingWindow(eqlClient, criteria, null, matcher, Collections.emptyList()); + TumblingWindow window = new TumblingWindow(eqlClient, criteria, null, matcher, Collections.emptyList(), randomBoolean()); window.execute(ActionListener.noop()); assertTrue(esClient.searchRequestsRemainingCount() == 0); // ensure all the search requests have been asked for @@ -276,7 +276,7 @@ public void testEqlCBCleanedUp_on_ParentCBBreak() { booleanArrayOf(sequenceFiltersCount, false), eqlCircuitBreaker ); - TumblingWindow window = new TumblingWindow(eqlClient, criteria, null, matcher, Collections.emptyList()); + TumblingWindow window = new TumblingWindow(eqlClient, criteria, null, matcher, Collections.emptyList(), randomBoolean()); window.execute(wrap(p -> fail(), ex -> assertTrue(ex instanceof CircuitBreakingException))); } assertCriticalWarnings("[indices.breaker.total.limit] setting of [0%] is below the recommended minimum of 50.0% of the heap"); @@ -334,6 +334,7 @@ private QueryClient buildQueryClient(ESMockClient esClient, CircuitBreaker eqlCi null, 123, 1, + randomBoolean(), "", new TaskId("test", 123), new EqlSearchTask( diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/PITFailureTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/PITFailureTests.java index 1a2f00463b49b..85df96645cf1f 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/PITFailureTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/PITFailureTests.java @@ -83,6 +83,7 @@ public void testHandlingPitFailure() { null, 123, 1, + randomBoolean(), "", new TaskId("test", 123), new EqlSearchTask( @@ -132,7 +133,7 @@ public void testHandlingPitFailure() { ); SequenceMatcher matcher = new SequenceMatcher(1, false, TimeValue.MINUS_ONE, null, booleanArrayOf(1, false), cb); - TumblingWindow window = new TumblingWindow(eqlClient, criteria, null, matcher, Collections.emptyList()); + TumblingWindow window = new TumblingWindow(eqlClient, criteria, null, matcher, Collections.emptyList(), randomBoolean()); window.execute( wrap( p -> { fail("Search succeeded despite PIT failure"); }, From 95f9a1b10c7573f5b999448512475abdbe65772f Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 7 Nov 2024 14:41:30 +0100 Subject: [PATCH 02/23] Add shard_failures to the response --- .../eql/action/PartialSearchResultsIT.java | 32 ++++++++++- .../xpack/eql/action/EqlSearchResponse.java | 53 +++++++++++++++++-- .../xpack/eql/action/EqlSearchTask.java | 4 +- .../execution/payload/AbstractPayload.java | 10 +++- .../eql/execution/payload/EventPayload.java | 2 +- .../eql/execution/sample/SampleIterator.java | 29 +++++++++- .../eql/execution/sample/SamplePayload.java | 11 +++- .../execution/sequence/SequencePayload.java | 11 +++- .../execution/sequence/TumblingWindow.java | 26 ++++++++- .../eql/plugin/TransportEqlSearchAction.java | 16 ++++-- .../xpack/eql/session/EmptyPayload.java | 13 ++++- .../xpack/eql/session/Payload.java | 3 ++ .../xpack/eql/session/Results.java | 19 ++++++- .../eql/action/EqlSearchResponseTests.java | 14 +++-- 14 files changed, 214 insertions(+), 29 deletions(-) diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java index 60e7c897f45ee..05a16c9d37d8b 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java @@ -22,6 +22,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; public class PartialSearchResultsIT extends AbstractEqlIntegTestCase { @@ -96,6 +97,7 @@ public void testPartialResults() throws Exception { for (int i = 0; i < 10; i++) { assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + i)); } + assertThat(response.shardFailures().length, is(0)); // sequence query on both shards request = new EqlSearchRequest().indices("test-*") @@ -106,6 +108,7 @@ public void testPartialResults() throws Exception { EqlSearchResponse.Sequence sequence = response.hits().sequences().get(0); assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 2")); + assertThat(response.shardFailures().length, is(0)); // sequence query on the available shard only request = new EqlSearchRequest().indices("test-*") @@ -116,6 +119,7 @@ public void testPartialResults() throws Exception { sequence = response.hits().sequences().get(0); assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 3")); + assertThat(response.shardFailures().length, is(0)); // sequence query on the unavailable shard only request = new EqlSearchRequest().indices("test-*") @@ -126,6 +130,7 @@ public void testPartialResults() throws Exception { sequence = response.hits().sequences().get(0); assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 0")); assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 2")); + assertThat(response.shardFailures().length, is(0)); // sequence query with missing event on unavailable shard request = new EqlSearchRequest().indices("test-*") @@ -133,6 +138,7 @@ public void testPartialResults() throws Exception { .allowPartialSearchResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(0)); // sample query on both shards request = new EqlSearchRequest().indices("test-*") @@ -143,6 +149,7 @@ public void testPartialResults() throws Exception { EqlSearchResponse.Sequence sample = response.hits().sequences().get(0); assertThat(sample.events().get(0).toString(), containsString("\"value\" : 2")); assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); + assertThat(response.shardFailures().length, is(0)); // sample query on the available shard only request = new EqlSearchRequest().indices("test-*") @@ -153,6 +160,7 @@ public void testPartialResults() throws Exception { sample = response.hits().sequences().get(0); assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); + assertThat(response.shardFailures().length, is(0)); // sample query on the unavailable shard only request = new EqlSearchRequest().indices("test-*") @@ -163,6 +171,7 @@ public void testPartialResults() throws Exception { sample = response.hits().sequences().get(0); assertThat(sample.events().get(0).toString(), containsString("\"value\" : 2")); assertThat(sample.events().get(1).toString(), containsString("\"value\" : 0")); + assertThat(response.shardFailures().length, is(0)); // ------------------------------------------------------------------------ // stop one of the nodes, make one of the shards unavailable @@ -181,6 +190,7 @@ public void testPartialResults() throws Exception { for (int i = 0; i < 5; i++) { assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1))); } + assertThat(response.shardFailures().length, is(1)); // sequence query on both shards request = new EqlSearchRequest().indices("test-*") @@ -188,6 +198,9 @@ public void testPartialResults() throws Exception { .allowPartialSearchResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); // sequence query on the available shard only request = new EqlSearchRequest().indices("test-*") @@ -198,6 +211,9 @@ public void testPartialResults() throws Exception { sequence = response.hits().sequences().get(0); assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 3")); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); // sequence query on the unavailable shard only request = new EqlSearchRequest().indices("test-*") @@ -205,6 +221,9 @@ public void testPartialResults() throws Exception { .allowPartialSearchResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE request = new EqlSearchRequest().indices("test-*") @@ -215,6 +234,9 @@ public void testPartialResults() throws Exception { sequence = response.hits().sequences().get(0); assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); assertThat(sequence.events().get(2).toString(), containsString("\"value\" : 3")); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); // sample query on both shards request = new EqlSearchRequest().indices("test-*") @@ -222,6 +244,9 @@ public void testPartialResults() throws Exception { .allowPartialSearchResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); // sample query on the available shard only request = new EqlSearchRequest().indices("test-*") @@ -232,6 +257,9 @@ public void testPartialResults() throws Exception { sample = response.hits().sequences().get(0); assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); // sample query on the unavailable shard only request = new EqlSearchRequest().indices("test-*") @@ -239,7 +267,9 @@ public void testPartialResults() throws Exception { .allowPartialSearchResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); - + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java index 2b7b8b074fa71..ebe96310f74b5 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java @@ -7,8 +7,11 @@ package org.elasticsearch.xpack.eql.action; import org.apache.lucene.search.TotalHits; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ShardOperationFailedException; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -17,6 +20,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.core.Nullable; @@ -42,6 +46,7 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.action.search.ShardSearchFailure.readShardSearchFailure; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.xpack.eql.util.SearchHitUtils.qualifiedIndex; @@ -54,6 +59,7 @@ public class EqlSearchResponse extends ActionResponse implements ToXContentObjec private final String asyncExecutionId; private final boolean isRunning; private final boolean isPartial; + private final ShardSearchFailure[] shardFailures; private static final class Fields { static final String TOOK = "took"; @@ -62,6 +68,7 @@ private static final class Fields { static final String ID = "id"; static final String IS_RUNNING = "is_running"; static final String IS_PARTIAL = "is_partial"; + static final String SHARD_FAILURES = "shard_failures"; } private static final ParseField TOOK = new ParseField(Fields.TOOK); @@ -70,8 +77,10 @@ private static final class Fields { private static final ParseField ID = new ParseField(Fields.ID); private static final ParseField IS_RUNNING = new ParseField(Fields.IS_RUNNING); private static final ParseField IS_PARTIAL = new ParseField(Fields.IS_PARTIAL); + private static final ParseField SHARD_FAILURES = new ParseField(Fields.SHARD_FAILURES); private static final InstantiatingObjectParser PARSER; + static { InstantiatingObjectParser.Builder parser = InstantiatingObjectParser.builder( "eql/search_response", @@ -84,11 +93,12 @@ private static final class Fields { parser.declareString(optionalConstructorArg(), ID); parser.declareBoolean(constructorArg(), IS_RUNNING); parser.declareBoolean(constructorArg(), IS_PARTIAL); + parser.declareObjectArray(optionalConstructorArg(), (p, c) -> ShardSearchFailure.EMPTY_ARRAY, SHARD_FAILURES); // TODO fix this PARSER = parser.build(); } - public EqlSearchResponse(Hits hits, long tookInMillis, boolean isTimeout) { - this(hits, tookInMillis, isTimeout, null, false, false); + public EqlSearchResponse(Hits hits, long tookInMillis, boolean isTimeout, ShardSearchFailure[] shardFailures) { + this(hits, tookInMillis, isTimeout, null, false, false, shardFailures); } public EqlSearchResponse( @@ -97,7 +107,8 @@ public EqlSearchResponse( boolean isTimeout, String asyncExecutionId, boolean isRunning, - boolean isPartial + boolean isPartial, + ShardSearchFailure[] shardFailures ) { super(); this.hits = hits == null ? Hits.EMPTY : hits; @@ -106,6 +117,7 @@ public EqlSearchResponse( this.asyncExecutionId = asyncExecutionId; this.isRunning = isRunning; this.isPartial = isPartial; + this.shardFailures = shardFailures; } public EqlSearchResponse(StreamInput in) throws IOException { @@ -116,6 +128,19 @@ public EqlSearchResponse(StreamInput in) throws IOException { asyncExecutionId = in.readOptionalString(); isPartial = in.readBoolean(); isRunning = in.readBoolean(); + if (in.getTransportVersion().onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS)) { + int size = in.readVInt(); + if (size == 0) { + shardFailures = ShardSearchFailure.EMPTY_ARRAY; + } else { + shardFailures = new ShardSearchFailure[size]; + for (int i = 0; i < shardFailures.length; i++) { + shardFailures[i] = readShardSearchFailure(in); + } + } + } else { + shardFailures = ShardSearchFailure.EMPTY_ARRAY; + } } public static EqlSearchResponse fromXContent(XContentParser parser) { @@ -130,6 +155,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(asyncExecutionId); out.writeBoolean(isPartial); out.writeBoolean(isRunning); + if (out.getTransportVersion().onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS)) { + out.writeVInt(shardFailures.length); + for (ShardSearchFailure shardSearchFailure : shardFailures) { + shardSearchFailure.writeTo(out); + } + } } @Override @@ -147,6 +178,13 @@ private XContentBuilder innerToXContent(XContentBuilder builder, Params params) builder.field(IS_RUNNING.getPreferredName(), isRunning); builder.field(TOOK.getPreferredName(), tookInMillis); builder.field(TIMED_OUT.getPreferredName(), isTimeout); + if (CollectionUtils.isEmpty(shardFailures) == false) { + builder.startArray(SHARD_FAILURES.getPreferredName()); + for (ShardOperationFailedException shardFailure : ExceptionsHelper.groupBy(shardFailures)) { + shardFailure.toXContent(builder, params); + } + builder.endArray(); + } hits.toXContent(builder, params); return builder; } @@ -178,6 +216,10 @@ public boolean isPartial() { return isPartial; } + public ShardSearchFailure[] shardFailures() { + return shardFailures; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -190,12 +232,13 @@ public boolean equals(Object o) { return Objects.equals(hits, that.hits) && Objects.equals(tookInMillis, that.tookInMillis) && Objects.equals(isTimeout, that.isTimeout) - && Objects.equals(asyncExecutionId, that.asyncExecutionId); + && Objects.equals(asyncExecutionId, that.asyncExecutionId) + && Objects.equals(shardFailures, that.shardFailures); } @Override public int hashCode() { - return Objects.hash(hits, tookInMillis, isTimeout, asyncExecutionId); + return Objects.hash(hits, tookInMillis, isTimeout, asyncExecutionId, shardFailures); } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchTask.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchTask.java index 2a1bc3b7adb67..0fc8e8c88d7d9 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchTask.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchTask.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.eql.action; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.xpack.core.async.AsyncExecutionId; @@ -39,7 +40,8 @@ public EqlSearchResponse getCurrentResult() { false, getExecutionId().getEncoded(), true, - true + true, + ShardSearchFailure.EMPTY_ARRAY ); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/AbstractPayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/AbstractPayload.java index 823cd04d25f45..9fecf958b9714 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/AbstractPayload.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/AbstractPayload.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.eql.execution.payload; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.eql.session.Payload; @@ -14,10 +15,12 @@ public abstract class AbstractPayload implements Payload { private final boolean timedOut; private final TimeValue timeTook; + private ShardSearchFailure[] shardFailures; - protected AbstractPayload(boolean timedOut, TimeValue timeTook) { + protected AbstractPayload(boolean timedOut, TimeValue timeTook, ShardSearchFailure[] shardFailures) { this.timedOut = timedOut; this.timeTook = timeTook; + this.shardFailures = shardFailures; } @Override @@ -29,4 +32,9 @@ public boolean timedOut() { public TimeValue timeTook() { return timeTook; } + + @Override + public ShardSearchFailure[] shardFailures() { + return shardFailures; + } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/EventPayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/EventPayload.java index a7845ca62dccc..6471bc0814f70 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/EventPayload.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/EventPayload.java @@ -20,7 +20,7 @@ public class EventPayload extends AbstractPayload { private final List values; public EventPayload(SearchResponse response) { - super(response.isTimedOut(), response.getTook()); + super(response.isTimedOut(), response.getTook(), response.getShardFailures()); SearchHits hits = response.getHits(); values = new ArrayList<>(hits.getHits().length); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java index 5127edcdb4b7d..3b6b2ebaf2088 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java @@ -14,6 +14,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.core.TimeValue; import org.elasticsearch.search.SearchHit; @@ -35,6 +37,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -58,6 +61,7 @@ public class SampleIterator implements Executable { private final Limit limit; private final int maxSamplesPerKey; private long startTime; + private Map shardFailures = new HashMap<>(); // ---------- CIRCUIT BREAKER ----------- @@ -151,6 +155,7 @@ private void advance(ActionListener listener) { private void queryForCompositeAggPage(ActionListener listener, final SampleQueryRequest request) { client.query(request, listener.delegateFailureAndWrap((delegate, r) -> { + addShardFailures(r); // either the fields values or the fields themselves are missing // or the filter applied on the eql query matches no documents if (r.hasAggregations() == false) { @@ -178,6 +183,14 @@ private void queryForCompositeAggPage(ActionListener listener, final Sa })); } + private void addShardFailures(SearchResponse r) { + if (r.getShardFailures() != null) { + for (ShardSearchFailure shardFailure : r.getShardFailures()) { + shardFailures.put(shardFailure.toString(), shardFailure); // TODO find a better way to deduplicate + } + } + } + protected void pushToStack(Page nextPage) { stack.push(nextPage); totalPageSize += nextPage.size; @@ -220,6 +233,9 @@ private void finalStep(ActionListener listener) { int initialSize = samples.size(); client.multiQuery(searches, listener.delegateFailureAndWrap((delegate, r) -> { + for (MultiSearchResponse.Item item : r) { + addShardFailures(item.getResponse()); + } List> sample = new ArrayList<>(maxCriteria); MultiSearchResponse.Item[] response = r.getResponses(); int docGroupsCounter = 1; @@ -284,14 +300,23 @@ private void payload(ActionListener listener) { log.trace("Sending payload for [{}] samples", samples.size()); if (samples.isEmpty()) { - listener.onResponse(new EmptyPayload(Type.SAMPLE, timeTook())); + listener.onResponse(new EmptyPayload(Type.SAMPLE, timeTook(), shardFailures.values().toArray(new ShardSearchFailure[0]))); return; } // get results through search (to keep using PIT) client.fetchHits( hits(samples), - ActionListeners.map(listener, listOfHits -> new SamplePayload(samples, listOfHits, false, timeTook())) + ActionListeners.map( + listener, + listOfHits -> new SamplePayload( + samples, + listOfHits, + false, + timeTook(), + shardFailures.values().toArray(new ShardSearchFailure[0]) + ) + ) ); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SamplePayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SamplePayload.java index 121f4c208273b..aee084dd88734 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SamplePayload.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SamplePayload.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.eql.execution.sample; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.core.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event; @@ -19,8 +20,14 @@ class SamplePayload extends AbstractPayload { private final List values; - SamplePayload(List samples, List> docs, boolean timedOut, TimeValue timeTook) { - super(timedOut, timeTook); + SamplePayload( + List samples, + List> docs, + boolean timedOut, + TimeValue timeTook, + ShardSearchFailure[] shardFailures + ) { + super(timedOut, timeTook, shardFailures); values = new ArrayList<>(samples.size()); for (int i = 0; i < samples.size(); i++) { diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java index 45083babddbb4..b4a8edc79b3ad 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.eql.execution.sequence; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.core.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event; @@ -19,8 +20,14 @@ class SequencePayload extends AbstractPayload { private final List values; - SequencePayload(List sequences, List> docs, boolean timedOut, TimeValue timeTook) { - super(timedOut, timeTook); + SequencePayload( + List sequences, + List> docs, + boolean timedOut, + TimeValue timeTook, + ShardSearchFailure[] shardFailures + ) { + super(timedOut, timeTook, shardFailures); values = new ArrayList<>(sequences.size()); for (int i = 0; i < sequences.size(); i++) { diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java index 271b0fdb3d40b..136bf80003345 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.Strings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; @@ -41,6 +42,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -104,6 +106,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) { private final boolean hasKeys; private final List> listOfKeys; private final boolean allowPartialSearchResults; + private Map shardFailures = new HashMap<>(); // flag used for DESC sequences to indicate whether // the window needs to restart (since the DESC query still has results) @@ -227,6 +230,9 @@ public void checkMissingEvents(Runnable next, ActionListener listener) private void doCheckMissingEvents(List batchToCheck, MultiSearchResponse p, ActionListener listener, Runnable next) { MultiSearchResponse.Item[] responses = p.getResponses(); + for (MultiSearchResponse.Item response : responses) { + addShardFailures(response.getResponse()); + } int nextResponse = 0; for (Sequence sequence : batchToCheck) { boolean leading = true; @@ -371,6 +377,7 @@ private void advance(int stage, ActionListener listener) { * Execute the base query. */ private void baseCriterion(int baseStage, SearchResponse r, ActionListener listener) { + addShardFailures(r); SequenceCriterion base = criteria.get(baseStage); SearchHits hits = r.getHits(); @@ -742,7 +749,7 @@ private void doPayload(ActionListener listener) { log.trace("Sending payload for [{}] sequences", completed.size()); if (completed.isEmpty()) { - listener.onResponse(new EmptyPayload(Type.SEQUENCE, timeTook())); + listener.onResponse(new EmptyPayload(Type.SEQUENCE, timeTook(), shardFailures.values().toArray(new ShardSearchFailure[0]))); return; } @@ -751,7 +758,13 @@ private void doPayload(ActionListener listener) { if (criteria.get(matcher.firstPositiveStage).descending()) { Collections.reverse(completed); } - return new SequencePayload(completed, addMissingEventPlaceholders(listOfHits), false, timeTook()); + return new SequencePayload( + completed, + addMissingEventPlaceholders(listOfHits), + false, + timeTook(), + shardFailures.values().toArray(new ShardSearchFailure[0]) + ); })); } @@ -928,4 +941,13 @@ public KeyAndOrdinal next() { }; }; } + + private void addShardFailures(SearchResponse r) { + if (r.getShardFailures() != null) { + for (ShardSearchFailure shardFailure : r.getShardFailures()) { + shardFailures.put(shardFailure.toString(), shardFailure); // TODO find a better way to deduplicate + } + } + } + } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java index ca55e6f7a7e54..3e28f78f3a542 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.internal.Client; @@ -144,7 +145,8 @@ public EqlSearchResponse initialResponse(EqlSearchTask task) { false, task.getExecutionId().getEncoded(), true, - true + true, + ShardSearchFailure.EMPTY_ARRAY ); } @@ -248,9 +250,17 @@ public static void operation( static EqlSearchResponse createResponse(Results results, AsyncExecutionId id) { EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(results.events(), results.sequences(), results.totalHits()); if (id != null) { - return new EqlSearchResponse(hits, results.tookTime().getMillis(), results.timedOut(), id.getEncoded(), false, false); + return new EqlSearchResponse( + hits, + results.tookTime().getMillis(), + results.timedOut(), + id.getEncoded(), + false, + false, + results.shardFailures() + ); } else { - return new EqlSearchResponse(hits, results.tookTime().getMillis(), results.timedOut()); + return new EqlSearchResponse(hits, results.tookTime().getMillis(), results.timedOut(), results.shardFailures()); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EmptyPayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EmptyPayload.java index 9822285465087..33ed5799cd073 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EmptyPayload.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EmptyPayload.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.eql.session; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.core.TimeValue; import java.util.List; @@ -17,14 +18,16 @@ public class EmptyPayload implements Payload { private final Type type; private final TimeValue timeTook; + private final ShardSearchFailure[] shardFailures; public EmptyPayload(Type type) { - this(type, TimeValue.ZERO); + this(type, TimeValue.ZERO, ShardSearchFailure.EMPTY_ARRAY); } - public EmptyPayload(Type type, TimeValue timeTook) { + public EmptyPayload(Type type, TimeValue timeTook, ShardSearchFailure[] shardFailures) { this.type = type; this.timeTook = timeTook; + this.shardFailures = shardFailures; } @Override @@ -46,4 +49,10 @@ public TimeValue timeTook() { public List values() { return emptyList(); } + + @Override + public ShardSearchFailure[] shardFailures() { + return shardFailures; + } + } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Payload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Payload.java index 1d82478e6db26..05e614714a5aa 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Payload.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Payload.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.eql.session; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.core.TimeValue; import java.util.List; @@ -29,4 +30,6 @@ enum Type { TimeValue timeTook(); List values(); + + ShardSearchFailure[] shardFailures(); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Results.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Results.java index bb76c08c801cb..13886470f21f5 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Results.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Results.java @@ -9,6 +9,7 @@ import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits.Relation; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event; import org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence; @@ -23,18 +24,28 @@ public class Results { private final boolean timedOut; private final TimeValue tookTime; private final Type type; + private ShardSearchFailure[] shardFailures; public static Results fromPayload(Payload payload) { List values = payload.values(); - return new Results(new TotalHits(values.size(), Relation.EQUAL_TO), payload.timeTook(), false, values, payload.resultType()); + payload.shardFailures(); + return new Results( + new TotalHits(values.size(), Relation.EQUAL_TO), + payload.timeTook(), + false, + values, + payload.resultType(), + payload.shardFailures() + ); } - Results(TotalHits totalHits, TimeValue tookTime, boolean timedOut, List results, Type type) { + Results(TotalHits totalHits, TimeValue tookTime, boolean timedOut, List results, Type type, ShardSearchFailure[] shardFailures) { this.totalHits = totalHits; this.tookTime = tookTime; this.timedOut = timedOut; this.results = results; this.type = type; + this.shardFailures = shardFailures; } public TotalHits totalHits() { @@ -51,6 +62,10 @@ public List sequences() { return (type == Type.SEQUENCE || type == Type.SAMPLE) ? (List) results : null; } + public ShardSearchFailure[] shardFailures() { + return shardFailures; + } + public TimeValue tookTime() { return tookTime; } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java index 6cb283d11848e..fa118a5256df1 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java @@ -9,6 +9,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.document.DocumentField; @@ -190,7 +191,7 @@ public static EqlSearchResponse createRandomEventsResponse(TotalHits totalHits, hits = new EqlSearchResponse.Hits(randomEvents(xType), null, totalHits); } if (randomBoolean()) { - return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean(), ShardSearchFailure.EMPTY_ARRAY); } else { return new EqlSearchResponse( hits, @@ -198,7 +199,8 @@ public static EqlSearchResponse createRandomEventsResponse(TotalHits totalHits, randomBoolean(), randomAlphaOfLength(10), randomBoolean(), - randomBoolean() + randomBoolean(), + ShardSearchFailure.EMPTY_ARRAY ); } } @@ -222,7 +224,7 @@ public static EqlSearchResponse createRandomSequencesResponse(TotalHits totalHit hits = new EqlSearchResponse.Hits(null, seq, totalHits); } if (randomBoolean()) { - return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean()); + return new EqlSearchResponse(hits, randomIntBetween(0, 1001), randomBoolean(), ShardSearchFailure.EMPTY_ARRAY); } else { return new EqlSearchResponse( hits, @@ -230,7 +232,8 @@ public static EqlSearchResponse createRandomSequencesResponse(TotalHits totalHit randomBoolean(), randomAlphaOfLength(10), randomBoolean(), - randomBoolean() + randomBoolean(), + ShardSearchFailure.EMPTY_ARRAY ); } } @@ -273,7 +276,8 @@ protected EqlSearchResponse mutateInstanceForVersion(EqlSearchResponse instance, instance.isTimeout(), instance.id(), instance.isRunning(), - instance.isPartial() + instance.isPartial(), + ShardSearchFailure.EMPTY_ARRAY ); } From d14c1f61e97f5836b01e084b75f621d8775086a2 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 7 Nov 2024 17:15:36 +0100 Subject: [PATCH 03/23] Docs + request param --- docs/reference/eql/eql-search-api.asciidoc | 16 ++++++++++++++++ .../xpack/eql/plugin/RestEqlSearchAction.java | 3 +++ 2 files changed, 19 insertions(+) diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc index d7f10f4627f6c..beb9b40875b8f 100644 --- a/docs/reference/eql/eql-search-api.asciidoc +++ b/docs/reference/eql/eql-search-api.asciidoc @@ -88,6 +88,22 @@ request that targets only `bar*` still returns an error. + Defaults to `true`. +`allow_partial_search_results`:: +(Optional, Boolean) + +If `false`, the request returns an error if one or more shards involved in the query are unavailable. ++ +If `true`, the query is executed only on the available shards, ignoring shard request timeouts and +<>. ++ +Defaults to `false`. + +[IMPORTANT] +==== +You can also specify this value using the `allow_partial_search_results` request body parameter. +If both parameters are specified, only the query parameter is used. +==== + `ccs_minimize_roundtrips`:: (Optional, Boolean) If `true`, network round-trips between the local and the remote cluster are minimized when running cross-cluster search (CCS) requests. diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java index e24a4749f45cd..126462797141e 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java @@ -64,6 +64,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli } eqlRequest.keepOnCompletion(request.paramAsBoolean("keep_on_completion", eqlRequest.keepOnCompletion())); eqlRequest.ccsMinimizeRoundtrips(request.paramAsBoolean("ccs_minimize_roundtrips", eqlRequest.ccsMinimizeRoundtrips())); + eqlRequest.allowPartialSearchResults( + request.paramAsBoolean("allow_partial_search_results", eqlRequest.allowPartialSearchResults()) + ); } return channel -> { From b72fd3761cb9b2a52473deaac1a4c93d2a4fa034 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Fri, 8 Nov 2024 11:03:31 +0100 Subject: [PATCH 04/23] Add cluster setting --- docs/reference/eql/eql-search-api.asciidoc | 4 + .../eql/action/PartialSearchResultsIT.java | 148 +++++++++++++++++- .../xpack/eql/action/EqlSearchRequest.java | 12 +- .../xpack/eql/plugin/EqlPlugin.java | 9 +- .../eql/plugin/TransportEqlSearchAction.java | 11 +- 5 files changed, 176 insertions(+), 8 deletions(-) diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc index beb9b40875b8f..70d1f865c1ecd 100644 --- a/docs/reference/eql/eql-search-api.asciidoc +++ b/docs/reference/eql/eql-search-api.asciidoc @@ -97,6 +97,10 @@ If `true`, the query is executed only on the available shards, ignoring shard re <>. + Defaults to `false`. ++ +To override the default for this field, set the +`xpack.eql.default_allow_partial_results` cluster setting to `true`. + [IMPORTANT] ==== diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java index 05a16c9d37d8b..3a573bf09b907 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java @@ -7,6 +7,9 @@ package org.elasticsearch.xpack.eql.action; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsAction; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Settings; @@ -15,13 +18,16 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchService; import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.xpack.eql.plugin.EqlPlugin; import java.util.Collection; import java.util.List; +import java.util.concurrent.ExecutionException; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; public class PartialSearchResultsIT extends AbstractEqlIntegTestCase { @@ -180,7 +186,44 @@ public void testPartialResults() throws Exception { internalCluster().stopNode(assignedNodeForIndex1); // ------------------------------------------------------------------------ - // same queries, with missing shards + // same queries, with missing shards. Let them fail + // ------------------------------------------------------------------------ + + // event query + request = new EqlSearchRequest().indices("test-*").query("process where true"); + shouldFail(request); + + // sequence query on both shards + request = new EqlSearchRequest().indices("test-*").query("sequence [process where value == 1] [process where value == 2]"); + shouldFail(request); + + // sequence query on the available shard only + request = new EqlSearchRequest().indices("test-*").query("sequence [process where value == 1] [process where value == 3]"); + shouldFail(request); + + // sequence query on the unavailable shard only + request = new EqlSearchRequest().indices("test-*").query("sequence [process where value == 0] [process where value == 2]"); + shouldFail(request); + + // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE + request = new EqlSearchRequest().indices("test-*") + .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]"); + shouldFail(request); + + // sample query on both shards + request = new EqlSearchRequest().indices("test-*").query("sample by key [process where value == 2] [process where value == 1]"); + shouldFail(request); + + // sample query on the available shard only + request = new EqlSearchRequest().indices("test-*").query("sample by key [process where value == 3] [process where value == 1]"); + shouldFail(request); + + // sample query on the unavailable shard only + request = new EqlSearchRequest().indices("test-*").query("sample by key [process where value == 2] [process where value == 0]"); + shouldFail(request); + + // ------------------------------------------------------------------------ + // same queries, with missing shards and allow_partial_search_results=true // ------------------------------------------------------------------------ // event query @@ -270,6 +313,109 @@ public void testPartialResults() throws Exception { assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // ------------------------------------------------------------------------ + // same queries, with missing shards and with default xpack.eql.default_allow_partial_results=true + // ------------------------------------------------------------------------ + + client().execute( + ClusterUpdateSettingsAction.INSTANCE, + new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings( + Settings.builder().put(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey(), true) + ) + ).get(); + + // event query + request = new EqlSearchRequest().indices("test-*").query("process where true"); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().events().size(), equalTo(5)); + for (int i = 0; i < 5; i++) { + assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1))); + } + assertThat(response.shardFailures().length, is(1)); + + // sequence query on both shards + request = new EqlSearchRequest().indices("test-*").query("sequence [process where value == 1] [process where value == 2]"); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query on the available shard only + request = new EqlSearchRequest().indices("test-*").query("sequence [process where value == 1] [process where value == 3]"); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sequence = response.hits().sequences().get(0); + assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); + assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 3")); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query on the unavailable shard only + request = new EqlSearchRequest().indices("test-*").query("sequence [process where value == 0] [process where value == 2]"); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE + request = new EqlSearchRequest().indices("test-*") + .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]"); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sequence = response.hits().sequences().get(0); + assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); + assertThat(sequence.events().get(2).toString(), containsString("\"value\" : 3")); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on both shards + request = new EqlSearchRequest().indices("test-*").query("sample by key [process where value == 2] [process where value == 1]"); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on the available shard only + request = new EqlSearchRequest().indices("test-*").query("sample by key [process where value == 3] [process where value == 1]"); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sample = response.hits().sequences().get(0); + assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); + assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on the unavailable shard only + request = new EqlSearchRequest().indices("test-*").query("sample by key [process where value == 2] [process where value == 0]"); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + client().execute( + ClusterUpdateSettingsAction.INSTANCE, + new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings( + Settings.builder().putNull(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey()) + ) + ).get(); + + } + + private static void shouldFail(EqlSearchRequest request) throws InterruptedException { + try { + client().execute(EqlSearchAction.INSTANCE, request).get(); + fail(); + } catch (ExecutionException e) { + assertThat(e.getCause(), instanceOf(SearchPhaseExecutionException.class)); + } } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java index 68d325ed6ee6c..3914c683b0953 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java @@ -63,7 +63,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re private List fetchFields; private Map runtimeMappings = emptyMap(); private int maxSamplesPerKey = RequestDefaults.MAX_SAMPLES_PER_KEY; - private boolean allowPartialSearchResults; + private Boolean allowPartialSearchResults; // Async settings private TimeValue waitForCompletionTimeout = null; @@ -139,7 +139,9 @@ public EqlSearchRequest(StreamInput in) throws IOException { maxSamplesPerKey = in.readInt(); } if (in.getTransportVersion().onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS)) { - allowPartialSearchResults = in.readBoolean(); + allowPartialSearchResults = in.readOptionalBoolean(); + } else { + allowPartialSearchResults = false; } } @@ -435,12 +437,12 @@ public EqlSearchRequest maxSamplesPerKey(int maxSamplesPerKey) { return this; } - public boolean allowPartialSearchResults() { + public Boolean allowPartialSearchResults() { return allowPartialSearchResults; } public EqlSearchRequest allowPartialSearchResults(Boolean val) { - this.allowPartialSearchResults = Boolean.TRUE.equals(val); + this.allowPartialSearchResults = val; return this; } @@ -488,7 +490,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeInt(maxSamplesPerKey); } if (out.getTransportVersion().onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS)) { - out.writeBoolean(allowPartialSearchResults); + out.writeOptionalBoolean(allowPartialSearchResults); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java index 084a5e74a47e8..8b15d230cabb7 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java @@ -60,6 +60,13 @@ public class EqlPlugin extends Plugin implements ActionPlugin, CircuitBreakerPlu Setting.Property.DeprecatedWarning ); + public static final Setting DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS = Setting.boolSetting( + "xpack.eql.default_allow_partial_results", + false, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + public EqlPlugin() {} @Override @@ -86,7 +93,7 @@ private Collection createComponents(Client client, Settings settings, Cl */ @Override public List> getSettings() { - return List.of(EQL_ENABLED_SETTING); + return List.of(EQL_ENABLED_SETTING, DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS); } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java index 3e28f78f3a542..f8f8a2885afb8 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java @@ -233,7 +233,9 @@ public static void operation( request.indicesOptions(), request.fetchSize(), request.maxSamplesPerKey(), - request.allowPartialSearchResults(), + request.allowPartialSearchResults() == null + ? defaultAllowPartialSearchResults(clusterService) + : request.allowPartialSearchResults(), clientId, new TaskId(nodeId, task.getId()), task @@ -247,6 +249,13 @@ public static void operation( } } + private static boolean defaultAllowPartialSearchResults(ClusterService clusterService) { + if (clusterService.getClusterSettings() == null) { + return EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getDefault(Settings.EMPTY); + } + return clusterService.getClusterSettings().get(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS); + } + static EqlSearchResponse createResponse(Results results, AsyncExecutionId id) { EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(results.events(), results.sequences(), results.totalHits()); if (id != null) { From 8777fc866c2807e1d6e45ce6eb510e7fcc73d578 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Mon, 11 Nov 2024 10:47:34 +0100 Subject: [PATCH 05/23] Randomize spec tests --- .../org/elasticsearch/test/eql/BaseEqlSpecTestCase.java | 6 ++++++ .../xpack/eql/action/PartialSearchResultsIT.java | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java index 90244d9b2c019..7455bb3f7c2e7 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java @@ -163,6 +163,9 @@ protected ObjectPath runQuery(String index, String query) throws Exception { if (maxSamplesPerKey > 0) { builder.field("max_samples_per_key", maxSamplesPerKey); } + if (randomBoolean()) { + builder.field("allow_partial_search_results", randomBoolean()); + } builder.endObject(); Request request = new Request("POST", "/" + index + "/_eql/search"); @@ -170,6 +173,9 @@ protected ObjectPath runQuery(String index, String query) throws Exception { if (ccsMinimizeRoundtrips != null) { request.addParameter("ccs_minimize_roundtrips", ccsMinimizeRoundtrips.toString()); } + if (randomBoolean()) { + request.addParameter("allow_partial_search_results", String.valueOf(randomBoolean())); + } int timeout = Math.toIntExact(timeout().millis()); RequestConfig config = RequestConfig.copy(RequestConfig.DEFAULT) .setConnectionRequestTimeout(timeout) diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java index 3a573bf09b907..4008e9e87e729 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java @@ -205,7 +205,7 @@ public void testPartialResults() throws Exception { request = new EqlSearchRequest().indices("test-*").query("sequence [process where value == 0] [process where value == 2]"); shouldFail(request); - // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE + // sequence query with missing event on unavailable shard. request = new EqlSearchRequest().indices("test-*") .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]"); shouldFail(request); From 4e63b3caf4466c0e69001013ad0ccba101c9d049 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Mon, 11 Nov 2024 11:18:56 +0100 Subject: [PATCH 06/23] More tests --- .../rest-api-spec/test/eql/10_basic.yml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml index e49264d76d5e9..c40b2d7dac107 100644 --- a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml +++ b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml @@ -478,3 +478,34 @@ setup: query: 'sequence with maxspan=10d [network where user == "ADMIN"] ![network where used == "SYSTEM"]' - match: { error.root_cause.0.type: "verification_exception" } - match: { error.root_cause.0.reason: "Found 1 problem\nline 1:75: Unknown column [used], did you mean [user]?" } + + +--- +"Execute query with allow_partial_search_results": + - do: + eql.search: + index: eql_test + body: + query: 'process where user == "SYSTEM"' + fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"] + allow_partial_search_results: true + + - match: {timed_out: false} + - match: {hits.total.value: 3} + - match: {hits.total.relation: "eq"} + - match: {hits.events.0._source.user: "SYSTEM"} + - match: {hits.events.0._id: "1"} + - match: {hits.events.0.fields.@timestamp: ["1580733296000"]} + - match: {hits.events.0.fields.id: [123]} + - match: {hits.events.0.fields.valid: [false]} + - match: {hits.events.0.fields.day_of_week: ["Monday"]} + - match: {hits.events.1._id: "2"} + - match: {hits.events.1.fields.@timestamp: ["1580819696000"]} + - match: {hits.events.1.fields.id: [123]} + - match: {hits.events.1.fields.valid: [true]} + - match: {hits.events.1.fields.day_of_week: ["Tuesday"]} + - match: {hits.events.2._id: "3"} + - match: {hits.events.2.fields.@timestamp: ["1580906096000"]} + - match: {hits.events.2.fields.id: [123]} + - match: {hits.events.2.fields.valid: [true]} + - match: {hits.events.2.fields.day_of_week: ["Wednesday"]} From b6501ccdfac3c5843ac628d2c1835eb7c85acd26 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Mon, 11 Nov 2024 12:38:50 +0100 Subject: [PATCH 07/23] Update docs/changelog/116388.yaml --- docs/changelog/116388.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/116388.yaml diff --git a/docs/changelog/116388.yaml b/docs/changelog/116388.yaml new file mode 100644 index 0000000000000..59cdafc9ec337 --- /dev/null +++ b/docs/changelog/116388.yaml @@ -0,0 +1,5 @@ +pr: 116388 +summary: Add support for partial shard results +area: EQL +type: enhancement +issues: [] From f05278290596b9e3d87ab7e5ac1f6d35ceaf6e4a Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Mon, 18 Nov 2024 18:02:50 +0100 Subject: [PATCH 08/23] Fine tune sequences behavior using allow_partial_sequence_results If true (default), the sequences are calculated on the available shards If false, in case of shard failures, the query does not return any results, but does not throw errors either --- docs/reference/eql/eql-search-api.asciidoc | 24 +++ .../eql/action/PartialSearchResultsIT.java | 152 ++++++++++++++++-- .../xpack/eql/action/EqlSearchRequest.java | 23 ++- .../execution/assembler/ExecutionManager.java | 3 +- .../execution/sequence/TumblingWindow.java | 11 +- .../xpack/eql/plugin/EqlPlugin.java | 9 +- .../eql/plugin/TransportEqlSearchAction.java | 10 ++ .../xpack/eql/session/EqlConfiguration.java | 7 + .../elasticsearch/xpack/eql/EqlTestUtils.java | 2 + .../eql/action/EqlSearchRequestTests.java | 4 + .../assembler/ImplicitTiebreakerTests.java | 10 +- .../assembler/SequenceSpecTests.java | 10 +- .../search/PITAwareQueryClientTests.java | 11 +- .../sequence/CircuitBreakerTests.java | 31 +++- .../execution/sequence/PITFailureTests.java | 11 +- 15 files changed, 289 insertions(+), 29 deletions(-) diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc index 70d1f865c1ecd..047aa298dd223 100644 --- a/docs/reference/eql/eql-search-api.asciidoc +++ b/docs/reference/eql/eql-search-api.asciidoc @@ -108,6 +108,30 @@ You can also specify this value using the `allow_partial_search_results` request If both parameters are specified, only the query parameter is used. ==== + +`allow_partial_sequence_results`:: +(Optional, Boolean) + + +Used together with `allow_partial_search_results=true`, controls the behavior of sequence queries specifically +(if `allow_partial_search_results=false` this setting has no effect). +If `true` and if some shards are unavailable, the sequences are calculated on available shards. ++ +If `false` and if some shards are unavailable, the query only returns information about the shard failures, +but no further results. ++ +Defaults to `true`. ++ +To override the default for this field, set the +`xpack.eql.default_allow_partial_sequence_results` cluster setting to `false`. + + +[IMPORTANT] +==== +You can also specify this value using the `allow_partial_sequence_results` request body parameter. +If both parameters are specified, only the query parameter is used. +==== + `ccs_minimize_roundtrips`:: (Optional, Boolean) If `true`, network round-trips between the local and the remote cluster are minimized when running cross-cluster search (CCS) requests. diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java index 4008e9e87e729..5918b84d03410 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java @@ -97,7 +97,10 @@ public void testPartialResults() throws Exception { // ------------------------------------------------------------------------ // event query - var request = new EqlSearchRequest().indices("test-*").query("process where true").allowPartialSearchResults(true); + var request = new EqlSearchRequest().indices("test-*") + .query("process where true") + .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()); EqlSearchResponse response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().events().size(), equalTo(10)); for (int i = 0; i < 10; i++) { @@ -108,7 +111,8 @@ public void testPartialResults() throws Exception { // sequence query on both shards request = new EqlSearchRequest().indices("test-*") .query("sequence [process where value == 1] [process where value == 2]") - .allowPartialSearchResults(true); + .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); EqlSearchResponse.Sequence sequence = response.hits().sequences().get(0); @@ -119,7 +123,8 @@ public void testPartialResults() throws Exception { // sequence query on the available shard only request = new EqlSearchRequest().indices("test-*") .query("sequence [process where value == 1] [process where value == 3]") - .allowPartialSearchResults(true); + .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); sequence = response.hits().sequences().get(0); @@ -130,7 +135,8 @@ public void testPartialResults() throws Exception { // sequence query on the unavailable shard only request = new EqlSearchRequest().indices("test-*") .query("sequence [process where value == 0] [process where value == 2]") - .allowPartialSearchResults(true); + .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); sequence = response.hits().sequences().get(0); @@ -141,7 +147,8 @@ public void testPartialResults() throws Exception { // sequence query with missing event on unavailable shard request = new EqlSearchRequest().indices("test-*") .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]") - .allowPartialSearchResults(true); + .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(0)); @@ -149,7 +156,8 @@ public void testPartialResults() throws Exception { // sample query on both shards request = new EqlSearchRequest().indices("test-*") .query("sample by key [process where value == 2] [process where value == 1]") - .allowPartialSearchResults(true); + .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); EqlSearchResponse.Sequence sample = response.hits().sequences().get(0); @@ -160,7 +168,8 @@ public void testPartialResults() throws Exception { // sample query on the available shard only request = new EqlSearchRequest().indices("test-*") .query("sample by key [process where value == 3] [process where value == 1]") - .allowPartialSearchResults(true); + .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); sample = response.hits().sequences().get(0); @@ -171,7 +180,8 @@ public void testPartialResults() throws Exception { // sample query on the unavailable shard only request = new EqlSearchRequest().indices("test-*") .query("sample by key [process where value == 2] [process where value == 0]") - .allowPartialSearchResults(true); + .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); sample = response.hits().sequences().get(0); @@ -187,39 +197,53 @@ public void testPartialResults() throws Exception { // ------------------------------------------------------------------------ // same queries, with missing shards. Let them fail + // allow_partial_sequence_results has no effect if allow_partial_sequence_results is not set to true. // ------------------------------------------------------------------------ // event query - request = new EqlSearchRequest().indices("test-*").query("process where true"); + request = new EqlSearchRequest().indices("test-*").query("process where true").allowPartialSequenceResults(randomBoolean()); shouldFail(request); // sequence query on both shards - request = new EqlSearchRequest().indices("test-*").query("sequence [process where value == 1] [process where value == 2]"); + request = new EqlSearchRequest().indices("test-*") + .query("sequence [process where value == 1] [process where value == 2]") + .allowPartialSequenceResults(randomBoolean()); shouldFail(request); // sequence query on the available shard only - request = new EqlSearchRequest().indices("test-*").query("sequence [process where value == 1] [process where value == 3]"); + request = new EqlSearchRequest().indices("test-*") + .query("sequence [process where value == 1] [process where value == 3]") + .allowPartialSequenceResults(randomBoolean()); shouldFail(request); // sequence query on the unavailable shard only - request = new EqlSearchRequest().indices("test-*").query("sequence [process where value == 0] [process where value == 2]"); + request = new EqlSearchRequest().indices("test-*") + .query("sequence [process where value == 0] [process where value == 2]") + .allowPartialSequenceResults(randomBoolean()); shouldFail(request); // sequence query with missing event on unavailable shard. request = new EqlSearchRequest().indices("test-*") - .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]"); + .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]") + .allowPartialSequenceResults(randomBoolean()); shouldFail(request); // sample query on both shards - request = new EqlSearchRequest().indices("test-*").query("sample by key [process where value == 2] [process where value == 1]"); + request = new EqlSearchRequest().indices("test-*") + .query("sample by key [process where value == 2] [process where value == 1]") + .allowPartialSequenceResults(randomBoolean()); shouldFail(request); // sample query on the available shard only - request = new EqlSearchRequest().indices("test-*").query("sample by key [process where value == 3] [process where value == 1]"); + request = new EqlSearchRequest().indices("test-*") + .query("sample by key [process where value == 3] [process where value == 1]") + .allowPartialSequenceResults(randomBoolean()); shouldFail(request); // sample query on the unavailable shard only - request = new EqlSearchRequest().indices("test-*").query("sample by key [process where value == 2] [process where value == 0]"); + request = new EqlSearchRequest().indices("test-*") + .query("sample by key [process where value == 2] [process where value == 0]") + .allowPartialSequenceResults(randomBoolean()); shouldFail(request); // ------------------------------------------------------------------------ @@ -314,6 +338,102 @@ public void testPartialResults() throws Exception { assertThat(response.shardFailures()[0].index(), is("test-1")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + // ------------------------------------------------------------------------ + // same queries, with missing shards and allow_partial_search_results=true and allow_partial_sequence_results=false + // ------------------------------------------------------------------------ + + // event query + request = new EqlSearchRequest().indices("test-*") + .query("process where true") + .allowPartialSearchResults(true) + .allowPartialSequenceResults(false); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().events().size(), equalTo(5)); + for (int i = 0; i < 5; i++) { + assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1))); + } + assertThat(response.shardFailures().length, is(1)); + + // sequence query on both shards + request = new EqlSearchRequest().indices("test-*") + .query("sequence [process where value == 1] [process where value == 2]") + .allowPartialSearchResults(true) + .allowPartialSequenceResults(false); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query on the available shard only + request = new EqlSearchRequest().indices("test-*") + .query("sequence [process where value == 1] [process where value == 3]") + .allowPartialSearchResults(true) + .allowPartialSequenceResults(false); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query on the unavailable shard only + request = new EqlSearchRequest().indices("test-*") + .query("sequence [process where value == 0] [process where value == 2]") + .allowPartialSearchResults(true) + .allowPartialSequenceResults(false); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE + request = new EqlSearchRequest().indices("test-*") + .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]") + .allowPartialSearchResults(true) + .allowPartialSequenceResults(false); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on both shards + request = new EqlSearchRequest().indices("test-*") + .query("sample by key [process where value == 2] [process where value == 1]") + .allowPartialSearchResults(true) + .allowPartialSequenceResults(false); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on the available shard only + request = new EqlSearchRequest().indices("test-*") + .query("sample by key [process where value == 3] [process where value == 1]") + .allowPartialSearchResults(true) + .allowPartialSequenceResults(false); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sample = response.hits().sequences().get(0); + assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); + assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on the unavailable shard only + request = new EqlSearchRequest().indices("test-*") + .query("sample by key [process where value == 2] [process where value == 0]") + .allowPartialSearchResults(true) + .allowPartialSequenceResults(false); + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + // ------------------------------------------------------------------------ // same queries, with missing shards and with default xpack.eql.default_allow_partial_results=true // ------------------------------------------------------------------------ diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java index 3914c683b0953..239187b365794 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java @@ -64,6 +64,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re private Map runtimeMappings = emptyMap(); private int maxSamplesPerKey = RequestDefaults.MAX_SAMPLES_PER_KEY; private Boolean allowPartialSearchResults; + private Boolean allowPartialSequenceResults; // Async settings private TimeValue waitForCompletionTimeout = null; @@ -85,6 +86,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re static final String KEY_RUNTIME_MAPPINGS = "runtime_mappings"; static final String KEY_MAX_SAMPLES_PER_KEY = "max_samples_per_key"; static final String KEY_ALLOW_PARTIAL_SEARCH_RESULTS = "allow_partial_search_results"; + static final String KEY_ALLOW_PARTIAL_SEQUENCE_RESULTS = "allow_partial_sequence_results"; static final ParseField FILTER = new ParseField(KEY_FILTER); static final ParseField TIMESTAMP_FIELD = new ParseField(KEY_TIMESTAMP_FIELD); @@ -100,6 +102,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re static final ParseField FETCH_FIELDS_FIELD = SearchSourceBuilder.FETCH_FIELDS_FIELD; static final ParseField MAX_SAMPLES_PER_KEY = new ParseField(KEY_MAX_SAMPLES_PER_KEY); static final ParseField ALLOW_PARTIAL_SEARCH_RESULTS = new ParseField(KEY_ALLOW_PARTIAL_SEARCH_RESULTS); + static final ParseField ALLOW_PARTIAL_SEQUENCE_RESULTS = new ParseField(KEY_ALLOW_PARTIAL_SEQUENCE_RESULTS); private static final ObjectParser PARSER = objectParser(EqlSearchRequest::new); @@ -140,8 +143,10 @@ public EqlSearchRequest(StreamInput in) throws IOException { } if (in.getTransportVersion().onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS)) { allowPartialSearchResults = in.readOptionalBoolean(); + allowPartialSequenceResults = in.readOptionalBoolean(); } else { allowPartialSearchResults = false; + allowPartialSequenceResults = true; } } @@ -254,6 +259,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.field(KEY_MAX_SAMPLES_PER_KEY, maxSamplesPerKey); builder.field(KEY_ALLOW_PARTIAL_SEARCH_RESULTS, allowPartialSearchResults); + builder.field(KEY_ALLOW_PARTIAL_SEQUENCE_RESULTS, allowPartialSequenceResults); return builder; } @@ -289,6 +295,7 @@ protected static ObjectParser objectParser parser.declareObject(EqlSearchRequest::runtimeMappings, (p, c) -> p.map(), SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD); parser.declareInt(EqlSearchRequest::maxSamplesPerKey, MAX_SAMPLES_PER_KEY); parser.declareBoolean(EqlSearchRequest::allowPartialSearchResults, ALLOW_PARTIAL_SEARCH_RESULTS); + parser.declareBoolean(EqlSearchRequest::allowPartialSequenceResults, ALLOW_PARTIAL_SEQUENCE_RESULTS); return parser; } @@ -446,6 +453,15 @@ public EqlSearchRequest allowPartialSearchResults(Boolean val) { return this; } + public Boolean allowPartialSequenceResults() { + return allowPartialSequenceResults; + } + + public EqlSearchRequest allowPartialSequenceResults(Boolean val) { + this.allowPartialSequenceResults = val; + return this; + } + private static List parseFetchFields(XContentParser parser) throws IOException { List result = new ArrayList<>(); Token token = parser.currentToken(); @@ -491,6 +507,7 @@ public void writeTo(StreamOutput out) throws IOException { } if (out.getTransportVersion().onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS)) { out.writeOptionalBoolean(allowPartialSearchResults); + out.writeOptionalBoolean(allowPartialSequenceResults); } } @@ -519,7 +536,8 @@ public boolean equals(Object o) { && Objects.equals(fetchFields, that.fetchFields) && Objects.equals(runtimeMappings, that.runtimeMappings) && Objects.equals(maxSamplesPerKey, that.maxSamplesPerKey) - && Objects.equals(allowPartialSearchResults, that.allowPartialSearchResults); + && Objects.equals(allowPartialSearchResults, that.allowPartialSearchResults) + && Objects.equals(allowPartialSequenceResults, that.allowPartialSequenceResults); } @Override @@ -541,7 +559,8 @@ public int hashCode() { fetchFields, runtimeMappings, maxSamplesPerKey, - allowPartialSearchResults + allowPartialSearchResults, + allowPartialSequenceResults ); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java index 951a169d94ee3..672d6b87a8dbb 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java @@ -168,7 +168,8 @@ public Executable assemble( criteria.get(completionStage), matcher, listOfKeys, - cfg.allowPartialSearchResults() + cfg.allowPartialSearchResults(), + cfg.allowPartialSequenceResults() ); return w; diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java index 136bf80003345..eb39d3432aa7c 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java @@ -106,6 +106,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) { private final boolean hasKeys; private final List> listOfKeys; private final boolean allowPartialSearchResults; + private final boolean allowPartialSequenceResults; private Map shardFailures = new HashMap<>(); // flag used for DESC sequences to indicate whether @@ -132,7 +133,9 @@ public TumblingWindow( SequenceCriterion until, SequenceMatcher matcher, List> listOfKeys, - boolean allowPartialSearchResults + boolean allowPartialSearchResults, + boolean allowPartialSequenceResults + ) { this.client = client; @@ -147,6 +150,7 @@ public TumblingWindow( this.restartWindowFromTailQuery = baseRequest.descending(); this.listOfKeys = listOfKeys; this.allowPartialSearchResults = allowPartialSearchResults; + this.allowPartialSequenceResults = allowPartialSequenceResults; } @Override @@ -164,6 +168,9 @@ public void execute(ActionListener listener) { * Move the window while preserving the same base. */ private void tumbleWindow(int currentStage, ActionListener listener) { + if (allowPartialSequenceResults == false && shardFailures.isEmpty() == false) { + doPayload(listener); + } if (currentStage > matcher.firstPositiveStage && matcher.hasCandidates() == false) { if (restartWindowFromTailQuery) { currentStage = matcher.firstPositiveStage; @@ -748,7 +755,7 @@ private void doPayload(ActionListener listener) { log.trace("Sending payload for [{}] sequences", completed.size()); - if (completed.isEmpty()) { + if (completed.isEmpty() || (allowPartialSequenceResults == false && shardFailures.isEmpty() == false)) { listener.onResponse(new EmptyPayload(Type.SEQUENCE, timeTook(), shardFailures.values().toArray(new ShardSearchFailure[0]))); return; } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java index 8b15d230cabb7..8829c72d69388 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java @@ -67,6 +67,13 @@ public class EqlPlugin extends Plugin implements ActionPlugin, CircuitBreakerPlu Setting.Property.Dynamic ); + public static final Setting DEFAULT_ALLOW_PARTIAL_SEQUENCE_RESULTS = Setting.boolSetting( + "xpack.eql.default_allow_partial_sequence_results", + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + public EqlPlugin() {} @Override @@ -93,7 +100,7 @@ private Collection createComponents(Client client, Settings settings, Cl */ @Override public List> getSettings() { - return List.of(EQL_ENABLED_SETTING, DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS); + return List.of(EQL_ENABLED_SETTING, DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS, DEFAULT_ALLOW_PARTIAL_SEQUENCE_RESULTS); } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java index f8f8a2885afb8..582352722fc58 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java @@ -236,6 +236,9 @@ public static void operation( request.allowPartialSearchResults() == null ? defaultAllowPartialSearchResults(clusterService) : request.allowPartialSearchResults(), + request.allowPartialSequenceResults() == null + ? defaultAllowPartialSequenceResults(clusterService) + : request.allowPartialSequenceResults(), clientId, new TaskId(nodeId, task.getId()), task @@ -256,6 +259,13 @@ private static boolean defaultAllowPartialSearchResults(ClusterService clusterSe return clusterService.getClusterSettings().get(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS); } + private static boolean defaultAllowPartialSequenceResults(ClusterService clusterService) { + if (clusterService.getClusterSettings() == null) { + return EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEQUENCE_RESULTS.getDefault(Settings.EMPTY); + } + return clusterService.getClusterSettings().get(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEQUENCE_RESULTS); + } + static EqlSearchResponse createResponse(Results results, AsyncExecutionId id) { EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(results.events(), results.sequences(), results.totalHits()); if (id != null) { diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java index 590b484399436..8242b0b533ad3 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java @@ -31,6 +31,7 @@ public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu private final int fetchSize; private final int maxSamplesPerKey; private final boolean allowPartialSearchResults; + private final boolean allowPartialSequenceResults; @Nullable private final QueryBuilder filter; @@ -52,6 +53,7 @@ public EqlConfiguration( int fetchSize, int maxSamplesPerKey, boolean allowPartialSearchResults, + boolean allowPartialSequenceResults, String clientId, TaskId taskId, EqlSearchTask task @@ -70,6 +72,7 @@ public EqlConfiguration( this.fetchSize = fetchSize; this.maxSamplesPerKey = maxSamplesPerKey; this.allowPartialSearchResults = allowPartialSearchResults; + this.allowPartialSequenceResults = allowPartialSequenceResults; } public String[] indices() { @@ -96,6 +99,10 @@ public boolean allowPartialSearchResults() { return allowPartialSearchResults; } + public boolean allowPartialSequenceResults() { + return allowPartialSequenceResults; + } + public QueryBuilder filter() { return filter; } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java index 99906935bcd59..75884fab4dbb3 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java @@ -52,6 +52,7 @@ private EqlTestUtils() {} 123, 1, false, + true, "", new TaskId("test", 123), null @@ -71,6 +72,7 @@ public static EqlConfiguration randomConfiguration() { randomIntBetween(1, 1000), randomIntBetween(1, 1000), randomBoolean(), + randomBoolean(), randomAlphaOfLength(16), new TaskId(randomAlphaOfLength(10), randomNonNegativeLong()), randomTask() diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java index eae32902b941b..0d1256b1aab8b 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java @@ -81,6 +81,7 @@ protected EqlSearchRequest createTestInstance() { .keepAlive(randomTimeValue()) .keepOnCompletion(randomBoolean()) .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()) .fetchFields(randomFetchFields) .runtimeMappings(randomRuntimeMappings()) .resultPosition(randomFrom("tail", "head")) @@ -140,6 +141,9 @@ protected EqlSearchRequest mutateInstanceForVersion(EqlSearchRequest instance, T mutatedInstance.allowPartialSearchResults( version.onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS) ? instance.allowPartialSearchResults() : false ); + mutatedInstance.allowPartialSequenceResults( + version.onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS) ? instance.allowPartialSequenceResults() : true + ); return mutatedInstance; } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java index 28f5d042a8eef..abd928b04a9c7 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java @@ -141,7 +141,15 @@ public void testImplicitTiebreakerBeingSet() { booleanArrayOf(stages, false), NOOP_CIRCUIT_BREAKER ); - TumblingWindow window = new TumblingWindow(client, criteria, null, matcher, Collections.emptyList(), randomBoolean()); + TumblingWindow window = new TumblingWindow( + client, + criteria, + null, + matcher, + Collections.emptyList(), + randomBoolean(), + randomBoolean() + ); window.execute(wrap(p -> {}, ex -> { throw ExceptionsHelper.convertToRuntime(ex); })); } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java index 40c88eeac3214..f6aa851b2fff0 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java @@ -277,7 +277,15 @@ public void test() throws Exception { ); QueryClient testClient = new TestQueryClient(); - TumblingWindow window = new TumblingWindow(testClient, criteria, null, matcher, Collections.emptyList(), randomBoolean()); + TumblingWindow window = new TumblingWindow( + testClient, + criteria, + null, + matcher, + Collections.emptyList(), + randomBoolean(), + randomBoolean() + ); // finally make the assertion at the end of the listener window.execute(ActionTestUtils.assertNoFailureListener(this::checkResults)); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java index 462d13f1bb125..9a78dd4573133 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java @@ -103,6 +103,7 @@ public void testQueryFilterUsedInPitAndSearches() { 123, 1, randomBoolean(), + randomBoolean(), "", new TaskId("test", 123), new EqlSearchTask( @@ -169,7 +170,15 @@ public void fetchHits(Iterable> refs, ActionListener { // do nothing, we don't care about the query results }, ex -> { fail("Shouldn't have failed"); })); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java index b6eed31893a77..d05abc8e284d6 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java @@ -146,7 +146,15 @@ public void testCircuitBreakerTumblingWindow() { booleanArrayOf(stages, false), CIRCUIT_BREAKER ); - TumblingWindow window = new TumblingWindow(client, criteria, null, matcher, Collections.emptyList(), randomBoolean()); + TumblingWindow window = new TumblingWindow( + client, + criteria, + null, + matcher, + Collections.emptyList(), + randomBoolean(), + randomBoolean() + ); window.execute(ActionTestUtils.assertNoFailureListener(p -> {})); CIRCUIT_BREAKER.startBreaking(); @@ -230,7 +238,15 @@ private void assertMemoryCleared( booleanArrayOf(sequenceFiltersCount, false), eqlCircuitBreaker ); - TumblingWindow window = new TumblingWindow(eqlClient, criteria, null, matcher, Collections.emptyList(), randomBoolean()); + TumblingWindow window = new TumblingWindow( + eqlClient, + criteria, + null, + matcher, + Collections.emptyList(), + randomBoolean(), + randomBoolean() + ); window.execute(ActionListener.noop()); assertTrue(esClient.searchRequestsRemainingCount() == 0); // ensure all the search requests have been asked for @@ -276,7 +292,15 @@ public void testEqlCBCleanedUp_on_ParentCBBreak() { booleanArrayOf(sequenceFiltersCount, false), eqlCircuitBreaker ); - TumblingWindow window = new TumblingWindow(eqlClient, criteria, null, matcher, Collections.emptyList(), randomBoolean()); + TumblingWindow window = new TumblingWindow( + eqlClient, + criteria, + null, + matcher, + Collections.emptyList(), + randomBoolean(), + randomBoolean() + ); window.execute(wrap(p -> fail(), ex -> assertTrue(ex instanceof CircuitBreakingException))); } assertCriticalWarnings("[indices.breaker.total.limit] setting of [0%] is below the recommended minimum of 50.0% of the heap"); @@ -335,6 +359,7 @@ private QueryClient buildQueryClient(ESMockClient esClient, CircuitBreaker eqlCi 123, 1, randomBoolean(), + randomBoolean(), "", new TaskId("test", 123), new EqlSearchTask( diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/PITFailureTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/PITFailureTests.java index 85df96645cf1f..2eee6a262e73c 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/PITFailureTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/PITFailureTests.java @@ -84,6 +84,7 @@ public void testHandlingPitFailure() { 123, 1, randomBoolean(), + randomBoolean(), "", new TaskId("test", 123), new EqlSearchTask( @@ -133,7 +134,15 @@ public void testHandlingPitFailure() { ); SequenceMatcher matcher = new SequenceMatcher(1, false, TimeValue.MINUS_ONE, null, booleanArrayOf(1, false), cb); - TumblingWindow window = new TumblingWindow(eqlClient, criteria, null, matcher, Collections.emptyList(), randomBoolean()); + TumblingWindow window = new TumblingWindow( + eqlClient, + criteria, + null, + matcher, + Collections.emptyList(), + randomBoolean(), + randomBoolean() + ); window.execute( wrap( p -> { fail("Search succeeded despite PIT failure"); }, From b003c1c99eae97cd5be44d3b4d317bade8d218b4 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 21 Nov 2024 15:30:15 +0100 Subject: [PATCH 09/23] allow_partial_sequence_results=false by default --- docs/reference/eql/eql-search-api.asciidoc | 11 ++-- .../eql/action/PartialSearchResultsIT.java | 61 ++++++++----------- .../xpack/eql/action/EqlSearchRequest.java | 2 +- .../xpack/eql/plugin/EqlPlugin.java | 2 +- .../eql/action/EqlSearchRequestTests.java | 2 +- 5 files changed, 37 insertions(+), 41 deletions(-) diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc index 047aa298dd223..0fd490609277f 100644 --- a/docs/reference/eql/eql-search-api.asciidoc +++ b/docs/reference/eql/eql-search-api.asciidoc @@ -114,16 +114,19 @@ If both parameters are specified, only the query parameter is used. Used together with `allow_partial_search_results=true`, controls the behavior of sequence queries specifically -(if `allow_partial_search_results=false` this setting has no effect). -If `true` and if some shards are unavailable, the sequences are calculated on available shards. +(if `allow_partial_search_results=false`, this setting has no effect). +If `true` and if some shards are unavailable, the sequences are calculated on available shards only. + If `false` and if some shards are unavailable, the query only returns information about the shard failures, but no further results. + -Defaults to `true`. +Defaults to `false`. ++ +Consider that sequences calculated with `allow_partial_search_results=true` can return incorrect results +(eg. if a <> clause matches records in unavailable shards) + To override the default for this field, set the -`xpack.eql.default_allow_partial_sequence_results` cluster setting to `false`. +`xpack.eql.default_allow_partial_sequence_results` cluster setting to `true`. [IMPORTANT] diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java index 5918b84d03410..d0ca0d3da1da3 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java @@ -248,6 +248,7 @@ public void testPartialResults() throws Exception { // ------------------------------------------------------------------------ // same queries, with missing shards and allow_partial_search_results=true + // and allow_partial_sequence_result=true // ------------------------------------------------------------------------ // event query @@ -262,7 +263,8 @@ public void testPartialResults() throws Exception { // sequence query on both shards request = new EqlSearchRequest().indices("test-*") .query("sequence [process where value == 1] [process where value == 2]") - .allowPartialSearchResults(true); + .allowPartialSearchResults(true) + .allowPartialSequenceResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); @@ -272,7 +274,8 @@ public void testPartialResults() throws Exception { // sequence query on the available shard only request = new EqlSearchRequest().indices("test-*") .query("sequence [process where value == 1] [process where value == 3]") - .allowPartialSearchResults(true); + .allowPartialSearchResults(true) + .allowPartialSequenceResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); sequence = response.hits().sequences().get(0); @@ -285,7 +288,8 @@ public void testPartialResults() throws Exception { // sequence query on the unavailable shard only request = new EqlSearchRequest().indices("test-*") .query("sequence [process where value == 0] [process where value == 2]") - .allowPartialSearchResults(true); + .allowPartialSearchResults(true) + .allowPartialSequenceResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); @@ -295,7 +299,8 @@ public void testPartialResults() throws Exception { // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE request = new EqlSearchRequest().indices("test-*") .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]") - .allowPartialSearchResults(true); + .allowPartialSearchResults(true) + .allowPartialSequenceResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); sequence = response.hits().sequences().get(0); @@ -308,7 +313,8 @@ public void testPartialResults() throws Exception { // sample query on both shards request = new EqlSearchRequest().indices("test-*") .query("sample by key [process where value == 2] [process where value == 1]") - .allowPartialSearchResults(true); + .allowPartialSearchResults(true) + .allowPartialSequenceResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); @@ -318,7 +324,8 @@ public void testPartialResults() throws Exception { // sample query on the available shard only request = new EqlSearchRequest().indices("test-*") .query("sample by key [process where value == 3] [process where value == 1]") - .allowPartialSearchResults(true); + .allowPartialSearchResults(true) + .allowPartialSequenceResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); sample = response.hits().sequences().get(0); @@ -331,7 +338,8 @@ public void testPartialResults() throws Exception { // sample query on the unavailable shard only request = new EqlSearchRequest().indices("test-*") .query("sample by key [process where value == 2] [process where value == 0]") - .allowPartialSearchResults(true); + .allowPartialSearchResults(true) + .allowPartialSequenceResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); @@ -339,14 +347,12 @@ public void testPartialResults() throws Exception { assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); // ------------------------------------------------------------------------ - // same queries, with missing shards and allow_partial_search_results=true and allow_partial_sequence_results=false + // same queries, with missing shards and allow_partial_search_results=true + // and default allow_partial_sequence_results (ie. false) // ------------------------------------------------------------------------ // event query - request = new EqlSearchRequest().indices("test-*") - .query("process where true") - .allowPartialSearchResults(true) - .allowPartialSequenceResults(false); + request = new EqlSearchRequest().indices("test-*").query("process where true").allowPartialSearchResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().events().size(), equalTo(5)); for (int i = 0; i < 5; i++) { @@ -357,8 +363,7 @@ public void testPartialResults() throws Exception { // sequence query on both shards request = new EqlSearchRequest().indices("test-*") .query("sequence [process where value == 1] [process where value == 2]") - .allowPartialSearchResults(true) - .allowPartialSequenceResults(false); + .allowPartialSearchResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); @@ -368,8 +373,7 @@ public void testPartialResults() throws Exception { // sequence query on the available shard only request = new EqlSearchRequest().indices("test-*") .query("sequence [process where value == 1] [process where value == 3]") - .allowPartialSearchResults(true) - .allowPartialSequenceResults(false); + .allowPartialSearchResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); @@ -379,8 +383,7 @@ public void testPartialResults() throws Exception { // sequence query on the unavailable shard only request = new EqlSearchRequest().indices("test-*") .query("sequence [process where value == 0] [process where value == 2]") - .allowPartialSearchResults(true) - .allowPartialSequenceResults(false); + .allowPartialSearchResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); @@ -390,8 +393,7 @@ public void testPartialResults() throws Exception { // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE request = new EqlSearchRequest().indices("test-*") .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]") - .allowPartialSearchResults(true) - .allowPartialSequenceResults(false); + .allowPartialSearchResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); @@ -401,8 +403,7 @@ public void testPartialResults() throws Exception { // sample query on both shards request = new EqlSearchRequest().indices("test-*") .query("sample by key [process where value == 2] [process where value == 1]") - .allowPartialSearchResults(true) - .allowPartialSequenceResults(false); + .allowPartialSearchResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); @@ -412,8 +413,7 @@ public void testPartialResults() throws Exception { // sample query on the available shard only request = new EqlSearchRequest().indices("test-*") .query("sample by key [process where value == 3] [process where value == 1]") - .allowPartialSearchResults(true) - .allowPartialSequenceResults(false); + .allowPartialSearchResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); sample = response.hits().sequences().get(0); @@ -426,8 +426,7 @@ public void testPartialResults() throws Exception { // sample query on the unavailable shard only request = new EqlSearchRequest().indices("test-*") .query("sample by key [process where value == 2] [process where value == 0]") - .allowPartialSearchResults(true) - .allowPartialSequenceResults(false); + .allowPartialSearchResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); @@ -465,10 +464,7 @@ public void testPartialResults() throws Exception { // sequence query on the available shard only request = new EqlSearchRequest().indices("test-*").query("sequence [process where value == 1] [process where value == 3]"); response = client().execute(EqlSearchAction.INSTANCE, request).get(); - assertThat(response.hits().sequences().size(), equalTo(1)); - sequence = response.hits().sequences().get(0); - assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); - assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 3")); + assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); @@ -485,10 +481,7 @@ public void testPartialResults() throws Exception { request = new EqlSearchRequest().indices("test-*") .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]"); response = client().execute(EqlSearchAction.INSTANCE, request).get(); - assertThat(response.hits().sequences().size(), equalTo(1)); - sequence = response.hits().sequences().get(0); - assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); - assertThat(sequence.events().get(2).toString(), containsString("\"value\" : 3")); + assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java index 239187b365794..5804e11b72ff5 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java @@ -146,7 +146,7 @@ public EqlSearchRequest(StreamInput in) throws IOException { allowPartialSequenceResults = in.readOptionalBoolean(); } else { allowPartialSearchResults = false; - allowPartialSequenceResults = true; + allowPartialSequenceResults = false; } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java index 8829c72d69388..210f88c991539 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java @@ -69,7 +69,7 @@ public class EqlPlugin extends Plugin implements ActionPlugin, CircuitBreakerPlu public static final Setting DEFAULT_ALLOW_PARTIAL_SEQUENCE_RESULTS = Setting.boolSetting( "xpack.eql.default_allow_partial_sequence_results", - true, + false, Setting.Property.NodeScope, Setting.Property.Dynamic ); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java index 0d1256b1aab8b..1a06aead910c8 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java @@ -142,7 +142,7 @@ protected EqlSearchRequest mutateInstanceForVersion(EqlSearchRequest instance, T version.onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS) ? instance.allowPartialSearchResults() : false ); mutatedInstance.allowPartialSequenceResults( - version.onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS) ? instance.allowPartialSequenceResults() : true + version.onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS) ? instance.allowPartialSequenceResults() : false ); return mutatedInstance; From 4c421c04eba66a52dc84a91cfccbe2ccdb94515f Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Mon, 25 Nov 2024 15:57:41 +0100 Subject: [PATCH 10/23] Add CCS test --- .../elasticsearch/test/ESIntegTestCase.java | 2 +- .../xpack/eql/action/CCSPartialResultsIT.java | 659 ++++++++++++++++++ 2 files changed, 660 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index d7c5c598ce978..414d6384d5688 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -2453,7 +2453,7 @@ public static void afterClass() throws Exception { /** * After the cluster is stopped, there are a few netty threads that can linger, so we make sure we don't leak any tasks on them. */ - static void awaitGlobalNettyThreadsFinish() throws Exception { + public static void awaitGlobalNettyThreadsFinish() throws Exception { // Don't use GlobalEventExecutor#awaitInactivity. It will waste up to 1s for every call and we expect no tasks queued for it // except for the odd scheduled shutdown task. assertBusy(() -> assertEquals(0, GlobalEventExecutor.INSTANCE.pendingTasks())); diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java new file mode 100644 index 0000000000000..4ff80a975e6e3 --- /dev/null +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java @@ -0,0 +1,659 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.xpack.eql.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.cluster.remote.RemoteInfoRequest; +import org.elasticsearch.action.admin.cluster.remote.TransportRemoteInfoAction; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsAction; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.CloseableTestClusterWrapper; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.test.MockHttpTransport; +import org.elasticsearch.test.NodeConfigurationSource; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.transport.RemoteClusterAware; +import org.elasticsearch.transport.RemoteConnectionInfo; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.eql.plugin.EqlPlugin; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING; +import static org.elasticsearch.discovery.SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; + +public class CCSPartialResultsIT extends ESTestCase { + + + public static final String LOCAL_CLUSTER = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; + public static final String REMOTE_CLUSTER = "remote_cluster"; + + private static volatile ClusterGroup clusterGroup; + + protected Collection> nodePlugins() { + return Collections.singletonList(LocalStateEQLXPackPlugin.class); + } + + protected Settings nodeSettings() { + return Settings.builder() + .put("cluster.routing.rebalance.enable", "none") + .build(); + } + + protected final Client localClient() { + return client(LOCAL_CLUSTER); + } + + protected final Client client(String clusterAlias) { + return cluster(clusterAlias).client(); + } + + protected final InternalTestCluster cluster(String clusterAlias) { + return clusterGroup.getCluster(clusterAlias); + } + + protected final Map clusters() { + return Collections.unmodifiableMap(clusterGroup.clusters); + } + + + @Before + public final void startClusters() throws Exception { + if (clusterGroup != null) { + return; + } + stopClusters(); + final Map clusters = new HashMap<>(); + final List clusterAliases = List.of(REMOTE_CLUSTER, LOCAL_CLUSTER); + for (String clusterAlias : clusterAliases) { + final String clusterName = clusterAlias.equals(LOCAL_CLUSTER) ? "main-cluster" : clusterAlias; + final int numberOfNodes = 2; + final List> mockPlugins = List.of( + MockHttpTransport.TestPlugin.class, + MockTransportService.TestPlugin.class, + getTestTransportPlugin() + ); + final Collection> nodePlugins = nodePlugins(); + + final NodeConfigurationSource nodeConfigurationSource = nodeConfigurationSource(nodeSettings(), nodePlugins); + final InternalTestCluster cluster = new InternalTestCluster( + randomLong(), + createTempDir(), + true, + true, + numberOfNodes, + numberOfNodes, + clusterName, + nodeConfigurationSource, + 0, + clusterName + "-", + mockPlugins, + Function.identity() + ); + cluster.getNodeNames(); + cluster.beforeTest(random()); + clusters.put(clusterAlias, cluster); + } + clusterGroup = new ClusterGroup(clusters); + configureAndConnectsToRemoteClusters(); + } + + @After + public void assertAfterTest() throws Exception { + for (InternalTestCluster cluster : clusters().values()) { + cluster.wipe(Set.of()); + cluster.assertAfterTest(); + } + ESIntegTestCase.awaitGlobalNettyThreadsFinish(); + } + + @AfterClass + public static void stopClusters() throws IOException { + IOUtils.close(clusterGroup); + clusterGroup = null; + } + + protected void configureAndConnectsToRemoteClusters() throws Exception { + final InternalTestCluster cluster = clusterGroup.getCluster(REMOTE_CLUSTER); + final String[] allNodes = cluster.getNodeNames(); + configureRemoteCluster(REMOTE_CLUSTER, allNodes[1]); + } + + protected void configureRemoteCluster(String clusterAlias, String seedNode) throws Exception { + final String remoteClusterSettingPrefix = "cluster.remote." + clusterAlias + "."; + Settings.Builder settings = Settings.builder(); + final TransportService transportService = cluster(clusterAlias).getInstance(TransportService.class, seedNode); + String seedAddress = transportService.boundAddress().publishAddress().toString(); + + Settings.Builder builder; + if (randomBoolean()) { + builder = settings.putNull(remoteClusterSettingPrefix + "proxy_address") + .put(remoteClusterSettingPrefix + "mode", "sniff") + .put(remoteClusterSettingPrefix + "seeds", seedAddress); + } else { + builder = settings.putNull(remoteClusterSettingPrefix + "seeds") + .put(remoteClusterSettingPrefix + "mode", "proxy") + .put(remoteClusterSettingPrefix + "proxy_address", seedAddress); + } + + localClient().admin() + .cluster() + .prepareUpdateSettings(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT) + .setPersistentSettings(builder) + .get(); + + assertBusy(() -> { + List remoteConnectionInfos = localClient().execute(TransportRemoteInfoAction.TYPE, new RemoteInfoRequest()) + .actionGet() + .getInfos() + .stream() + .filter(c -> c.isConnected() && c.getClusterAlias().equals(clusterAlias)) + .collect(Collectors.toList()); + assertThat(remoteConnectionInfos, not(empty())); + }); + } + + static class ClusterGroup implements Closeable { + private final Map clusters; + + ClusterGroup(Map clusters) { + this.clusters = Collections.unmodifiableMap(clusters); + } + + InternalTestCluster getCluster(String clusterAlias) { + assertThat(clusters, hasKey(clusterAlias)); + return clusters.get(clusterAlias); + } + + @Override + public void close() throws IOException { + IOUtils.close(CloseableTestClusterWrapper.wrap(clusters.values())); + } + } + + static NodeConfigurationSource nodeConfigurationSource(Settings nodeSettings, Collection> nodePlugins) { + final Settings.Builder builder = Settings.builder(); + builder.putList(DISCOVERY_SEED_HOSTS_SETTING.getKey()); // empty list disables a port scan for other nodes + builder.putList(DISCOVERY_SEED_PROVIDERS_SETTING.getKey(), "file"); + builder.put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType()); + builder.put(nodeSettings); + + return new NodeConfigurationSource() { + @Override + public Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return builder.build(); + } + + @Override + public Path nodeConfigPath(int nodeOrdinal) { + return null; + } + + @Override + public Collection> nodePlugins() { + return nodePlugins; + } + }; + } + + + public void testFailuresFromRemote() throws ExecutionException, InterruptedException, IOException { + final Client localClient = localClient(); + final Client remoteClient = client(REMOTE_CLUSTER); + + assertAcked( + remoteClient.admin().indices().prepareCreate("test-1-remote") + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.routing.allocation.include._name", clusterGroup.clusters.get(REMOTE_CLUSTER).getNodeNames()[0]) + .build() + ) + .setMapping("@timestamp", "type=date"), + TimeValue.timeValueSeconds(60) + ); + + assertAcked( + remoteClient.admin().indices().prepareCreate("test-2-remote") + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.routing.allocation.exclude._name", clusterGroup.clusters.get(REMOTE_CLUSTER).getNodeNames()[0]) + .build() + ) + .setMapping("@timestamp", "type=date"), + TimeValue.timeValueSeconds(60) + ); + + for (int i = 0; i < 5; i++) { + int val = i * 2; + remoteClient.prepareIndex("test-1-remote").setId(Integer.toString(i)) + .setSource("@timestamp", 100000 + val, "event.category", "process", "key", "same", "value", val) + .get(); + } + for (int i = 0; i < 5; i++) { + int val = i * 2 + 1; + remoteClient.prepareIndex("test-2-remote").setId(Integer.toString(i)) + .setSource("@timestamp", 100000 + val, "event.category", "process", "key", "same", "value", val) + .get(); + } + + remoteClient.admin().indices().prepareRefresh().get(); + localClient.admin().indices().prepareRefresh().get(); + + // ------------------------------------------------------------------------ + // queries with full cluster (no missing shards) + // ------------------------------------------------------------------------ + + // event query + var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("process where true") + .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()); + EqlSearchResponse response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().events().size(), equalTo(10)); + for (int i = 0; i < 10; i++) { + assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + i)); + } + assertThat(response.shardFailures().length, is(0)); + + // sequence query on both shards + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sequence [process where value == 1] [process where value == 2]") + .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + EqlSearchResponse.Sequence sequence = response.hits().sequences().get(0); + assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); + assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 2")); + assertThat(response.shardFailures().length, is(0)); + + // sequence query on the available shard only + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sequence [process where value == 1] [process where value == 3]") + .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sequence = response.hits().sequences().get(0); + assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); + assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 3")); + assertThat(response.shardFailures().length, is(0)); + + // sequence query on the unavailable shard only + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sequence [process where value == 0] [process where value == 2]") + .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sequence = response.hits().sequences().get(0); + assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 0")); + assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 2")); + assertThat(response.shardFailures().length, is(0)); + + // sequence query with missing event on unavailable shard + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]") + .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(0)); + + // sample query on both shards + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sample by key [process where value == 2] [process where value == 1]") + .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + EqlSearchResponse.Sequence sample = response.hits().sequences().get(0); + assertThat(sample.events().get(0).toString(), containsString("\"value\" : 2")); + assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); + assertThat(response.shardFailures().length, is(0)); + + // sample query on the available shard only + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sample by key [process where value == 3] [process where value == 1]") + .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sample = response.hits().sequences().get(0); + assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); + assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); + assertThat(response.shardFailures().length, is(0)); + + // sample query on the unavailable shard only + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sample by key [process where value == 2] [process where value == 0]") + .allowPartialSearchResults(randomBoolean()) + .allowPartialSequenceResults(randomBoolean()); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sample = response.hits().sequences().get(0); + assertThat(sample.events().get(0).toString(), containsString("\"value\" : 2")); + assertThat(sample.events().get(1).toString(), containsString("\"value\" : 0")); + assertThat(response.shardFailures().length, is(0)); + + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + cluster(REMOTE_CLUSTER).stopNode(clusterGroup.clusters.get(REMOTE_CLUSTER).getNodeNames()[0]); + + // ------------------------------------------------------------------------ + // same queries, with missing shards and allow_partial_search_results=true + // and allow_partial_sequence_result=true + // ------------------------------------------------------------------------ + + // event query + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("process where true").allowPartialSearchResults(true); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().events().size(), equalTo(5)); + for (int i = 0; i < 5; i++) { + assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1))); + } + assertThat(response.shardFailures().length, is(1)); + + // sequence query on both shards + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sequence [process where value == 1] [process where value == 2]") + .allowPartialSearchResults(true) + .allowPartialSequenceResults(true); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query on the available shard only + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sequence [process where value == 1] [process where value == 3]") + .allowPartialSearchResults(true) + .allowPartialSequenceResults(true); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sequence = response.hits().sequences().get(0); + assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); + assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 3")); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query on the unavailable shard only + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sequence [process where value == 0] [process where value == 2]") + .allowPartialSearchResults(true) + .allowPartialSequenceResults(true); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]") + .allowPartialSearchResults(true) + .allowPartialSequenceResults(true); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sequence = response.hits().sequences().get(0); + assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); + assertThat(sequence.events().get(2).toString(), containsString("\"value\" : 3")); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on both shards + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sample by key [process where value == 2] [process where value == 1]") + .allowPartialSearchResults(true) + .allowPartialSequenceResults(true); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on the available shard only + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sample by key [process where value == 3] [process where value == 1]") + .allowPartialSearchResults(true) + .allowPartialSequenceResults(true); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sample = response.hits().sequences().get(0); + assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); + assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on the unavailable shard only + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sample by key [process where value == 2] [process where value == 0]") + .allowPartialSearchResults(true) + .allowPartialSequenceResults(true); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // ------------------------------------------------------------------------ + // same queries, with missing shards and allow_partial_search_results=true + // and default allow_partial_sequence_results (ie. false) + // ------------------------------------------------------------------------ + + // event query + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("process where true").allowPartialSearchResults(true); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().events().size(), equalTo(5)); + for (int i = 0; i < 5; i++) { + assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1))); + } + assertThat(response.shardFailures().length, is(1)); + + // sequence query on both shards + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sequence [process where value == 1] [process where value == 2]") + .allowPartialSearchResults(true); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query on the available shard only + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sequence [process where value == 1] [process where value == 3]") + .allowPartialSearchResults(true); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query on the unavailable shard only + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sequence [process where value == 0] [process where value == 2]") + .allowPartialSearchResults(true); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]") + .allowPartialSearchResults(true); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on both shards + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sample by key [process where value == 2] [process where value == 1]") + .allowPartialSearchResults(true); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on the available shard only + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sample by key [process where value == 3] [process where value == 1]") + .allowPartialSearchResults(true); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sample = response.hits().sequences().get(0); + assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); + assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on the unavailable shard only + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sample by key [process where value == 2] [process where value == 0]") + .allowPartialSearchResults(true); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // ------------------------------------------------------------------------ + // same queries, with missing shards and with default xpack.eql.default_allow_partial_results=true + // ------------------------------------------------------------------------ + + cluster(REMOTE_CLUSTER).client().execute( + ClusterUpdateSettingsAction.INSTANCE, + new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings( + Settings.builder().put(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey(), true) + ) + ).get(); + + // event query + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("process where true"); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().events().size(), equalTo(5)); + for (int i = 0; i < 5; i++) { + assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1))); + } + assertThat(response.shardFailures().length, is(1)); + + // sequence query on both shards + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("sequence [process where value == 1] [process where value == 2]"); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query on the available shard only + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("sequence [process where value == 1] [process where value == 3]"); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query on the unavailable shard only + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("sequence [process where value == 0] [process where value == 2]"); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]"); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on both shards + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("sample by key [process where value == 2] [process where value == 1]"); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on the available shard only + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("sample by key [process where value == 3] [process where value == 1]"); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(1)); + sample = response.hits().sequences().get(0); + assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); + assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on the unavailable shard only + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("sample by key [process where value == 2] [process where value == 0]"); + response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1-remote")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + localClient().execute( + ClusterUpdateSettingsAction.INSTANCE, + new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings( + Settings.builder().putNull(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey()) + ) + ).get(); + } +} From 3ab9740a88d51db73bf779ef9e4ad3a04afdfb6c Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Mon, 25 Nov 2024 16:15:21 +0100 Subject: [PATCH 11/23] Fix license --- .../xpack/eql/action/CCSPartialResultsIT.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java index 4ff80a975e6e3..1d27ed21c8273 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java @@ -1,12 +1,9 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ - package org.elasticsearch.xpack.eql.action; import org.apache.logging.log4j.LogManager; From 8207ea042cb88733fbba07486131edcc7ef5d741 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Mon, 25 Nov 2024 17:01:47 +0100 Subject: [PATCH 12/23] Refactor test --- .../xpack/eql/action/CCSPartialResultsIT.java | 253 +++--------------- 1 file changed, 39 insertions(+), 214 deletions(-) diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java index 1d27ed21c8273..81840f1d0907f 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java @@ -4,244 +4,57 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + package org.elasticsearch.xpack.eql.action; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.cluster.remote.RemoteInfoRequest; -import org.elasticsearch.action.admin.cluster.remote.TransportRemoteInfoAction; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsAction; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.TimeValue; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.CloseableTestClusterWrapper; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.InternalTestCluster; -import org.elasticsearch.test.MockHttpTransport; -import org.elasticsearch.test.NodeConfigurationSource; -import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.transport.RemoteClusterAware; -import org.elasticsearch.transport.RemoteConnectionInfo; -import org.elasticsearch.transport.TransportService; +import org.elasticsearch.test.AbstractMultiClustersTestCase; import org.elasticsearch.xpack.eql.plugin.EqlPlugin; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import java.io.Closeable; import java.io.IOException; -import java.nio.file.Path; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Set; import java.util.concurrent.ExecutionException; -import java.util.function.Function; -import java.util.stream.Collectors; -import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING; -import static org.elasticsearch.discovery.SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; - -public class CCSPartialResultsIT extends ESTestCase { - - - public static final String LOCAL_CLUSTER = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; - public static final String REMOTE_CLUSTER = "remote_cluster"; - private static volatile ClusterGroup clusterGroup; +public class CCSPartialResultsIT extends AbstractMultiClustersTestCase { - protected Collection> nodePlugins() { + protected Collection> nodePlugins(String cluster) { return Collections.singletonList(LocalStateEQLXPackPlugin.class); } - protected Settings nodeSettings() { - return Settings.builder() - .put("cluster.routing.rebalance.enable", "none") - .build(); - } - protected final Client localClient() { return client(LOCAL_CLUSTER); } - protected final Client client(String clusterAlias) { - return cluster(clusterAlias).client(); - } - - protected final InternalTestCluster cluster(String clusterAlias) { - return clusterGroup.getCluster(clusterAlias); - } - - protected final Map clusters() { - return Collections.unmodifiableMap(clusterGroup.clusters); - } - - - @Before - public final void startClusters() throws Exception { - if (clusterGroup != null) { - return; - } - stopClusters(); - final Map clusters = new HashMap<>(); - final List clusterAliases = List.of(REMOTE_CLUSTER, LOCAL_CLUSTER); - for (String clusterAlias : clusterAliases) { - final String clusterName = clusterAlias.equals(LOCAL_CLUSTER) ? "main-cluster" : clusterAlias; - final int numberOfNodes = 2; - final List> mockPlugins = List.of( - MockHttpTransport.TestPlugin.class, - MockTransportService.TestPlugin.class, - getTestTransportPlugin() - ); - final Collection> nodePlugins = nodePlugins(); - - final NodeConfigurationSource nodeConfigurationSource = nodeConfigurationSource(nodeSettings(), nodePlugins); - final InternalTestCluster cluster = new InternalTestCluster( - randomLong(), - createTempDir(), - true, - true, - numberOfNodes, - numberOfNodes, - clusterName, - nodeConfigurationSource, - 0, - clusterName + "-", - mockPlugins, - Function.identity() - ); - cluster.getNodeNames(); - cluster.beforeTest(random()); - clusters.put(clusterAlias, cluster); - } - clusterGroup = new ClusterGroup(clusters); - configureAndConnectsToRemoteClusters(); - } - - @After - public void assertAfterTest() throws Exception { - for (InternalTestCluster cluster : clusters().values()) { - cluster.wipe(Set.of()); - cluster.assertAfterTest(); - } - ESIntegTestCase.awaitGlobalNettyThreadsFinish(); - } - - @AfterClass - public static void stopClusters() throws IOException { - IOUtils.close(clusterGroup); - clusterGroup = null; - } - - protected void configureAndConnectsToRemoteClusters() throws Exception { - final InternalTestCluster cluster = clusterGroup.getCluster(REMOTE_CLUSTER); - final String[] allNodes = cluster.getNodeNames(); - configureRemoteCluster(REMOTE_CLUSTER, allNodes[1]); - } - - protected void configureRemoteCluster(String clusterAlias, String seedNode) throws Exception { - final String remoteClusterSettingPrefix = "cluster.remote." + clusterAlias + "."; - Settings.Builder settings = Settings.builder(); - final TransportService transportService = cluster(clusterAlias).getInstance(TransportService.class, seedNode); - String seedAddress = transportService.boundAddress().publishAddress().toString(); - - Settings.Builder builder; - if (randomBoolean()) { - builder = settings.putNull(remoteClusterSettingPrefix + "proxy_address") - .put(remoteClusterSettingPrefix + "mode", "sniff") - .put(remoteClusterSettingPrefix + "seeds", seedAddress); - } else { - builder = settings.putNull(remoteClusterSettingPrefix + "seeds") - .put(remoteClusterSettingPrefix + "mode", "proxy") - .put(remoteClusterSettingPrefix + "proxy_address", seedAddress); - } - - localClient().admin() - .cluster() - .prepareUpdateSettings(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT) - .setPersistentSettings(builder) - .get(); - - assertBusy(() -> { - List remoteConnectionInfos = localClient().execute(TransportRemoteInfoAction.TYPE, new RemoteInfoRequest()) - .actionGet() - .getInfos() - .stream() - .filter(c -> c.isConnected() && c.getClusterAlias().equals(clusterAlias)) - .collect(Collectors.toList()); - assertThat(remoteConnectionInfos, not(empty())); - }); - } - - static class ClusterGroup implements Closeable { - private final Map clusters; - - ClusterGroup(Map clusters) { - this.clusters = Collections.unmodifiableMap(clusters); - } - - InternalTestCluster getCluster(String clusterAlias) { - assertThat(clusters, hasKey(clusterAlias)); - return clusters.get(clusterAlias); - } - - @Override - public void close() throws IOException { - IOUtils.close(CloseableTestClusterWrapper.wrap(clusters.values())); - } - } - - static NodeConfigurationSource nodeConfigurationSource(Settings nodeSettings, Collection> nodePlugins) { - final Settings.Builder builder = Settings.builder(); - builder.putList(DISCOVERY_SEED_HOSTS_SETTING.getKey()); // empty list disables a port scan for other nodes - builder.putList(DISCOVERY_SEED_PROVIDERS_SETTING.getKey(), "file"); - builder.put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType()); - builder.put(nodeSettings); - - return new NodeConfigurationSource() { - @Override - public Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { - return builder.build(); - } - - @Override - public Path nodeConfigPath(int nodeOrdinal) { - return null; - } - - @Override - public Collection> nodePlugins() { - return nodePlugins; - } - }; + protected Collection remoteClusterAlias() { + return List.of("cluster-a", "cluster-b"); } - public void testFailuresFromRemote() throws ExecutionException, InterruptedException, IOException { final Client localClient = localClient(); + String REMOTE_CLUSTER = "cluster-b"; final Client remoteClient = client(REMOTE_CLUSTER); assertAcked( - remoteClient.admin().indices().prepareCreate("test-1-remote") + remoteClient.admin() + .indices() + .prepareCreate("test-1-remote") .setSettings( Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.routing.allocation.include._name", clusterGroup.clusters.get(REMOTE_CLUSTER).getNodeNames()[0]) + .put("index.routing.allocation.include._name", clusters().get(REMOTE_CLUSTER).getNodeNames()[0]) .build() ) .setMapping("@timestamp", "type=date"), @@ -249,11 +62,13 @@ public void testFailuresFromRemote() throws ExecutionException, InterruptedExcep ); assertAcked( - remoteClient.admin().indices().prepareCreate("test-2-remote") + remoteClient.admin() + .indices() + .prepareCreate("test-2-remote") .setSettings( Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.routing.allocation.exclude._name", clusterGroup.clusters.get(REMOTE_CLUSTER).getNodeNames()[0]) + .put("index.routing.allocation.exclude._name", clusters().get(REMOTE_CLUSTER).getNodeNames()[0]) .build() ) .setMapping("@timestamp", "type=date"), @@ -262,13 +77,15 @@ public void testFailuresFromRemote() throws ExecutionException, InterruptedExcep for (int i = 0; i < 5; i++) { int val = i * 2; - remoteClient.prepareIndex("test-1-remote").setId(Integer.toString(i)) + remoteClient.prepareIndex("test-1-remote") + .setId(Integer.toString(i)) .setSource("@timestamp", 100000 + val, "event.category", "process", "key", "same", "value", val) .get(); } for (int i = 0; i < 5; i++) { int val = i * 2 + 1; - remoteClient.prepareIndex("test-2-remote").setId(Integer.toString(i)) + remoteClient.prepareIndex("test-2-remote") + .setId(Integer.toString(i)) .setSource("@timestamp", 100000 + val, "event.category", "process", "key", "same", "value", val) .get(); } @@ -377,7 +194,7 @@ public void testFailuresFromRemote() throws ExecutionException, InterruptedExcep // stop one of the nodes, make one of the shards unavailable // ------------------------------------------------------------------------ - cluster(REMOTE_CLUSTER).stopNode(clusterGroup.clusters.get(REMOTE_CLUSTER).getNodeNames()[0]); + cluster(REMOTE_CLUSTER).stopNode(clusters().get(REMOTE_CLUSTER).getNodeNames()[0]); // ------------------------------------------------------------------------ // same queries, with missing shards and allow_partial_search_results=true @@ -570,12 +387,14 @@ public void testFailuresFromRemote() throws ExecutionException, InterruptedExcep // same queries, with missing shards and with default xpack.eql.default_allow_partial_results=true // ------------------------------------------------------------------------ - cluster(REMOTE_CLUSTER).client().execute( - ClusterUpdateSettingsAction.INSTANCE, - new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings( - Settings.builder().put(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey(), true) + cluster(REMOTE_CLUSTER).client() + .execute( + ClusterUpdateSettingsAction.INSTANCE, + new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings( + Settings.builder().put(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey(), true) + ) ) - ).get(); + .get(); // event query request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("process where true"); @@ -587,7 +406,8 @@ public void testFailuresFromRemote() throws ExecutionException, InterruptedExcep assertThat(response.shardFailures().length, is(1)); // sequence query on both shards - request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("sequence [process where value == 1] [process where value == 2]"); + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sequence [process where value == 1] [process where value == 2]"); response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); @@ -595,7 +415,8 @@ public void testFailuresFromRemote() throws ExecutionException, InterruptedExcep assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); // sequence query on the available shard only - request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("sequence [process where value == 1] [process where value == 3]"); + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sequence [process where value == 1] [process where value == 3]"); response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); @@ -603,7 +424,8 @@ public void testFailuresFromRemote() throws ExecutionException, InterruptedExcep assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); // sequence query on the unavailable shard only - request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("sequence [process where value == 0] [process where value == 2]"); + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sequence [process where value == 0] [process where value == 2]"); response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); @@ -620,7 +442,8 @@ public void testFailuresFromRemote() throws ExecutionException, InterruptedExcep assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); // sample query on both shards - request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("sample by key [process where value == 2] [process where value == 1]"); + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sample by key [process where value == 2] [process where value == 1]"); response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); @@ -628,7 +451,8 @@ public void testFailuresFromRemote() throws ExecutionException, InterruptedExcep assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); // sample query on the available shard only - request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("sample by key [process where value == 3] [process where value == 1]"); + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sample by key [process where value == 3] [process where value == 1]"); response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); sample = response.hits().sequences().get(0); @@ -639,7 +463,8 @@ public void testFailuresFromRemote() throws ExecutionException, InterruptedExcep assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); // sample query on the unavailable shard only - request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("sample by key [process where value == 2] [process where value == 0]"); + request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("sample by key [process where value == 2] [process where value == 0]"); response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); From f3a1a65225db8d1f7ad0a4377ba1e782c8baba20 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Mon, 25 Nov 2024 19:13:38 +0100 Subject: [PATCH 13/23] Make CCS test more deterministic --- .../xpack/eql/action/CCSPartialResultsIT.java | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java index 81840f1d0907f..b74347e436a63 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java @@ -30,6 +30,8 @@ public class CCSPartialResultsIT extends AbstractMultiClustersTestCase { + static String REMOTE_CLUSTER = "cluster_a"; + protected Collection> nodePlugins(String cluster) { return Collections.singletonList(LocalStateEQLXPackPlugin.class); } @@ -38,14 +40,20 @@ protected final Client localClient() { return client(LOCAL_CLUSTER); } + @Override protected Collection remoteClusterAlias() { - return List.of("cluster-a", "cluster-b"); + return List.of(REMOTE_CLUSTER); + } + + @Override + protected boolean reuseClusters() { + return false; } public void testFailuresFromRemote() throws ExecutionException, InterruptedException, IOException { - final Client localClient = localClient(); - String REMOTE_CLUSTER = "cluster-b"; final Client remoteClient = client(REMOTE_CLUSTER); + final String remoteNode = cluster(REMOTE_CLUSTER).startDataOnlyNode(); + final String remoteNode2 = cluster(REMOTE_CLUSTER).startDataOnlyNode(); assertAcked( remoteClient.admin() @@ -53,8 +61,8 @@ public void testFailuresFromRemote() throws ExecutionException, InterruptedExcep .prepareCreate("test-1-remote") .setSettings( Settings.builder() + .put("index.routing.allocation.require._name", remoteNode) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.routing.allocation.include._name", clusters().get(REMOTE_CLUSTER).getNodeNames()[0]) .build() ) .setMapping("@timestamp", "type=date"), @@ -67,8 +75,8 @@ public void testFailuresFromRemote() throws ExecutionException, InterruptedExcep .prepareCreate("test-2-remote") .setSettings( Settings.builder() + .put("index.routing.allocation.require._name", remoteNode2) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.routing.allocation.exclude._name", clusters().get(REMOTE_CLUSTER).getNodeNames()[0]) .build() ) .setMapping("@timestamp", "type=date"), @@ -91,7 +99,6 @@ public void testFailuresFromRemote() throws ExecutionException, InterruptedExcep } remoteClient.admin().indices().prepareRefresh().get(); - localClient.admin().indices().prepareRefresh().get(); // ------------------------------------------------------------------------ // queries with full cluster (no missing shards) @@ -194,7 +201,7 @@ public void testFailuresFromRemote() throws ExecutionException, InterruptedExcep // stop one of the nodes, make one of the shards unavailable // ------------------------------------------------------------------------ - cluster(REMOTE_CLUSTER).stopNode(clusters().get(REMOTE_CLUSTER).getNodeNames()[0]); + cluster(REMOTE_CLUSTER).stopNode(remoteNode); // ------------------------------------------------------------------------ // same queries, with missing shards and allow_partial_search_results=true From a8f5fb511269cd18dae1e115a7a8da8b83eb09ef Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Tue, 26 Nov 2024 10:53:02 +0100 Subject: [PATCH 14/23] Cleanup --- .../org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java index b74347e436a63..0e67049729e3c 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java @@ -50,7 +50,7 @@ protected boolean reuseClusters() { return false; } - public void testFailuresFromRemote() throws ExecutionException, InterruptedException, IOException { + public void testPartialResults() throws ExecutionException, InterruptedException, IOException { final Client remoteClient = client(REMOTE_CLUSTER); final String remoteNode = cluster(REMOTE_CLUSTER).startDataOnlyNode(); final String remoteNode2 = cluster(REMOTE_CLUSTER).startDataOnlyNode(); From 706935c2452c169d2188657d238abad9a8182d6f Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Wed, 27 Nov 2024 17:12:53 +0100 Subject: [PATCH 15/23] Cleanup serialization code --- .../xpack/eql/action/EqlSearchResponse.java | 23 +++++-------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java index ebe96310f74b5..a4d93b7659970 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java @@ -40,13 +40,13 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import static org.elasticsearch.action.search.ShardSearchFailure.readShardSearchFailure; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.xpack.eql.util.SearchHitUtils.qualifiedIndex; @@ -93,7 +93,7 @@ private static final class Fields { parser.declareString(optionalConstructorArg(), ID); parser.declareBoolean(constructorArg(), IS_RUNNING); parser.declareBoolean(constructorArg(), IS_PARTIAL); - parser.declareObjectArray(optionalConstructorArg(), (p, c) -> ShardSearchFailure.EMPTY_ARRAY, SHARD_FAILURES); // TODO fix this + parser.declareObjectArray(optionalConstructorArg(), (p, c) -> ShardSearchFailure.EMPTY_ARRAY, SHARD_FAILURES); PARSER = parser.build(); } @@ -129,15 +129,7 @@ public EqlSearchResponse(StreamInput in) throws IOException { isPartial = in.readBoolean(); isRunning = in.readBoolean(); if (in.getTransportVersion().onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS)) { - int size = in.readVInt(); - if (size == 0) { - shardFailures = ShardSearchFailure.EMPTY_ARRAY; - } else { - shardFailures = new ShardSearchFailure[size]; - for (int i = 0; i < shardFailures.length; i++) { - shardFailures[i] = readShardSearchFailure(in); - } - } + shardFailures = in.readArray(ShardSearchFailure::readShardSearchFailure, ShardSearchFailure[]::new); } else { shardFailures = ShardSearchFailure.EMPTY_ARRAY; } @@ -156,10 +148,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(isPartial); out.writeBoolean(isRunning); if (out.getTransportVersion().onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS)) { - out.writeVInt(shardFailures.length); - for (ShardSearchFailure shardSearchFailure : shardFailures) { - shardSearchFailure.writeTo(out); - } + out.writeArray(shardFailures); } } @@ -233,12 +222,12 @@ public boolean equals(Object o) { && Objects.equals(tookInMillis, that.tookInMillis) && Objects.equals(isTimeout, that.isTimeout) && Objects.equals(asyncExecutionId, that.asyncExecutionId) - && Objects.equals(shardFailures, that.shardFailures); + && Arrays.equals(shardFailures, that.shardFailures); } @Override public int hashCode() { - return Objects.hash(hits, tookInMillis, isTimeout, asyncExecutionId, shardFailures); + return Objects.hash(hits, tookInMillis, isTimeout, asyncExecutionId, Arrays.hashCode(shardFailures)); } @Override From 32a7aef09253907da716836f4e759037637b188d Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 28 Nov 2024 18:20:56 +0100 Subject: [PATCH 16/23] More yaml tests --- .../rest-api-spec/test/eql/10_basic.yml | 90 ++++++++++++++++--- 1 file changed, 76 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml index c40b2d7dac107..2c42dd55981be 100644 --- a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml +++ b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml @@ -83,6 +83,34 @@ setup: id: 123 valid: true + - do: + indices.create: + index: eql_test_rebel + body: + mappings: + properties: + some_keyword: + type: keyword + runtime: + day_of_week: + type: keyword + script: + source: "throw new IllegalArgumentException(\"rebel shards\")" + - do: + bulk: + refresh: true + body: + - index: + _index: eql_test_rebel + _id: "1" + - event: + - category: process + "@timestamp": 2020-02-03T12:34:56Z + user: SYSTEM + id: 123 + valid: false + some_keyword: longer than normal + --- # Testing round-trip and the basic shape of the response "Execute some EQL.": @@ -481,17 +509,17 @@ setup: --- -"Execute query with allow_partial_search_results": +"Execute query shard failures and with allow_partial_search_results": - do: eql.search: - index: eql_test + index: eql_test* body: - query: 'process where user == "SYSTEM"' + query: 'process where user == "SYSTEM" and day_of_week == "Monday"' fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"] allow_partial_search_results: true - match: {timed_out: false} - - match: {hits.total.value: 3} + - match: {hits.total.value: 1} - match: {hits.total.relation: "eq"} - match: {hits.events.0._source.user: "SYSTEM"} - match: {hits.events.0._id: "1"} @@ -499,13 +527,47 @@ setup: - match: {hits.events.0.fields.id: [123]} - match: {hits.events.0.fields.valid: [false]} - match: {hits.events.0.fields.day_of_week: ["Monday"]} - - match: {hits.events.1._id: "2"} - - match: {hits.events.1.fields.@timestamp: ["1580819696000"]} - - match: {hits.events.1.fields.id: [123]} - - match: {hits.events.1.fields.valid: [true]} - - match: {hits.events.1.fields.day_of_week: ["Tuesday"]} - - match: {hits.events.2._id: "3"} - - match: {hits.events.2.fields.@timestamp: ["1580906096000"]} - - match: {hits.events.2.fields.id: [123]} - - match: {hits.events.2.fields.valid: [true]} - - match: {hits.events.2.fields.day_of_week: ["Wednesday"]} + - match: {shard_failures.0.index: "eql_test_rebel"} + + +--- +"Execute sequence with shard failures and allow_partial_search_results=true": + - do: + eql.search: + index: eql_test* + body: + query: 'sequence [process where user == "SYSTEM" and day_of_week == "Monday"] [process where user == "SYSTEM" and day_of_week == "Tuesday"]' + fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"] + allow_partial_search_results: true + + - match: {timed_out: false} + - match: {hits.total.value: 0} + - match: {shard_failures.0.index: "eql_test_rebel"} + + +--- +"Execute sequence with shard failures, allow_partial_search_results=true and allow_partial_sequence_results=true": + - do: + eql.search: + index: eql_test* + body: + query: 'sequence [process where user == "SYSTEM" and day_of_week == "Monday"] [process where user == "SYSTEM" and day_of_week == "Tuesday"]' + fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"] + allow_partial_search_results: true + allow_partial_sequence_results: true + + - match: {timed_out: false} + - match: {hits.total.value: 1} + - match: {hits.total.relation: "eq"} + - match: {hits.sequences.0.events.0._source.user: "SYSTEM"} + - match: {hits.sequences.0.events.0._id: "1"} + - match: {hits.sequences.0.events.0.fields.@timestamp: ["1580733296000"]} + - match: {hits.sequences.0.events.0.fields.id: [123]} + - match: {hits.sequences.0.events.0.fields.valid: [false]} + - match: {hits.sequences.0.events.0.fields.day_of_week: ["Monday"]} + - match: {hits.sequences.0.events.1._id: "2"} + - match: {hits.sequences.0.events.1.fields.@timestamp: ["1580819696000"]} + - match: {hits.sequences.0.events.1.fields.id: [123]} + - match: {hits.sequences.0.events.1.fields.valid: [true]} + - match: {hits.sequences.0.events.1.fields.day_of_week: ["Tuesday"]} + - match: {shard_failures.0.index: "eql_test_rebel"} From 1e97b85595772ed0b1d0e45fb7b4cf85169687fd Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Fri, 29 Nov 2024 13:40:08 +0100 Subject: [PATCH 17/23] Add toml tests and fix REST params --- .../test/eql/BaseEqlSpecTestCase.java | 58 +++++++++- .../elasticsearch/test/eql/DataLoader.java | 6 + .../test/eql/EqlDateNanosSpecTestCase.java | 36 +++++- .../test/eql/EqlExtraSpecTestCase.java | 36 +++++- .../eql/EqlMissingEventsSpecTestCase.java | 36 +++++- .../eql/EqlSampleMultipleEntriesTestCase.java | 36 +++++- .../test/eql/EqlSampleTestCase.java | 43 ++++++- .../org/elasticsearch/test/eql/EqlSpec.java | 51 ++++++++- .../eql/EqlSpecFailingShardsTestCase.java | 83 ++++++++++++++ .../elasticsearch/test/eql/EqlSpecLoader.java | 7 ++ .../test/eql/EqlSpecTestCase.java | 43 ++++++- .../data/endgame-shard-failures.data | 14 +++ .../data/endgame-shard-failures.mapping | 105 ++++++++++++++++++ .../main/resources/test_failing_shards.toml | 78 +++++++++++++ .../xpack/eql/EqlDateNanosIT.java | 25 ++++- .../elasticsearch/xpack/eql/EqlExtraIT.java | 25 ++++- .../elasticsearch/xpack/eql/EqlSampleIT.java | 25 ++++- .../xpack/eql/EqlSampleMultipleEntriesIT.java | 18 ++- .../elasticsearch/xpack/eql/EqlSpecIT.java | 25 ++++- .../xpack/eql/EqlDateNanosIT.java | 24 +++- .../elasticsearch/xpack/eql/EqlExtraIT.java | 24 +++- .../xpack/eql/EqlMissingEventsIT.java | 24 +++- .../elasticsearch/xpack/eql/EqlSampleIT.java | 24 +++- .../xpack/eql/EqlSampleMultipleEntriesIT.java | 17 ++- .../xpack/eql/EqlSpecFailingShardsIT.java | 53 +++++++++ .../elasticsearch/xpack/eql/EqlSpecIT.java | 24 +++- .../xpack/eql/plugin/RestEqlSearchAction.java | 3 + 27 files changed, 890 insertions(+), 53 deletions(-) create mode 100644 x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecFailingShardsTestCase.java create mode 100644 x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.data create mode 100644 x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.mapping create mode 100644 x-pack/plugin/eql/qa/common/src/main/resources/test_failing_shards.toml create mode 100644 x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecFailingShardsIT.java diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java index 7455bb3f7c2e7..4e33ecc2c4d50 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java @@ -33,6 +33,9 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; + public abstract class BaseEqlSpecTestCase extends RemoteClusterAwareEqlRestTestCase { protected static final String PARAM_FORMATTING = "%2$s"; @@ -52,6 +55,9 @@ public abstract class BaseEqlSpecTestCase extends RemoteClusterAwareEqlRestTestC */ private final int size; private final int maxSamplesPerKey; + private final Boolean allowPartialSearchResults; + private final Boolean allowPartialSequenceResults; + private final Boolean expectShardFailures; @Before public void setup() throws Exception { @@ -104,7 +110,16 @@ protected static List asArray(List specs) { } results.add( - new Object[] { spec.query(), name, spec.expectedEventIds(), spec.joinKeys(), spec.size(), spec.maxSamplesPerKey() } + new Object[] { + spec.query(), + name, + spec.expectedEventIds(), + spec.joinKeys(), + spec.size(), + spec.maxSamplesPerKey(), + spec.allowPartialSearchResults(), + spec.allowPartialSequenceResults(), + spec.expectShardFailures() } ); } @@ -118,7 +133,10 @@ protected static List asArray(List specs) { List eventIds, String[] joinKeys, Integer size, - Integer maxSamplesPerKey + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures ) { this.index = index; @@ -128,6 +146,9 @@ protected static List asArray(List specs) { this.joinKeys = joinKeys; this.size = size == null ? -1 : size; this.maxSamplesPerKey = maxSamplesPerKey == null ? -1 : maxSamplesPerKey; + this.allowPartialSearchResults = allowPartialSearchResults; + this.allowPartialSequenceResults = allowPartialSequenceResults; + this.expectShardFailures = expectShardFailures; } public void test() throws Exception { @@ -137,6 +158,7 @@ public void test() throws Exception { private void assertResponse(ObjectPath response) throws Exception { List> events = response.evaluate("hits.events"); List> sequences = response.evaluate("hits.sequences"); + Object shardFailures = response.evaluate("shard_failures"); if (events != null) { assertEvents(events); @@ -145,6 +167,7 @@ private void assertResponse(ObjectPath response) throws Exception { } else { fail("No events or sequences found"); } + assertShardFailures(shardFailures); } protected ObjectPath runQuery(String index, String query) throws Exception { @@ -163,7 +186,15 @@ protected ObjectPath runQuery(String index, String query) throws Exception { if (maxSamplesPerKey > 0) { builder.field("max_samples_per_key", maxSamplesPerKey); } - if (randomBoolean()) { + boolean allowPartialResultsInBody = randomBoolean(); + if (allowPartialSearchResults != null) { + if (allowPartialResultsInBody) { + builder.field("allow_partial_search_results", String.valueOf(allowPartialSearchResults)); + if (allowPartialSequenceResults != null) { + builder.field("allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults)); + } + } + } else if (randomBoolean()) { builder.field("allow_partial_search_results", randomBoolean()); } builder.endObject(); @@ -173,7 +204,14 @@ protected ObjectPath runQuery(String index, String query) throws Exception { if (ccsMinimizeRoundtrips != null) { request.addParameter("ccs_minimize_roundtrips", ccsMinimizeRoundtrips.toString()); } - if (randomBoolean()) { + if (allowPartialSearchResults != null) { + if (allowPartialResultsInBody == false) { + request.addParameter("allow_partial_search_results", String.valueOf(allowPartialSearchResults)); + if (allowPartialSequenceResults != null) { + request.addParameter("allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults)); + } + } + } else if (randomBoolean()) { request.addParameter("allow_partial_search_results", String.valueOf(randomBoolean())); } int timeout = Math.toIntExact(timeout().millis()); @@ -188,6 +226,18 @@ protected ObjectPath runQuery(String index, String query) throws Exception { return ObjectPath.createFromResponse(client().performRequest(request)); } + private void assertShardFailures(Object shardFailures) { + if (expectShardFailures != null) { + if (expectShardFailures) { + assertNotNull(shardFailures); + List list = (List) shardFailures; + assertThat(list.size(), is(greaterThan(0))); + } else { + assertNull(shardFailures); + } + } + } + private void assertEvents(List> events) { assertNotNull(events); logger.debug("Events {}", new Object() { diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java index 1d51af574c810..4618bd8f4ff3d 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java @@ -52,6 +52,7 @@ */ public class DataLoader { public static final String TEST_INDEX = "endgame-140"; + public static final String TEST_SHARD_FAILURES_INDEX = "endgame-shard-failures"; public static final String TEST_EXTRA_INDEX = "extra"; public static final String TEST_NANOS_INDEX = "endgame-140-nanos"; public static final String TEST_SAMPLE = "sample1,sample2,sample3"; @@ -103,6 +104,11 @@ public static void loadDatasetIntoEs(RestClient client, CheckedBiFunction eventIds, String[] joinKeys, Integer size, - Integer maxSamplesPerKey + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures ) { - this(TEST_NANOS_INDEX, query, name, eventIds, joinKeys, size, maxSamplesPerKey); + this( + TEST_NANOS_INDEX, + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } // constructor for multi-cluster tests @@ -40,9 +54,23 @@ public EqlDateNanosSpecTestCase( List eventIds, String[] joinKeys, Integer size, - Integer maxSamplesPerKey + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures ) { - super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey); + super( + index, + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } @Override diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java index 292fe6c895cee..cc858ded25f37 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java @@ -27,9 +27,23 @@ public EqlExtraSpecTestCase( List eventIds, String[] joinKeys, Integer size, - Integer maxSamplesPerKey + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures ) { - this(TEST_EXTRA_INDEX, query, name, eventIds, joinKeys, size, maxSamplesPerKey); + this( + TEST_EXTRA_INDEX, + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } // constructor for multi-cluster tests @@ -40,9 +54,23 @@ public EqlExtraSpecTestCase( List eventIds, String[] joinKeys, Integer size, - Integer maxSamplesPerKey + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures ) { - super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey); + super( + index, + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } @Override diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlMissingEventsSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlMissingEventsSpecTestCase.java index cdda9e9e068f5..f62c2b29101db 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlMissingEventsSpecTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlMissingEventsSpecTestCase.java @@ -27,9 +27,23 @@ public EqlMissingEventsSpecTestCase( List eventIds, String[] joinKeys, Integer size, - Integer maxSamplesPerKey + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures ) { - this(TEST_MISSING_EVENTS_INDEX, query, name, eventIds, joinKeys, size, maxSamplesPerKey); + this( + TEST_MISSING_EVENTS_INDEX, + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } // constructor for multi-cluster tests @@ -40,9 +54,23 @@ public EqlMissingEventsSpecTestCase( List eventIds, String[] joinKeys, Integer size, - Integer maxSamplesPerKey + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures ) { - super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey); + super( + index, + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } @Override diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleMultipleEntriesTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleMultipleEntriesTestCase.java index 6471e264a92fa..a38ccacb42f5f 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleMultipleEntriesTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleMultipleEntriesTestCase.java @@ -21,9 +21,23 @@ public EqlSampleMultipleEntriesTestCase( List eventIds, String[] joinKeys, Integer size, - Integer maxSamplesPerKey + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures ) { - this(TEST_SAMPLE_MULTI, query, name, eventIds, joinKeys, size, maxSamplesPerKey); + this( + TEST_SAMPLE_MULTI, + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } public EqlSampleMultipleEntriesTestCase( @@ -33,9 +47,23 @@ public EqlSampleMultipleEntriesTestCase( List eventIds, String[] joinKeys, Integer size, - Integer maxSamplesPerKey + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures ) { - super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey); + super( + index, + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } @ParametersFactory(shuffle = false, argumentFormatting = PARAM_FORMATTING) diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleTestCase.java index dfae73b3602a7..4748bd0e3307b 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleTestCase.java @@ -15,8 +15,29 @@ public abstract class EqlSampleTestCase extends BaseEqlSpecTestCase { - public EqlSampleTestCase(String query, String name, List eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) { - this(TEST_SAMPLE, query, name, eventIds, joinKeys, size, maxSamplesPerKey); + public EqlSampleTestCase( + String query, + String name, + List eventIds, + String[] joinKeys, + Integer size, + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures + ) { + this( + TEST_SAMPLE, + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } public EqlSampleTestCase( @@ -26,9 +47,23 @@ public EqlSampleTestCase( List eventIds, String[] joinKeys, Integer size, - Integer maxSamplesPerKey + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures ) { - super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey); + super( + index, + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } @ParametersFactory(shuffle = false, argumentFormatting = PARAM_FORMATTING) diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java index db7ee05ff2239..4dd617bac0abd 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java @@ -30,6 +30,9 @@ public class EqlSpec { private Integer size; private Integer maxSamplesPerKey; + private Boolean allowPartialSearchResults; + private Boolean allowPartialSequenceResults; + private Boolean expectShardFailures; public String name() { return name; @@ -103,6 +106,30 @@ public void maxSamplesPerKey(Integer maxSamplesPerKey) { this.maxSamplesPerKey = maxSamplesPerKey; } + public Boolean allowPartialSearchResults() { + return allowPartialSearchResults; + } + + public void allowPartialSearchResults(Boolean allowPartialSearchResults) { + this.allowPartialSearchResults = allowPartialSearchResults; + } + + public Boolean allowPartialSequenceResults() { + return allowPartialSequenceResults; + } + + public void allowPartialSequenceResults(Boolean allowPartialSequenceResults) { + this.allowPartialSequenceResults = allowPartialSequenceResults; + } + + public Boolean expectShardFailures() { + return expectShardFailures; + } + + public void expectShardFailures(Boolean expectShardFailures) { + this.expectShardFailures = expectShardFailures; + } + @Override public String toString() { String str = ""; @@ -132,7 +159,15 @@ public String toString() { if (maxSamplesPerKey != null) { str = appendWithComma(str, "max_samples_per_key", "" + maxSamplesPerKey); } - + if (allowPartialSearchResults != null) { + str = appendWithComma(str, "allow_partial_search_results", String.valueOf(allowPartialSearchResults)); + } + if (allowPartialSequenceResults != null) { + str = appendWithComma(str, "allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults)); + } + if (expectShardFailures != null) { + str = appendWithComma(str, "expect_shard_failures", String.valueOf(expectShardFailures)); + } return str; } @@ -150,12 +185,22 @@ public boolean equals(Object other) { return Objects.equals(this.query(), that.query()) && Objects.equals(size, that.size) - && Objects.equals(maxSamplesPerKey, that.maxSamplesPerKey); + && Objects.equals(maxSamplesPerKey, that.maxSamplesPerKey) + && Objects.equals(allowPartialSearchResults, that.allowPartialSearchResults) + && Objects.equals(allowPartialSequenceResults, that.allowPartialSequenceResults) + && Objects.equals(expectShardFailures, that.expectShardFailures); } @Override public int hashCode() { - return Objects.hash(this.query, size, maxSamplesPerKey); + return Objects.hash( + this.query, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } private static String appendWithComma(String str, String name, String append) { diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecFailingShardsTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecFailingShardsTestCase.java new file mode 100644 index 0000000000000..c490a2f703dcc --- /dev/null +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecFailingShardsTestCase.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.test.eql; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import java.util.List; + +import static org.elasticsearch.test.eql.DataLoader.TEST_INDEX; +import static org.elasticsearch.test.eql.DataLoader.TEST_SHARD_FAILURES_INDEX; + +public abstract class EqlSpecFailingShardsTestCase extends BaseEqlSpecTestCase { + + @ParametersFactory(shuffle = false, argumentFormatting = PARAM_FORMATTING) + public static List readTestSpecs() throws Exception { + + // Load EQL validation specs + return asArray(EqlSpecLoader.load("/test_failing_shards.toml")); + } + + @Override + protected String tiebreaker() { + return "serial_event_id"; + } + + // constructor for "local" rest tests + public EqlSpecFailingShardsTestCase( + String query, + String name, + List eventIds, + String[] joinKeys, + Integer size, + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures + ) { + this( + TEST_INDEX + "," + TEST_SHARD_FAILURES_INDEX, + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); + } + + // constructor for multi-cluster tests + public EqlSpecFailingShardsTestCase( + String index, + String query, + String name, + List eventIds, + String[] joinKeys, + Integer size, + Integer maxSamplesPerKey, + Boolean allowPartialSearch, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures + ) { + super( + index, + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearch, + allowPartialSequenceResults, + expectShardFailures + ); + } +} diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java index a1f555563e29c..f86107cf3bac5 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java @@ -76,6 +76,10 @@ private static Integer getInteger(TomlTable table, String key) { return null; } + private static Boolean getBoolean(TomlTable table, String key) { + return table.getBoolean(key); + } + private static List readFromStream(InputStream is, Set uniqueTestNames) throws Exception { List testSpecs = new ArrayList<>(); @@ -90,6 +94,9 @@ private static List readFromStream(InputStream is, Set uniqueTe spec.note(getTrimmedString(table, "note")); spec.description(getTrimmedString(table, "description")); spec.size(getInteger(table, "size")); + spec.allowPartialSearchResults(getBoolean(table, "allow_partial_search_results")); + spec.allowPartialSequenceResults(getBoolean(table, "allow_partial_sequence_results")); + spec.expectShardFailures(getBoolean(table, "expect_shard_failures")); List arr = table.getList("tags"); if (arr != null) { diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java index 7113924f79029..62a3ea72fe51f 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java @@ -28,8 +28,29 @@ protected String tiebreaker() { } // constructor for "local" rest tests - public EqlSpecTestCase(String query, String name, List eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) { - this(TEST_INDEX, query, name, eventIds, joinKeys, size, maxSamplesPerKey); + public EqlSpecTestCase( + String query, + String name, + List eventIds, + String[] joinKeys, + Integer size, + Integer maxSamplesPerKey, + Boolean allowPartialSearch, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures + ) { + this( + TEST_INDEX, + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearch, + allowPartialSequenceResults, + expectShardFailures + ); } // constructor for multi-cluster tests @@ -40,8 +61,22 @@ public EqlSpecTestCase( List eventIds, String[] joinKeys, Integer size, - Integer maxSamplesPerKey + Integer maxSamplesPerKey, + Boolean allowPartialSearch, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures ) { - super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey); + super( + index, + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearch, + allowPartialSequenceResults, + expectShardFailures + ); } } diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.data b/x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.data new file mode 100644 index 0000000000000..e2d552a19a923 --- /dev/null +++ b/x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.data @@ -0,0 +1,14 @@ +[ + { + "event_subtype_full": "already_running", + "event_type": "process", + "event_type_full": "process_event", + "opcode": 3, + "pid": 0, + "process_name": "System Idle Process", + "serial_event_id": 10000, + "subtype": "create", + "timestamp": 116444736000000000, + "unique_pid": 1 + } +] diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.mapping b/x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.mapping new file mode 100644 index 0000000000000..3b5039f4098af --- /dev/null +++ b/x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.mapping @@ -0,0 +1,105 @@ +# Text patterns like "[runtime_random_keyword_type]" will get replaced at runtime with a random string type. +# See DataLoader class for pattern replacements. +{ + "runtime":{ + "broken":{ + "type": "long", + "script": { + "lang": "painless", + "source": "emit(doc['non_existing'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))" + } + } + }, + "properties" : { + "command_line" : { + "type" : "[runtime_random_keyword_type]" + }, + "event_type" : { + "type" : "[runtime_random_keyword_type]" + }, + "event" : { + "properties" : { + "category" : { + "type" : "alias", + "path" : "event_type" + }, + "sequence" : { + "type" : "alias", + "path" : "serial_event_id" + } + } + }, + "md5" : { + "type" : "[runtime_random_keyword_type]" + }, + "parent_process_name": { + "type" : "[runtime_random_keyword_type]" + }, + "parent_process_path": { + "type" : "[runtime_random_keyword_type]" + }, + "pid" : { + "type" : "long" + }, + "ppid" : { + "type" : "long" + }, + "process_name": { + "type" : "[runtime_random_keyword_type]" + }, + "process_path": { + "type" : "[runtime_random_keyword_type]" + }, + "subtype" : { + "type" : "[runtime_random_keyword_type]" + }, + "timestamp" : { + "type" : "date" + }, + "@timestamp" : { + "type" : "date" + }, + "user" : { + "type" : "[runtime_random_keyword_type]" + }, + "user_name" : { + "type" : "[runtime_random_keyword_type]" + }, + "user_domain": { + "type" : "[runtime_random_keyword_type]" + }, + "hostname" : { + "type" : "text", + "fields" : { + "[runtime_random_keyword_type]" : { + "type" : "[runtime_random_keyword_type]", + "ignore_above" : 256 + } + } + }, + "opcode" : { + "type" : "long" + }, + "file_name" : { + "type" : "text", + "fields" : { + "[runtime_random_keyword_type]" : { + "type" : "[runtime_random_keyword_type]", + "ignore_above" : 256 + } + } + }, + "file_path" : { + "type" : "[runtime_random_keyword_type]" + }, + "serial_event_id" : { + "type" : "long" + }, + "source_address" : { + "type" : "ip" + }, + "exit_code" : { + "type" : "long" + } + } +} diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_failing_shards.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_failing_shards.toml new file mode 100644 index 0000000000000..98bd7da8bd4e8 --- /dev/null +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_failing_shards.toml @@ -0,0 +1,78 @@ +# this query doesn't touch the "broken" field, so it should not fail +[[queries]] +name = "eventQueryNoShardFailures" +query = 'process where serial_event_id == 1' +allow_partial_search_results = true +expected_event_ids = [1] +expect_shard_failures = false + + +[[queries]] +name = "eventQueryShardFailures" +query = 'process where serial_event_id == 1 or broken == 1' +allow_partial_search_results = true +expected_event_ids = [1] +expect_shard_failures = true + + +# this query doesn't touch the "broken" field, so it should not fail +[[queries]] +name = "sequenceQueryNoShardFailures" +query = ''' +sequence + [process where serial_event_id == 1] + [process where serial_event_id == 2] +''' +expected_event_ids = [1, 2] +expect_shard_failures = false + + +# this query doesn't touch the "broken" field, so it should not fail +[[queries]] +name = "sequenceQueryNoShardFailuresAllowFalse" +query = ''' +sequence + [process where serial_event_id == 1] + [process where serial_event_id == 2] +''' +allow_partial_search_results = false +expected_event_ids = [1, 2] +expect_shard_failures = false + + +# this query doesn't touch the "broken" field, so it should not fail +[[queries]] +name = "sequenceQueryNoShardFailuresAllowTrue" +query = ''' +sequence + [process where serial_event_id == 1] + [process where serial_event_id == 2] +''' +allow_partial_search_results = true +expected_event_ids = [1, 2] +expect_shard_failures = false + + +[[queries]] +name = "sequenceQueryMissingShards" +query = ''' +sequence + [process where serial_event_id == 1 or broken == 1] + [process where serial_event_id == 2] +''' +allow_partial_search_results = true +expected_event_ids = [] +expect_shard_failures = true + + +[[queries]] +name = "sequenceQueryMissingShardsPartialResults" +query = ''' +sequence + [process where serial_event_id == 1 or broken == 1] + [process where serial_event_id == 2] +''' +allow_partial_search_results = true +allow_partial_sequence_results = true +expected_event_ids = [1, 2] +expect_shard_failures = true diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java index c20968871472f..5d6824232d80f 100644 --- a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java +++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java @@ -37,7 +37,28 @@ protected String getRemoteCluster() { return REMOTE_CLUSTER.getHttpAddresses(); } - public EqlDateNanosIT(String query, String name, List eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) { - super(remoteClusterIndex(TEST_NANOS_INDEX), query, name, eventIds, joinKeys, size, maxSamplesPerKey); + public EqlDateNanosIT( + String query, + String name, + List eventIds, + String[] joinKeys, + Integer size, + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures + ) { + super( + remoteClusterIndex(TEST_NANOS_INDEX), + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } } diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java index 774c19d02adf0..79b095434814b 100644 --- a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java +++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java @@ -37,7 +37,28 @@ protected String getRemoteCluster() { return REMOTE_CLUSTER.getHttpAddresses(); } - public EqlExtraIT(String query, String name, List eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) { - super(remoteClusterIndex(TEST_EXTRA_INDEX), query, name, eventIds, joinKeys, size, maxSamplesPerKey); + public EqlExtraIT( + String query, + String name, + List eventIds, + String[] joinKeys, + Integer size, + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures + ) { + super( + remoteClusterIndex(TEST_EXTRA_INDEX), + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } } diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java index 1502c250bd058..7673eec32ec55 100644 --- a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java +++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java @@ -37,7 +37,28 @@ protected String getRemoteCluster() { return REMOTE_CLUSTER.getHttpAddresses(); } - public EqlSampleIT(String query, String name, List eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) { - super(remoteClusterPattern(TEST_SAMPLE), query, name, eventIds, joinKeys, size, maxSamplesPerKey); + public EqlSampleIT( + String query, + String name, + List eventIds, + String[] joinKeys, + Integer size, + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures + ) { + super( + remoteClusterPattern(TEST_SAMPLE), + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } } diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java index 795fe4e103a31..ac6f7fe508c99 100644 --- a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java +++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java @@ -43,8 +43,22 @@ public EqlSampleMultipleEntriesIT( List eventIds, String[] joinKeys, Integer size, - Integer maxSamplesPerKey + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures ) { - super(remoteClusterPattern(TEST_SAMPLE_MULTI), query, name, eventIds, joinKeys, size, maxSamplesPerKey); + super( + remoteClusterPattern(TEST_SAMPLE_MULTI), + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } } diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java index 2cddecb644a1a..db0c03e8fdb6f 100644 --- a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java +++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java @@ -37,7 +37,28 @@ protected String getRemoteCluster() { return REMOTE_CLUSTER.getHttpAddresses(); } - public EqlSpecIT(String query, String name, List eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) { - super(remoteClusterIndex(TEST_INDEX), query, name, eventIds, joinKeys, size, maxSamplesPerKey); + public EqlSpecIT( + String query, + String name, + List eventIds, + String[] joinKeys, + Integer size, + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures + ) { + super( + remoteClusterIndex(TEST_INDEX), + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } } diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java index 1df10fde7fde5..5e1fa224de58d 100644 --- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java +++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java @@ -27,7 +27,27 @@ protected String getTestRestCluster() { return cluster.getHttpAddresses(); } - public EqlDateNanosIT(String query, String name, List eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) { - super(query, name, eventIds, joinKeys, size, maxSamplesPerKey); + public EqlDateNanosIT( + String query, + String name, + List eventIds, + String[] joinKeys, + Integer size, + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures + ) { + super( + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } } diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java index 8af8fcac087b5..cb92eddeb0410 100644 --- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java +++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java @@ -27,7 +27,27 @@ protected String getTestRestCluster() { return cluster.getHttpAddresses(); } - public EqlExtraIT(String query, String name, List eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) { - super(query, name, eventIds, joinKeys, size, maxSamplesPerKey); + public EqlExtraIT( + String query, + String name, + List eventIds, + String[] joinKeys, + Integer size, + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures + ) { + super( + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } } diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlMissingEventsIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlMissingEventsIT.java index 05557fb4883b3..4f1faf3322e7f 100644 --- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlMissingEventsIT.java +++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlMissingEventsIT.java @@ -27,8 +27,28 @@ protected String getTestRestCluster() { return cluster.getHttpAddresses(); } - public EqlMissingEventsIT(String query, String name, List eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) { - super(query, name, eventIds, joinKeys, size, maxSamplesPerKey); + public EqlMissingEventsIT( + String query, + String name, + List eventIds, + String[] joinKeys, + Integer size, + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures + ) { + super( + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } } diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java index dc2c653fad89e..c0bce3ffc9e4f 100644 --- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java +++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java @@ -27,8 +27,28 @@ protected String getTestRestCluster() { return cluster.getHttpAddresses(); } - public EqlSampleIT(String query, String name, List eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) { - super(query, name, eventIds, joinKeys, size, maxSamplesPerKey); + public EqlSampleIT( + String query, + String name, + List eventIds, + String[] joinKeys, + Integer size, + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures + ) { + super( + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } } diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java index af1ade9120bbd..f50ee36095ae0 100644 --- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java +++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java @@ -33,9 +33,22 @@ public EqlSampleMultipleEntriesIT( List eventIds, String[] joinKeys, Integer size, - Integer maxSamplesPerKey + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures ) { - super(query, name, eventIds, joinKeys, size, maxSamplesPerKey); + super( + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } } diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecFailingShardsIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecFailingShardsIT.java new file mode 100644 index 0000000000000..cf05811a77857 --- /dev/null +++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecFailingShardsIT.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.eql; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.elasticsearch.test.TestClustersThreadFilter; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.eql.EqlSpecFailingShardsTestCase; +import org.junit.ClassRule; + +import java.util.List; + +@ThreadLeakFilters(filters = TestClustersThreadFilter.class) +public class EqlSpecFailingShardsIT extends EqlSpecFailingShardsTestCase { + + @ClassRule + public static final ElasticsearchCluster cluster = EqlTestCluster.CLUSTER; + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + public EqlSpecFailingShardsIT( + String query, + String name, + List eventIds, + String[] joinKeys, + Integer size, + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures + ) { + super( + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); + } +} diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java index 7aac0ae336c8a..0aad5cc1b73da 100644 --- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java +++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java @@ -27,7 +27,27 @@ protected String getTestRestCluster() { return cluster.getHttpAddresses(); } - public EqlSpecIT(String query, String name, List eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) { - super(query, name, eventIds, joinKeys, size, maxSamplesPerKey); + public EqlSpecIT( + String query, + String name, + List eventIds, + String[] joinKeys, + Integer size, + Integer maxSamplesPerKey, + Boolean allowPartialSearchResults, + Boolean allowPartialSequenceResults, + Boolean expectShardFailures + ) { + super( + query, + name, + eventIds, + joinKeys, + size, + maxSamplesPerKey, + allowPartialSearchResults, + allowPartialSequenceResults, + expectShardFailures + ); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java index 126462797141e..65def24563e5e 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java @@ -67,6 +67,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli eqlRequest.allowPartialSearchResults( request.paramAsBoolean("allow_partial_search_results", eqlRequest.allowPartialSearchResults()) ); + eqlRequest.allowPartialSequenceResults( + request.paramAsBoolean("allow_partial_sequence_results", eqlRequest.allowPartialSequenceResults()) + ); } return channel -> { From 045d8da53674c66436a06b475b1f98d240ae8f1d Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Fri, 6 Dec 2024 14:28:46 +0100 Subject: [PATCH 18/23] More tests --- .../test/eql/BaseEqlSpecTestCase.java | 26 +++- .../eql/action/PartialSearchResultsIT.java | 133 +++++++++++++----- 2 files changed, 123 insertions(+), 36 deletions(-) diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java index 4e33ecc2c4d50..483f81201e374 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java @@ -193,9 +193,22 @@ protected ObjectPath runQuery(String index, String query) throws Exception { if (allowPartialSequenceResults != null) { builder.field("allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults)); } + } else { + // these will be overwritten by the query params, that have higher priority than the body params + if (allowPartialSearchResults != null) { + builder.field("allow_partial_search_results", randomBoolean()); + } + if (allowPartialSequenceResults != null) { + builder.field("allow_partial_sequence_results", randomBoolean()); + } + } + } else { + if (randomBoolean()) { + builder.field("allow_partial_search_results", randomBoolean()); + } + if (randomBoolean()) { + builder.field("allow_partial_sequence_results", randomBoolean()); } - } else if (randomBoolean()) { - builder.field("allow_partial_search_results", randomBoolean()); } builder.endObject(); @@ -211,8 +224,13 @@ protected ObjectPath runQuery(String index, String query) throws Exception { request.addParameter("allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults)); } } - } else if (randomBoolean()) { - request.addParameter("allow_partial_search_results", String.valueOf(randomBoolean())); + } else { + if (randomBoolean()) { + request.addParameter("allow_partial_search_results", String.valueOf(randomBoolean())); + } + if (randomBoolean()) { + request.addParameter("allow_partial_sequence_results", String.valueOf(randomBoolean())); + } } int timeout = Math.toIntExact(timeout().millis()); RequestConfig config = RequestConfig.copy(RequestConfig.DEFAULT) diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java index d0ca0d3da1da3..1435b40730ab7 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java @@ -18,6 +18,8 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchService; import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.xpack.core.async.GetAsyncResultRequest; +import org.elasticsearch.xpack.eql.plugin.EqlAsyncGetResultAction; import org.elasticsearch.xpack.eql.plugin.EqlPlugin; import java.util.Collection; @@ -201,50 +203,28 @@ public void testPartialResults() throws Exception { // ------------------------------------------------------------------------ // event query - request = new EqlSearchRequest().indices("test-*").query("process where true").allowPartialSequenceResults(randomBoolean()); - shouldFail(request); + shouldFail("process where true"); // sequence query on both shards - request = new EqlSearchRequest().indices("test-*") - .query("sequence [process where value == 1] [process where value == 2]") - .allowPartialSequenceResults(randomBoolean()); - shouldFail(request); + shouldFail("sequence [process where value == 1] [process where value == 2]"); // sequence query on the available shard only - request = new EqlSearchRequest().indices("test-*") - .query("sequence [process where value == 1] [process where value == 3]") - .allowPartialSequenceResults(randomBoolean()); - shouldFail(request); + shouldFail("sequence [process where value == 1] [process where value == 3]"); // sequence query on the unavailable shard only - request = new EqlSearchRequest().indices("test-*") - .query("sequence [process where value == 0] [process where value == 2]") - .allowPartialSequenceResults(randomBoolean()); - shouldFail(request); + shouldFail("sequence [process where value == 0] [process where value == 2]"); // sequence query with missing event on unavailable shard. - request = new EqlSearchRequest().indices("test-*") - .query("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]") - .allowPartialSequenceResults(randomBoolean()); - shouldFail(request); + shouldFail("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]"); // sample query on both shards - request = new EqlSearchRequest().indices("test-*") - .query("sample by key [process where value == 2] [process where value == 1]") - .allowPartialSequenceResults(randomBoolean()); - shouldFail(request); + shouldFail("sample by key [process where value == 2] [process where value == 1]"); // sample query on the available shard only - request = new EqlSearchRequest().indices("test-*") - .query("sample by key [process where value == 3] [process where value == 1]") - .allowPartialSequenceResults(randomBoolean()); - shouldFail(request); + shouldFail("sample by key [process where value == 3] [process where value == 1]"); // sample query on the unavailable shard only - request = new EqlSearchRequest().indices("test-*") - .query("sample by key [process where value == 2] [process where value == 0]") - .allowPartialSequenceResults(randomBoolean()); - shouldFail(request); + shouldFail("sample by key [process where value == 2] [process where value == 0]"); // ------------------------------------------------------------------------ // same queries, with missing shards and allow_partial_search_results=true @@ -433,6 +413,74 @@ public void testPartialResults() throws Exception { assertThat(response.shardFailures()[0].index(), is("test-1")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + // ------------------------------------------------------------------------ + // same queries, this time async, with missing shards and allow_partial_search_results=true + // and default allow_partial_sequence_results (ie. false) + // ------------------------------------------------------------------------ + + // event query + response = runAsync("process where true", true); + assertThat(response.hits().events().size(), equalTo(5)); + for (int i = 0; i < 5; i++) { + assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1))); + } + assertThat(response.shardFailures().length, is(1)); + + // sequence query on both shards + response = runAsync("sequence [process where value == 1] [process where value == 2]", true); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query on the available shard only + response = runAsync("sequence [process where value == 1] [process where value == 3]", true); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query on the unavailable shard only + response = runAsync("sequence [process where value == 0] [process where value == 2]", true); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sequence query with missing event on unavailable shard. THIS IS A FALSE POSITIVE + response = runAsync( + "sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]", + true + ); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on both shards + response = runAsync("sample by key [process where value == 2] [process where value == 1]", true); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on the available shard only + response = runAsync("sample by key [process where value == 3] [process where value == 1]", true); + assertThat(response.hits().sequences().size(), equalTo(1)); + sample = response.hits().sequences().get(0); + assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); + assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + + // sample query on the unavailable shard only + response = runAsync("sample by key [process where value == 2] [process where value == 0]", true); + assertThat(response.hits().sequences().size(), equalTo(0)); + assertThat(response.shardFailures().length, is(1)); + assertThat(response.shardFailures()[0].index(), is("test-1")); + assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + // ------------------------------------------------------------------------ // same queries, with missing shards and with default xpack.eql.default_allow_partial_results=true // ------------------------------------------------------------------------ @@ -522,7 +570,29 @@ public void testPartialResults() throws Exception { } - private static void shouldFail(EqlSearchRequest request) throws InterruptedException { + private static EqlSearchResponse runAsync(String query, Boolean allowPartialSearchResults) throws InterruptedException, + ExecutionException { + EqlSearchRequest request; + EqlSearchResponse response; + request = new EqlSearchRequest().indices("test-*").query(query).waitForCompletionTimeout(TimeValue.ZERO); + if (allowPartialSearchResults != null) { + request = request.allowPartialSearchResults(allowPartialSearchResults); + } + response = client().execute(EqlSearchAction.INSTANCE, request).get(); + GetAsyncResultRequest getResultsRequest = new GetAsyncResultRequest(response.id()).setKeepAlive(TimeValue.timeValueMinutes(10)) + .setWaitForCompletionTimeout(TimeValue.timeValueMillis(10)); + response = client().execute(EqlAsyncGetResultAction.INSTANCE, getResultsRequest).get(); + return response; + } + + private static void shouldFail(String query) throws InterruptedException { + EqlSearchRequest request = new EqlSearchRequest().indices("test-*").query(query); + if (randomBoolean()) { + request = request.allowPartialSearchResults(false); + } + if (randomBoolean()) { + request = request.allowPartialSequenceResults(randomBoolean()); + } try { client().execute(EqlSearchAction.INSTANCE, request).get(); fail(); @@ -530,5 +600,4 @@ private static void shouldFail(EqlSearchRequest request) throws InterruptedExcep assertThat(e.getCause(), instanceOf(SearchPhaseExecutionException.class)); } } - } From 672e5121d06e2cde3a0f86174f50023407e89eef Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Mon, 9 Dec 2024 10:28:41 +0100 Subject: [PATCH 19/23] Refactoring --- .../eql/execution/sample/SampleIterator.java | 14 +++----------- .../eql/execution/search/BasicQueryClient.java | 2 +- .../eql/execution/sequence/TumblingWindow.java | 18 ++++++------------ .../xpack/eql/util/SearchHitUtils.java | 12 ++++++++++++ 4 files changed, 22 insertions(+), 24 deletions(-) diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java index 3b6b2ebaf2088..b9b7cfd6b615a 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java @@ -14,7 +14,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.core.TimeValue; @@ -47,6 +46,7 @@ import static org.elasticsearch.common.Strings.EMPTY_ARRAY; import static org.elasticsearch.xpack.eql.execution.assembler.SampleQueryRequest.COMPOSITE_AGG_NAME; import static org.elasticsearch.xpack.eql.execution.search.RuntimeUtils.prepareRequest; +import static org.elasticsearch.xpack.eql.util.SearchHitUtils.addShardFailures; public class SampleIterator implements Executable { @@ -155,7 +155,7 @@ private void advance(ActionListener listener) { private void queryForCompositeAggPage(ActionListener listener, final SampleQueryRequest request) { client.query(request, listener.delegateFailureAndWrap((delegate, r) -> { - addShardFailures(r); + addShardFailures(shardFailures, r); // either the fields values or the fields themselves are missing // or the filter applied on the eql query matches no documents if (r.hasAggregations() == false) { @@ -183,14 +183,6 @@ private void queryForCompositeAggPage(ActionListener listener, final Sa })); } - private void addShardFailures(SearchResponse r) { - if (r.getShardFailures() != null) { - for (ShardSearchFailure shardFailure : r.getShardFailures()) { - shardFailures.put(shardFailure.toString(), shardFailure); // TODO find a better way to deduplicate - } - } - } - protected void pushToStack(Page nextPage) { stack.push(nextPage); totalPageSize += nextPage.size; @@ -234,7 +226,7 @@ private void finalStep(ActionListener listener) { int initialSize = samples.size(); client.multiQuery(searches, listener.delegateFailureAndWrap((delegate, r) -> { for (MultiSearchResponse.Item item : r) { - addShardFailures(item.getResponse()); + addShardFailures(shardFailures, item.getResponse()); } List> sample = new ArrayList<>(maxCriteria); MultiSearchResponse.Item[] response = r.getResponses(); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java index 38ae983900206..18623c17dcffb 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java @@ -53,7 +53,7 @@ public BasicQueryClient(EqlSession eqlSession) { this.client = eqlSession.client(); this.indices = cfg.indices(); this.fetchFields = cfg.fetchFields(); - this.allowPartialSearchResults = eqlSession.configuration().allowPartialSearchResults(); + this.allowPartialSearchResults = cfg.allowPartialSearchResults(); } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java index eb39d3432aa7c..fac8788db0f95 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/TumblingWindow.java @@ -53,6 +53,7 @@ import static org.elasticsearch.action.ActionListener.runAfter; import static org.elasticsearch.xpack.eql.execution.ExecutionUtils.copySource; import static org.elasticsearch.xpack.eql.execution.search.RuntimeUtils.combineFilters; +import static org.elasticsearch.xpack.eql.util.SearchHitUtils.addShardFailures; import static org.elasticsearch.xpack.eql.util.SearchHitUtils.qualifiedIndex; /** @@ -238,7 +239,7 @@ public void checkMissingEvents(Runnable next, ActionListener listener) private void doCheckMissingEvents(List batchToCheck, MultiSearchResponse p, ActionListener listener, Runnable next) { MultiSearchResponse.Item[] responses = p.getResponses(); for (MultiSearchResponse.Item response : responses) { - addShardFailures(response.getResponse()); + addShardFailures(shardFailures, response.getResponse()); } int nextResponse = 0; for (Sequence sequence : batchToCheck) { @@ -384,7 +385,7 @@ private void advance(int stage, ActionListener listener) { * Execute the base query. */ private void baseCriterion(int baseStage, SearchResponse r, ActionListener listener) { - addShardFailures(r); + addShardFailures(shardFailures, r); SequenceCriterion base = criteria.get(baseStage); SearchHits hits = r.getHits(); @@ -756,7 +757,9 @@ private void doPayload(ActionListener listener) { log.trace("Sending payload for [{}] sequences", completed.size()); if (completed.isEmpty() || (allowPartialSequenceResults == false && shardFailures.isEmpty() == false)) { - listener.onResponse(new EmptyPayload(Type.SEQUENCE, timeTook(), shardFailures.values().toArray(new ShardSearchFailure[0]))); + listener.onResponse( + new EmptyPayload(Type.SEQUENCE, timeTook(), shardFailures.values().toArray(new ShardSearchFailure[shardFailures.size()])) + ); return; } @@ -948,13 +951,4 @@ public KeyAndOrdinal next() { }; }; } - - private void addShardFailures(SearchResponse r) { - if (r.getShardFailures() != null) { - for (ShardSearchFailure shardFailure : r.getShardFailures()) { - shardFailures.put(shardFailure.toString(), shardFailure); // TODO find a better way to deduplicate - } - } - } - } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/SearchHitUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/SearchHitUtils.java index 91795ac15b53e..2b5ec9718cfc4 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/SearchHitUtils.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/SearchHitUtils.java @@ -7,8 +7,12 @@ package org.elasticsearch.xpack.eql.util; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.search.SearchHit; +import java.util.Map; + import static org.elasticsearch.transport.RemoteClusterAware.buildRemoteIndexName; public final class SearchHitUtils { @@ -16,4 +20,12 @@ public final class SearchHitUtils { public static String qualifiedIndex(SearchHit hit) { return buildRemoteIndexName(hit.getClusterAlias(), hit.getIndex()); } + + public static void addShardFailures(Map shardFailures, SearchResponse r) { + if (r.getShardFailures() != null) { + for (ShardSearchFailure shardFailure : r.getShardFailures()) { + shardFailures.put(shardFailure.toString(), shardFailure); + } + } + } } From e1e83a68b878ef158068675aed18dd623ff6903b Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Mon, 9 Dec 2024 18:27:24 +0100 Subject: [PATCH 20/23] More tests --- .../data/endgame-shard-failures.data | 2 +- .../main/resources/test_failing_shards.toml | 95 +++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.data b/x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.data index e2d552a19a923..18a1d05656d09 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.data +++ b/x-pack/plugin/eql/qa/common/src/main/resources/data/endgame-shard-failures.data @@ -8,7 +8,7 @@ "process_name": "System Idle Process", "serial_event_id": 10000, "subtype": "create", - "timestamp": 116444736000000000, + "timestamp": 117444736000000000, "unique_pid": 1 } ] diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_failing_shards.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_failing_shards.toml index 98bd7da8bd4e8..a551c66fd48bd 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_failing_shards.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_failing_shards.toml @@ -15,6 +15,22 @@ expected_event_ids = [1] expect_shard_failures = true +[[queries]] +name = "eventQueryShardFailuresOptionalField" +query = 'process where serial_event_id == 1 and ?optional_field_default_null == null or broken == 1' +allow_partial_search_results = true +expected_event_ids = [1] +expect_shard_failures = true + + +[[queries]] +name = "eventQueryShardFailuresOptionalFieldMatching" +query = 'process where serial_event_id == 2 and ?subtype == "create" or broken == 1' +allow_partial_search_results = true +expected_event_ids = [2] +expect_shard_failures = true + + # this query doesn't touch the "broken" field, so it should not fail [[queries]] name = "sequenceQueryNoShardFailures" @@ -76,3 +92,82 @@ allow_partial_search_results = true allow_partial_sequence_results = true expected_event_ids = [1, 2] expect_shard_failures = true + + +[[queries]] +name = "sequenceQueryMissingShardsPartialResultsOptional" +query = ''' +sequence + [process where ?serial_event_id == 1 or broken == 1] + [process where serial_event_id == 2] +''' +allow_partial_search_results = true +allow_partial_sequence_results = true +expected_event_ids = [1, 2] +expect_shard_failures = true + + +[[queries]] +name = "sequenceQueryMissingShardsPartialResultsOptional2" +query = ''' +sequence with maxspan=100000d + [process where serial_event_id == 1 and ?subtype == "create" or broken == 1] + [process where serial_event_id == 2] +''' +allow_partial_search_results = true +allow_partial_sequence_results = true +expected_event_ids = [1, 2] +expect_shard_failures = true + + +[[queries]] +name = "sequenceQueryMissingShardsPartialResultsOptionalMissing" +query = ''' +sequence with maxspan=100000d + [process where serial_event_id == 1 and ?subtype == "create"] + ![process where broken == 1] + [process where serial_event_id == 2] +''' +allow_partial_search_results = true +allow_partial_sequence_results = true +expected_event_ids = [1, -1, 2] +expect_shard_failures = true + + +[[queries]] +name = "sequenceQueryMissingShardsPartialResultsOptionalMissing2" +query = ''' +sequence with maxspan=100000d + [process where serial_event_id == 1 and ?subtype == "create" or broken == 1] + ![process where broken == 1] + [process where serial_event_id == 2] +''' +allow_partial_search_results = true +allow_partial_sequence_results = true +expected_event_ids = [1, -1, 2] +expect_shard_failures = true + + +[[queries]] +name = "sampleQueryMissingShardsPartialResults" +query = ''' +sample by event_subtype_full + [process where serial_event_id == 1 or broken == 1] + [process where serial_event_id == 2] +''' +allow_partial_search_results = true +expected_event_ids = [1, 2] +expect_shard_failures = true + + +[[queries]] +name = "sampleQueryMissingShardsPartialResultsOptional" +query = ''' +sample by event_subtype_full + [process where serial_event_id == 1 and ?subtype == "create" or broken == 1] + [process where serial_event_id == 2] +''' +allow_partial_search_results = true +expected_event_ids = [1, 2] +expect_shard_failures = true + From 625acf45502c56eb46af85303addf2e19a6f4504 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Tue, 10 Dec 2024 09:36:25 +0100 Subject: [PATCH 21/23] API spec + more tests --- .../rest-api-spec/api/eql.search.json | 10 ++++ .../rest-api-spec/test/eql/10_basic.yml | 50 +++++++++++++++++++ .../eql/action/PartialSearchResultsIT.java | 8 +-- 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/eql.search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/eql.search.json index c854c44d9d761..0f9af508f4c16 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/eql.search.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/eql.search.json @@ -41,6 +41,16 @@ "type": "time", "description": "Update the time interval in which the results (partial or final) for this search will be available", "default": "5d" + }, + "allow_partial_search_results": { + "type":"boolean", + "description":"Control whether the query should keep running in case of shard failures, and return partial results", + "default":false + }, + "allow_partial_sequence_results": { + "type":"boolean", + "description":"Control whether a sequence query should return partial results or no results at all in case of shard failures. This option has effect only if [allow_partial_search_results] is true.", + "default":false } }, "body":{ diff --git a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml index 2c42dd55981be..c7974f3b584b4 100644 --- a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml +++ b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml @@ -530,6 +530,28 @@ setup: - match: {shard_failures.0.index: "eql_test_rebel"} +--- +"Execute query shard failures and with allow_partial_search_results as request param": + - do: + eql.search: + index: eql_test* + allow_partial_search_results: true + body: + query: 'process where user == "SYSTEM" and day_of_week == "Monday"' + fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"] + + - match: {timed_out: false} + - match: {hits.total.value: 1} + - match: {hits.total.relation: "eq"} + - match: {hits.events.0._source.user: "SYSTEM"} + - match: {hits.events.0._id: "1"} + - match: {hits.events.0.fields.@timestamp: ["1580733296000"]} + - match: {hits.events.0.fields.id: [123]} + - match: {hits.events.0.fields.valid: [false]} + - match: {hits.events.0.fields.day_of_week: ["Monday"]} + - match: {shard_failures.0.index: "eql_test_rebel"} + + --- "Execute sequence with shard failures and allow_partial_search_results=true": - do: @@ -571,3 +593,31 @@ setup: - match: {hits.sequences.0.events.1.fields.valid: [true]} - match: {hits.sequences.0.events.1.fields.day_of_week: ["Tuesday"]} - match: {shard_failures.0.index: "eql_test_rebel"} + + +--- +"Execute sequence with shard failures, allow_partial_search_results=true and allow_partial_sequence_results=true as query params": + - do: + eql.search: + index: eql_test* + allow_partial_search_results: true + allow_partial_sequence_results: true + body: + query: 'sequence [process where user == "SYSTEM" and day_of_week == "Monday"] [process where user == "SYSTEM" and day_of_week == "Tuesday"]' + fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"] + + - match: {timed_out: false} + - match: {hits.total.value: 1} + - match: {hits.total.relation: "eq"} + - match: {hits.sequences.0.events.0._source.user: "SYSTEM"} + - match: {hits.sequences.0.events.0._id: "1"} + - match: {hits.sequences.0.events.0.fields.@timestamp: ["1580733296000"]} + - match: {hits.sequences.0.events.0.fields.id: [123]} + - match: {hits.sequences.0.events.0.fields.valid: [false]} + - match: {hits.sequences.0.events.0.fields.day_of_week: ["Monday"]} + - match: {hits.sequences.0.events.1._id: "2"} + - match: {hits.sequences.0.events.1.fields.@timestamp: ["1580819696000"]} + - match: {hits.sequences.0.events.1.fields.id: [123]} + - match: {hits.sequences.0.events.1.fields.valid: [true]} + - match: {hits.sequences.0.events.1.fields.day_of_week: ["Tuesday"]} + - match: {shard_failures.0.index: "eql_test_rebel"} diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java index 1435b40730ab7..46f94db09aad0 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java @@ -579,9 +579,11 @@ private static EqlSearchResponse runAsync(String query, Boolean allowPartialSear request = request.allowPartialSearchResults(allowPartialSearchResults); } response = client().execute(EqlSearchAction.INSTANCE, request).get(); - GetAsyncResultRequest getResultsRequest = new GetAsyncResultRequest(response.id()).setKeepAlive(TimeValue.timeValueMinutes(10)) - .setWaitForCompletionTimeout(TimeValue.timeValueMillis(10)); - response = client().execute(EqlAsyncGetResultAction.INSTANCE, getResultsRequest).get(); + while (response.isRunning()) { + GetAsyncResultRequest getResultsRequest = new GetAsyncResultRequest(response.id()).setKeepAlive(TimeValue.timeValueMinutes(10)) + .setWaitForCompletionTimeout(TimeValue.timeValueMillis(10)); + response = client().execute(EqlAsyncGetResultAction.INSTANCE, getResultsRequest).get(); + } return response; } From 984fe024364255784ddd7020f29e62cc8a97c3fc Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Tue, 10 Dec 2024 09:51:40 +0100 Subject: [PATCH 22/23] Fix conflict --- .../org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java index 0e67049729e3c..eb418be32d55d 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java @@ -41,7 +41,7 @@ protected final Client localClient() { } @Override - protected Collection remoteClusterAlias() { + protected List remoteClusterAlias() { return List.of(REMOTE_CLUSTER); } From f58fd1c84e6b034f94e8cc032aecd2c800bfeda1 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Mon, 16 Dec 2024 12:07:13 +0100 Subject: [PATCH 23/23] Refactor tests --- .../elasticsearch/test/ESIntegTestCase.java | 2 +- .../test/eql/BaseEqlSpecTestCase.java | 8 +- .../xpack/eql/qa/mixed_node/EqlSearchIT.java | 11 +- .../xpack/eql/action/CCSPartialResultsIT.java | 193 ++++++++++--- .../eql/action/PartialSearchResultsIT.java | 253 +++++++++++++++--- .../eql/execution/search/RuntimeUtils.java | 8 +- 6 files changed, 396 insertions(+), 79 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index d2ca4078873f6..b9a097b4e76f3 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -2432,7 +2432,7 @@ public static void afterClass() throws Exception { /** * After the cluster is stopped, there are a few netty threads that can linger, so we make sure we don't leak any tasks on them. */ - public static void awaitGlobalNettyThreadsFinish() throws Exception { + static void awaitGlobalNettyThreadsFinish() throws Exception { // Don't use GlobalEventExecutor#awaitInactivity. It will waste up to 1s for every call and we expect no tasks queued for it // except for the odd scheduled shutdown task. assertBusy(() -> assertEquals(0, GlobalEventExecutor.INSTANCE.pendingTasks())); diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java index 483f81201e374..3557114e2f4c7 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java @@ -194,7 +194,7 @@ protected ObjectPath runQuery(String index, String query) throws Exception { builder.field("allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults)); } } else { - // these will be overwritten by the query params, that have higher priority than the body params + // these will be overwritten by the path params, that have higher priority than the query (JSON body) params if (allowPartialSearchResults != null) { builder.field("allow_partial_search_results", randomBoolean()); } @@ -203,6 +203,8 @@ protected ObjectPath runQuery(String index, String query) throws Exception { } } } else { + // Tests that don't specify a setting for these parameters should always pass. + // These params should be irrelevant. if (randomBoolean()) { builder.field("allow_partial_search_results", randomBoolean()); } @@ -225,6 +227,8 @@ protected ObjectPath runQuery(String index, String query) throws Exception { } } } else { + // Tests that don't specify a setting for these parameters should always pass. + // These params should be irrelevant. if (randomBoolean()) { request.addParameter("allow_partial_search_results", String.valueOf(randomBoolean())); } @@ -253,6 +257,8 @@ private void assertShardFailures(Object shardFailures) { } else { assertNull(shardFailures); } + } else { + assertNull(shardFailures); } } diff --git a/x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java b/x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java index 2a29572374fa8..60c7fb1c7ad25 100644 --- a/x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java +++ b/x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java @@ -407,7 +407,16 @@ private void assertMultiValueFunctionQuery( for (int id : ids) { eventIds.add(String.valueOf(id)); } - request.setJsonEntity("{\"query\":\"" + query + "\"}"); + + StringBuilder payload = new StringBuilder("{\"query\":\"" + query + "\""); + if (randomBoolean()) { + payload.append(", \"allow_partial_search_results\": true"); + } + if (randomBoolean()) { + payload.append(", \"allow_partial_sequence_results\": true"); + } + payload.append("}"); + request.setJsonEntity(payload.toString()); assertResponse(query, eventIds, runEql(client, request)); testedFunctions.add(functionName); } diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java index eb418be32d55d..da6bb6180428b 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/CCSPartialResultsIT.java @@ -50,7 +50,11 @@ protected boolean reuseClusters() { return false; } - public void testPartialResults() throws ExecutionException, InterruptedException, IOException { + /** + * + * @return remote node name + */ + private String createSchema() { final Client remoteClient = client(REMOTE_CLUSTER); final String remoteNode = cluster(REMOTE_CLUSTER).startDataOnlyNode(); final String remoteNode2 = cluster(REMOTE_CLUSTER).startDataOnlyNode(); @@ -99,10 +103,15 @@ public void testPartialResults() throws ExecutionException, InterruptedException } remoteClient.admin().indices().prepareRefresh().get(); + return remoteNode; + } - // ------------------------------------------------------------------------ - // queries with full cluster (no missing shards) - // ------------------------------------------------------------------------ + // ------------------------------------------------------------------------ + // queries with full cluster (no missing shards) + // ------------------------------------------------------------------------ + + public void testNoFailures() throws ExecutionException, InterruptedException, IOException { + createSchema(); // event query var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") @@ -197,32 +206,47 @@ public void testPartialResults() throws ExecutionException, InterruptedException assertThat(sample.events().get(1).toString(), containsString("\"value\" : 0")); assertThat(response.shardFailures().length, is(0)); + } + + // ------------------------------------------------------------------------ + // same queries, with missing shards and allow_partial_search_results=true + // and allow_partial_sequence_result=true + // ------------------------------------------------------------------------ + + public void testAllowPartialSearchAndSequence_event() throws ExecutionException, InterruptedException, IOException { + var remoteNode = createSchema(); // ------------------------------------------------------------------------ // stop one of the nodes, make one of the shards unavailable // ------------------------------------------------------------------------ cluster(REMOTE_CLUSTER).stopNode(remoteNode); - // ------------------------------------------------------------------------ - // same queries, with missing shards and allow_partial_search_results=true - // and allow_partial_sequence_result=true - // ------------------------------------------------------------------------ - // event query - request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("process where true").allowPartialSearchResults(true); - response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("process where true") + .allowPartialSearchResults(true); + var response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().events().size(), equalTo(5)); for (int i = 0; i < 5; i++) { assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1))); } assertThat(response.shardFailures().length, is(1)); + } + + public void testAllowPartialSearchAndSequence_sequence() throws ExecutionException, InterruptedException, IOException { + var remoteNode = createSchema(); + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + cluster(REMOTE_CLUSTER).stopNode(remoteNode); // sequence query on both shards - request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") .query("sequence [process where value == 1] [process where value == 2]") .allowPartialSearchResults(true) .allowPartialSequenceResults(true); - response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + var response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1-remote")); @@ -235,7 +259,7 @@ public void testPartialResults() throws ExecutionException, InterruptedException .allowPartialSequenceResults(true); response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); - sequence = response.hits().sequences().get(0); + var sequence = response.hits().sequences().get(0); assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 3")); assertThat(response.shardFailures().length, is(1)); @@ -267,12 +291,22 @@ public void testPartialResults() throws ExecutionException, InterruptedException assertThat(response.shardFailures()[0].index(), is("test-1-remote")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + } + + public void testAllowPartialSearchAndSequence_sample() throws ExecutionException, InterruptedException, IOException { + var remoteNode = createSchema(); + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + cluster(REMOTE_CLUSTER).stopNode(remoteNode); + // sample query on both shards - request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") .query("sample by key [process where value == 2] [process where value == 1]") .allowPartialSearchResults(true) .allowPartialSequenceResults(true); - response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + var response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1-remote")); @@ -285,7 +319,7 @@ public void testPartialResults() throws ExecutionException, InterruptedException .allowPartialSequenceResults(true); response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); - sample = response.hits().sequences().get(0); + var sample = response.hits().sequences().get(0); assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); assertThat(response.shardFailures().length, is(1)); @@ -303,25 +337,47 @@ public void testPartialResults() throws ExecutionException, InterruptedException assertThat(response.shardFailures()[0].index(), is("test-1-remote")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + } + + // ------------------------------------------------------------------------ + // same queries, with missing shards and allow_partial_search_results=true + // and default allow_partial_sequence_results (ie. false) + // ------------------------------------------------------------------------ + + public void testAllowPartialSearch_event() throws ExecutionException, InterruptedException, IOException { + var remoteNode = createSchema(); // ------------------------------------------------------------------------ - // same queries, with missing shards and allow_partial_search_results=true - // and default allow_partial_sequence_results (ie. false) + // stop one of the nodes, make one of the shards unavailable // ------------------------------------------------------------------------ + cluster(REMOTE_CLUSTER).stopNode(remoteNode); + // event query - request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("process where true").allowPartialSearchResults(true); - response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + .query("process where true") + .allowPartialSearchResults(true); + var response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().events().size(), equalTo(5)); for (int i = 0; i < 5; i++) { assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1))); } assertThat(response.shardFailures().length, is(1)); + } + + public void testAllowPartialSearch_sequence() throws ExecutionException, InterruptedException, IOException { + var remoteNode = createSchema(); + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + cluster(REMOTE_CLUSTER).stopNode(remoteNode); + // sequence query on both shards - request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") .query("sequence [process where value == 1] [process where value == 2]") .allowPartialSearchResults(true); - response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + var response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1-remote")); @@ -357,11 +413,21 @@ public void testPartialResults() throws ExecutionException, InterruptedException assertThat(response.shardFailures()[0].index(), is("test-1-remote")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + } + + public void testAllowPartialSearch_sample() throws ExecutionException, InterruptedException, IOException { + var remoteNode = createSchema(); + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + cluster(REMOTE_CLUSTER).stopNode(remoteNode); + // sample query on both shards - request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") .query("sample by key [process where value == 2] [process where value == 1]") .allowPartialSearchResults(true); - response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + var response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1-remote")); @@ -373,7 +439,7 @@ public void testPartialResults() throws ExecutionException, InterruptedException .allowPartialSearchResults(true); response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); - sample = response.hits().sequences().get(0); + var sample = response.hits().sequences().get(0); assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); assertThat(response.shardFailures().length, is(1)); @@ -390,10 +456,20 @@ public void testPartialResults() throws ExecutionException, InterruptedException assertThat(response.shardFailures()[0].index(), is("test-1-remote")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + } + + // ------------------------------------------------------------------------ + // same queries, with missing shards and with default xpack.eql.default_allow_partial_results=true + // ------------------------------------------------------------------------ + + public void testClusterSetting_event() throws ExecutionException, InterruptedException, IOException { + var remoteNode = createSchema(); // ------------------------------------------------------------------------ - // same queries, with missing shards and with default xpack.eql.default_allow_partial_results=true + // stop one of the nodes, make one of the shards unavailable // ------------------------------------------------------------------------ + cluster(REMOTE_CLUSTER).stopNode(remoteNode); + cluster(REMOTE_CLUSTER).client() .execute( ClusterUpdateSettingsAction.INSTANCE, @@ -404,18 +480,42 @@ public void testPartialResults() throws ExecutionException, InterruptedException .get(); // event query - request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("process where true"); - response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*").query("process where true"); + var response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().events().size(), equalTo(5)); for (int i = 0; i < 5; i++) { assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1))); } assertThat(response.shardFailures().length, is(1)); + localClient().execute( + ClusterUpdateSettingsAction.INSTANCE, + new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings( + Settings.builder().putNull(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey()) + ) + ).get(); + } + + public void testClusterSetting_sequence() throws ExecutionException, InterruptedException, IOException { + var remoteNode = createSchema(); + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + cluster(REMOTE_CLUSTER).stopNode(remoteNode); + + cluster(REMOTE_CLUSTER).client() + .execute( + ClusterUpdateSettingsAction.INSTANCE, + new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings( + Settings.builder().put(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey(), true) + ) + ) + .get(); // sequence query on both shards - request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") .query("sequence [process where value == 1] [process where value == 2]"); - response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + var response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1-remote")); @@ -448,10 +548,35 @@ public void testPartialResults() throws ExecutionException, InterruptedException assertThat(response.shardFailures()[0].index(), is("test-1-remote")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + localClient().execute( + ClusterUpdateSettingsAction.INSTANCE, + new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings( + Settings.builder().putNull(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey()) + ) + ).get(); + } + + public void testClusterSetting_sample() throws ExecutionException, InterruptedException, IOException { + var remoteNode = createSchema(); + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + cluster(REMOTE_CLUSTER).stopNode(remoteNode); + + cluster(REMOTE_CLUSTER).client() + .execute( + ClusterUpdateSettingsAction.INSTANCE, + new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings( + Settings.builder().put(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey(), true) + ) + ) + .get(); + // sample query on both shards - request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") + var request = new EqlSearchRequest().indices(REMOTE_CLUSTER + ":test-*") .query("sample by key [process where value == 2] [process where value == 1]"); - response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); + var response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1-remote")); @@ -462,7 +587,7 @@ public void testPartialResults() throws ExecutionException, InterruptedException .query("sample by key [process where value == 3] [process where value == 1]"); response = localClient().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); - sample = response.hits().sequences().get(0); + var sample = response.hits().sequences().get(0); assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); assertThat(response.shardFailures().length, is(1)); diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java index 46f94db09aad0..9048d11f4eddf 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/PartialSearchResultsIT.java @@ -47,7 +47,11 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .build(); } - public void testPartialResults() throws Exception { + /** + * + * @return node name where the first index is + */ + private String createSchema() { internalCluster().ensureAtLeastNumDataNodes(2); final List dataNodes = internalCluster().clusterService() .state() @@ -93,6 +97,11 @@ public void testPartialResults() throws Exception { .get(); } refresh(); + return assignedNodeForIndex1; + } + + public void testNoFailures() throws Exception { + createSchema(); // ------------------------------------------------------------------------ // queries with full cluster (no missing shards) @@ -191,19 +200,33 @@ public void testPartialResults() throws Exception { assertThat(sample.events().get(1).toString(), containsString("\"value\" : 0")); assertThat(response.shardFailures().length, is(0)); + } + + // ------------------------------------------------------------------------ + // same queries, with missing shards. Let them fail + // allow_partial_sequence_results has no effect if allow_partial_sequence_results is not set to true. + // ------------------------------------------------------------------------ + + public void testFailures_event() throws Exception { + final String assignedNodeForIndex1 = createSchema(); // ------------------------------------------------------------------------ // stop one of the nodes, make one of the shards unavailable // ------------------------------------------------------------------------ internalCluster().stopNode(assignedNodeForIndex1); + // event query + shouldFail("process where true"); + + } + + public void testFailures_sequence() throws Exception { + final String assignedNodeForIndex1 = createSchema(); // ------------------------------------------------------------------------ - // same queries, with missing shards. Let them fail - // allow_partial_sequence_results has no effect if allow_partial_sequence_results is not set to true. + // stop one of the nodes, make one of the shards unavailable // ------------------------------------------------------------------------ - // event query - shouldFail("process where true"); + internalCluster().stopNode(assignedNodeForIndex1); // sequence query on both shards shouldFail("sequence [process where value == 1] [process where value == 2]"); @@ -216,6 +239,15 @@ public void testPartialResults() throws Exception { // sequence query with missing event on unavailable shard. shouldFail("sequence with maxspan=10s [process where value == 1] ![process where value == 2] [process where value == 3]"); + } + + public void testFailures_sample() throws Exception { + final String assignedNodeForIndex1 = createSchema(); + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + internalCluster().stopNode(assignedNodeForIndex1); // sample query on both shards shouldFail("sample by key [process where value == 2] [process where value == 1]"); @@ -226,26 +258,46 @@ public void testPartialResults() throws Exception { // sample query on the unavailable shard only shouldFail("sample by key [process where value == 2] [process where value == 0]"); + } + + // ------------------------------------------------------------------------ + // same queries, with missing shards and allow_partial_search_results=true + // and allow_partial_sequence_result=true + // ------------------------------------------------------------------------ + + public void testAllowPartialSearchAndSequenceResults_event() throws Exception { + final String assignedNodeForIndex1 = createSchema(); // ------------------------------------------------------------------------ - // same queries, with missing shards and allow_partial_search_results=true - // and allow_partial_sequence_result=true + // stop one of the nodes, make one of the shards unavailable // ------------------------------------------------------------------------ + internalCluster().stopNode(assignedNodeForIndex1); + // event query - request = new EqlSearchRequest().indices("test-*").query("process where true").allowPartialSearchResults(true); - response = client().execute(EqlSearchAction.INSTANCE, request).get(); + var request = new EqlSearchRequest().indices("test-*").query("process where true").allowPartialSearchResults(true); + var response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().events().size(), equalTo(5)); for (int i = 0; i < 5; i++) { assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1))); } assertThat(response.shardFailures().length, is(1)); + } + + public void testAllowPartialSearchAndSequenceResults_sequence() throws Exception { + final String assignedNodeForIndex1 = createSchema(); + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + internalCluster().stopNode(assignedNodeForIndex1); + // sequence query on both shards - request = new EqlSearchRequest().indices("test-*") + var request = new EqlSearchRequest().indices("test-*") .query("sequence [process where value == 1] [process where value == 2]") .allowPartialSearchResults(true) .allowPartialSequenceResults(true); - response = client().execute(EqlSearchAction.INSTANCE, request).get(); + var response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1")); @@ -258,7 +310,7 @@ public void testPartialResults() throws Exception { .allowPartialSequenceResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); - sequence = response.hits().sequences().get(0); + var sequence = response.hits().sequences().get(0); assertThat(sequence.events().get(0).toString(), containsString("\"value\" : 1")); assertThat(sequence.events().get(1).toString(), containsString("\"value\" : 3")); assertThat(response.shardFailures().length, is(1)); @@ -290,12 +342,22 @@ public void testPartialResults() throws Exception { assertThat(response.shardFailures()[0].index(), is("test-1")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + } + + public void testAllowPartialSearchAndSequenceResults_sample() throws Exception { + final String assignedNodeForIndex1 = createSchema(); + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + internalCluster().stopNode(assignedNodeForIndex1); + // sample query on both shards - request = new EqlSearchRequest().indices("test-*") + var request = new EqlSearchRequest().indices("test-*") .query("sample by key [process where value == 2] [process where value == 1]") .allowPartialSearchResults(true) .allowPartialSequenceResults(true); - response = client().execute(EqlSearchAction.INSTANCE, request).get(); + var response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1")); @@ -308,7 +370,7 @@ public void testPartialResults() throws Exception { .allowPartialSequenceResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); - sample = response.hits().sequences().get(0); + var sample = response.hits().sequences().get(0); assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); assertThat(response.shardFailures().length, is(1)); @@ -326,25 +388,45 @@ public void testPartialResults() throws Exception { assertThat(response.shardFailures()[0].index(), is("test-1")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + } + + // ------------------------------------------------------------------------ + // same queries, with missing shards and allow_partial_search_results=true + // and default allow_partial_sequence_results (ie. false) + // ------------------------------------------------------------------------ + + public void testAllowPartialSearchResults_event() throws Exception { + final String assignedNodeForIndex1 = createSchema(); // ------------------------------------------------------------------------ - // same queries, with missing shards and allow_partial_search_results=true - // and default allow_partial_sequence_results (ie. false) + // stop one of the nodes, make one of the shards unavailable // ------------------------------------------------------------------------ + internalCluster().stopNode(assignedNodeForIndex1); + // event query - request = new EqlSearchRequest().indices("test-*").query("process where true").allowPartialSearchResults(true); - response = client().execute(EqlSearchAction.INSTANCE, request).get(); + var request = new EqlSearchRequest().indices("test-*").query("process where true").allowPartialSearchResults(true); + var response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().events().size(), equalTo(5)); for (int i = 0; i < 5; i++) { assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1))); } assertThat(response.shardFailures().length, is(1)); + } + + public void testAllowPartialSearchResults_sequence() throws Exception { + final String assignedNodeForIndex1 = createSchema(); + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + internalCluster().stopNode(assignedNodeForIndex1); + // sequence query on both shards - request = new EqlSearchRequest().indices("test-*") + var request = new EqlSearchRequest().indices("test-*") .query("sequence [process where value == 1] [process where value == 2]") .allowPartialSearchResults(true); - response = client().execute(EqlSearchAction.INSTANCE, request).get(); + var response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1")); @@ -380,11 +462,21 @@ public void testPartialResults() throws Exception { assertThat(response.shardFailures()[0].index(), is("test-1")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + } + + public void testAllowPartialSearchResults_sample() throws Exception { + final String assignedNodeForIndex1 = createSchema(); + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + internalCluster().stopNode(assignedNodeForIndex1); + // sample query on both shards - request = new EqlSearchRequest().indices("test-*") + var request = new EqlSearchRequest().indices("test-*") .query("sample by key [process where value == 2] [process where value == 1]") .allowPartialSearchResults(true); - response = client().execute(EqlSearchAction.INSTANCE, request).get(); + var response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1")); @@ -396,7 +488,7 @@ public void testPartialResults() throws Exception { .allowPartialSearchResults(true); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); - sample = response.hits().sequences().get(0); + var sample = response.hits().sequences().get(0); assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); assertThat(response.shardFailures().length, is(1)); @@ -413,21 +505,41 @@ public void testPartialResults() throws Exception { assertThat(response.shardFailures()[0].index(), is("test-1")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + } + + // ------------------------------------------------------------------------ + // same queries, this time async, with missing shards and allow_partial_search_results=true + // and default allow_partial_sequence_results (ie. false) + // ------------------------------------------------------------------------ + + public void testAsyncAllowPartialSearchResults_event() throws Exception { + final String assignedNodeForIndex1 = createSchema(); // ------------------------------------------------------------------------ - // same queries, this time async, with missing shards and allow_partial_search_results=true - // and default allow_partial_sequence_results (ie. false) + // stop one of the nodes, make one of the shards unavailable // ------------------------------------------------------------------------ + internalCluster().stopNode(assignedNodeForIndex1); + // event query - response = runAsync("process where true", true); + var response = runAsync("process where true", true); assertThat(response.hits().events().size(), equalTo(5)); for (int i = 0; i < 5; i++) { assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1))); } assertThat(response.shardFailures().length, is(1)); + } + + public void testAsyncAllowPartialSearchResults_sequence() throws Exception { + final String assignedNodeForIndex1 = createSchema(); + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + internalCluster().stopNode(assignedNodeForIndex1); + // sequence query on both shards - response = runAsync("sequence [process where value == 1] [process where value == 2]", true); + var response = runAsync("sequence [process where value == 1] [process where value == 2]", true); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1")); @@ -457,8 +569,17 @@ public void testPartialResults() throws Exception { assertThat(response.shardFailures()[0].index(), is("test-1")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + } + + public void testAsyncAllowPartialSearchResults_sample() throws Exception { + final String assignedNodeForIndex1 = createSchema(); + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + internalCluster().stopNode(assignedNodeForIndex1); // sample query on both shards - response = runAsync("sample by key [process where value == 2] [process where value == 1]", true); + var response = runAsync("sample by key [process where value == 2] [process where value == 1]", true); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1")); @@ -467,7 +588,7 @@ public void testPartialResults() throws Exception { // sample query on the available shard only response = runAsync("sample by key [process where value == 3] [process where value == 1]", true); assertThat(response.hits().sequences().size(), equalTo(1)); - sample = response.hits().sequences().get(0); + var sample = response.hits().sequences().get(0); assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); assertThat(response.shardFailures().length, is(1)); @@ -481,10 +602,20 @@ public void testPartialResults() throws Exception { assertThat(response.shardFailures()[0].index(), is("test-1")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + } + + // ------------------------------------------------------------------------ + // same queries, with missing shards and with default xpack.eql.default_allow_partial_results=true + // ------------------------------------------------------------------------ + + public void testClusterSetting_event() throws Exception { + final String assignedNodeForIndex1 = createSchema(); // ------------------------------------------------------------------------ - // same queries, with missing shards and with default xpack.eql.default_allow_partial_results=true + // stop one of the nodes, make one of the shards unavailable // ------------------------------------------------------------------------ + internalCluster().stopNode(assignedNodeForIndex1); + client().execute( ClusterUpdateSettingsAction.INSTANCE, new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings( @@ -493,17 +624,39 @@ public void testPartialResults() throws Exception { ).get(); // event query - request = new EqlSearchRequest().indices("test-*").query("process where true"); - response = client().execute(EqlSearchAction.INSTANCE, request).get(); + var request = new EqlSearchRequest().indices("test-*").query("process where true"); + var response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().events().size(), equalTo(5)); for (int i = 0; i < 5; i++) { assertThat(response.hits().events().get(i).toString(), containsString("\"value\" : " + (i * 2 + 1))); } assertThat(response.shardFailures().length, is(1)); + client().execute( + ClusterUpdateSettingsAction.INSTANCE, + new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings( + Settings.builder().putNull(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey()) + ) + ).get(); + } + + public void testClusterSetting_sequence() throws Exception { + final String assignedNodeForIndex1 = createSchema(); + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + internalCluster().stopNode(assignedNodeForIndex1); + + client().execute( + ClusterUpdateSettingsAction.INSTANCE, + new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings( + Settings.builder().put(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey(), true) + ) + ).get(); // sequence query on both shards - request = new EqlSearchRequest().indices("test-*").query("sequence [process where value == 1] [process where value == 2]"); - response = client().execute(EqlSearchAction.INSTANCE, request).get(); + var request = new EqlSearchRequest().indices("test-*").query("sequence [process where value == 1] [process where value == 2]"); + var response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1")); @@ -534,9 +687,32 @@ public void testPartialResults() throws Exception { assertThat(response.shardFailures()[0].index(), is("test-1")); assertThat(response.shardFailures()[0].reason(), containsString("NoShardAvailableActionException")); + client().execute( + ClusterUpdateSettingsAction.INSTANCE, + new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings( + Settings.builder().putNull(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey()) + ) + ).get(); + } + + public void testClusterSetting_sample() throws Exception { + final String assignedNodeForIndex1 = createSchema(); + // ------------------------------------------------------------------------ + // stop one of the nodes, make one of the shards unavailable + // ------------------------------------------------------------------------ + + internalCluster().stopNode(assignedNodeForIndex1); + + client().execute( + ClusterUpdateSettingsAction.INSTANCE, + new ClusterUpdateSettingsRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).persistentSettings( + Settings.builder().put(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey(), true) + ) + ).get(); + // sample query on both shards - request = new EqlSearchRequest().indices("test-*").query("sample by key [process where value == 2] [process where value == 1]"); - response = client().execute(EqlSearchAction.INSTANCE, request).get(); + var request = new EqlSearchRequest().indices("test-*").query("sample by key [process where value == 2] [process where value == 1]"); + var response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(0)); assertThat(response.shardFailures().length, is(1)); assertThat(response.shardFailures()[0].index(), is("test-1")); @@ -546,7 +722,7 @@ public void testPartialResults() throws Exception { request = new EqlSearchRequest().indices("test-*").query("sample by key [process where value == 3] [process where value == 1]"); response = client().execute(EqlSearchAction.INSTANCE, request).get(); assertThat(response.hits().sequences().size(), equalTo(1)); - sample = response.hits().sequences().get(0); + var sample = response.hits().sequences().get(0); assertThat(sample.events().get(0).toString(), containsString("\"value\" : 3")); assertThat(sample.events().get(1).toString(), containsString("\"value\" : 1")); assertThat(response.shardFailures().length, is(1)); @@ -567,7 +743,6 @@ public void testPartialResults() throws Exception { Settings.builder().putNull(EqlPlugin.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS.getKey()) ) ).get(); - } private static EqlSearchResponse runAsync(String query, Boolean allowPartialSearchResults) throws InterruptedException, diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java index 94504cad02633..92af8c562f840 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java @@ -85,9 +85,11 @@ public static ActionListener multiSearchLogListener( SearchResponse response = item.getResponse(); if (failure == null) { - ShardSearchFailure[] failures = response.getShardFailures(); - if (CollectionUtils.isEmpty(failures) == false && allowPartialSearchResults == false) { - failure = new EqlIllegalArgumentException(failures[0].reason(), failures[0].getCause()); + if (allowPartialSearchResults == false) { + ShardSearchFailure[] failures = response.getShardFailures(); + if (CollectionUtils.isEmpty(failures) == false) { + failure = new EqlIllegalArgumentException(failures[0].reason(), failures[0].getCause()); + } } } if (failure != null) {