Skip to content

Commit 9d017d6

Browse files
luigidellaquilaluigidellaquila
authored
[8.x] EQL: set allow_partial_search_results=true by default (#120267) (#120887)
1 parent 4cd345f commit 9d017d6

File tree

18 files changed

+334
-237
lines changed

18 files changed

+334
-237
lines changed

‎docs/changelog/120267.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
pr: 120267
2+
summary: Set allow_partial_search_results=true by default
3+
area: EQL
4+
type: breaking
5+
issues: []
6+
breaking:
7+
title: Set allow_partial_search_results=true by default
8+
area: REST API
9+
details:
10+
Before this change, in case of shard failures, EQL queries always returned an error.
11+
With this change, they will keep running and will return partial results.
12+
impact:
13+
EQL queries that would previously fail due to shard failures, will now succeed and return partial results.
14+
The previous defaults can be restored by setting `xpack.eql.default_allow_partial_results` cluster setting to `false`
15+
or setting with `allow_partial_search_results` to `false` in the query request.
16+
notable: false

‎docs/reference/eql/eql-search-api.asciidoc

+2-2
Original file line numberDiff line numberDiff line change
@@ -102,10 +102,10 @@ If `false`, the request returns an error if one or more shards involved in the q
102102
If `true`, the query is executed only on the available shards, ignoring shard request timeouts and
103103
<<shard-failures,shard failures>>.
104104
+
105-
Defaults to `false`.
105+
Defaults to `true`.
106106
+
107107
To override the default for this field, set the
108-
`xpack.eql.default_allow_partial_results` cluster setting to `true`.
108+
`xpack.eql.default_allow_partial_results` cluster setting to `false`.
109109

110110

111111
[IMPORTANT]

‎rest-api-spec/src/main/resources/rest-api-spec/api/eql.search.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
"allow_partial_search_results": {
4646
"type":"boolean",
4747
"description":"Control whether the query should keep running in case of shard failures, and return partial results",
48-
"default":false
48+
"default":true
4949
},
5050
"allow_partial_sequence_results": {
5151
"type":"boolean",

‎x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java

+24-30
Original file line numberDiff line numberDiff line change
@@ -187,29 +187,29 @@ protected ObjectPath runQuery(String index, String query) throws Exception {
187187
builder.field("max_samples_per_key", maxSamplesPerKey);
188188
}
189189
boolean allowPartialResultsInBody = randomBoolean();
190-
if (allowPartialSearchResults != null) {
191-
if (allowPartialResultsInBody) {
190+
191+
if (allowPartialResultsInBody) {
192+
if (allowPartialSearchResults != null) {
192193
builder.field("allow_partial_search_results", String.valueOf(allowPartialSearchResults));
193-
if (allowPartialSequenceResults != null) {
194-
builder.field("allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults));
195-
}
196-
} else {
197-
// these will be overwritten by the path params, that have higher priority than the query (JSON body) params
198-
if (allowPartialSearchResults != null) {
199-
builder.field("allow_partial_search_results", randomBoolean());
200-
}
201-
if (allowPartialSequenceResults != null) {
202-
builder.field("allow_partial_sequence_results", randomBoolean());
203-
}
194+
} else if (randomBoolean()) {
195+
builder.field("allow_partial_search_results", true);
196+
}
197+
if (allowPartialSequenceResults != null) {
198+
builder.field("allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults));
199+
} else if (randomBoolean()) {
200+
builder.field("allow_partial_sequence_results", false);
204201
}
205202
} else {
206-
// Tests that don't specify a setting for these parameters should always pass.
207-
// These params should be irrelevant.
208-
if (randomBoolean()) {
203+
// these will be overwritten by the path params, that have higher priority than the query (JSON body) params
204+
if (allowPartialSearchResults != null) {
209205
builder.field("allow_partial_search_results", randomBoolean());
206+
} else if (randomBoolean()) {
207+
builder.field("allow_partial_search_results", true);
210208
}
211-
if (randomBoolean()) {
209+
if (allowPartialSequenceResults != null) {
212210
builder.field("allow_partial_sequence_results", randomBoolean());
211+
} else if (randomBoolean()) {
212+
builder.field("allow_partial_sequence_results", false);
213213
}
214214
}
215215
builder.endObject();
@@ -219,23 +219,17 @@ protected ObjectPath runQuery(String index, String query) throws Exception {
219219
if (ccsMinimizeRoundtrips != null) {
220220
request.addParameter("ccs_minimize_roundtrips", ccsMinimizeRoundtrips.toString());
221221
}
222-
if (allowPartialSearchResults != null) {
223-
if (allowPartialResultsInBody == false) {
222+
if (allowPartialResultsInBody == false) {
223+
if (allowPartialSearchResults != null) {
224224
request.addParameter("allow_partial_search_results", String.valueOf(allowPartialSearchResults));
225-
if (allowPartialSequenceResults != null) {
226-
request.addParameter("allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults));
227-
}
225+
} else if (randomBoolean()) {
226+
request.addParameter("allow_partial_search_results", String.valueOf(true));
228227
}
229-
} else {
230-
// Tests that don't specify a setting for these parameters should always pass.
231-
// These params should be irrelevant.
232-
if (randomBoolean()) {
233-
request.addParameter("allow_partial_search_results", String.valueOf(randomBoolean()));
234-
}
235-
if (randomBoolean()) {
236-
request.addParameter("allow_partial_sequence_results", String.valueOf(randomBoolean()));
228+
if (allowPartialSequenceResults != null) {
229+
request.addParameter("allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults));
237230
}
238231
}
232+
239233
int timeout = Math.toIntExact(timeout().millis());
240234
RequestConfig config = RequestConfig.copy(RequestConfig.DEFAULT)
241235
.setConnectionRequestTimeout(timeout)

‎x-pack/plugin/eql/qa/common/src/main/resources/test_failing_shards.toml

+41-8
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,13 @@ expect_shard_failures = false
1010
[[queries]]
1111
name = "eventQueryShardFailures"
1212
query = 'process where serial_event_id == 1 or broken == 1'
13+
expected_event_ids = [1]
14+
expect_shard_failures = true
15+
16+
17+
[[queries]]
18+
name = "eventQueryShardFailuresTrue"
19+
query = 'process where serial_event_id == 1 or broken == 1'
1320
allow_partial_search_results = true
1421
expected_event_ids = [1]
1522
expect_shard_failures = true
@@ -18,15 +25,13 @@ expect_shard_failures = true
1825
[[queries]]
1926
name = "eventQueryShardFailuresOptionalField"
2027
query = 'process where serial_event_id == 1 and ?optional_field_default_null == null or broken == 1'
21-
allow_partial_search_results = true
2228
expected_event_ids = [1]
2329
expect_shard_failures = true
2430

2531

2632
[[queries]]
2733
name = "eventQueryShardFailuresOptionalFieldMatching"
2834
query = 'process where serial_event_id == 2 and ?subtype == "create" or broken == 1'
29-
allow_partial_search_results = true
3035
expected_event_ids = [2]
3136
expect_shard_failures = true
3237

@@ -64,7 +69,6 @@ sequence
6469
[process where serial_event_id == 1]
6570
[process where serial_event_id == 2]
6671
'''
67-
allow_partial_search_results = true
6872
expected_event_ids = [1, 2]
6973
expect_shard_failures = false
7074

@@ -76,6 +80,17 @@ sequence
7680
[process where serial_event_id == 1 or broken == 1]
7781
[process where serial_event_id == 2]
7882
'''
83+
expected_event_ids = []
84+
expect_shard_failures = true
85+
86+
87+
[[queries]]
88+
name = "sequenceQueryMissingShardsTrue"
89+
query = '''
90+
sequence
91+
[process where serial_event_id == 1 or broken == 1]
92+
[process where serial_event_id == 2]
93+
'''
7994
allow_partial_search_results = true
8095
expected_event_ids = []
8196
expect_shard_failures = true
@@ -88,6 +103,18 @@ sequence
88103
[process where serial_event_id == 1 or broken == 1]
89104
[process where serial_event_id == 2]
90105
'''
106+
allow_partial_sequence_results = true
107+
expected_event_ids = [1, 2]
108+
expect_shard_failures = true
109+
110+
111+
[[queries]]
112+
name = "sequenceQueryMissingShardsPartialResultsTrue"
113+
query = '''
114+
sequence
115+
[process where serial_event_id == 1 or broken == 1]
116+
[process where serial_event_id == 2]
117+
'''
91118
allow_partial_search_results = true
92119
allow_partial_sequence_results = true
93120
expected_event_ids = [1, 2]
@@ -101,7 +128,6 @@ sequence
101128
[process where ?serial_event_id == 1 or broken == 1]
102129
[process where serial_event_id == 2]
103130
'''
104-
allow_partial_search_results = true
105131
allow_partial_sequence_results = true
106132
expected_event_ids = [1, 2]
107133
expect_shard_failures = true
@@ -114,7 +140,6 @@ sequence with maxspan=100000d
114140
[process where serial_event_id == 1 and ?subtype == "create" or broken == 1]
115141
[process where serial_event_id == 2]
116142
'''
117-
allow_partial_search_results = true
118143
allow_partial_sequence_results = true
119144
expected_event_ids = [1, 2]
120145
expect_shard_failures = true
@@ -128,7 +153,6 @@ sequence with maxspan=100000d
128153
![process where broken == 1]
129154
[process where serial_event_id == 2]
130155
'''
131-
allow_partial_search_results = true
132156
allow_partial_sequence_results = true
133157
expected_event_ids = [1, -1, 2]
134158
expect_shard_failures = true
@@ -142,7 +166,6 @@ sequence with maxspan=100000d
142166
![process where broken == 1]
143167
[process where serial_event_id == 2]
144168
'''
145-
allow_partial_search_results = true
146169
allow_partial_sequence_results = true
147170
expected_event_ids = [1, -1, 2]
148171
expect_shard_failures = true
@@ -155,6 +178,17 @@ sample by event_subtype_full
155178
[process where serial_event_id == 1 or broken == 1]
156179
[process where serial_event_id == 2]
157180
'''
181+
expected_event_ids = [1, 2]
182+
expect_shard_failures = true
183+
184+
185+
[[queries]]
186+
name = "sampleQueryMissingShardsPartialResultsTrue"
187+
query = '''
188+
sample by event_subtype_full
189+
[process where serial_event_id == 1 or broken == 1]
190+
[process where serial_event_id == 2]
191+
'''
158192
allow_partial_search_results = true
159193
expected_event_ids = [1, 2]
160194
expect_shard_failures = true
@@ -167,7 +201,6 @@ sample by event_subtype_full
167201
[process where serial_event_id == 1 and ?subtype == "create" or broken == 1]
168202
[process where serial_event_id == 2]
169203
'''
170-
allow_partial_search_results = true
171204
expected_event_ids = [1, 2]
172205
expect_shard_failures = true
173206

‎x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java

+34-4
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
package org.elasticsearch.xpack.eql.qa.mixed_node;
99

1010
import org.apache.http.HttpHost;
11+
import org.elasticsearch.TransportVersions;
1112
import org.elasticsearch.client.Request;
1213
import org.elasticsearch.client.Response;
1314
import org.elasticsearch.client.RestClient;
@@ -278,7 +279,21 @@ private void assertEventsQueryOnNodes(List<TestNode> nodesList) throws Exception
278279
String filterPath = "filter_path=hits.events._source.@timestamp,hits.events._source.event_type,hits.events._source.sequence";
279280

280281
Request request = new Request("POST", index + "/_eql/search?" + filterPath);
281-
request.setJsonEntity("{\"query\":\"" + event + " where true\",\"size\":15}");
282+
StringBuilder payload = new StringBuilder("{\"query\":\"" + event + " where true\",\"size\":15");
283+
// Older versions don't support this option
284+
if (nodesList.stream()
285+
.allMatch(
286+
x -> x.transportVersion() != null && x.transportVersion().onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS)
287+
)) {
288+
if (randomBoolean()) {
289+
payload.append(", \"allow_partial_search_results\": " + randomBoolean());
290+
}
291+
if (randomBoolean()) {
292+
payload.append(", \"allow_partial_sequence_results\": " + randomBoolean());
293+
}
294+
}
295+
payload.append("}");
296+
request.setJsonEntity(payload.toString());
282297
assertBusy(() -> { assertResponse(expectedResponse, runEql(client, request)); });
283298
}
284299
}
@@ -294,7 +309,22 @@ private void assertSequncesQueryOnNodes(List<TestNode> nodesList) throws Excepti
294309
String filter = "{\"range\":{\"@timestamp\":{\"gte\":\"1970-05-01\"}}}";
295310

296311
Request request = new Request("POST", index + "/_eql/search?" + filterPath);
297-
request.setJsonEntity("{\"query\":\"" + query + "\",\"filter\":" + filter + "}");
312+
313+
StringBuilder payload = new StringBuilder("{\"query\":\"" + query + "\",\"filter\":" + filter);
314+
// Older versions don't support this option
315+
if (nodesList.stream()
316+
.allMatch(
317+
x -> x.transportVersion() != null && x.transportVersion().onOrAfter(TransportVersions.EQL_ALLOW_PARTIAL_SEARCH_RESULTS)
318+
)) {
319+
if (randomBoolean()) {
320+
payload.append(", \"allow_partial_search_results\": " + randomBoolean());
321+
}
322+
if (randomBoolean()) {
323+
payload.append(", \"allow_partial_sequence_results\": " + randomBoolean());
324+
}
325+
}
326+
payload.append("}");
327+
request.setJsonEntity(payload.toString());
298328
assertBusy(() -> { assertResponse(expectedResponse, runEql(client, request)); });
299329
}
300330
}
@@ -410,10 +440,10 @@ private void assertMultiValueFunctionQuery(
410440

411441
StringBuilder payload = new StringBuilder("{\"query\":\"" + query + "\"");
412442
if (randomBoolean()) {
413-
payload.append(", \"allow_partial_search_results\": true");
443+
payload.append(", \"allow_partial_search_results\": " + randomBoolean());
414444
}
415445
if (randomBoolean()) {
416-
payload.append(", \"allow_partial_sequence_results\": true");
446+
payload.append(", \"allow_partial_sequence_results\": " + randomBoolean());
417447
}
418448
payload.append("}");
419449
request.setJsonEntity(payload.toString());

‎x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml

+24-3
Original file line numberDiff line numberDiff line change
@@ -509,6 +509,30 @@ setup:
509509

510510

511511
---
512+
513+
"Execute query shard failures":
514+
- do:
515+
eql.search:
516+
index: eql_test*
517+
body:
518+
query: 'process where user == "SYSTEM" and day_of_week == "Monday"'
519+
fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
520+
allow_partial_search_results: true
521+
522+
- match: {timed_out: false}
523+
- match: {hits.total.value: 1}
524+
- match: {hits.total.relation: "eq"}
525+
- match: {hits.events.0._source.user: "SYSTEM"}
526+
- match: {hits.events.0._id: "1"}
527+
- match: {hits.events.0.fields.@timestamp: ["1580733296000"]}
528+
- match: {hits.events.0.fields.id: [123]}
529+
- match: {hits.events.0.fields.valid: [false]}
530+
- match: {hits.events.0.fields.day_of_week: ["Monday"]}
531+
- match: {shard_failures.0.index: "eql_test_rebel"}
532+
533+
534+
---
535+
512536
"Execute query shard failures and with allow_partial_search_results":
513537
- do:
514538
eql.search:
@@ -535,7 +559,6 @@ setup:
535559
- do:
536560
eql.search:
537561
index: eql_test*
538-
allow_partial_search_results: true
539562
body:
540563
query: 'process where user == "SYSTEM" and day_of_week == "Monday"'
541564
fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
@@ -575,7 +598,6 @@ setup:
575598
body:
576599
query: 'sequence [process where user == "SYSTEM" and day_of_week == "Monday"] [process where user == "SYSTEM" and day_of_week == "Tuesday"]'
577600
fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
578-
allow_partial_search_results: true
579601
allow_partial_sequence_results: true
580602

581603
- match: {timed_out: false}
@@ -600,7 +622,6 @@ setup:
600622
- do:
601623
eql.search:
602624
index: eql_test*
603-
allow_partial_search_results: true
604625
allow_partial_sequence_results: true
605626
body:
606627
query: 'sequence [process where user == "SYSTEM" and day_of_week == "Monday"] [process where user == "SYSTEM" and day_of_week == "Tuesday"]'

0 commit comments

Comments
 (0)