Skip to content

Commit aa7a690

Browse files
LantaoJinLantaoJin
authored
Change the default value of plugins.query.size_limit to MAX_RESULT_WINDOW (10000) (#2860)
* Change the default value of plugins.query.size_limit to MAX_RESULT_WINDOW (10000) Signed-off-by: Lantao Jin <[email protected]> * fix ut Signed-off-by: Lantao Jin <[email protected]> * fix spotless Signed-off-by: Lantao Jin <[email protected]> --------- Signed-off-by: Lantao Jin <[email protected]>
1 parent 103c416 commit aa7a690

File tree

7 files changed

+21
-20
lines changed

7 files changed

+21
-20
lines changed

‎docs/user/admin/settings.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ plugins.query.size_limit
202202
Description
203203
-----------
204204

205-
The new engine fetches a default size of index from OpenSearch set by this setting, the default value is 200. You can change the value to any value not greater than the max result window value in index level (10000 by default), here is an example::
205+
The new engine fetches a default size of index from OpenSearch set by this setting, the default value equals to max result window in index level (10000 by default). You can change the value to any value not greater than the max result window value in index level (`index.max_result_window`), here is an example::
206206

207207
>> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{
208208
"transient" : {

‎docs/user/optimization/optimization.rst

+11-11
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ The consecutive Filter operator will be merged as one Filter operator::
4444
{
4545
"name": "OpenSearchIndexScan",
4646
"description": {
47-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\":{\"age\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":10,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, searchDone=false)"
47+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\":{\"age\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":10,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, searchDone=false)"
4848
},
4949
"children": []
5050
}
@@ -71,7 +71,7 @@ The Filter operator should be push down under Sort operator::
7171
{
7272
"name": "OpenSearchIndexScan",
7373
"description": {
74-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, searchDone=false)"
74+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, searchDone=false)"
7575
},
7676
"children": []
7777
}
@@ -102,7 +102,7 @@ The Project list will push down to Query DSL to `filter the source <https://www.
102102
{
103103
"name": "OpenSearchIndexScan",
104104
"description": {
105-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, searchDone=false)"
105+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, searchDone=false)"
106106
},
107107
"children": []
108108
}
@@ -128,7 +128,7 @@ The Filter operator will merge into OpenSearch Query DSL::
128128
{
129129
"name": "OpenSearchIndexScan",
130130
"description": {
131-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, searchDone=false)"
131+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, searchDone=false)"
132132
},
133133
"children": []
134134
}
@@ -154,7 +154,7 @@ The Sort operator will merge into OpenSearch Query DSL::
154154
{
155155
"name": "OpenSearchIndexScan",
156156
"description": {
157-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, searchDone=false)"
157+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, searchDone=false)"
158158
},
159159
"children": []
160160
}
@@ -188,7 +188,7 @@ Because the OpenSearch Script Based Sorting can't handle NULL/MISSING value, the
188188
{
189189
"name": "OpenSearchIndexScan",
190190
"description": {
191-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\"}, searchDone=false)"
191+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, searchDone=false)"
192192
},
193193
"children": []
194194
}
@@ -257,7 +257,7 @@ If sort that includes expression, which cannot be merged into query DSL, also ex
257257
{
258258
"name": "OpenSearchIndexScan",
259259
"description": {
260-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\"}, searchDone=false)"
260+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, searchDone=false)"
261261
},
262262
"children": []
263263
}
@@ -287,7 +287,7 @@ The Aggregation operator will merge into OpenSearch Aggregation::
287287
{
288288
"name": "OpenSearchIndexScan",
289289
"description": {
290-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone=false)"
290+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone=false)"
291291
},
292292
"children": []
293293
}
@@ -313,7 +313,7 @@ The Sort operator will merge into OpenSearch Aggregation.::
313313
{
314314
"name": "OpenSearchIndexScan",
315315
"description": {
316-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"last\",\"order\":\"desc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone=false)"
316+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"last\",\"order\":\"desc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone=false)"
317317
},
318318
"children": []
319319
}
@@ -348,7 +348,7 @@ Because the OpenSearch Composite Aggregation doesn't support order by metrics fi
348348
{
349349
"name": "OpenSearchIndexScan",
350350
"description": {
351-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone=false)"
351+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone=false)"
352352
},
353353
"children": []
354354
}
@@ -373,4 +373,4 @@ At the moment there is no optimization to merge similar sort operators to avoid
373373

374374
Sort Push Down
375375
--------------
376-
Without sort push down optimization, the sort operator will sort the result from child operator. By default, only 200 docs will extracted from the source index, `you can change this value by using size_limit setting <../admin/settings.rst#opensearch-query-size-limit>`_.
376+
Without sort push down optimization, the sort operator will sort the result from child operator. By default, only 10000 docs will extracted from the source index, `you can change this value by using size_limit setting <../admin/settings.rst#opensearch-query-size-limit>`_.

‎docs/user/ppl/admin/settings.rst

+2-2
Original file line numberDiff line numberDiff line change
@@ -125,9 +125,9 @@ plugins.query.size_limit
125125
Description
126126
-----------
127127

128-
The size configure the maximum amount of documents to be pull from OpenSearch. The default value is: 200
128+
The size configure the maximum amount of documents to be pull from OpenSearch. The default value is: 10000
129129

130-
Notes: This setting will impact the correctness of the aggregation operation, for example, there are 1000 docs in the index, by default, only 200 docs will be extract from index and do aggregation.
130+
Notes: This setting will impact the correctness of the aggregation operation, for example, there are 1000 docs in the index, if you change the value to 200, only 200 docs will be extract from index and do aggregation.
131131

132132
Example
133133
-------

‎docs/user/ppl/interfaces/endpoint.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ The following PPL query demonstrated that where and stats command were pushed do
9191
{
9292
"name": "OpenSearchIndexScan",
9393
"description": {
94-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":200,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":10,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}, searchDone=false)"
94+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":10,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}, searchDone=false)"
9595
},
9696
"children": []
9797
}

‎integ-test/src/test/java/org/opensearch/sql/legacy/ExplainIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ public void orderByOnNestedFieldTest() throws Exception {
185185
Assert.assertThat(
186186
result.replaceAll("\\s+", ""),
187187
equalTo(
188-
"{\"from\":0,\"size\":200,\"sort\":[{\"message.info\":"
188+
"{\"from\":0,\"size\":10000,\"sort\":[{\"message.info\":"
189189
+ "{\"order\":\"asc\",\"nested\":{\"path\":\"message\"}}}]}"));
190190
}
191191

‎opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.opensearch.common.settings.Setting;
2929
import org.opensearch.common.unit.MemorySizeValue;
3030
import org.opensearch.common.unit.TimeValue;
31+
import org.opensearch.index.IndexSettings;
3132
import org.opensearch.sql.common.setting.LegacySettings;
3233
import org.opensearch.sql.common.setting.Settings;
3334

@@ -90,7 +91,7 @@ public class OpenSearchSettings extends Settings {
9091
public static final Setting<?> QUERY_SIZE_LIMIT_SETTING =
9192
Setting.intSetting(
9293
Key.QUERY_SIZE_LIMIT.getKeyValue(),
93-
LegacyOpenDistroSettings.QUERY_SIZE_LIMIT_SETTING,
94+
IndexSettings.MAX_RESULT_WINDOW_SETTING,
9495
0,
9596
Setting.Property.NodeScope,
9697
Setting.Property.Dynamic);

‎opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.opensearch.common.settings.ClusterSettings;
3535
import org.opensearch.common.settings.Setting;
3636
import org.opensearch.core.common.unit.ByteSizeValue;
37+
import org.opensearch.index.IndexSettings;
3738
import org.opensearch.monitor.jvm.JvmInfo;
3839
import org.opensearch.sql.common.setting.LegacySettings;
3940
import org.opensearch.sql.common.setting.Settings;
@@ -132,8 +133,7 @@ void settingsFallback() {
132133
org.opensearch.common.settings.Settings.EMPTY));
133134
assertEquals(
134135
settings.getSettingValue(Settings.Key.QUERY_SIZE_LIMIT),
135-
LegacyOpenDistroSettings.QUERY_SIZE_LIMIT_SETTING.get(
136-
org.opensearch.common.settings.Settings.EMPTY));
136+
IndexSettings.MAX_RESULT_WINDOW_SETTING.get(org.opensearch.common.settings.Settings.EMPTY));
137137
assertEquals(
138138
settings.getSettingValue(Settings.Key.METRICS_ROLLING_WINDOW),
139139
LegacyOpenDistroSettings.METRICS_ROLLING_WINDOW_SETTING.get(
@@ -165,7 +165,7 @@ public void updateLegacySettingsFallback() {
165165
assertEquals(
166166
QUERY_MEMORY_LIMIT_SETTING.get(settings),
167167
new ByteSizeValue((int) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.2)));
168-
assertEquals(QUERY_SIZE_LIMIT_SETTING.get(settings), 100);
168+
assertEquals(QUERY_SIZE_LIMIT_SETTING.get(settings), 10000);
169169
assertEquals(METRICS_ROLLING_WINDOW_SETTING.get(settings), 2000L);
170170
assertEquals(METRICS_ROLLING_INTERVAL_SETTING.get(settings), 100L);
171171
}

0 commit comments

Comments
 (0)