Skip to content

Commit 1964be5

Browse files
authored
Allow querying index_mode (elastic#110676)
This change allows querying the `index.mode` setting via a new `_index_mode` metadata field, enabling APIs such as `field_caps` or `resolve_indices` to target indices that are either time_series or logs only. This approach avoids adding and handling a new parameter for `index_mode` in these APIs. Both ES|QL and the `_search` API should also work with this new field.
1 parent 3c35fdc commit 1964be5

File tree

32 files changed

+469
-45
lines changed

32 files changed

+469
-45
lines changed

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java

+6
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator;
4242
import org.elasticsearch.compute.operator.topn.TopNOperator;
4343
import org.elasticsearch.core.IOUtils;
44+
import org.elasticsearch.index.IndexSettings;
4445
import org.elasticsearch.index.IndexVersion;
4546
import org.elasticsearch.index.mapper.BlockLoader;
4647
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
@@ -189,6 +190,11 @@ public String indexName() {
189190
return "benchmark";
190191
}
191192

193+
@Override
194+
public IndexSettings indexSettings() {
195+
throw new UnsupportedOperationException();
196+
}
197+
192198
@Override
193199
public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
194200
return MappedFieldType.FieldExtractPreference.NONE;

docs/changelog/110676.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 110676
2+
summary: Allow querying `index_mode`
3+
area: Mapping
4+
type: enhancement
5+
issues: []

modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java

+1
Original file line numberDiff line numberDiff line change
@@ -655,6 +655,7 @@ public <IFD extends IndexFieldData<?>> IFD getForField(
655655
IndexFieldData.Builder builder = fieldType.fielddataBuilder(
656656
new FieldDataContext(
657657
delegate.getFullyQualifiedIndex().getName(),
658+
delegate.getIndexSettings(),
658659
delegate::lookup,
659660
this::sourcePath,
660661
fielddataOperation
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
---
2+
setup:
3+
- requires:
4+
cluster_features: "mapper.query_index_mode"
5+
reason: "require index_mode"
6+
7+
- do:
8+
indices.create:
9+
index: test_metrics
10+
body:
11+
settings:
12+
index:
13+
mode: time_series
14+
routing_path: [container]
15+
time_series:
16+
start_time: 2021-04-28T00:00:00Z
17+
end_time: 2021-04-29T00:00:00Z
18+
mappings:
19+
properties:
20+
"@timestamp":
21+
type: date
22+
container:
23+
type: keyword
24+
time_series_dimension: true
25+
26+
- do:
27+
indices.create:
28+
index: test
29+
body:
30+
mappings:
31+
properties:
32+
"@timestamp":
33+
type: date
34+
35+
---
36+
Field-caps:
37+
- do:
38+
field_caps:
39+
index: "test*"
40+
fields: "*"
41+
body: { index_filter: { term: { _index_mode: "time_series" } } }
42+
- match: { indices: [ "test_metrics" ] }
43+
- do:
44+
field_caps:
45+
index: "test*"
46+
fields: "*"
47+
body: { index_filter: { term: { _index_mode: "logs" } } }
48+
- match: { indices: [ ] }
49+
- do:
50+
field_caps:
51+
index: "test*"
52+
fields: "*"
53+
body: { index_filter: { term: { _index_mode: "standard" } } }
54+
- match: { indices: [ "test" ] }
55+
- do:
56+
field_caps:
57+
index: "test*"
58+
fields: "*"
59+
- match: { indices: [ "test" , "test_metrics" ] }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml

+24-23
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,7 @@
417417

418418
- requires:
419419
test_runner_features: [arbitrary_key]
420-
cluster_features: ["mapper.track_ignored_source"]
420+
cluster_features: ["mapper.query_index_mode"]
421421
reason: "_ignored_source added to mappings"
422422

423423
- do:
@@ -478,34 +478,35 @@
478478
# 6. _ignored
479479
# 7. _ignored_source
480480
# 8. _index
481-
# 9. _nested_path
482-
# 10. _routing
483-
# 11. _seq_no
484-
# 12. _source
485-
# 13. _tier
486-
# 14. _version
487-
# 15. @timestamp
488-
# 16. authors.age
489-
# 17. authors.company
490-
# 18. authors.company.keyword
491-
# 19. authors.name.last_name
492-
# 20. authors.name.first_name
493-
# 21. authors.name.full_name
494-
# 22. link
495-
# 23. title
496-
# 24. url
481+
# 9. _index_mode
482+
# 10. _nested_path
483+
# 11. _routing
484+
# 12. _seq_no
485+
# 13. _source
486+
# 14. _tier
487+
# 15. _version
488+
# 16. @timestamp
489+
# 17. authors.age
490+
# 18. authors.company
491+
# 19. authors.company.keyword
492+
# 20. authors.name.last_name
493+
# 21. authors.name.first_name
494+
# 22. authors.name.full_name
495+
# 23. link
496+
# 24. title
497+
# 25. url
497498
# Object mappers:
498-
# 25. authors
499-
# 26. authors.name
499+
# 26. authors
500+
# 27. authors.name
500501
# Runtime field mappers:
501-
# 27. a_source_field
502+
# 28. a_source_field
502503

503-
- gte: { nodes.$node_id.indices.mappings.total_count: 27 }
504+
- gte: { nodes.$node_id.indices.mappings.total_count: 28 }
504505
- is_true: nodes.$node_id.indices.mappings.total_estimated_overhead
505506
- gte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 26624 }
506-
- match: { nodes.$node_id.indices.indices.index1.mappings.total_count: 27 }
507+
- match: { nodes.$node_id.indices.indices.index1.mappings.total_count: 28 }
507508
- is_true: nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead
508-
- match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 27648 }
509+
- match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 28672 }
509510

510511
---
511512
"indices mappings does not exist in shards level":

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml

+17
Original file line numberDiff line numberDiff line change
@@ -337,3 +337,20 @@ sort by tsid:
337337

338338
- match: {hits.hits.7.sort: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o", 1619635864467]}
339339
- match: {hits.hits.7.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]}
340+
341+
---
342+
aggs by index_mode:
343+
- requires:
344+
cluster_features: ["mapper.query_index_mode"]
345+
reason: require _index_mode metadata field
346+
- do:
347+
search:
348+
index: test
349+
body:
350+
aggs:
351+
modes:
352+
terms:
353+
field: "_index_mode"
354+
- match: {aggregations.modes.buckets.0.key: "time_series"}
355+
- match: {aggregations.modes.buckets.0.doc_count: 8}
356+

server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java

+5-6
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
package org.elasticsearch.index.fielddata;
1010

11+
import org.elasticsearch.index.IndexSettings;
1112
import org.elasticsearch.index.mapper.MappedFieldType;
1213
import org.elasticsearch.search.lookup.SearchLookup;
1314

@@ -25,6 +26,7 @@
2526
*/
2627
public record FieldDataContext(
2728
String fullyQualifiedIndexName,
29+
IndexSettings indexSettings,
2830
Supplier<SearchLookup> lookupSupplier,
2931
Function<String, Set<String>> sourcePathsLookup,
3032
MappedFieldType.FielddataOperation fielddataOperation
@@ -38,11 +40,8 @@ public record FieldDataContext(
3840
* @param reason the reason that runtime fields are not supported
3941
*/
4042
public static FieldDataContext noRuntimeFields(String reason) {
41-
return new FieldDataContext(
42-
"",
43-
() -> { throw new UnsupportedOperationException("Runtime fields not supported for [" + reason + "]"); },
44-
Set::of,
45-
MappedFieldType.FielddataOperation.SEARCH
46-
);
43+
return new FieldDataContext("", null, () -> {
44+
throw new UnsupportedOperationException("Runtime fields not supported for [" + reason + "]");
45+
}, Set::of, MappedFieldType.FielddataOperation.SEARCH);
4746
}
4847
}

server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

+7-1
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,13 @@ private static void executeIndexTimeScripts(DocumentParserContext context) {
161161
SearchLookup searchLookup = new SearchLookup(
162162
context.mappingLookup().indexTimeLookup()::get,
163163
(ft, lookup, fto) -> ft.fielddataBuilder(
164-
new FieldDataContext(context.indexSettings().getIndex().getName(), lookup, context.mappingLookup()::sourcePaths, fto)
164+
new FieldDataContext(
165+
context.indexSettings().getIndex().getName(),
166+
context.indexSettings(),
167+
lookup,
168+
context.mappingLookup()::sourcePaths,
169+
fto
170+
)
165171
).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()),
166172
(ctx, doc) -> Source.fromBytes(context.sourceToParse().source())
167173
);
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
package org.elasticsearch.index.mapper;
10+
11+
import org.apache.lucene.search.MatchAllDocsQuery;
12+
import org.apache.lucene.search.Query;
13+
import org.apache.lucene.util.BytesRef;
14+
import org.elasticsearch.common.regex.Regex;
15+
import org.elasticsearch.features.NodeFeature;
16+
import org.elasticsearch.index.fielddata.FieldData;
17+
import org.elasticsearch.index.fielddata.FieldDataContext;
18+
import org.elasticsearch.index.fielddata.IndexFieldData;
19+
import org.elasticsearch.index.fielddata.ScriptDocValues;
20+
import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData;
21+
import org.elasticsearch.index.query.QueryRewriteContext;
22+
import org.elasticsearch.index.query.SearchExecutionContext;
23+
import org.elasticsearch.script.field.DelegateDocValuesField;
24+
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
25+
import org.elasticsearch.search.fetch.StoredFieldsSpec;
26+
import org.elasticsearch.search.lookup.Source;
27+
28+
import java.util.Collections;
29+
import java.util.List;
30+
31+
public class IndexModeFieldMapper extends MetadataFieldMapper {
32+
33+
static final NodeFeature QUERYING_INDEX_MODE = new NodeFeature("mapper.query_index_mode");
34+
35+
public static final String NAME = "_index_mode";
36+
37+
public static final String CONTENT_TYPE = "_index_mode";
38+
39+
private static final IndexModeFieldMapper INSTANCE = new IndexModeFieldMapper();
40+
41+
public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE);
42+
43+
static final class IndexModeFieldType extends ConstantFieldType {
44+
45+
static final IndexModeFieldType INSTANCE = new IndexModeFieldType();
46+
47+
private IndexModeFieldType() {
48+
super(NAME, Collections.emptyMap());
49+
}
50+
51+
@Override
52+
public String typeName() {
53+
return CONTENT_TYPE;
54+
}
55+
56+
@Override
57+
protected boolean matches(String pattern, boolean caseInsensitive, QueryRewriteContext context) {
58+
final String indexMode = context.getIndexSettings().getMode().getName();
59+
return Regex.simpleMatch(pattern, indexMode, caseInsensitive);
60+
}
61+
62+
@Override
63+
public Query existsQuery(SearchExecutionContext context) {
64+
return new MatchAllDocsQuery();
65+
}
66+
67+
@Override
68+
public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
69+
final String indexMode = fieldDataContext.indexSettings().getMode().getName();
70+
return new ConstantIndexFieldData.Builder(
71+
indexMode,
72+
name(),
73+
CoreValuesSourceType.KEYWORD,
74+
(dv, n) -> new DelegateDocValuesField(
75+
new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))),
76+
n
77+
)
78+
);
79+
}
80+
81+
@Override
82+
public BlockLoader blockLoader(BlockLoaderContext blContext) {
83+
final String indexMode = blContext.indexSettings().getMode().getName();
84+
return BlockLoader.constantBytes(new BytesRef(indexMode));
85+
}
86+
87+
@Override
88+
public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
89+
return new ValueFetcher() {
90+
private final List<Object> indexMode = List.of(context.getIndexSettings().getMode().getName());
91+
92+
@Override
93+
public List<Object> fetchValues(Source source, int doc, List<Object> ignoredValues) {
94+
return indexMode;
95+
}
96+
97+
@Override
98+
public StoredFieldsSpec storedFieldsSpec() {
99+
return StoredFieldsSpec.NO_REQUIREMENTS;
100+
}
101+
};
102+
}
103+
104+
}
105+
106+
public IndexModeFieldMapper() {
107+
super(IndexModeFieldType.INSTANCE);
108+
}
109+
110+
@Override
111+
protected String contentType() {
112+
return CONTENT_TYPE;
113+
}
114+
115+
@Override
116+
public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
117+
return SourceLoader.SyntheticFieldLoader.NOTHING;
118+
}
119+
}

server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java

+6
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.elasticsearch.common.time.DateMathParser;
3636
import org.elasticsearch.common.unit.Fuzziness;
3737
import org.elasticsearch.core.Nullable;
38+
import org.elasticsearch.index.IndexSettings;
3839
import org.elasticsearch.index.fielddata.FieldDataContext;
3940
import org.elasticsearch.index.fielddata.IndexFieldData;
4041
import org.elasticsearch.index.query.DistanceFeatureQueryBuilder;
@@ -693,6 +694,11 @@ public interface BlockLoaderContext {
693694
*/
694695
String indexName();
695696

697+
/**
698+
* The index settings of the index
699+
*/
700+
IndexSettings indexSettings();
701+
696702
/**
697703
* How the field should be extracted into the BlockLoader. The default is {@link FieldExtractPreference#NONE}, which means
698704
* that the field type can choose where to load the field from. However, in some cases, the caller may have a preference.

server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ public Set<NodeFeature> getFeatures() {
2828
DenseVectorFieldMapper.INT4_QUANTIZATION,
2929
DenseVectorFieldMapper.BIT_VECTORS,
3030
DocumentMapper.INDEX_SORTING_ON_NESTED,
31-
KeywordFieldMapper.KEYWORD_DIMENSION_IGNORE_ABOVE
31+
KeywordFieldMapper.KEYWORD_DIMENSION_IGNORE_ABOVE,
32+
IndexModeFieldMapper.QUERYING_INDEX_MODE
3233
);
3334
}
3435
}

0 commit comments

Comments
 (0)