Skip to content

Commit 71e74bd

Browse files
jordan-powersjordan-powers
authored
Store arrays offsets for scaled float fields natively with synthetic source (elastic#125793)
This patch builds on the work in elastic#113757, elastic#122999, elastic#124594, elastic#125529, and elastic#125709 to natively store array offsets for scaled float fields instead of falling back to ignored source when synthetic_source_keep: arrays.
1 parent 848a678 commit 71e74bd

File tree

8 files changed

+298
-153
lines changed

8 files changed

+298
-153
lines changed

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java

Lines changed: 111 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
import org.elasticsearch.common.settings.Settings;
2020
import org.elasticsearch.common.xcontent.support.XContentMapValues;
2121
import org.elasticsearch.index.IndexMode;
22+
import org.elasticsearch.index.IndexVersion;
23+
import org.elasticsearch.index.IndexVersions;
2224
import org.elasticsearch.index.fielddata.FieldData;
2325
import org.elasticsearch.index.fielddata.FieldDataContext;
2426
import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -32,6 +34,7 @@
3234
import org.elasticsearch.index.mapper.BlockDocValuesReader;
3335
import org.elasticsearch.index.mapper.BlockLoader;
3436
import org.elasticsearch.index.mapper.BlockSourceReader;
37+
import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
3538
import org.elasticsearch.index.mapper.DocumentParserContext;
3639
import org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader;
3740
import org.elasticsearch.index.mapper.FieldMapper;
@@ -40,6 +43,8 @@
4043
import org.elasticsearch.index.mapper.NumberFieldMapper;
4144
import org.elasticsearch.index.mapper.SimpleMappedFieldType;
4245
import org.elasticsearch.index.mapper.SortedNumericDocValuesSyntheticFieldLoader;
46+
import org.elasticsearch.index.mapper.SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer;
47+
import org.elasticsearch.index.mapper.SourceLoader;
4348
import org.elasticsearch.index.mapper.SourceValueFetcher;
4449
import org.elasticsearch.index.mapper.TextSearchInfo;
4550
import org.elasticsearch.index.mapper.TimeSeriesParams;
@@ -67,6 +72,8 @@
6772
import java.util.Objects;
6873
import java.util.Set;
6974

75+
import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
76+
7077
/** A {@link FieldMapper} for scaled floats. Values are internally multiplied
7178
* by a scaling factor and rounded to the closest long. */
7279
public class ScaledFloatFieldMapper extends FieldMapper {
@@ -125,12 +132,34 @@ public static class Builder extends FieldMapper.Builder {
125132
private final Parameter<TimeSeriesParams.MetricType> metric;
126133

127134
private final IndexMode indexMode;
135+
private final IndexVersion indexCreatedVersion;
136+
private final SourceKeepMode indexSourceKeepMode;
128137

129-
public Builder(String name, Settings settings, IndexMode indexMode) {
130-
this(name, IGNORE_MALFORMED_SETTING.get(settings), COERCE_SETTING.get(settings), indexMode);
138+
public Builder(
139+
String name,
140+
Settings settings,
141+
IndexMode indexMode,
142+
IndexVersion indexCreatedVersion,
143+
SourceKeepMode indexSourceKeepMode
144+
) {
145+
this(
146+
name,
147+
IGNORE_MALFORMED_SETTING.get(settings),
148+
COERCE_SETTING.get(settings),
149+
indexMode,
150+
indexCreatedVersion,
151+
indexSourceKeepMode
152+
);
131153
}
132154

133-
public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDefault, IndexMode indexMode) {
155+
public Builder(
156+
String name,
157+
boolean ignoreMalformedByDefault,
158+
boolean coerceByDefault,
159+
IndexMode indexMode,
160+
IndexVersion indexCreatedVersion,
161+
SourceKeepMode indexSourceKeepMode
162+
) {
134163
super(name);
135164
this.ignoreMalformed = Parameter.explicitBoolParam(
136165
"ignore_malformed",
@@ -159,6 +188,8 @@ public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDe
159188
);
160189
}
161190
});
191+
this.indexCreatedVersion = indexCreatedVersion;
192+
this.indexSourceKeepMode = indexSourceKeepMode;
162193
}
163194

164195
Builder scalingFactor(double scalingFactor) {
@@ -200,11 +231,35 @@ public ScaledFloatFieldMapper build(MapperBuilderContext context) {
200231
coerce.getValue().value(),
201232
context.isSourceSynthetic()
202233
);
203-
return new ScaledFloatFieldMapper(leafName(), type, builderParams(this, context), context.isSourceSynthetic(), this);
234+
String offsetsFieldName = getOffsetsFieldName(
235+
context,
236+
indexSourceKeepMode,
237+
hasDocValues.getValue(),
238+
stored.getValue(),
239+
this,
240+
indexCreatedVersion,
241+
IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_SCALED_FLOAT
242+
);
243+
return new ScaledFloatFieldMapper(
244+
leafName(),
245+
type,
246+
builderParams(this, context),
247+
context.isSourceSynthetic(),
248+
this,
249+
offsetsFieldName
250+
);
204251
}
205252
}
206253

207-
public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings(), c.getIndexSettings().getMode()));
254+
public static final TypeParser PARSER = new TypeParser(
255+
(n, c) -> new Builder(
256+
n,
257+
c.getSettings(),
258+
c.getIndexSettings().getMode(),
259+
c.indexVersionCreated(),
260+
c.getIndexSettings().sourceKeepMode()
261+
)
262+
);
208263

209264
public static final class ScaledFloatFieldType extends SimpleMappedFieldType {
210265

@@ -533,12 +588,17 @@ public String toString() {
533588
private final TimeSeriesParams.MetricType metricType;
534589
private final IndexMode indexMode;
535590

591+
private final IndexVersion indexCreatedVersion;
592+
private final String offsetsFieldName;
593+
private final SourceKeepMode indexSourceKeepMode;
594+
536595
private ScaledFloatFieldMapper(
537596
String simpleName,
538597
ScaledFloatFieldType mappedFieldType,
539598
BuilderParams builderParams,
540599
boolean isSourceSynthetic,
541-
Builder builder
600+
Builder builder,
601+
String offsetsFieldName
542602
) {
543603
super(simpleName, mappedFieldType, builderParams);
544604
this.isSourceSynthetic = isSourceSynthetic;
@@ -553,6 +613,9 @@ private ScaledFloatFieldMapper(
553613
this.coerceByDefault = builder.coerce.getDefaultValue().value();
554614
this.metricType = builder.metric.getValue();
555615
this.indexMode = builder.indexMode;
616+
this.indexCreatedVersion = builder.indexCreatedVersion;
617+
this.offsetsFieldName = offsetsFieldName;
618+
this.indexSourceKeepMode = builder.indexSourceKeepMode;
556619
}
557620

558621
boolean coerce() {
@@ -564,6 +627,11 @@ public boolean ignoreMalformed() {
564627
return ignoreMalformed.value();
565628
}
566629

630+
@Override
631+
public String getOffsetFieldName() {
632+
return offsetsFieldName;
633+
}
634+
567635
@Override
568636
public ScaledFloatFieldType fieldType() {
569637
return (ScaledFloatFieldType) super.fieldType();
@@ -576,7 +644,9 @@ protected String contentType() {
576644

577645
@Override
578646
public FieldMapper.Builder getMergeBuilder() {
579-
return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault, indexMode).metric(metricType).init(this);
647+
return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault, indexMode, indexCreatedVersion, indexSourceKeepMode)
648+
.metric(metricType)
649+
.init(this);
580650
}
581651

582652
@Override
@@ -606,11 +676,16 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
606676
value = numericValue;
607677
}
608678

679+
boolean shouldStoreOffsets = offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField();
680+
609681
if (value == null) {
610682
value = nullValue;
611683
}
612684

613685
if (value == null) {
686+
if (shouldStoreOffsets) {
687+
context.getOffSetContext().recordNull(offsetsFieldName);
688+
}
614689
return;
615690
}
616691

@@ -636,6 +711,10 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
636711

637712
NumberFieldMapper.NumberType.LONG.addFields(context.doc(), fieldType().name(), scaledValue, indexed, hasDocValues, stored);
638713

714+
if (shouldStoreOffsets) {
715+
context.getOffSetContext().recordOffset(offsetsFieldName, scaledValue);
716+
}
717+
639718
if (hasDocValues == false && (indexed || stored)) {
640719
context.addToFieldNames(fieldType().name());
641720
}
@@ -777,17 +856,34 @@ public int docValueCount() {
777856
}
778857
}
779858

859+
private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() {
860+
if (offsetsFieldName != null) {
861+
var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>(2);
862+
layers.add(
863+
new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(
864+
fullPath(),
865+
offsetsFieldName,
866+
(b, value) -> b.value(decodeForSyntheticSource(value, scalingFactor))
867+
)
868+
);
869+
if (ignoreMalformed.value()) {
870+
layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));
871+
}
872+
return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers);
873+
} else {
874+
return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) {
875+
@Override
876+
protected void writeValue(XContentBuilder b, long value) throws IOException {
877+
b.value(decodeForSyntheticSource(value, scalingFactor));
878+
}
879+
};
880+
}
881+
}
882+
780883
@Override
781884
protected SyntheticSourceSupport syntheticSourceSupport() {
782885
if (hasDocValues) {
783-
return new SyntheticSourceSupport.Native(
784-
() -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) {
785-
@Override
786-
protected void writeValue(XContentBuilder b, long value) throws IOException {
787-
b.value(decodeForSyntheticSource(value, scalingFactor));
788-
}
789-
}
790-
);
886+
return new SyntheticSourceSupport.Native(this::docValuesSyntheticFieldLoader);
791887
}
792888

793889
return super.syntheticSourceSupport();

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.elasticsearch.index.mapper.DocumentMapper;
2020
import org.elasticsearch.index.mapper.DocumentParsingException;
2121
import org.elasticsearch.index.mapper.MappedFieldType;
22+
import org.elasticsearch.index.mapper.Mapper;
2223
import org.elasticsearch.index.mapper.MapperParsingException;
2324
import org.elasticsearch.index.mapper.MapperService;
2425
import org.elasticsearch.index.mapper.NumberFieldMapperTests;
@@ -365,6 +366,24 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed)
365366
return new ScaledFloatSyntheticSourceSupport(ignoreMalformed);
366367
}
367368

369+
@Override
370+
protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) {
371+
return new ScaledFloatSyntheticSourceSupport(ignoreMalformed) {
372+
@Override
373+
public SyntheticSourceExample example(int maxVals) {
374+
var example = super.example(maxVals);
375+
// Need the expectedForSyntheticSource as inputValue since MapperTestCase#testSyntheticSourceKeepArrays
376+
// uses the inputValue as both the input and expected.
377+
return new SyntheticSourceExample(
378+
example.expectedForSyntheticSource(),
379+
example.expectedForSyntheticSource(),
380+
example.expectedForBlockLoader(),
381+
example.mapping()
382+
);
383+
}
384+
};
385+
}
386+
368387
private static class ScaledFloatSyntheticSourceSupport implements SyntheticSourceSupport {
369388
private final boolean ignoreMalformedEnabled;
370389
private final double scalingFactor = randomDoubleBetween(0, Double.MAX_VALUE, false);

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -222,14 +222,14 @@ public void testFieldData() throws IOException {
222222
}
223223

224224
public void testFetchSourceValue() throws IOException {
225-
MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false, null).scalingFactor(100)
225+
MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null).scalingFactor(100)
226226
.build(MapperBuilderContext.root(false, false))
227227
.fieldType();
228228
assertEquals(List.of(3.14), fetchSourceValue(mapper, 3.1415926));
229229
assertEquals(List.of(3.14), fetchSourceValue(mapper, "3.1415"));
230230
assertEquals(List.of(), fetchSourceValue(mapper, ""));
231231

232-
MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false, null).scalingFactor(100)
232+
MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null).scalingFactor(100)
233233
.nullValue(2.71)
234234
.build(MapperBuilderContext.root(false, false))
235235
.fieldType();

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper.extras;
11+
12+
import org.elasticsearch.index.mapper.OffsetDocValuesLoaderTestCase;
13+
import org.elasticsearch.plugins.Plugin;
14+
import org.elasticsearch.xcontent.XContentBuilder;
15+
16+
import java.io.IOException;
17+
import java.util.Collection;
18+
19+
import static java.util.Collections.singletonList;
20+
21+
public class ScaledFloatOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase {
22+
private static final double TEST_SCALING_FACTOR = 10.0;
23+
24+
@Override
25+
protected Collection<? extends Plugin> getPlugins() {
26+
return singletonList(new MapperExtrasPlugin());
27+
}
28+
29+
@Override
30+
protected void minimalMapping(XContentBuilder b) throws IOException {
31+
b.field("type", "scaled_float").field("scaling_factor", TEST_SCALING_FACTOR);
32+
}
33+
34+
public void testOffsetArray() throws Exception {
35+
verifyOffsets("{\"field\":[1.0,10.0,100.0,0.1,10.0,1.0,0.1,100.0]}");
36+
verifyOffsets("{\"field\":[10.0,null,1.0,null,5.0,null,null,6.3,1.5]}");
37+
}
38+
39+
@Override
40+
protected String getFieldTypeName() {
41+
fail("Should not be called because minimalMapping is overridden");
42+
return null;
43+
}
44+
45+
@Override
46+
protected Double randomValue() {
47+
return randomLong() / TEST_SCALING_FACTOR;
48+
}
49+
}

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper.extras;
11+
12+
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
13+
14+
import org.elasticsearch.index.mapper.NativeArrayIntegrationTestCase;
15+
import org.elasticsearch.plugins.Plugin;
16+
import org.elasticsearch.xcontent.XContentBuilder;
17+
18+
import java.io.IOException;
19+
import java.util.Collection;
20+
21+
import static java.util.Collections.singletonList;
22+
23+
public class ScaledFloatSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase {
24+
private static final double TEST_SCALING_FACTOR = 10.0;
25+
26+
@Override
27+
protected Collection<Class<? extends Plugin>> getPlugins() {
28+
return singletonList(MapperExtrasPlugin.class);
29+
}
30+
31+
@Override
32+
protected void minimalMapping(XContentBuilder b) throws IOException {
33+
b.field("type", "scaled_float").field("scaling_factor", TEST_SCALING_FACTOR);
34+
}
35+
36+
@Override
37+
protected String getFieldTypeName() {
38+
fail("Should not be called because minimalMapping is overridden");
39+
return null;
40+
}
41+
42+
@Override
43+
protected Object getRandomValue() {
44+
return randomLong() / TEST_SCALING_FACTOR;
45+
}
46+
47+
@Override
48+
protected Object getMalformedValue() {
49+
return randomBoolean() ? RandomStrings.randomAsciiOfLength(random(), 8) : Double.POSITIVE_INFINITY;
50+
}
51+
}

0 commit comments

Comments
 (0)