Skip to content

Commit 0e1cdd5

Browse files
parkertimminsmartijnvg
parkertimmins
and
martijnvg
authored
Fix false-positive in boolean queries with must_not queries on not indexed fields (#155936) (#155992)
Fork Lucene DocValuesRangeIterator into XDocValuesRangeIterator and fix docIDRunEnd() for MAYBE and YES_IF_PRESENT blocks to return the current doc rather than the block end. The over-report caused bulk scorers to skip per-doc matches() calls, collecting false positives. Also fork SortedSetDocValuesRangeQuery and DocValuesRewriteMethod to wire the fixed iterator into all ordinal DV range/rewrite paths, and update SortedNumericDocValuesRangeQuery to use XDocValuesRangeIterator for the numeric path. Repoint all mapper and x-pack call sites to the ES forks. Closes #155653 (cherry picked from commit 2cba386) # Conflicts: # server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java Co-authored-by: Martijn van Groningen <martijn.v.groningen@gmail.com>
1 parent 9b692d5 commit 0e1cdd5

20 files changed

Lines changed: 1138 additions & 72 deletions

File tree

‎docs/changelog/155936.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
area: Codec
2+
issues:
3+
- 155653
4+
pr: 155936
5+
summary: "Fix false-positive in boolean queries with must_not queries on not indexed fields - #155936"
6+
type: bug

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,9 @@
9696
import org.elasticsearch.lucene.queries.ScanningBinaryDocValuesTermInSetQuery;
9797
import org.elasticsearch.lucene.queries.ScanningBinaryDocValuesTermQuery;
9898
import org.elasticsearch.lucene.queries.ScanningBinaryDocValuesWildcardQuery;
99+
import org.elasticsearch.lucene.queries.SortedSetDocValuesRangeQuery;
99100
import org.elasticsearch.lucene.search.FuzzyQueries;
101+
import org.elasticsearch.lucene.search.XDocValuesRewriteMethod;
100102
import org.elasticsearch.script.Script;
101103
import org.elasticsearch.script.SortedSetDocValuesStringFieldScript;
102104
import org.elasticsearch.script.field.TextDocValuesField;
@@ -707,7 +709,7 @@ public Query termQuery(Object value, SearchExecutionContext context) {
707709
if (usesBinaryDocValues) {
708710
return new ScanningBinaryDocValuesTermQuery(name(), indexedValueForSearch(value), useArrayOrderBinaryDocValues);
709711
} else {
710-
return SortedSetDocValuesField.newSlowExactQuery(name(), indexedValueForSearch(value));
712+
return SortedSetDocValuesRangeQuery.newSlowExactQuery(name(), indexedValueForSearch(value));
711713
}
712714
}
713715

@@ -742,7 +744,7 @@ public Query prefixQuery(
742744
return new ScanningBinaryDocValuesPrefixQuery(name(), value, caseInsensitive, useArrayOrderBinaryDocValues);
743745
}
744746
if (caseInsensitive == false) {
745-
return new PrefixQuery(new Term(name(), value), MultiTermQuery.DOC_VALUES_REWRITE);
747+
return new PrefixQuery(new Term(name(), value), XDocValuesRewriteMethod.DOC_VALUES_REWRITE);
746748
}
747749
return new StringScriptFieldPrefixQuery(
748750
new Script(""),
@@ -771,9 +773,9 @@ public Query wildcardQuery(
771773
Term term = new Term(name(), value);
772774
if (context.getCircuitBreaker() != null) {
773775
Automaton dfa = AutomatonQueries.toWildcardAutomaton(term, context.getCircuitBreaker());
774-
return new AutomatonQuery(term, dfa, false, MultiTermQuery.DOC_VALUES_REWRITE);
776+
return new AutomatonQuery(term, dfa, false, XDocValuesRewriteMethod.DOC_VALUES_REWRITE);
775777
}
776-
return new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, MultiTermQuery.DOC_VALUES_REWRITE);
778+
return new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, XDocValuesRewriteMethod.DOC_VALUES_REWRITE);
777779
}
778780
return new StringScriptFieldWildcardQuery(
779781
new Script(""),
@@ -818,15 +820,15 @@ public Query regexpQuery(
818820
maxDeterminizedStates,
819821
context.getCircuitBreaker()
820822
);
821-
return new AutomatonQuery(term, dfa, false, MultiTermQuery.DOC_VALUES_REWRITE);
823+
return new AutomatonQuery(term, dfa, false, XDocValuesRewriteMethod.DOC_VALUES_REWRITE);
822824
}
823825
return new RegexpQuery(
824826
new Term(name(), value),
825827
syntaxFlags,
826828
matchFlags,
827829
RegexpQuery.DEFAULT_PROVIDER,
828830
maxDeterminizedStates,
829-
MultiTermQuery.DOC_VALUES_REWRITE
831+
XDocValuesRewriteMethod.DOC_VALUES_REWRITE
830832
);
831833
}
832834

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
import org.apache.lucene.search.FuzzyQuery;
1818
import org.apache.lucene.search.IndexSearcher;
1919
import org.apache.lucene.search.MultiPhraseQuery;
20-
import org.apache.lucene.search.MultiTermQuery;
2120
import org.apache.lucene.search.PhraseQuery;
2221
import org.apache.lucene.search.PrefixQuery;
2322
import org.apache.lucene.search.Query;
@@ -64,6 +63,7 @@
6463
import org.elasticsearch.lucene.queries.ScanningBinaryDocValuesPrefixQuery;
6564
import org.elasticsearch.lucene.queries.ScanningBinaryDocValuesRegexpQuery;
6665
import org.elasticsearch.lucene.queries.ScanningBinaryDocValuesWildcardQuery;
66+
import org.elasticsearch.lucene.search.XDocValuesRewriteMethod;
6767
import org.elasticsearch.script.ScriptCompiler;
6868
import org.elasticsearch.search.lookup.SearchLookup;
6969
import org.elasticsearch.search.runtime.StringScriptFieldPrefixQuery;
@@ -595,7 +595,7 @@ public void testPrefixQueryDocValuesOnly() {
595595
// SortedSet DV, case-sensitive: native PrefixQuery with DOC_VALUES_REWRITE
596596
assertThat(
597597
sortedSet.prefixQuery("foo", null, false, MOCK_CONTEXT),
598-
Matchers.equalTo(new PrefixQuery(new Term("field", "foo"), MultiTermQuery.DOC_VALUES_REWRITE))
598+
Matchers.equalTo(new PrefixQuery(new Term("field", "foo"), XDocValuesRewriteMethod.DOC_VALUES_REWRITE))
599599
);
600600

601601
// SortedSet DV, case-insensitive: script-backed query
@@ -620,7 +620,11 @@ public void testWildcardQueryDocValuesOnly() {
620620
assertThat(
621621
sortedSet.wildcardQuery("foo*", null, false, MOCK_CONTEXT),
622622
Matchers.equalTo(
623-
new WildcardQuery(new Term("field", "foo*"), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, MultiTermQuery.DOC_VALUES_REWRITE)
623+
new WildcardQuery(
624+
new Term("field", "foo*"),
625+
Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
626+
XDocValuesRewriteMethod.DOC_VALUES_REWRITE
627+
)
624628
)
625629
);
626630

@@ -649,7 +653,14 @@ public void testRegexpQueryDocValuesOnly() {
649653
assertThat(
650654
sortedSet.regexpQuery("foo.*", 0, 0, 10, null, MOCK_CONTEXT),
651655
Matchers.equalTo(
652-
new RegexpQuery(new Term("field", "foo.*"), 0, 0, RegexpQuery.DEFAULT_PROVIDER, 10, MultiTermQuery.DOC_VALUES_REWRITE)
656+
new RegexpQuery(
657+
new Term("field", "foo.*"),
658+
0,
659+
0,
660+
RegexpQuery.DEFAULT_PROVIDER,
661+
10,
662+
XDocValuesRewriteMethod.DOC_VALUES_REWRITE
663+
)
653664
)
654665
);
655666

@@ -671,7 +682,7 @@ public void testRegexpQueryDocValuesOnlyCaseInsensitive() {
671682
// SortedSet DV → RegexpQuery with DOC_VALUES_REWRITE and ASCII_CASE_INSENSITIVE matchFlag
672683
Query q = sortedSetDocValuesOnly().regexpQuery("foo.*", 0, RegExp.ASCII_CASE_INSENSITIVE, 10, null, MOCK_CONTEXT);
673684
assertThat(q, Matchers.instanceOf(RegexpQuery.class));
674-
assertEquals(MultiTermQuery.DOC_VALUES_REWRITE, ((RegexpQuery) q).getRewriteMethod());
685+
assertEquals(XDocValuesRewriteMethod.DOC_VALUES_REWRITE, ((RegexpQuery) q).getRewriteMethod());
675686

676687
// Binary DV → ScanningBinaryDocValuesRegexpQuery with ASCII_CASE_INSENSITIVE matchFlag
677688
assertThat(

‎server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
package org.elasticsearch.index.mapper;
1111

1212
import org.apache.lucene.document.InetAddressPoint;
13-
import org.apache.lucene.document.SortedSetDocValuesField;
1413
import org.apache.lucene.document.StoredField;
1514
import org.apache.lucene.index.IndexReader;
1615
import org.apache.lucene.index.LeafReaderContext;
@@ -47,6 +46,7 @@
4746
import org.elasticsearch.index.mapper.blockloader.docvalues.fn.MvMinBytesRefsFromOrdsBlockLoader;
4847
import org.elasticsearch.index.query.SearchExecutionContext;
4948
import org.elasticsearch.lucene.queries.ScanningBinaryDocValuesRangeQuery;
49+
import org.elasticsearch.lucene.queries.SortedSetDocValuesRangeQuery;
5050
import org.elasticsearch.script.IpFieldScript;
5151
import org.elasticsearch.script.Script;
5252
import org.elasticsearch.script.ScriptCompiler;
@@ -485,7 +485,7 @@ static Query convertToDocValuesQuery(
485485
if (usesBinaryDocValues) {
486486
return new ScanningBinaryDocValuesRangeQuery(field, lower, upper, arrayOrderInlineNull);
487487
} else {
488-
return SortedSetDocValuesField.newSlowRangeQuery(field, lower, upper, true, true);
488+
return SortedSetDocValuesRangeQuery.newSlowRangeQuery(field, lower, upper, true, true);
489489
}
490490
}
491491

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,9 @@
8080
import org.elasticsearch.lucene.queries.ScanningBinaryDocValuesTermInSetQuery;
8181
import org.elasticsearch.lucene.queries.ScanningBinaryDocValuesTermQuery;
8282
import org.elasticsearch.lucene.queries.ScanningBinaryDocValuesWildcardQuery;
83+
import org.elasticsearch.lucene.queries.SortedSetDocValuesRangeQuery;
8384
import org.elasticsearch.lucene.search.FuzzyQueries;
85+
import org.elasticsearch.lucene.search.XDocValuesRewriteMethod;
8486
import org.elasticsearch.script.Script;
8587
import org.elasticsearch.script.ScriptCompiler;
8688
import org.elasticsearch.script.SortedBinaryDocValuesStringFieldScript;
@@ -767,7 +769,7 @@ public Query termQuery(Object value, SearchExecutionContext context) {
767769
} else if (usesBinaryDocValues) {
768770
return new ScanningBinaryDocValuesTermQuery(name(), indexedValueForSearch(value), useArrayOrderBinaryDocValues);
769771
} else {
770-
return SortedSetDocValuesField.newSlowExactQuery(name(), indexedValueForSearch(value));
772+
return SortedSetDocValuesRangeQuery.newSlowExactQuery(name(), indexedValueForSearch(value));
771773
}
772774
}
773775

@@ -807,7 +809,7 @@ public Query rangeQuery(
807809
includeUpper
808810
);
809811
} else {
810-
return SortedSetDocValuesField.newSlowRangeQuery(
812+
return SortedSetDocValuesRangeQuery.newSlowRangeQuery(
811813
name(),
812814
lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
813815
upperTerm == null ? null : indexedValueForSearch(upperTerm),
@@ -848,7 +850,7 @@ public Query fuzzyQuery(
848850
prefixLength,
849851
maxExpansions,
850852
transpositions,
851-
MultiTermQuery.DOC_VALUES_REWRITE,
853+
XDocValuesRewriteMethod.DOC_VALUES_REWRITE,
852854
context,
853855
name()
854856
);
@@ -875,7 +877,7 @@ public Query prefixQuery(
875877
} else {
876878
if (caseInsensitive == false) {
877879
Term prefix = new Term(name(), indexedValueForSearch(value));
878-
return new PrefixQuery(prefix, MultiTermQuery.DOC_VALUES_REWRITE);
880+
return new PrefixQuery(prefix, XDocValuesRewriteMethod.DOC_VALUES_REWRITE);
879881
}
880882
return new StringScriptFieldPrefixQuery(
881883
new Script(""),
@@ -1233,9 +1235,9 @@ public Query wildcardQuery(
12331235
Term term = new Term(name(), value);
12341236
if (context.getCircuitBreaker() != null) {
12351237
Automaton dfa = AutomatonQueries.toWildcardAutomaton(term, context.getCircuitBreaker());
1236-
return new AutomatonQuery(term, dfa, false, MultiTermQuery.DOC_VALUES_REWRITE);
1238+
return new AutomatonQuery(term, dfa, false, XDocValuesRewriteMethod.DOC_VALUES_REWRITE);
12371239
}
1238-
return new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, MultiTermQuery.DOC_VALUES_REWRITE);
1240+
return new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, XDocValuesRewriteMethod.DOC_VALUES_REWRITE);
12391241
}
12401242

12411243
StringFieldScript.LeafFactory leafFactory = ctx -> new SortedSetDocValuesStringFieldScript(name(), context.lookup(), ctx);
@@ -1267,9 +1269,9 @@ public Query normalizedWildcardQuery(String value, MultiTermQuery.RewriteMethod
12671269
Term term = new Term(name(), value);
12681270
if (context.getCircuitBreaker() != null) {
12691271
Automaton dfa = AutomatonQueries.toWildcardAutomaton(term, context.getCircuitBreaker());
1270-
return new AutomatonQuery(term, dfa, false, MultiTermQuery.DOC_VALUES_REWRITE);
1272+
return new AutomatonQuery(term, dfa, false, XDocValuesRewriteMethod.DOC_VALUES_REWRITE);
12711273
}
1272-
return new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, MultiTermQuery.DOC_VALUES_REWRITE);
1274+
return new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, XDocValuesRewriteMethod.DOC_VALUES_REWRITE);
12731275
}
12741276
}
12751277
}
@@ -1308,15 +1310,15 @@ public Query regexpQuery(
13081310
maxDeterminizedStates,
13091311
context.getCircuitBreaker()
13101312
);
1311-
return new AutomatonQuery(term, dfa, false, MultiTermQuery.DOC_VALUES_REWRITE);
1313+
return new AutomatonQuery(term, dfa, false, XDocValuesRewriteMethod.DOC_VALUES_REWRITE);
13121314
}
13131315
return new RegexpQuery(
13141316
new Term(name(), indexedValueForSearch(value)),
13151317
syntaxFlags,
13161318
matchFlags,
13171319
RegexpQuery.DEFAULT_PROVIDER,
13181320
maxDeterminizedStates,
1319-
MultiTermQuery.DOC_VALUES_REWRITE
1321+
XDocValuesRewriteMethod.DOC_VALUES_REWRITE
13201322
);
13211323
}
13221324
}

‎server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.elasticsearch.common.lucene.Lucene;
1515
import org.elasticsearch.index.analysis.NamedAnalyzer;
1616
import org.elasticsearch.index.query.SearchExecutionContext;
17+
import org.elasticsearch.lucene.queries.SortedSetDocValuesRangeQuery;
1718

1819
import java.util.Collection;
1920
import java.util.Collections;
@@ -64,7 +65,7 @@ public boolean isSearchable() {
6465

6566
@Override
6667
public Query termQuery(Object value, SearchExecutionContext context) {
67-
return SortedSetDocValuesField.newSlowExactQuery(name(), indexedValueForSearch(value));
68+
return SortedSetDocValuesRangeQuery.newSlowExactQuery(name(), indexedValueForSearch(value));
6869
}
6970

7071
@Override
@@ -81,7 +82,7 @@ public Query rangeQuery(
8182
boolean includeUpper,
8283
SearchExecutionContext context
8384
) {
84-
return SortedSetDocValuesField.newSlowRangeQuery(
85+
return SortedSetDocValuesRangeQuery.newSlowRangeQuery(
8586
name(),
8687
lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
8788
upperTerm == null ? null : indexedValueForSearch(upperTerm),

‎server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,9 @@
3434
import org.elasticsearch.index.fielddata.plain.SortedOrdinalsIndexFieldData;
3535
import org.elasticsearch.index.mapper.blockloader.docvalues.BytesRefsFromOrdsBlockLoader;
3636
import org.elasticsearch.index.query.SearchExecutionContext;
37+
import org.elasticsearch.lucene.queries.SortedSetDocValuesRangeQuery;
3738
import org.elasticsearch.lucene.search.FuzzyQueries;
39+
import org.elasticsearch.lucene.search.XDocValuesRewriteMethod;
3840
import org.elasticsearch.script.Script;
3941
import org.elasticsearch.script.SortedSetDocValuesStringFieldScript;
4042
import org.elasticsearch.script.StringFieldScript;
@@ -147,7 +149,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
147149
public Query termQuery(Object value, SearchExecutionContext context) {
148150
failIfNotIndexedNorDocValuesFallback(context);
149151
if (indexType.hasDocValues()) {
150-
return SortedDocValuesField.newSlowExactQuery(name(), indexedValueForSearch(value));
152+
return SortedSetDocValuesRangeQuery.newSlowExactQuery(name(), indexedValueForSearch(value));
151153
} else {
152154
return super.termQuery(value, context);
153155
}
@@ -174,7 +176,7 @@ public Query rangeQuery(
174176
) {
175177
failIfNotIndexedNorDocValuesFallback(context);
176178
if (indexType.hasDocValues()) {
177-
return SortedDocValuesField.newSlowRangeQuery(
179+
return SortedSetDocValuesRangeQuery.newSlowRangeQuery(
178180
name(),
179181
lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
180182
upperTerm == null ? null : indexedValueForSearch(upperTerm),
@@ -204,7 +206,7 @@ public Query fuzzyQuery(
204206
prefixLength,
205207
maxExpansions,
206208
transpositions,
207-
MultiTermQuery.DOC_VALUES_REWRITE,
209+
XDocValuesRewriteMethod.DOC_VALUES_REWRITE,
208210
context,
209211
name()
210212
);
@@ -224,7 +226,7 @@ public Query prefixQuery(
224226
if (indexType.hasDocValues()) {
225227
if (caseInsensitive == false) {
226228
Term prefix = new Term(name(), indexedValueForSearch(value));
227-
return new PrefixQuery(prefix, MultiTermQuery.DOC_VALUES_REWRITE);
229+
return new PrefixQuery(prefix, XDocValuesRewriteMethod.DOC_VALUES_REWRITE);
228230
}
229231
return new StringScriptFieldPrefixQuery(
230232
new Script(""),
@@ -256,9 +258,9 @@ public Query wildcardQuery(
256258
Term term = new Term(name(), value);
257259
if (context.getCircuitBreaker() != null) {
258260
Automaton dfa = AutomatonQueries.toWildcardAutomaton(term, context.getCircuitBreaker());
259-
return new AutomatonQuery(term, dfa, false, MultiTermQuery.DOC_VALUES_REWRITE);
261+
return new AutomatonQuery(term, dfa, false, XDocValuesRewriteMethod.DOC_VALUES_REWRITE);
260262
}
261-
return new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, MultiTermQuery.DOC_VALUES_REWRITE);
263+
return new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, XDocValuesRewriteMethod.DOC_VALUES_REWRITE);
262264
}
263265

264266
StringFieldScript.LeafFactory leafFactory = ctx -> new SortedSetDocValuesStringFieldScript(name(), context.lookup(), ctx);
@@ -289,15 +291,15 @@ public Query regexpQuery(
289291
maxDeterminizedStates,
290292
context.getCircuitBreaker()
291293
);
292-
return new AutomatonQuery(term, dfa, false, MultiTermQuery.DOC_VALUES_REWRITE);
294+
return new AutomatonQuery(term, dfa, false, XDocValuesRewriteMethod.DOC_VALUES_REWRITE);
293295
}
294296
return new RegexpQuery(
295297
term,
296298
syntaxFlags,
297299
matchFlags,
298300
RegexpQuery.DEFAULT_PROVIDER,
299301
maxDeterminizedStates,
300-
MultiTermQuery.DOC_VALUES_REWRITE
302+
XDocValuesRewriteMethod.DOC_VALUES_REWRITE
301303
);
302304
}
303305
return super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context);

0 commit comments

Comments
 (0)