Support synthetic _source for _doc_count field (#91465)

This add synthetic `_source` support for the `_doc_count` field so
downsampling should play nicely with sythetic `_source`.

Author: nik9000

docs/changelog/91465.yaml

@@ -0,0 +1,5 @@
+pr: 91465
+summary: Support synthetic `_source` for `_doc_count` field
+area: TSDB
+type: enhancement
+issues: []

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java

@@ -11,6 +11,7 @@
 import org.apache.lucene.search.Query;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
+import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.mapper.ValueFetcher;
 import org.elasticsearch.index.query.SearchExecutionContext;
@@ -68,4 +69,8 @@ protected String contentType() {
         return CONTENT_TYPE;
     }
 
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
+import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.ValueFetcher;
 import org.elasticsearch.index.query.SearchExecutionContext;
 
@@ -96,4 +97,9 @@ public void postParse(DocumentParserContext context) {
     public FieldMapper.Builder getMergeBuilder() {
         return new Builder().init(this);
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml

@@ -584,6 +584,45 @@ _source filtering:
         kwd: foo
   - is_false: fields
 
+---
+_doc_count:
+  - skip:
+      version: " - 8.5.99"
+      reason: introduced in 8.6.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_replicas: 0
+          mappings:
+            _source:
+              mode: synthetic
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        refresh: true
+        body:
+          _doc_count: 3
+          foo: bar
+
+  - do:
+      get:
+        index: test
+        id:    1
+  - match: {_index: "test"}
+  - match: {_id: "1"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        _doc_count: 3
+        foo: bar
+  - is_false: fields
+
 ---
 ip with ignore_malformed:
   - skip:

server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
+import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -718,6 +719,11 @@ protected String contentType() {
             return CONTENT_TYPE;
         }
 
+        @Override
+        public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+            throw new UnsupportedOperationException();
+        }
+
         private static final TypeParser PARSER = new FixedTypeParser(c -> new TestMetadataMapper());
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java

@@ -268,4 +268,9 @@ protected String contentType() {
     public boolean isEnabled() {
         return enabled;
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java

@@ -8,14 +8,20 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.Map;
+import java.util.stream.Stream;
 
 /** Mapper for the doc_count field. */
 public class DocCountFieldMapper extends MetadataFieldMapper {
@@ -25,6 +31,11 @@ public class DocCountFieldMapper extends MetadataFieldMapper {
 
     private static final DocCountFieldMapper INSTANCE = new DocCountFieldMapper();
 
+    /**
+     * The term that is the key to the postings list that stores the doc counts.
+     */
+    private static final Term TERM = new Term(NAME, NAME);
+
     public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE);
 
     public static final class DocCountFieldType extends MappedFieldType {
@@ -115,4 +126,49 @@ protected String contentType() {
     public static IndexableField field(int count) {
         return new CustomTermFreqField(NAME, NAME, count);
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return new SyntheticFieldLoader();
+    }
+
+    /**
+     * The lookup for loading values.
+     */
+    public static PostingsEnum leafLookup(LeafReader reader) throws IOException {
+        return reader.postings(TERM);
+    }
+
+    private class SyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader {
+        private PostingsEnum postings;
+        private boolean hasValue;
+
+        @Override
+        public Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
+            return Stream.empty();
+        }
+
+        @Override
+        public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException {
+            postings = leafLookup(leafReader);
+            if (postings == null) {
+                hasValue = false;
+                return null;
+            }
+            return docId -> hasValue = docId == postings.advance(docId);
+        }
+
+        @Override
+        public boolean hasValue() {
+            return hasValue;
+        }
+
+        @Override
+        public void write(XContentBuilder b) throws IOException {
+            if (hasValue == false) {
+                return;
+            }
+            b.field(NAME, postings.freq());
+        }
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java

@@ -196,4 +196,8 @@ protected String contentType() {
         return CONTENT_TYPE;
     }
 
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java

@@ -41,6 +41,11 @@ protected final String contentType() {
         return CONTENT_TYPE;
     }
 
+    @Override
+    public final SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
+
     /**
      * Description of the document being parsed used in error messages. Not
      * called unless there is an error.

server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java

@@ -88,4 +88,8 @@ protected String contentType() {
         return CONTENT_TYPE;
     }
 
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java

@@ -102,4 +102,9 @@ public IndexFieldMapper() {
     protected String contentType() {
         return CONTENT_TYPE;
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java

@@ -106,4 +106,9 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
     protected String contentType() {
         return CONTENT_TYPE;
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.common.Strings;
 
 import java.util.Objects;
+import java.util.function.BooleanSupplier;
 
 /**
  * Holds context for building Mapper objects from their Builders
@@ -21,19 +22,28 @@ public final class MapperBuilderContext {
      * The root context, to be used when building a tree of mappers
      */
     public static MapperBuilderContext root(boolean isSourceSynthetic) {
-        return new MapperBuilderContext(isSourceSynthetic);
+        return new MapperBuilderContext(null, () -> isSourceSynthetic);
+    }
+
+    /**
+     * A context to use to build metadata fields.
+     */
+    public static MapperBuilderContext forMetadata() {
+        return new MapperBuilderContext(
+            null,
+            () -> { throw new UnsupportedOperationException("metadata fields can't check if _source is synthetic"); }
+        );
     }
 
     private final String path;
-    private final boolean isSourceSynthetic;
+    private final BooleanSupplier isSourceSynthetic;
 
-    private MapperBuilderContext(boolean isSourceSynthetic) {
-        this.path = null;
-        this.isSourceSynthetic = isSourceSynthetic;
+    MapperBuilderContext(String path, boolean isSourceSynthetic) {
+        this(Objects.requireNonNull(path), () -> isSourceSynthetic);
     }
 
-    MapperBuilderContext(String path, boolean isSourceSynthetic) {
-        this.path = Objects.requireNonNull(path);
+    private MapperBuilderContext(String path, BooleanSupplier isSourceSynthetic) {
+        this.path = path;
         this.isSourceSynthetic = isSourceSynthetic;
     }
 
@@ -56,7 +66,10 @@ public String buildFullName(String name) {
         return path + "." + name;
     }
 
+    /**
+     * Is the {@code _source} field being reconstructed on the fly?
+     */
     public boolean isSourceSynthetic() {
-        return isSourceSynthetic;
+        return isSourceSynthetic.getAsBoolean();
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/Mapping.java

@@ -124,6 +124,10 @@ private boolean isSourceSynthetic() {
         return sfm != null && sfm.isSynthetic();
     }
 
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return root.syntheticFieldLoader(Arrays.stream(metadataMappers));
+    }
+
     /**
      * Merges a new mapping into the existing one.
      *

server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java

@@ -120,7 +120,7 @@ private Mapping parse(String type, Map<String, Object> mapping) throws MapperPar
                 @SuppressWarnings("unchecked")
                 Map<String, Object> fieldNodeMap = (Map<String, Object>) fieldNode;
                 MetadataFieldMapper metadataFieldMapper = typeParser.parse(fieldName, fieldNodeMap, parserContext)
-                    .build(MapperBuilderContext.root(false));
+                    .build(MapperBuilderContext.forMetadata());
                 metadataMappers.put(metadataFieldMapper.getClass(), metadataFieldMapper);
                 fieldNodeMap.remove("type");
                 checkNoRemainingFields(fieldName, fieldNodeMap);

server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java

@@ -155,7 +155,7 @@ public final XContentBuilder toXContent(XContentBuilder builder, Params params)
     @Override
     protected void parseCreateField(DocumentParserContext context) throws IOException {
         throw new MapperParsingException(
-            "Field [" + name() + "] is a metadata field and cannot be added inside" + " a document. Use the index API request parameters."
+            "Field [" + name() + "] is a metadata field and cannot be added inside a document. Use the index API request parameters."
         );
     }
 
@@ -173,4 +173,6 @@ public void postParse(DocumentParserContext context) throws IOException {
         // do nothing
     }
 
+    @Override
+    public abstract SourceLoader.SyntheticFieldLoader syntheticFieldLoader();
 }

server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java

@@ -97,4 +97,8 @@ protected String contentType() {
         return NAME;
     }
 
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java

@@ -578,18 +578,21 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
 
     }
 
-    @Override
-    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(Stream<Mapper> extra) {
         return new SyntheticSourceFieldLoader(
-            mappers.values()
-                .stream()
+            Stream.concat(extra, mappers.values().stream())
                 .sorted(Comparator.comparing(Mapper::name))
                 .map(Mapper::syntheticFieldLoader)
                 .filter(l -> l != null)
                 .toList()
         );
     }
 
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return syntheticFieldLoader(Stream.empty());
+    }
+
     private class SyntheticSourceFieldLoader implements SourceLoader.SyntheticFieldLoader {
         private final List<SourceLoader.SyntheticFieldLoader> fields;
         private boolean hasValue;

server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java

@@ -131,4 +131,8 @@ protected String contentType() {
         return CONTENT_TYPE;
     }
 
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java

@@ -239,4 +239,8 @@ protected String contentType() {
         return CONTENT_TYPE;
     }
 
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }