Support configuring reusing existing embedding for the semantic field. (opensearch-project#1480)

Signed-off-by: Bo Zhang <[email protected]>

Author: bzhangam

CHANGELOG.md

@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Semantic Field] Allow configuring prune strategies for sparse encoding in semantic fields. ([#1434](https://github.com/opensearch-project/neural-search/pull/1434))
 - Enable inner hits within collapse parameter for hybrid query ([#1447](https://github.com/opensearch-project/neural-search/pull/1447))
 - [Semantic Field] Support configuring the chunking strategies through the semantic field. ([#1446](https://github.com/opensearch-project/neural-search/pull/1446))
+- [Semantic Field] Support configuring reusing existing embedding for the semantic field. ([#1480](https://github.com/opensearch-project/neural-search/pull/1480/files))
 
 ### Bug Fixes
 - Fix for collapse bug with knn query not deduplicating results ([#1413](https://github.com/opensearch-project/neural-search/pull/1413))

src/main/java/org/opensearch/neuralsearch/constants/DocFieldNames.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.opensearch.neuralsearch.constants;
+
+/**
+ * Constants related to SourceAndMetadata of a doc
+ */
+public class DocFieldNames {
+    /**
+     * field name of the doc index
+     */
+    public static final String INDEX_FIELD = "_index";
+    /**
+     * field name of the doc id
+     */
+    public static final String ID_FIELD = "_id";
+}

src/main/java/org/opensearch/neuralsearch/constants/SemanticFieldConstants.java

@@ -59,4 +59,11 @@ public class SemanticFieldConstants {
      * {@link org.opensearch.neuralsearch.mapper.dto.SparseEncodingConfig}
      */
     public static final String SPARSE_ENCODING_CONFIG = "sparse_encoding_config";
+
+    /**
+     * Name of the field to configure if we should skip the embedding generation for the semantic field with existing
+     * reusable embedding. We think the embedding is reusable if it exists and the semantic field value and the
+     * model is not changed.
+     */
+    public static final String SKIP_EXISTING_EMBEDDING = "skip_existing_embedding";
 }

src/main/java/org/opensearch/neuralsearch/mapper/SemanticFieldMapper.java

@@ -39,6 +39,7 @@
 import static org.opensearch.neuralsearch.constants.SemanticFieldConstants.DENSE_EMBEDDING_CONFIG;
 import static org.opensearch.neuralsearch.constants.SemanticFieldConstants.MODEL_ID;
 import static org.opensearch.neuralsearch.constants.SemanticFieldConstants.RAW_FIELD_TYPE;
+import static org.opensearch.neuralsearch.constants.SemanticFieldConstants.SKIP_EXISTING_EMBEDDING;
 import static org.opensearch.neuralsearch.constants.SemanticFieldConstants.SEARCH_MODEL_ID;
 import static org.opensearch.neuralsearch.constants.SemanticFieldConstants.SEMANTIC_INFO_FIELD_NAME;
 import static org.opensearch.neuralsearch.constants.SemanticFieldConstants.SEMANTIC_FIELD_SEARCH_ANALYZER;
@@ -196,6 +197,14 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
             }
         }, (value) -> value == null ? null : value.toString());
 
+        @Getter
+        protected final Parameter<Boolean> skipExistingEmbedding = Parameter.boolParam(
+            SKIP_EXISTING_EMBEDDING,
+            true,
+            m -> ((SemanticFieldMapper) m).semanticParameters.getSkipExistingEmbedding(),
+            false
+        );
+
         @Setter
         protected ParametrizedFieldMapper.Builder delegateBuilder;
 
@@ -213,7 +222,8 @@ protected List<Parameter<?>> getParameters() {
                 chunkingConfig,
                 semanticFieldSearchAnalyzer,
                 denseEmbeddingConfig,
-                sparseEncodingConfig
+                sparseEncodingConfig,
+                skipExistingEmbedding
             );
         }
 
@@ -244,6 +254,7 @@ public SemanticParameters getSemanticParameters() {
                 .semanticFieldSearchAnalyzer(semanticFieldSearchAnalyzer.getValue())
                 .denseEmbeddingConfig(denseEmbeddingConfig.getValue())
                 .sparseEncodingConfig(sparseEncodingConfig.getValue())
+                .skipExistingEmbedding(skipExistingEmbedding.getValue())
                 .build();
         }
     }

src/main/java/org/opensearch/neuralsearch/mapper/dto/SemanticParameters.java

@@ -30,4 +30,6 @@ public boolean isChunkingEnabled() {
         }
         return chunkingConfig.isEnabled();
     }
+
+    private final Boolean skipExistingEmbedding;
 }

src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java

@@ -311,7 +311,8 @@ public Map<String, Processor.Factory> getSystemIngestProcessors(Processor.Parame
                 clientAccessor,
                 parameters.env,
                 parameters.ingestService.getClusterService(),
-                parameters.analysisRegistry
+                parameters.analysisRegistry,
+                parameters.client
             )
         );
     }

src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java

@@ -52,6 +52,9 @@
 import org.opensearch.neuralsearch.util.prune.PruneType;
 import org.opensearch.neuralsearch.util.prune.PruneUtils;
 
+import static org.opensearch.neuralsearch.constants.DocFieldNames.ID_FIELD;
+import static org.opensearch.neuralsearch.constants.DocFieldNames.INDEX_FIELD;
+
 /**
  * The abstract class for text processing use cases. Users provide a field name map and a model id.
  * During ingestion, the processor will use the corresponding model to inference the input texts,
@@ -62,8 +65,6 @@ public abstract class InferenceProcessor extends AbstractBatchingProcessor {
 
     public static final String MODEL_ID_FIELD = "model_id";
     public static final String FIELD_MAP_FIELD = "field_map";
-    public static final String INDEX_FIELD = "_index";
-    public static final String ID_FIELD = "_id";
     public static final String SKIP_EXISTING = "skip_existing";
     public static final boolean DEFAULT_SKIP_EXISTING = false;
     private static final BiFunction<Object, Object, Object> REMAPPING_FUNCTION = (v1, v2) -> {

src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java

@@ -32,6 +32,9 @@
 import org.opensearch.neuralsearch.util.prune.PruneUtils;
 import org.opensearch.transport.client.OpenSearchClient;
 
+import static org.opensearch.neuralsearch.constants.DocFieldNames.ID_FIELD;
+import static org.opensearch.neuralsearch.constants.DocFieldNames.INDEX_FIELD;
+
 /**
  * This processor is used for user input data text sparse encoding processing, model_id can be used to indicate which model user use,
  * and field_map can be used to indicate which fields needs text embedding and the corresponding keys for the sparse encoding results.

src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java

@@ -27,6 +27,9 @@
 import org.opensearch.neuralsearch.stats.events.EventStatsManager;
 import org.opensearch.neuralsearch.stats.events.EventStatName;
 
+import static org.opensearch.neuralsearch.constants.DocFieldNames.ID_FIELD;
+import static org.opensearch.neuralsearch.constants.DocFieldNames.INDEX_FIELD;
+
 /**
  * This processor is used for user input data text embedding processing, model_id can be used to indicate which model user use,
  * and field_map can be used to indicate which fields needs text embedding and the corresponding keys for the text embedding results.

src/main/java/org/opensearch/neuralsearch/processor/dto/SemanticFieldInfo.java

@@ -45,6 +45,10 @@ public class SemanticFieldInfo {
      * The full path to the semantic field in the index mapping
      */
     private String semanticFieldFullPathInMapping;
+    /**
+     * The full path to the semantic field in the doc
+     */
+    private String semanticFieldFullPathInDoc;
     /**
      * The full path to the semantic info fields in the doc. The path in the doc will contain the index of the inter
      * nested object.
@@ -67,6 +71,17 @@ public class SemanticFieldInfo {
 
     private SparseEncodingConfig sparseEncodingConfig;
 
+    /**
+     * If we should skip the embedding generation for the semantic field with exist reusable embedding. We think the
+     * embedding is reusable if it exists and semantic field value and model are not changed.
+     */
+    private Boolean skipExistingEmbedding;
+
+    /**
+     * The id of the doc of the semantic field
+     */
+    private String docId;
+
     /**
      * @return full path to the chunks field of the semantic field in a doc
      */