diff --git a/CHANGELOG.md b/CHANGELOG.md index c7e24a16d5be7..208081c3e9483 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add update and delete support in pull-based ingestion ([#17822](https://github.com/opensearch-project/OpenSearch/pull/17822)) - Allow maxPollSize and pollTimeout in IngestionSource to be configurable ([#17863](https://github.com/opensearch-project/OpenSearch/pull/17863)) - [Star Tree] [Search] Add query changes to support unsigned-long in star tree ([#17275](https://github.com/opensearch-project/OpenSearch/pull/17275)) +- Introduce mapping transformer to allow transform mappings during index create/update or index template create/update ([#17635](https://github.com/opensearch-project/OpenSearch/pull/17635)) ### Changed - Migrate BC libs to their FIPS counterparts ([#14912](https://github.com/opensearch-project/OpenSearch/pull/14912)) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java index 38dc1a418ec8b..dd0a815a5c497 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java @@ -43,6 +43,7 @@ import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.index.mapper.MappingTransformerRegistry; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -56,6 +57,7 @@ public class TransportCreateIndexAction extends TransportClusterManagerNodeAction { private final MetadataCreateIndexService createIndexService; + private final MappingTransformerRegistry mappingTransformerRegistry; @Inject public TransportCreateIndexAction( @@ -64,7 +66,9 @@ public TransportCreateIndexAction( ThreadPool threadPool, MetadataCreateIndexService createIndexService, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + MappingTransformerRegistry mappingTransformerRegistry + ) { super( CreateIndexAction.NAME, @@ -76,6 +80,7 @@ public TransportCreateIndexAction( indexNameExpressionResolver ); this.createIndexService = createIndexService; + this.mappingTransformerRegistry = mappingTransformerRegistry; } @Override @@ -112,25 +117,31 @@ protected void clusterManagerOperation( } final String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index()); - final CreateIndexClusterStateUpdateRequest updateRequest = new CreateIndexClusterStateUpdateRequest( - cause, - indexName, - request.index() - ).ackTimeout(request.timeout()) - .clusterManagerNodeTimeout(request.clusterManagerNodeTimeout()) - .settings(request.settings()) - .mappings(request.mappings()) - .aliases(request.aliases()) - .context(request.context()) - .waitForActiveShards(request.waitForActiveShards()); - - createIndexService.createIndex( - updateRequest, - ActionListener.map( - listener, - response -> new CreateIndexResponse(response.isAcknowledged(), response.isShardsAcknowledged(), indexName) - ) - ); + + final String finalCause = cause; + final ActionListener mappingTransformListener = ActionListener.wrap(transformedMappings -> { + final CreateIndexClusterStateUpdateRequest updateRequest = new CreateIndexClusterStateUpdateRequest( + finalCause, + indexName, + request.index() + ).ackTimeout(request.timeout()) + .clusterManagerNodeTimeout(request.clusterManagerNodeTimeout()) + .settings(request.settings()) + .mappings(transformedMappings) + .aliases(request.aliases()) + .context(request.context()) + .waitForActiveShards(request.waitForActiveShards()); + + createIndexService.createIndex( + updateRequest, + ActionListener.map( + listener, + response -> new CreateIndexResponse(response.isAcknowledged(), response.isShardsAcknowledged(), indexName) + ) + ); + }, listener::onFailure); + + mappingTransformerRegistry.applyTransformers(request.mappings(), null, mappingTransformListener); } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index ed936822bfdcd..4c37592714185 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -50,7 +50,9 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.index.Index; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.mapper.MappingTransformerRegistry; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -72,6 +74,7 @@ public class TransportPutMappingAction extends TransportClusterManagerNodeAction private final MetadataMappingService metadataMappingService; private final RequestValidators requestValidators; + private final MappingTransformerRegistry mappingTransformerRegistry; @Inject public TransportPutMappingAction( @@ -81,7 +84,8 @@ public TransportPutMappingAction( final MetadataMappingService metadataMappingService, final ActionFilters actionFilters, final IndexNameExpressionResolver indexNameExpressionResolver, - final RequestValidators requestValidators + final RequestValidators requestValidators, + final MappingTransformerRegistry mappingTransformerRegistry ) { super( PutMappingAction.NAME, @@ -94,6 +98,7 @@ public TransportPutMappingAction( ); this.metadataMappingService = metadataMappingService; this.requestValidators = Objects.requireNonNull(requestValidators); + this.mappingTransformerRegistry = mappingTransformerRegistry; } @Override @@ -132,7 +137,13 @@ protected void clusterManagerOperation( listener.onFailure(maybeValidationException.get()); return; } - performMappingUpdate(concreteIndices, request, listener, metadataMappingService); + + final ActionListener mappingTransformListener = ActionListener.wrap(transformedMapping -> { + request.source(transformedMapping, MediaTypeRegistry.JSON); + performMappingUpdate(concreteIndices, request, listener, metadataMappingService); + }, listener::onFailure); + + mappingTransformerRegistry.applyTransformers(request.source(), null, mappingTransformListener); } catch (IndexNotFoundException ex) { logger.debug(() -> new ParameterizedMessage("failed to put mappings on indices [{}]", Arrays.asList(request.indices())), ex); throw ex; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java index 66e2fe5c535db..a8002ec01d8cb 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java @@ -44,16 +44,20 @@ import org.opensearch.cluster.metadata.MetadataIndexTemplateService; import org.opensearch.cluster.metadata.Template; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.index.mapper.MappingTransformerRegistry; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; import java.io.IOException; +import reactor.util.annotation.NonNull; + /** * An action for putting a single component template into the cluster state * @@ -65,6 +69,7 @@ public class TransportPutComponentTemplateAction extends TransportClusterManager private final MetadataIndexTemplateService indexTemplateService; private final IndexScopedSettings indexScopedSettings; + private final MappingTransformerRegistry mappingTransformerRegistry; @Inject public TransportPutComponentTemplateAction( @@ -74,7 +79,8 @@ public TransportPutComponentTemplateAction( MetadataIndexTemplateService indexTemplateService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - IndexScopedSettings indexScopedSettings + IndexScopedSettings indexScopedSettings, + MappingTransformerRegistry mappingTransformerRegistry ) { super( PutComponentTemplateAction.NAME, @@ -87,6 +93,7 @@ public TransportPutComponentTemplateAction( ); this.indexTemplateService = indexTemplateService; this.indexScopedSettings = indexScopedSettings; + this.mappingTransformerRegistry = mappingTransformerRegistry; } @Override @@ -121,13 +128,37 @@ protected void clusterManagerOperation( template = new Template(settings, template.mappings(), template.aliases()); componentTemplate = new ComponentTemplate(template, componentTemplate.version(), componentTemplate.metadata()); } - indexTemplateService.putComponentTemplate( - request.cause(), - request.create(), - request.name(), - request.clusterManagerNodeTimeout(), - componentTemplate, - listener - ); + + final ActionListener mappingTransformListener = getMappingTransformListener(request, listener, componentTemplate); + + transformMapping(template, mappingTransformListener); + } + + private ActionListener getMappingTransformListener( + @NonNull final PutComponentTemplateAction.Request request, + @NonNull final ActionListener listener, + @NonNull final ComponentTemplate componentTemplate + ) { + return ActionListener.wrap(transformedMappings -> { + if (transformedMappings != null && componentTemplate.template() != null) { + componentTemplate.template().setMappings(new CompressedXContent(transformedMappings)); + } + indexTemplateService.putComponentTemplate( + request.cause(), + request.create(), + request.name(), + request.clusterManagerNodeTimeout(), + componentTemplate, + listener + ); + }, listener::onFailure); + } + + private void transformMapping(final Template template, @NonNull final ActionListener mappingTransformListener) { + if (template == null || template.mappings() == null) { + mappingTransformListener.onResponse(null); + } else { + mappingTransformerRegistry.applyTransformers(template.mappings().string(), null, mappingTransformListener); + } } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java index a5c3590a0a6d7..5554a540a57f2 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java @@ -41,15 +41,20 @@ import org.opensearch.cluster.metadata.ComposableIndexTemplate; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataIndexTemplateService; +import org.opensearch.cluster.metadata.Template; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.index.mapper.MappingTransformerRegistry; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; import java.io.IOException; +import reactor.util.annotation.NonNull; + /** * An action for putting a composable index template into the cluster state * @@ -60,6 +65,7 @@ public class TransportPutComposableIndexTemplateAction extends TransportClusterM AcknowledgedResponse> { private final MetadataIndexTemplateService indexTemplateService; + private final MappingTransformerRegistry mappingTransformerRegistry; @Inject public TransportPutComposableIndexTemplateAction( @@ -68,7 +74,8 @@ public TransportPutComposableIndexTemplateAction( ThreadPool threadPool, MetadataIndexTemplateService indexTemplateService, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + MappingTransformerRegistry mappingTransformerRegistry ) { super( PutComposableIndexTemplateAction.NAME, @@ -80,6 +87,7 @@ public TransportPutComposableIndexTemplateAction( indexNameExpressionResolver ); this.indexTemplateService = indexTemplateService; + this.mappingTransformerRegistry = mappingTransformerRegistry; } @Override @@ -103,15 +111,35 @@ protected void clusterManagerOperation( final PutComposableIndexTemplateAction.Request request, final ClusterState state, final ActionListener listener + ) throws IOException { + final ComposableIndexTemplate indexTemplate = request.indexTemplate(); + + final ActionListener mappingTransformListener = ActionListener.wrap(transformedMappings -> { + if (transformedMappings != null && indexTemplate.template() != null) { + indexTemplate.template().setMappings(new CompressedXContent(transformedMappings)); + } + indexTemplateService.putIndexTemplateV2( + request.cause(), + request.create(), + request.name(), + request.clusterManagerNodeTimeout(), + indexTemplate, + listener + ); + }, listener::onFailure); + + transformMapping(indexTemplate, mappingTransformListener); + } + + private void transformMapping( + @NonNull final ComposableIndexTemplate indexTemplate, + @NonNull final ActionListener mappingTransformListener ) { - ComposableIndexTemplate indexTemplate = request.indexTemplate(); - indexTemplateService.putIndexTemplateV2( - request.cause(), - request.create(), - request.name(), - request.clusterManagerNodeTimeout(), - indexTemplate, - listener - ); + final Template template = indexTemplate.template(); + if (template == null || template.mappings() == null) { + mappingTransformListener.onResponse(null); + } else { + mappingTransformerRegistry.applyTransformers(template.mappings().string(), null, mappingTransformListener); + } } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java index b9f27c00d0d98..2467fc0346313 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java @@ -49,6 +49,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.index.mapper.MappingTransformerRegistry; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -65,6 +66,7 @@ public class TransportPutIndexTemplateAction extends TransportClusterManagerNode private final MetadataIndexTemplateService indexTemplateService; private final IndexScopedSettings indexScopedSettings; + private final MappingTransformerRegistry mappingTransformerRegistry; @Inject public TransportPutIndexTemplateAction( @@ -74,7 +76,8 @@ public TransportPutIndexTemplateAction( MetadataIndexTemplateService indexTemplateService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - IndexScopedSettings indexScopedSettings + IndexScopedSettings indexScopedSettings, + MappingTransformerRegistry mappingTransformerRegistry ) { super( PutIndexTemplateAction.NAME, @@ -87,6 +90,7 @@ public TransportPutIndexTemplateAction( ); this.indexTemplateService = indexTemplateService; this.indexScopedSettings = indexScopedSettings; + this.mappingTransformerRegistry = mappingTransformerRegistry; } @Override @@ -118,28 +122,33 @@ protected void clusterManagerOperation( final Settings.Builder templateSettingsBuilder = Settings.builder(); templateSettingsBuilder.put(request.settings()).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX); indexScopedSettings.validate(templateSettingsBuilder.build(), true); // templates must be consistent with regards to dependencies - indexTemplateService.putTemplate( - new MetadataIndexTemplateService.PutRequest(cause, request.name()).patterns(request.patterns()) - .order(request.order()) - .settings(templateSettingsBuilder.build()) - .mappings(request.mappings()) - .aliases(request.aliases()) - .create(request.create()) - .clusterManagerTimeout(request.clusterManagerNodeTimeout()) - .version(request.version()), + final String finalCause = cause; + final ActionListener mappingTransformListener = ActionListener.wrap(transformedMappings -> { + indexTemplateService.putTemplate( + new MetadataIndexTemplateService.PutRequest(finalCause, request.name()).patterns(request.patterns()) + .order(request.order()) + .settings(templateSettingsBuilder.build()) + .mappings(transformedMappings) + .aliases(request.aliases()) + .create(request.create()) + .clusterManagerTimeout(request.clusterManagerNodeTimeout()) + .version(request.version()), - new MetadataIndexTemplateService.PutListener() { - @Override - public void onResponse(MetadataIndexTemplateService.PutResponse response) { - listener.onResponse(new AcknowledgedResponse(response.acknowledged())); - } + new MetadataIndexTemplateService.PutListener() { + @Override + public void onResponse(MetadataIndexTemplateService.PutResponse response) { + listener.onResponse(new AcknowledgedResponse(response.acknowledged())); + } - @Override - public void onFailure(Exception e) { - logger.debug(() -> new ParameterizedMessage("failed to put template [{}]", request.name()), e); - listener.onFailure(e); + @Override + public void onFailure(Exception e) { + logger.debug(() -> new ParameterizedMessage("failed to put template [{}]", request.name()), e); + listener.onFailure(e); + } } - } - ); + ); + }, listener::onFailure); + + mappingTransformerRegistry.applyTransformers(request.mappings(), null, mappingTransformListener); } } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java index b032ade720612..de106d14c6fd9 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1754,6 +1754,10 @@ public PutRequest mappings(String mappings) { return this; } + public String getMappings() { + return mappings; + } + public PutRequest aliases(Set aliases) { this.aliases.addAll(aliases); return this; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Template.java b/server/src/main/java/org/opensearch/cluster/metadata/Template.java index bd110c6af8975..be29a73e9c0ad 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Template.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Template.java @@ -94,7 +94,7 @@ public class Template extends AbstractDiffable