From fd520edb9e57dc1424f64d11dbfa26bea8e02c6f Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 3 Jul 2026 20:30:15 +0300 Subject: [PATCH 01/23] Add IndexMode.TSDB as a preferred alias for TIME_SERIES Users and docs already call time-series indices tsdb; add a new IndexMode.TSDB constant that behaves identically to TIME_SERIES so index.mode: tsdb is now also accepted, without renaming or changing anything about existing time_series indices, which keep emitting and persisting exactly as before. TSDB delegates every behavior to TIME_SERIES and is treated identically everywhere via new isTsdb()/isTsdb(IndexMode)/isTsdbName(String) helpers, replacing direct == TIME_SERIES comparisons across the codebase. Adds a transport version gating TSDB's wire serialization for mixed-version clusters, unit tests for the new alias, and a rolling upgrade test that switches a data stream from time_series to tsdb mid-upgrade and verifies both backing indices stay queryable together. --- .../datastreams/TSDBSyntheticIdsIT.java | 8 +- .../DataStreamIndexSettingsProvider.java | 6 +- .../UpdateTimeSeriesRangeService.java | 2 +- .../action/TransportGetDataStreamsAction.java | 4 +- ...portPastTimeSeriesIndexCreationAction.java | 6 +- .../lifecycle/DataStreamLifecycleService.java | 3 +- .../action/PainlessExecuteAction.java | 3 +- .../mapper/extras/ScaledFloatFieldMapper.java | 6 +- .../rollover/MetadataRolloverService.java | 2 +- .../cluster/metadata/DataStream.java | 11 +- .../cluster/metadata/IndexMetadata.java | 2 +- .../cluster/routing/IndexRouting.java | 6 +- .../TimeSeriesEligibleWriteWindowLocator.java | 2 +- .../org/elasticsearch/index/IndexMode.java | 113 +++++++++++ .../elasticsearch/index/IndexSettings.java | 21 ++- .../elasticsearch/index/IndexSortConfig.java | 6 +- .../index/codec/PerFieldFormatSupplier.java | 3 +- .../tsdb/TSDBDocValuesFormatSelector.java | 5 +- .../index/engine/InternalEngine.java | 7 +- .../index/mapper/DateFieldMapper.java | 2 +- .../index/mapper/DocumentMapper.java | 2 +- .../index/mapper/DocumentParser.java | 5 +- .../index/mapper/DocumentParserContext.java | 5 +- .../index/mapper/FieldMapper.java | 2 +- .../index/mapper/GeoPointFieldMapper.java | 4 +- .../index/mapper/IdFieldMapper.java | 5 +- .../elasticsearch/index/mapper/IdLoader.java | 3 +- .../index/mapper/MappingLookup.java | 2 +- .../index/mapper/NumberFieldMapper.java | 4 +- .../TimeSeriesMetadataFieldBlockLoader.java | 2 +- .../TimeSeriesRoutingHashFieldMapper.java | 3 +- .../referable/index_mode_tsdb_added.csv | 1 + .../resources/transport/upper_bounds/9.5.csv | 2 +- .../indices/resolve/ResolveIndexTests.java | 2 +- .../index/IndexSettingsTests.java | 9 +- .../index/TsdbIndexModeTests.java | 122 ++++++++++++ .../index/codec/PerFieldMapperCodecTests.java | 8 +- .../TSDBDocValuesFormatSelectorTests.java | 4 +- .../index/mapper/IndexModeFieldTypeTests.java | 2 +- .../index/mapper/MapperServiceTests.java | 4 +- .../index/mapper/MappingLookupTests.java | 4 +- .../index/mapper/SourceFieldMapperTests.java | 2 +- .../index/mapper/MapperServiceTestCase.java | 9 + .../elasticsearch/test/ESIntegTestCase.java | 4 +- .../index/engine/FollowingEngineTests.java | 5 +- .../TimeSeriesUsageTransportAction.java | 4 +- .../xpack/core/ilm/DownsampleAction.java | 3 +- .../WaitUntilTimeSeriesEndTimePassesStep.java | 3 +- .../downsample/TransportDownsampleAction.java | 2 +- .../qa/rest/AllSupportedFieldsTestCase.java | 10 +- .../xpack/esql/analysis/Analyzer.java | 8 +- .../xpack/esql/analysis/PreAnalyzer.java | 2 +- .../esql/analysis/rules/ResolveUnmapped.java | 2 +- .../esql/datasources/DatasetRewriter.java | 2 +- .../rules/logical/PruneUnusedIndexMode.java | 2 +- .../logical/local/IgnoreNullMetrics.java | 5 +- .../local/PushExpressionsToFieldLoad.java | 2 +- .../local/ReplaceRoundToWithQueryAndTags.java | 6 +- .../local/ReplaceSourceAttributes.java | 3 +- .../xpack/esql/parser/LogicalPlanBuilder.java | 2 +- .../xpack/esql/plan/logical/Aggregate.java | 3 +- .../xpack/esql/plan/logical/InlineStats.java | 3 +- .../xpack/esql/plan/logical/MetricsInfo.java | 3 +- .../xpack/esql/plan/logical/TsInfo.java | 3 +- .../esql/plan/logical/UnresolvedRelation.java | 2 +- .../xpack/esql/plan/physical/EsQueryExec.java | 4 +- .../planner/EsPhysicalOperationProviders.java | 3 +- .../esql/planner/LocalExecutionPlanner.java | 2 +- .../xpack/esql/session/EsqlSession.java | 19 +- .../xpack/esql/telemetry/FeatureMetric.java | 5 +- ...SeriesToTsdbIndexModeRollingUpgradeIT.java | 176 ++++++++++++++++++ .../LogsdbIndexModeSettingsProvider.java | 8 +- .../unsignedlong/UnsignedLongFieldMapper.java | 6 +- .../IndexCommitTimestampFieldRangeTests.java | 18 +- 74 files changed, 579 insertions(+), 165 deletions(-) create mode 100644 server/src/main/resources/transport/definitions/referable/index_mode_tsdb_added.csv create mode 100644 server/src/test/java/org/elasticsearch/index/TsdbIndexModeTests.java create mode 100644 x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TimeSeriesToTsdbIndexModeRollingUpgradeIT.java diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java index 496065e0b0f8d..bf2c52d517585 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java @@ -152,7 +152,7 @@ protected Collection> nodePlugins() { public void testInvalidIndexMode() { final var indexName = randomIdentifier(); - var randomNonTsdbIndexMode = randomValueOtherThan(IndexMode.TIME_SERIES, () -> randomFrom(IndexMode.availableModes())); + var randomNonTsdbIndexMode = randomValueOtherThanMany(m -> m.isTsdb(), () -> randomFrom(IndexMode.availableModes())); var exception = expectThrows( IllegalArgumentException.class, @@ -168,7 +168,11 @@ public void testInvalidIndexMode() { containsString( "The setting [" + IndexSettings.SYNTHETIC_ID.getKey() - + "] is only permitted when [index.mode] is set to [TIME_SERIES]. Current mode: [" + + "] is only permitted when [index.mode] is set to [" + + IndexMode.TIME_SERIES.name() + + "] or [" + + IndexMode.TSDB.name() + + "]. Current mode: [" + randomNonTsdbIndexMode.getName().toUpperCase(Locale.ROOT) + "]." ) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index 403ec320e8a5d..4f85096db7fb6 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -94,7 +94,7 @@ public void provideAdditionalSettings( // First backing index is created and then data stream is rolled over (in a single cluster state update). // So at this point we can't check index_mode==time_series, // so checking that index_mode==null|standard and templateIndexMode == TIME_SERIES - boolean isMigratingToTimeSeries = templateIndexMode == IndexMode.TIME_SERIES; + boolean isMigratingToTimeSeries = IndexMode.isTsdb(templateIndexMode); boolean migrating = dataStream != null && (dataStream.getIndexMode() == null || dataStream.getIndexMode() == IndexMode.STANDARD) && isMigratingToTimeSeries; @@ -109,7 +109,7 @@ public void provideAdditionalSettings( indexMode = null; } if (indexMode != null) { - if (indexMode == IndexMode.TIME_SERIES) { + if (indexMode.isTsdb()) { TimeValue lookAheadTime = DataStreamsPlugin.getLookAheadTime(indexTemplateAndCreateRequestSettings); TimeValue lookBackTime = DataStreamsPlugin.LOOK_BACK_TIME.get(indexTemplateAndCreateRequestSettings); final Instant start; @@ -197,7 +197,7 @@ public void onUpdateMappings(IndexMetadata indexMetadata, DocumentMapper documen if (indexDimensions.isEmpty()) { return; } - assert indexMetadata.getIndexMode() == IndexMode.TIME_SERIES; + assert IndexMode.isTsdb(indexMetadata.getIndexMode()); List newIndexDimensions = new ArrayList<>(indexDimensions.size()); boolean matchesAllDimensions = findDimensionFields(newIndexDimensions, documentMapper); boolean hasChanges = indexDimensions.size() != newIndexDimensions.size() diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeService.java index f967fc51890f3..4b4259b947839 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeService.java @@ -118,7 +118,7 @@ ClusterState updateTimeSeriesTemporalRange(ClusterState current, Instant now) { private ProjectMetadata.Builder updateTimeSeriesTemporalRange(ProjectMetadata project, Instant now) { ProjectMetadata.Builder mBuilder = null; for (DataStream dataStream : project.dataStreams().values()) { - if (dataStream.getIndexMode() != IndexMode.TIME_SERIES) { + if (IndexMode.isTsdb(dataStream.getIndexMode()) == false) { continue; } if (dataStream.isReplicated()) { diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java index c8b5504bdb0eb..756b5ac5f03ea 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java @@ -332,7 +332,7 @@ static GetDataStreamAction.Response innerOperation( } GetDataStreamAction.Response.TimeSeries timeSeries = null; - if (dataStream.getIndexMode() == IndexMode.TIME_SERIES) { + if (IndexMode.isTsdb(dataStream.getIndexMode())) { record IndexInfo(String name, Instant timeSeriesStart, Instant timeSeriesEnd) implements Comparable { @Override public int compareTo(IndexInfo o) { @@ -350,7 +350,7 @@ public int compareTo(IndexInfo o) { var sortedRanges = dataStream.getIndices() .stream() .map(metadata::index) - .filter(m -> m.getIndexMode() == IndexMode.TIME_SERIES) + .filter(m -> IndexMode.isTsdb(m.getIndexMode())) .map(m -> new IndexInfo(m.getIndex().getName(), m.getTimeSeriesStart(), m.getTimeSeriesEnd())) .sorted() .toList(); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportPastTimeSeriesIndexCreationAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportPastTimeSeriesIndexCreationAction.java index 186a107a88c1b..c28b2decdf5b4 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportPastTimeSeriesIndexCreationAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportPastTimeSeriesIndexCreationAction.java @@ -471,7 +471,7 @@ static Deque retrieveSortedTimeWindows(DataStream dataStream, List coveredTimeWindows = new ArrayList<>(); for (Index existingIndex : dataStream.getIndices()) { IndexMetadata im = currentProject.index(existingIndex); - if (im == null || im.getIndexMode() != IndexMode.TIME_SERIES) { + if (im == null || IndexMode.isTsdb(im.getIndexMode()) == false) { continue; } assert im.getTimeSeriesStart() != null && im.getTimeSeriesEnd() != null : "TSDB indices always have start and end time"; @@ -525,7 +525,7 @@ private static void validateDataStream(String dataStreamName, DataStream dataStr throw new IllegalArgumentException("Cannot create past TSDB backing index for system data stream [" + dataStreamName + "]"); } - if (dataStream.getIndexMode() != IndexMode.TIME_SERIES) { + if (IndexMode.isTsdb(dataStream.getIndexMode()) == false) { throw new IllegalArgumentException( "Cannot create past TSDB backing index for data stream [" + dataStreamName @@ -536,7 +536,7 @@ private static void validateDataStream(String dataStreamName, DataStream dataStr } Index writeIndex = dataStream.getWriteIndex(); IndexMetadata writeIndexMetadata = writeIndex == null ? null : project.index(writeIndex); - if (writeIndexMetadata == null || writeIndexMetadata.getIndexMode() != IndexMode.TIME_SERIES) { + if (writeIndexMetadata == null || IndexMode.isTsdb(writeIndexMetadata.getIndexMode()) == false) { throw new IllegalStateException( "Cannot create past TSDB backing index for data stream [" + dataStreamName diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index e10b8650b458c..34dc5e541e2d2 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -78,7 +78,6 @@ import org.elasticsearch.dlm.DataStreamLifecycleErrorStore; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.MergePolicyConfig; @@ -665,7 +664,7 @@ static Set timeSeriesIndicesStillWithinTimeBounds(ProjectMetadata project for (Index index : targetIndices) { IndexMetadata backingIndex = project.index(index); assert backingIndex != null : "the data stream backing indices must exist"; - if (IndexSettings.MODE.get(backingIndex.getSettings()) == IndexMode.TIME_SERIES) { + if (IndexSettings.MODE.get(backingIndex.getSettings()).isTsdb()) { Instant configuredEndTime = IndexSettings.TIME_SERIES_END_TIME.get(backingIndex.getSettings()); assert configuredEndTime != null : "a time series index must have an end time configured but [" + index.getName() + "] does not"; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index ab18ce4a58a72..2d10b8125567e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -56,7 +56,6 @@ import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Point; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DateFieldMapper; @@ -850,7 +849,7 @@ private static Response prepareRamIndex( BytesReference document = request.contextSetup.document; XContentType xContentType = request.contextSetup.xContentType; - SourceToParse sourceToParse = (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) + SourceToParse sourceToParse = (indexService.getIndexSettings().getMode().isTsdb()) ? new SourceToParse( null, document, diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index c41d182c4db39..2d6d4c8b6d760 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -161,7 +161,7 @@ public Builder(String name, IndexSettings indexSettings) { return false; } - if (indexSettings.getMode() == IndexMode.TIME_SERIES) { + if (indexSettings.getMode().isTsdb()) { var metricType = getMetric().getValue(); return metricType != TimeSeriesParams.MetricType.COUNTER && metricType != TimeSeriesParams.MetricType.GAUGE; } else { @@ -407,7 +407,7 @@ public Query rangeQuery( @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - if (indexMode == IndexMode.TIME_SERIES && metricType == TimeSeriesParams.MetricType.COUNTER) { + if (indexMode.isTsdb() && metricType == TimeSeriesParams.MetricType.COUNTER) { // Counters are not supported by ESQL so we load them in null return ConstantNull.INSTANCE; } @@ -495,7 +495,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext failIfNoDocValues(); } - ValuesSourceType valuesSourceType = indexMode == IndexMode.TIME_SERIES && metricType == TimeSeriesParams.MetricType.COUNTER + ValuesSourceType valuesSourceType = indexMode.isTsdb() && metricType == TimeSeriesParams.MetricType.COUNTER ? TimeSeriesValuesSourceType.COUNTER : IndexNumericFieldData.NumericType.LONG.getValuesSourceType(); if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index 3d901d12af79b..d9b5957f8f5df 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -476,7 +476,7 @@ private static void downgradeBrokenTsdbBackingIndices(DataStream dataStream, Pro var index = projectBuilder.getSafe(indexName); final Settings originalSettings = index.getSettings(); if (index.getCreationVersion().before(IndexVersions.FIRST_DETACHED_INDEX_VERSION) - && index.getIndexMode() == IndexMode.TIME_SERIES + && IndexMode.isTsdb(index.getIndexMode()) && originalSettings.keySet().contains(IndexSettings.TIME_SERIES_START_TIME.getKey()) == false && originalSettings.keySet().contains(IndexSettings.TIME_SERIES_END_TIME.getKey()) == false) { final Settings.Builder settingsBuilder = Settings.builder() diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 15248351928fe..65084e5cbacfc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -609,7 +609,7 @@ public Index selectTimeSeriesWriteIndex(Instant timestamp, ProjectMetadata proje Index index = backingIndices.indices.get(i); IndexMetadata im = project.index(index); - if (im.getIndexMode() != IndexMode.TIME_SERIES) { + if (IndexMode.isTsdb(im.getIndexMode()) == false) { // Not a tsdb backing index, so skip. // (This can happen if this is a migrated tsdb data stream) continue; @@ -633,7 +633,7 @@ public Index selectTimeSeriesWriteIndex(Instant timestamp, ProjectMetadata proje * @param imSupplier Function that supplies {@link IndexMetadata} instances based on the provided index name */ public void validate(Function imSupplier) { - if (indexMode == IndexMode.TIME_SERIES) { + if (IndexMode.isTsdb(indexMode)) { // Get a sorted overview of each backing index with there start and end time range: var startAndEndTimes = backingIndices.indices.stream().map(index -> { IndexMetadata im = imSupplier.apply(index.getName()); @@ -883,8 +883,7 @@ public DataStream unsafeRollover( ); } if (dsIndexMode != indexModeFromTemplate) { - if (indexModeFromTemplate == IndexMode.TIME_SERIES - && (dsIndexMode == IndexMode.LOGSDB || dsIndexMode == IndexMode.LOGSDB_COLUMNAR)) { + if (IndexMode.isTsdb(indexModeFromTemplate) && (dsIndexMode == IndexMode.LOGSDB || dsIndexMode == IndexMode.LOGSDB_COLUMNAR)) { LOGGER.warn("Changing [{}] index mode from [{}] to [{}]", name, indexModeFromTemplate, dsIndexMode); } dsIndexMode = indexModeFromTemplate; @@ -1327,7 +1326,7 @@ public List getDownsamplingRoundsFor( } IndexMetadata indexMetadata = indexMetadataSupplier.apply(index.getName()); - if (indexMetadata == null || IndexSettings.MODE.get(indexMetadata.getSettings()) != IndexMode.TIME_SERIES) { + if (indexMetadata == null || IndexSettings.MODE.get(indexMetadata.getSettings()).isTsdb() == false) { return List.of(); } TimeValue indexGenerationTime = getGenerationLifecycleDate(indexMetadata); @@ -1782,7 +1781,7 @@ public Index getWriteIndex(IndexRequest request, ProjectMetadata project) { return getWriteIndex(); } - if (getIndexMode() != IndexMode.TIME_SERIES) { + if (IndexMode.isTsdb(getIndexMode()) == false) { return getWriteIndex(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index cff115c24cea1..3db77b8762340 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -2584,7 +2584,7 @@ IndexMetadata build(boolean repair) { final boolean isSearchableSnapshot = SearchableSnapshotsSettings.isSearchableSnapshotStore(settings); String indexModeString = settings.get(IndexSettings.MODE.getKey()); final IndexMode indexMode = indexModeString != null ? IndexMode.fromString(indexModeString.toLowerCase(Locale.ROOT)) : null; - final boolean isTsdb = indexMode == IndexMode.TIME_SERIES; + final boolean isTsdb = IndexMode.isTsdb(indexMode); boolean useTimeSeriesSyntheticId = shouldUseTimeSeriesSyntheticId(isTsdb, indexCreatedVersion, settings); final boolean sequenceNumbersDisabled = indexCreatedVersion.onOrAfter( IndexVersions.TIME_SERIES_DISABLE_SEQUENCE_NUMBERS_DEFAULT diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 78cd995420dfd..74705da7f67aa 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -84,7 +84,7 @@ private static IndexRouting create( RoutingFunction routingFunction, IndexReshardingMetadata reshardingMetadata ) { - if (metadata.getIndexMode() == IndexMode.TIME_SERIES + if (IndexMode.isTsdb(metadata.getIndexMode()) && metadata.getTimeSeriesDimensions().isEmpty() == false && metadata.getCreationVersion().onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING)) { return new ExtractFromSource.ForIndexDimensions(metadata, routingFunction, reshardingMetadata); @@ -411,7 +411,7 @@ final void setRecordedHash(int h) { } indexMode = metadata.getIndexMode(); assert indexMode != null : "Index mode must be set for ExtractFromSource routing"; - this.trackTimeSeriesRoutingHash = indexMode == IndexMode.TIME_SERIES + this.trackTimeSeriesRoutingHash = indexMode.isTsdb() && metadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID); this.useTimeSeriesSyntheticId = metadata.useTimeSeriesSyntheticId(); addIdWithRoutingHash = (indexMode == IndexMode.LOGSDB @@ -629,7 +629,7 @@ public static class ForIndexDimensions extends ExtractFromSource { ForIndexDimensions(IndexMetadata metadata, RoutingFunction routingFunction, IndexReshardingMetadata reshardingMetadata) { super(metadata, routingFunction, reshardingMetadata, metadata.getTimeSeriesDimensions()); - assert metadata.getIndexMode() == IndexMode.TIME_SERIES : "Index mode must be time_series for ForIndexDimensions routing"; + assert IndexMode.isTsdb(metadata.getIndexMode()) : "Index mode must be time_series for ForIndexDimensions routing"; assert metadata.getCreationVersion().onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING) : "Index version must be at least " + IndexVersions.TSID_CREATED_DURING_ROUTING diff --git a/server/src/main/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocator.java b/server/src/main/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocator.java index 4393588fbc1bd..14bc333111ffb 100644 --- a/server/src/main/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocator.java +++ b/server/src/main/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocator.java @@ -58,7 +58,7 @@ public long getEligibleWriteWindowStart( long requestStartTimestamp ) { // Eligible write window is only applicable for time series data streams - if (dataStream.getIndexMode() != IndexMode.TIME_SERIES || dataStream.isSystem()) { + if (IndexMode.isTsdb(dataStream.getIndexMode()) == false || dataStream.isSystem()) { return -1; } String ilmPolicy = getEffectiveIlmPolicy(dataStream, projectMetadata); diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 2f7fff0ce04b7..1293f63795913 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -307,6 +307,87 @@ public boolean isColumnar() { return true; } }, + /** + * Alias of {@link #TIME_SERIES} with the preferred name {@code tsdb}. Behaves identically + * to {@link #TIME_SERIES} in every respect other than {@link #getName()}; existing + * {@code time_series} indices are unaffected and {@link #fromString} accepts both spellings. + */ + TSDB("tsdb") { + @Override + void validateWithOtherSettings(Map, Object> settings) { + TIME_SERIES.validateWithOtherSettings(settings); + } + + @Override + public void validateMapping(MappingLookup lookup, Settings settings) { + TIME_SERIES.validateMapping(lookup, settings); + } + + @Override + public void validateAlias(@Nullable String indexRouting, @Nullable String searchRouting) { + TIME_SERIES.validateAlias(indexRouting, searchRouting); + } + + @Override + public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup mappingLookup) throws IOException { + TIME_SERIES.validateTimestampFieldMapping(isDataStream, mappingLookup); + } + + @Override + public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) { + return TIME_SERIES.getDefaultMapping(indexSettings); + } + + @Override + public TimestampBounds getTimestampBound(IndexMetadata indexMetadata) { + return TIME_SERIES.getTimestampBound(indexMetadata); + } + + @Override + public MetadataFieldMapper timeSeriesIdFieldMapper(MappingParserContext c) { + return TIME_SERIES.timeSeriesIdFieldMapper(c); + } + + @Override + public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() { + return TIME_SERIES.timeSeriesRoutingHashFieldMapper(); + } + + @Override + public Function idTransformerForReindex() { + return TIME_SERIES.idTransformerForReindex(); + } + + @Override + public RoutingFields buildRoutingFields(IndexSettings settings) { + return TIME_SERIES.buildRoutingFields(settings); + } + + @Override + public boolean shouldValidateTimestamp() { + return TIME_SERIES.shouldValidateTimestamp(); + } + + @Override + public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) { + TIME_SERIES.validateSourceFieldMapper(sourceFieldMapper); + } + + @Override + public SourceFieldMapper.Mode defaultSourceMode() { + return TIME_SERIES.defaultSourceMode(); + } + + @Override + public boolean isColumnar() { + return TIME_SERIES.isColumnar(); + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return INDEX_MODE_TSDB_ADDED; + } + }, LOGSDB("logsdb") { @Override void validateWithOtherSettings(Map, Object> settings) { @@ -835,6 +916,7 @@ private static CompressedXContent createDefaultMapping(CheckedConsumer IndexMode.STANDARD; case "time_series" -> IndexMode.TIME_SERIES; + case "tsdb" -> IndexMode.TSDB; case "logsdb" -> IndexMode.LOGSDB; case "columnar" -> IndexMode.COLUMNAR; case "logsdb_columnar" -> IndexMode.LOGSDB_COLUMNAR; @@ -1015,6 +1126,7 @@ public static IndexMode readFrom(StreamInput in) throws IOException { case 4 -> COLUMNAR; case 5 -> LOGSDB_COLUMNAR; case 6 -> VECTORDB_DOCUMENT; + case 7 -> TSDB; default -> throw new IllegalStateException("unexpected index mode [" + mode + "]"); }; } @@ -1036,6 +1148,7 @@ public static void writeTo(IndexMode indexMode, StreamOutput out) throws IOExcep case COLUMNAR -> 4; case LOGSDB_COLUMNAR -> 5; case VECTORDB_DOCUMENT -> 6; + case TSDB -> 7; }; out.writeByte((byte) code); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 844883ff4a819..82a6278a26bca 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -724,14 +724,14 @@ public void validate(Boolean enabled) { public void validate(Boolean enabled, Map, Object> settings) { if (enabled) { var indexMode = (IndexMode) settings.get(MODE); - if (indexMode == IndexMode.TIME_SERIES) { + if (IndexMode.isTsdb(indexMode)) { throw new IllegalArgumentException( String.format( Locale.ROOT, "The setting [%s] cannot be used with [%s=%s].", SLICE_ENABLED.getKey(), MODE.getKey(), - IndexMode.TIME_SERIES.getName() + indexMode.getName() ) ); } @@ -767,7 +767,7 @@ public Iterator> settings() { public static final Setting SYNTHETIC_ID = Setting.boolSetting("index.mapping.synthetic_id", settings -> { IndexVersion indexVersion = SETTING_INDEX_VERSION_CREATED.get(settings); - boolean isTimeSeries = IndexMode.TIME_SERIES.equals(MODE.get(settings)); + boolean isTimeSeries = MODE.get(settings).isTsdb(); boolean isValidCodec = isValidCodecForSyntheticId(INDEX_CODEC_SETTING.get(settings), indexVersion); boolean onByDefault = indexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD); return isTimeSeries && isValidCodec && onByDefault ? Boolean.TRUE.toString() : Boolean.FALSE.toString(); @@ -778,16 +778,17 @@ public void validate(Boolean enabled) {} @Override public void validate(Boolean enabled, Map, Object> settings) { if (enabled) { - // Verify if index mode is TIME_SERIES + // Verify if index mode is a time series mode (TIME_SERIES or its alias TSDB) var indexMode = (IndexMode) settings.get(MODE); - if (indexMode != IndexMode.TIME_SERIES) { + if (indexMode.isTsdb() == false) { throw new IllegalArgumentException( String.format( Locale.ROOT, - "The setting [%s] is only permitted when [%s] is set to [%s]. Current mode: [%s].", + "The setting [%s] is only permitted when [%s] is set to [%s] or [%s]. Current mode: [%s].", SYNTHETIC_ID.getKey(), MODE.getKey(), IndexMode.TIME_SERIES.name(), + IndexMode.TSDB.name(), indexMode.name() ) ); @@ -850,7 +851,7 @@ public static boolean isValidCodecForSyntheticId(String codecName, IndexVersion public static final Setting USE_DOC_VALUES_SKIPPER = Setting.boolSetting("index.mapping.use_doc_values_skipper", s -> { IndexVersion iv = SETTING_INDEX_VERSION_CREATED.get(s); var indexMode = MODE.get(s); - if (indexMode == IndexMode.TIME_SERIES) { + if (indexMode.isTsdb()) { return Boolean.toString(iv.onOrAfter(IndexVersions.STATELESS_SKIPPERS_ENABLED_FOR_TSDB)); } if (indexMode == IndexMode.LOGSDB || indexMode.isStrictColumnar()) { @@ -985,7 +986,7 @@ public Iterator> settings() { return Boolean.FALSE.toString(); } var indexMode = IndexSettings.MODE.get(settings); - return Boolean.toString(indexMode == IndexMode.TIME_SERIES); + return Boolean.toString(indexMode.isTsdb()); }, Property.IndexScope, Property.Final @@ -1039,7 +1040,7 @@ public Iterator> settings() { return Boolean.FALSE.toString(); } IndexVersion indexVersion = SETTING_INDEX_VERSION_CREATED.get(settings); - boolean isTimeSeries = IndexMode.TIME_SERIES.equals(MODE.get(settings)); + boolean isTimeSeries = MODE.get(settings).isTsdb(); boolean onByDefault = indexVersion.onOrAfter(IndexVersions.TIME_SERIES_ES95_CODEC_DEFAULT_FEATURE_FLAG); return Boolean.toString(isTimeSeries && onByDefault); }, @@ -1145,7 +1146,7 @@ static boolean indexModeSupportsSeqNoDocValuesOnly(IndexMode indexMode, IndexVer if (indexMode.isStrictColumnar()) { return true; } - return (indexMode == IndexMode.LOGSDB || indexMode == IndexMode.TIME_SERIES) + return (indexMode == IndexMode.LOGSDB || indexMode.isTsdb()) && (indexVersionCreated.onOrAfter(IndexVersions.SEQ_NO_WITHOUT_POINTS) || indexVersionCreated.equals(IndexVersions.ZERO)); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index a53c0a349a643..31d317af8b039 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -182,7 +182,7 @@ static SortDefault getSortDefault(Settings settings) { indexMode = indexMode.toLowerCase(Locale.ROOT); } - if (IndexMode.TIME_SERIES.getName().equals(indexMode)) { + if (IndexMode.isTsdbName(indexMode)) { return TIME_SERIES_SORT; } else if (IndexMode.LOGSDB.getName().equals(indexMode) || IndexMode.LOGSDB_COLUMNAR.getName().equals(indexMode)) { var version = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings); @@ -426,8 +426,8 @@ public Sort buildIndexSort( ); if (ft == null) { String err = "unknown index sort field:[" + sortSpec.field + "]"; - if (this.indexMode == IndexMode.TIME_SERIES) { - err += " required by [" + IndexSettings.MODE.getKey() + "=time_series]"; + if (this.indexMode.isTsdb()) { + err += " required by [" + IndexSettings.MODE.getKey() + "=" + this.indexMode.getName() + "]"; } throw new IllegalArgumentException(err); } diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java index 08a48d21d9a68..1513b474baabc 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java @@ -15,7 +15,6 @@ import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat; @@ -197,7 +196,7 @@ boolean useBloomFilter(String field) { // but based on dimension fields and timestamp field, so during indexing // version/seq_no/term needs to be looked up and having a bloom filter // can speed this up significantly. - return indexSettings.getMode() == IndexMode.TIME_SERIES + return indexSettings.getMode().isTsdb() && IdFieldMapper.NAME.equals(field) && IndexSettings.BLOOM_FILTER_ID_FIELD_ENABLED_SETTING.get(indexSettings.getSettings()); } else { diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesFormatSelector.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesFormatSelector.java index 0b27b7d78bb39..68b7d6db0d931 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesFormatSelector.java +++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesFormatSelector.java @@ -12,7 +12,6 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.elasticsearch.cluster.routing.TsidBuilder; import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -42,7 +41,7 @@ public static DocValuesFormat select(final IndexSettings indexSettings, @Nullabl final IndexVersion indexCreatedVersion = indexSettings.getIndexVersionCreated(); final boolean useLargeNumericBlockSize = indexSettings.isUseTimeSeriesDocValuesFormatLargeNumericBlockSize(); final boolean useLargeBinaryBlockSize = indexSettings.isUseTimeSeriesDocValuesFormatLargeBinaryBlockSize(); - final boolean writePartitions = indexSettings.getMode() == IndexMode.TIME_SERIES + final boolean writePartitions = indexSettings.getMode().isTsdb() && TsidBuilder.useSingleBytePrefixLayout(indexCreatedVersion) && indexCreatedVersion.onOrAfter(IndexVersions.WRITE_TSID_PREFIX_PARTITION); @@ -63,7 +62,7 @@ public static DocValuesFormat select(final IndexSettings indexSettings, @Nullabl } static boolean useES95(final IndexSettings indexSettings) { - return indexSettings.getMode() == IndexMode.TIME_SERIES + return indexSettings.getMode().isTsdb() && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.ES95_TSDB_CODEC_FEATURE_FLAG) && indexSettings.isTimeSeriesEs95CodecEnabled(); } diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 00ad217e52140..5149920cf3ac0 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -77,7 +77,6 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.Tuple; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -1096,7 +1095,7 @@ private VersionValue resolveDocVersion(final Operation op, boolean loadSeqNo) th if (versionValue == null) { assert incrementIndexVersionLookup(); // used for asserting in tests final DocIdAndVersion docIdAndVersion = performActionWithDirectoryReader(SearcherScope.INTERNAL, directoryReader -> { - if (engineConfig.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { + if (engineConfig.getIndexSettings().getMode().isTsdb()) { assert engineConfig.getLeafSorter() == DataStream.TIMESERIES_LEAF_READERS_SORTER; return VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion( directoryReader, @@ -1707,7 +1706,7 @@ private IndexingStrategy[] planPrimarySubBatch(Index[] ops, int count) throws IO // Phase 2: single Lucene reader acquisition for all versionMap misses. // Collect misses into flat arrays and resolve them all in one sorted segment scan. if (anyNeedsLucene) { - final boolean isTimeSeries = engineConfig.getIndexSettings().getMode() == IndexMode.TIME_SERIES; + final boolean isTimeSeries = engineConfig.getIndexSettings().getMode().isTsdb(); int luceneCount = 0; for (int i = 0; i < count; i++) { if (needsLucene[i]) luceneCount++; @@ -3337,7 +3336,7 @@ private IndexWriterConfig getIndexWriterConfig() { engineConfig.getIndexSettings().isRecoverySourceSyntheticEnabled() ? SourceFieldMapper.RECOVERY_SOURCE_SIZE_NAME : SourceFieldMapper.RECOVERY_SOURCE_NAME, - engineConfig.getIndexSettings().getMode() == IndexMode.TIME_SERIES, + engineConfig.getIndexSettings().getMode().isTsdb(), pruneSeqNo, () -> softDeletesPolicy.getRetentionQuery(seqNoIndexOptions), new SoftDeletesRetentionMergePolicy( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index dba41fe933798..2089081255245 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -431,7 +431,7 @@ private IndexType indexType(String fullFieldName) { return IndexType.skippers(); } // Otherwise if field name @timestamp and it is part of index sorting and index mode is either logsdb and tsdb use skippers: - if ((indexSettings.getMode() == IndexMode.TIME_SERIES || indexSettings.getMode() == IndexMode.LOGSDB) + if ((indexSettings.getMode().isTsdb() || indexSettings.getMode() == IndexMode.LOGSDB) && indexSettings.getIndexSortConfig() != null && indexSettings.getIndexSortConfig().hasSortOnField(fullFieldName) && DataStreamTimestampFieldMapper.DEFAULT_PATH.equals(fullFieldName)) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index f6c8dc40e3df2..b422592a3f576 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -202,7 +202,7 @@ public void validate(IndexSettings settings, boolean checkLimits) { } List routingPaths = settings.getIndexMetadata().getRoutingPaths(); for (String path : routingPaths) { - if (settings.getMode() == IndexMode.TIME_SERIES) { + if (settings.getMode().isTsdb()) { for (String match : mappingLookup.getMatchingFieldNames(path)) { mappingLookup.getFieldType(match).validateMatchedRoutingPath(path); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index f206379999ee8..92cf0301aa284 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -1187,7 +1187,7 @@ private static class RootDocumentParserContext extends DocumentParserContext imp IndexSettings indexSettings = mappingParserContext.getIndexSettings(); BytesRef tsid = source.tsid(); if (tsid == null - && indexSettings.getMode() == IndexMode.TIME_SERIES + && indexSettings.getMode().isTsdb() && indexSettings.getIndexRouting() instanceof IndexRouting.ExtractFromSource.ForIndexDimensions forIndexDimensions) { // the tsid is normally set on the coordinating node during shard routing and passed to the data node via the index request // but when applying a translog operation, shard routing is not happening, and we have to create the tsid from source @@ -1196,8 +1196,7 @@ private static class RootDocumentParserContext extends DocumentParserContext imp tsid = forIndexDimensions.buildTsid(sourceObject.xContentType(), sourceObject.originalBytes()); } this.tsid = tsid; - assert this.tsid == null || indexSettings.getMode() == IndexMode.TIME_SERIES - : "tsid should only be set for time series indices"; + assert this.tsid == null || indexSettings.getMode().isTsdb() : "tsid should only be set for time series indices"; XContentParserDecorator parserDecorator = source.getMeteringParserDecorator(); Mapping mapping = mappingLookup.getMapping(); if (mapping.getRoot().subobjects() == ObjectMapper.Subobjects.ENABLED) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index b8404af0b9ce2..bba59fb199268 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.SliceIndexing; @@ -393,7 +392,7 @@ public final SourceToParse sourceToParse() { } public final String routing() { - return mappingParserContext.getIndexSettings().getMode() == IndexMode.TIME_SERIES ? null : sourceToParse.routing(); + return mappingParserContext.getIndexSettings().getMode().isTsdb() ? null : sourceToParse.routing(); } /** @@ -972,7 +971,7 @@ public final DocumentParserContext createNestedContext(NestedObjectMapper nested // We just need to store the id as indexed field, so that IndexWriter#deleteDocuments(term) can then // delete it when the root document is deleted too. doc.add(standardIdField(idField.binaryValue(), Field.Store.NO)); - } else if (indexSettings().getMode() == IndexMode.TIME_SERIES) { + } else if (indexSettings().getMode().isTsdb()) { // For time series indices, the _id is generated from the _tsid, which in turn is generated from the values of the configured // routing fields. At this point in document parsing, we can't guarantee that we've parsed all the routing fields yet, so the // parent document's _id is not yet available. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index c24f8d6c7c698..fbd6c9269cfa4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -1412,7 +1412,7 @@ public static boolean useTimeSeriesDocValuesSkippers(IndexSettings indexSettings if (indexSettings.useDocValuesSkipper() == false) { return false; } - if (indexSettings.getMode() == IndexMode.TIME_SERIES) { + if (indexSettings.getMode().isTsdb()) { if (isDimension) { return indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 5aed7bcbbd226..2dedda670b336 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -136,7 +136,7 @@ public Builder(String name, ScriptCompiler scriptCompiler, IndexSettings indexSe return false; } - return indexSettings.getMode() != IndexMode.TIME_SERIES || getMetric().getValue() != TimeSeriesParams.MetricType.POSITION; + return indexSettings.getMode().isTsdb() == false || getMetric().getValue() != TimeSeriesParams.MetricType.POSITION; }); addScriptValidation(script, indexed, hasDocValues); @@ -482,7 +482,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext failIfNoDocValues(); } - ValuesSourceType valuesSourceType = indexMode == IndexMode.TIME_SERIES && metricType == TimeSeriesParams.MetricType.POSITION + ValuesSourceType valuesSourceType = indexMode.isTsdb() && metricType == TimeSeriesParams.MetricType.POSITION ? TimeSeriesValuesSourceType.POSITION : CoreValuesSourceType.GEOPOINT; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index 25958cff8a48d..a8b8380a6b349 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -16,7 +16,6 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.Queries; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.SearchExecutionContext; @@ -36,7 +35,7 @@ public abstract class IdFieldMapper extends MetadataFieldMapper { public static final TypeParser PARSER = new ConfigurableTypeParser(mappingParserContext -> { var indexMode = mappingParserContext.getIndexSettings().getMode(); - if (indexMode == IndexMode.TIME_SERIES) { + if (indexMode.isTsdb()) { return new ConstantBuilder(TsidExtractingIdFieldMapper.INSTANCE); } else { boolean useColumnarIdByDefault = mappingParserContext.getIndexSettings().isUseColumnarIdByDefault(); @@ -47,7 +46,7 @@ public abstract class IdFieldMapper extends MetadataFieldMapper { @Override public Builder parse(String name, Map node, MappingParserContext parserContext) throws MapperParsingException { var indexMode = parserContext.getIndexSettings().getMode(); - if (indexMode == IndexMode.TIME_SERIES) { + if (indexMode.isTsdb()) { throw new MapperParsingException(name + " is not configurable if index mode is time_series"); } Builder builder = super.parse(name, node, parserContext); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java index ce484b185bb9a..dcd1a1d24ee6d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Releasables; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; @@ -51,7 +50,7 @@ public sealed interface IdLoader permits IdLoader.TsIdLoader, IdLoader.StoredIdL * @return returns an {@link IdLoader} instance to load the value of the _id field. */ static IdLoader create(IndexSettings indexSettings, MappingLookup mappingLookup) { - if (indexSettings.getMode() == IndexMode.TIME_SERIES) { + if (indexSettings.getMode().isTsdb()) { IndexRouting.ExtractFromSource.ForRoutingPath indexRouting = null; List routingPaths = null; if (indexSettings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID)) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index f910533f351d0..b3ab92845f37f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -694,7 +694,7 @@ public void validateDoesNotShadow(String name) { if (shadowed == null) { return; } - if (indexMode == IndexMode.TIME_SERIES) { + if (indexMode.isTsdb()) { if (shadowed.isDimension()) { throw new MapperParsingException("Field [" + name + "] attempted to shadow a time_series_dimension"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 097a13e02e72d..c7acdcf2f0e17 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -200,7 +200,7 @@ public Builder(String name, NumberType type, ScriptCompiler compiler, IndexSetti if (useTimeSeriesDocValuesSkippers(indexSettings, dimension.get())) { return false; } - if (indexSettings.getMode() == IndexMode.TIME_SERIES) { + if (indexSettings.getMode().isTsdb()) { var metricType = getMetric().getValue(); return metricType != MetricType.COUNTER && metricType != MetricType.GAUGE; } else { @@ -2341,7 +2341,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext failIfNoDocValues(); } - ValuesSourceType valuesSourceType = indexMode == IndexMode.TIME_SERIES && metricType == TimeSeriesParams.MetricType.COUNTER + ValuesSourceType valuesSourceType = indexMode.isTsdb() && metricType == TimeSeriesParams.MetricType.COUNTER ? TimeSeriesValuesSourceType.COUNTER : type.numericType.getValuesSourceType(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoader.java index 21d55af85c1b7..66e50df2d151b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoader.java @@ -43,7 +43,7 @@ public TimeSeriesMetadataFieldBlockLoader(MappedFieldType.BlockLoaderContext con private static Set lookupTimeSeriesMetadataFieldNames(MappedFieldType.BlockLoaderContext context, boolean loadMetrics) { assert context.blockLoaderFunctionConfig() instanceof BlockLoaderFunctionConfig.TimeSeriesMetadata; - if (context.indexSettings().getMode() != IndexMode.TIME_SERIES) { + if (context.indexSettings().getMode().isTsdb() == false) { throw new IllegalStateException("TimeSeriesMetadataFieldBlockLoader requires index mode: [ " + IndexMode.TIME_SERIES + " ]"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java index 23764322ee0a3..6941f9c40743f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.ByteUtils; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -120,7 +119,7 @@ private TimeSeriesRoutingHashFieldMapper() { @Override public void postParse(DocumentParserContext context) { - if (context.indexSettings().getMode() == IndexMode.TIME_SERIES + if (context.indexSettings().getMode().isTsdb() && context.indexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID)) { String routingHash = context.sourceToParse().routing(); if (routingHash == null) { diff --git a/server/src/main/resources/transport/definitions/referable/index_mode_tsdb_added.csv b/server/src/main/resources/transport/definitions/referable/index_mode_tsdb_added.csv new file mode 100644 index 0000000000000..57c782ffe24c6 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/index_mode_tsdb_added.csv @@ -0,0 +1 @@ +9450000 diff --git a/server/src/main/resources/transport/upper_bounds/9.5.csv b/server/src/main/resources/transport/upper_bounds/9.5.csv index c3fcefd133d31..1278b748b4333 100644 --- a/server/src/main/resources/transport/upper_bounds/9.5.csv +++ b/server/src/main/resources/transport/upper_bounds/9.5.csv @@ -1 +1 @@ -esql_datasource_privilege,9449000 +index_mode_tsdb_added,9450000 diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java index 6b95a52784634..f2579002d1afc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java @@ -573,7 +573,7 @@ private static IndexMetadata createIndexMetadata( .put("index.frozen", frozen) .put("index.mode", mode.toString()); - if (mode == IndexMode.TIME_SERIES) { + if (mode.isTsdb()) { settingsBuilder.put( randomBoolean() ? IndexMetadata.INDEX_DIMENSIONS.getKey() : IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dummy_value" diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 941ff0f05b16b..796e496a87cf8 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -1128,7 +1128,7 @@ public void testSyntheticIdBadMode() { IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94, IndexVersion.current() ); - IndexMode badMode = randomValueOtherThan(IndexMode.TIME_SERIES, () -> randomFrom(IndexMode.availableModes())); + IndexMode badMode = randomValueOtherThanMany(m -> m.isTsdb(), () -> randomFrom(IndexMode.availableModes())); String codec = CodecService.DEFAULT_CODEC; Settings settings = Settings.builder() @@ -1144,10 +1144,11 @@ public void testSyntheticIdBadMode() { Matchers.containsString( String.format( Locale.ROOT, - "The setting [%s] is only permitted when [%s] is set to [%s]. Current mode: [%s].", + "The setting [%s] is only permitted when [%s] is set to [%s] or [%s]. Current mode: [%s].", IndexSettings.SYNTHETIC_ID.getKey(), IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name(), + IndexMode.TSDB.name(), badMode.name() ) ) @@ -1213,7 +1214,7 @@ public void testDisableSequenceNumbersRequiresDocValuesOnlyForNonStandardModes() .put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexSettings.SEQ_NO_INDEX_OPTIONS_SETTING.getKey(), SeqNoFieldMapper.SeqNoIndexOptions.POINTS_AND_DOC_VALUES) .put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true); - if (mode == IndexMode.TIME_SERIES) { + if (mode.isTsdb()) { builder.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo"); } IndexMetadata indexMetadata = newIndexMeta("some-index", builder.build(), indexVersion); @@ -1300,7 +1301,7 @@ public void testDynamicStringsAutoTextDefaultByIndexMode() { // All other modes default to true (text + keyword subfield) for (IndexMode otherMode : List.of(IndexMode.STANDARD, IndexMode.LOGSDB, IndexMode.TIME_SERIES)) { Settings.Builder builder = Settings.builder().put(IndexSettings.MODE.getKey(), otherMode.getName()); - if (otherMode == IndexMode.TIME_SERIES) { + if (otherMode.isTsdb()) { builder.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo"); } IndexSettings indexSettings = new IndexSettings(newIndexMeta(otherMode.getName() + "-index", builder.build()), Settings.EMPTY); diff --git a/server/src/test/java/org/elasticsearch/index/TsdbIndexModeTests.java b/server/src/test/java/org/elasticsearch/index/TsdbIndexModeTests.java new file mode 100644 index 0000000000000..6d8d338912829 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/TsdbIndexModeTests.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index; + +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.TransportVersionUtils; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +/** + * {@link IndexMode#TSDB} is a preferred alias for {@link IndexMode#TIME_SERIES}: both parse from + * (and behave identically for) the same {@code index.mode} settings, but {@code time_series} + * remains the canonical emitted/persisted string so existing indices are unaffected. + */ +public class TsdbIndexModeTests extends ESTestCase { + + public void testFromStringAcceptsBothSpellings() { + assertThat(IndexMode.fromString("time_series"), equalTo(IndexMode.TIME_SERIES)); + assertThat(IndexMode.fromString("TIME_SERIES"), equalTo(IndexMode.TIME_SERIES)); + assertThat(IndexMode.fromString("tsdb"), equalTo(IndexMode.TSDB)); + assertThat(IndexMode.fromString("TSDB"), equalTo(IndexMode.TSDB)); + assertThat(IndexMode.fromString("Tsdb"), equalTo(IndexMode.TSDB)); + } + + public void testGetName() { + assertThat(IndexMode.TIME_SERIES.getName(), equalTo("time_series")); + assertThat(IndexMode.TIME_SERIES.toString(), equalTo("time_series")); + assertThat(IndexMode.TSDB.getName(), equalTo("tsdb")); + assertThat(IndexMode.TSDB.toString(), equalTo("tsdb")); + } + + public void testIsTsdb() { + assertTrue(IndexMode.TIME_SERIES.isTsdb()); + assertTrue(IndexMode.TSDB.isTsdb()); + for (IndexMode mode : IndexMode.values()) { + if (mode != IndexMode.TIME_SERIES && mode != IndexMode.TSDB) { + assertFalse(mode + " must not be tsdb", mode.isTsdb()); + } + } + } + + public void testIsTsdbNullSafeStatic() { + assertTrue(IndexMode.isTsdb(IndexMode.TIME_SERIES)); + assertTrue(IndexMode.isTsdb(IndexMode.TSDB)); + assertFalse(IndexMode.isTsdb(IndexMode.STANDARD)); + assertFalse(IndexMode.isTsdb(null)); + } + + public void testIsTsdbName() { + assertTrue(IndexMode.isTsdbName("time_series")); + assertTrue(IndexMode.isTsdbName("TIME_SERIES")); + assertTrue(IndexMode.isTsdbName("tsdb")); + assertTrue(IndexMode.isTsdbName("TSDB")); + assertFalse(IndexMode.isTsdbName("standard")); + assertFalse(IndexMode.isTsdbName(null)); + } + + public void testIndexModeSettingAcceptsBothSpellings() { + Settings timeSeriesSettings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("uid")) + .build(); + assertThat(IndexSettings.MODE.get(timeSeriesSettings), equalTo(IndexMode.TIME_SERIES)); + + Settings tsdbSettings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "tsdb") + .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("uid")) + .build(); + assertThat(IndexSettings.MODE.get(tsdbSettings), equalTo(IndexMode.TSDB)); + } + + public void testSerialization() throws IOException { + try (BytesStreamOutput out = new BytesStreamOutput()) { + IndexMode.writeTo(IndexMode.TSDB, out); + try (var in = out.bytes().streamInput()) { + assertThat(IndexMode.readFrom(in), equalTo(IndexMode.TSDB)); + } + } + // TIME_SERIES's own wire representation is unaffected by adding TSDB. + try (BytesStreamOutput out = new BytesStreamOutput()) { + IndexMode.writeTo(IndexMode.TIME_SERIES, out); + try (var in = out.bytes().streamInput()) { + assertThat(IndexMode.readFrom(in), equalTo(IndexMode.TIME_SERIES)); + } + } + } + + public void testSerializationFailsOnOlderTransportVersion() throws IOException { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setTransportVersion(TransportVersionUtils.getPreviousVersion(IndexMode.INDEX_MODE_TSDB_ADDED)); + IllegalStateException e = expectThrows(IllegalStateException.class, () -> IndexMode.writeTo(IndexMode.TSDB, out)); + assertThat(e.getMessage(), containsString("[tsdb] doesn't support serialization with transport version")); + } + } + + /** + * TSDB delegates every behavior to TIME_SERIES, so the two must remain indistinguishable + * other than their name/identity. + */ + public void testBehaviorParityWithTimeSeries() { + assertThat(IndexMode.TSDB.defaultSourceMode(), equalTo(IndexMode.TIME_SERIES.defaultSourceMode())); + assertThat(IndexMode.TSDB.isColumnar(), equalTo(IndexMode.TIME_SERIES.isColumnar())); + assertThat(IndexMode.TSDB.isStrictColumnar(), equalTo(IndexMode.TIME_SERIES.isStrictColumnar())); + assertThat(IndexMode.TSDB.shouldValidateTimestamp(), equalTo(IndexMode.TIME_SERIES.shouldValidateTimestamp())); + assertThat(IndexMode.TSDB.supportedSourceModes(), equalTo(IndexMode.TIME_SERIES.supportedSourceModes())); + assertThat(IndexMode.TSDB.getDefaultCodec(), equalTo(IndexMode.TIME_SERIES.getDefaultCodec())); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java index e6a47555a9746..e0729c240d1db 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java @@ -227,7 +227,7 @@ public void testUseEs812PostingsFormatForIdField() throws IOException { for (int i = 0; i < numIterations; i++) { var indexMode = randomFrom(IndexMode.STANDARD, IndexMode.LOGSDB, IndexMode.TIME_SERIES); String mapping = randomFrom(METRIC_MAPPING, MULTI_METRIC_MAPPING, LOGS_MAPPING); - final boolean randomSyntheticId = syntheticId(indexMode.equals(IndexMode.TIME_SERIES)); + final boolean randomSyntheticId = syntheticId(indexMode.isTsdb()); PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier( randomBoolean(), randomBoolean(), @@ -414,7 +414,7 @@ private PerFieldFormatSupplier createFormatSupplier( ) throws IOException { Settings.Builder settings = Settings.builder(); settings.put(IndexSettings.MODE.getKey(), mode); - if (mode == IndexMode.TIME_SERIES) { + if (mode.isTsdb()) { settings.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field"); } if (syntheticId != null) { @@ -449,7 +449,7 @@ public void testES95OnlyUsedForTimeSeriesWhenSettingEnabled() throws IOException for (IndexMode mode : INDEX_MODES_UNDER_TEST) { final PerFieldFormatSupplier supplier = createFormatSupplierWithVersion(mode, mappingFor(mode), IndexVersion.current()); final DocValuesFormat format = supplier.getDocValuesFormatForField(fieldFor(mode)); - if (mode == IndexMode.TIME_SERIES) { + if (mode.isTsdb()) { assertThat("mode=" + mode, format, instanceOf(ES95TSDBDocValuesFormat.class)); } else { assertFalse("mode=" + mode + " expected non-ES95", format instanceof ES95TSDBDocValuesFormat); @@ -525,7 +525,7 @@ private PerFieldFormatSupplier createFormatSupplierWithVersion( final Settings.Builder settings = Settings.builder(); settings.put(IndexSettings.MODE.getKey(), mode); settings.put(IndexMetadata.SETTING_VERSION_CREATED, indexVersion); - if (mode == IndexMode.TIME_SERIES) { + if (mode.isTsdb()) { settings.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field"); } if (useTimeSeriesDocValuesFormat) { diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesFormatSelectorTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesFormatSelectorTests.java index 9cb44fc2215a3..dba593093b556 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesFormatSelectorTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesFormatSelectorTests.java @@ -55,7 +55,7 @@ public void testES95OnlySelectedForTimeSeriesWhenSettingEnabled() { ); for (IndexMode mode : indexModesUnderTest()) { final DocValuesFormat format = TSDBDocValuesFormatSelector.select(indexSettings(mode, version, true), null); - if (mode == IndexMode.TIME_SERIES) { + if (mode.isTsdb()) { assertThat("mode=" + mode + " version=" + version, format.getName(), equalTo(ES95_CODEC_NAME)); } else { assertThat("mode=" + mode + " version=" + version, format.getName(), startsWith("ES819")); @@ -100,7 +100,7 @@ private static IndexSettings indexSettings(final IndexMode mode, final IndexVers if (mode != IndexMode.STANDARD) { builder.put("index.mode", mode.getName()); } - if (mode == IndexMode.TIME_SERIES) { + if (mode.isTsdb()) { builder.put("index.routing_path", "dimension"); } if (IndexSettings.ES95_CODEC_FEATURE_FLAG.isEnabled()) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexModeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexModeFieldTypeTests.java index a5d47ad08b1f5..3fc886ab0c923 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexModeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexModeFieldTypeTests.java @@ -88,7 +88,7 @@ private SearchExecutionContext createContext(IndexMode mode) { if (mode != null) { settings.put(IndexSettings.MODE.getKey(), mode); } - if (mode == IndexMode.TIME_SERIES) { + if (IndexMode.isTsdb(mode)) { settings.putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("a,b,c")); } IndexMetadata indexMetadata = IndexMetadata.builder("index").settings(settings).numberOfShards(1).numberOfReplicas(0).build(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index 0f3d760d9a882..e576149cc8537 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -382,7 +382,7 @@ public void testIsMetadataField() throws IOException { .put(IndexMetadata.SETTING_VERSION_CREATED, version) .put(IndexSettings.MODE.getKey(), indexMode); - if (indexMode == IndexMode.TIME_SERIES) { + if (indexMode.isTsdb()) { settingsBuilder.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo"); } @@ -397,7 +397,7 @@ public void testIsMetadataField() throws IOException { continue; // Nested field does not exist in the 7x line } boolean isTimeSeriesField = builtIn.equals("_tsid") || builtIn.equals("_ts_routing_hash"); - boolean isTimeSeriesMode = mapperService.getIndexSettings().getMode().equals(IndexMode.TIME_SERIES); + boolean isTimeSeriesMode = mapperService.getIndexSettings().getMode().isTsdb(); if (isTimeSeriesField && isTimeSeriesMode == false) { assertFalse( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java index fa5344d5e4843..2678d42f85c38 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java @@ -180,7 +180,7 @@ public MetricType getMetricType() { List.of(dimMapper, metricMapper, plainMapper), emptyList(), emptyList(), - randomValueOtherThan(IndexMode.TIME_SERIES, () -> randomFrom(IndexMode.availableModes())) + randomValueOtherThanMany(m -> m.isTsdb(), () -> randomFrom(IndexMode.availableModes())) ); mappingLookup.validateDoesNotShadow("not_mapped"); mappingLookup.validateDoesNotShadow("dim"); @@ -226,7 +226,7 @@ public MetricType getMetricType() { List.of(dimMapper, metricMapper), emptyList(), List.of(shadowing), - randomValueOtherThan(IndexMode.TIME_SERIES, () -> randomFrom(IndexMode.availableModes())) + randomValueOtherThanMany(m -> m.isTsdb(), () -> randomFrom(IndexMode.availableModes())) ); Exception e = expectThrows( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index 95b4a1ebf127f..ecc73b26ce364 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -514,7 +514,7 @@ public void testColumnarStoredModeRequiresColumnarIndex() { Settings.Builder builder = Settings.builder() .put(IndexSettings.MODE.getKey(), nonColumnarMode.toString()) .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.COLUMNAR_STORED.toString()); - if (nonColumnarMode == IndexMode.TIME_SERIES) { + if (nonColumnarMode.isTsdb()) { // time_series requires a routing_path; provide one so its own validation passes and our source mode validator fires builder.putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim"); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index a241eb1035dea..733bdbf6ae78f 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -159,6 +159,7 @@ protected final DocumentMapper createDocumentMapper(XContentBuilder mappings, In case STANDARD, LOOKUP -> createDocumentMapper(mappings); case VECTORDB_DOCUMENT -> createVectordbDocumentModeDocumentMapper(mappings); case TIME_SERIES -> createTimeSeriesModeDocumentMapper(mappings); + case TSDB -> createTsdbModeDocumentMapper(mappings); case LOGSDB -> createLogsModeDocumentMapper(mappings); case COLUMNAR -> createColumnarModeDocumentMapper(mappings); case LOGSDB_COLUMNAR -> createColumnarLogsdbModeDocumentMapper(mappings); @@ -177,6 +178,14 @@ protected final DocumentMapper createTimeSeriesModeDocumentMapper(XContentBuilde return createMapperService(settings, mappings).documentMapper(); } + protected final DocumentMapper createTsdbModeDocumentMapper(XContentBuilder mappings) throws IOException { + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "tsdb") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "uid") + .build(); + return createMapperService(settings, mappings).documentMapper(); + } + protected final DocumentMapper createLogsModeDocumentMapper(XContentBuilder mappings) throws IOException { Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()).build(); return createMapperService(settings, mappings).documentMapper(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 0f5aa75229da1..9fc932a454c11 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1387,7 +1387,7 @@ private static List getLiveDocs(Engine engine, boolean refr final var storedFieldLoader = StoredFieldLoader.create(true, sourceLoader.requiredStoredFields()); // Some indices merge away the _id field - final var pruneIdField = engineConfig.getIndexSettings().getMode() == IndexMode.TIME_SERIES; + final var pruneIdField = engineConfig.getIndexSettings().getMode().isTsdb(); final var idLoader = IdLoader.create(mapperService.getIndexSettings(), mapperService.mappingLookup()); // Some integration tests merge away the _seq_no field, in which case this method sets all _seq_no to UNASSIGNED_SEQ_NO @@ -2910,7 +2910,7 @@ public Collection getAdditionalIndexSettingProviders(Index combinedTemplateMappings, indexVersion, additionalSettings) -> { - if (templateIndexMode == IndexMode.TIME_SERIES) { + if (IndexMode.isTsdb(templateIndexMode)) { // Don't randomly enable columnar id mode, if time series index mode has been enabled. // Enabling columnar id isn't possible because tsdb always uses synthetic id. return; diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java index 15632c34a7bbb..63be684caca78 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java @@ -774,12 +774,13 @@ private CheckedBiFunction nestedPa public void testProcessOnceOnPrimary() throws Exception { final Settings.Builder settingsBuilder = indexSettings(IndexVersion.current(), 1, 0).put("index.xpack.ccr.following_index", true); - boolean useSyntheticId = indexMode == IndexMode.TIME_SERIES && randomBoolean(); + boolean useSyntheticId = indexMode.isTsdb() && randomBoolean(); switch (indexMode) { case STANDARD: break; case TIME_SERIES: - settingsBuilder.put("index.mode", "time_series").put("index.routing_path", "foo"); + case TSDB: + settingsBuilder.put("index.mode", indexMode.getName()).put("index.routing_path", "foo"); settingsBuilder.put("index.seq_no.index_options", "points_and_doc_values"); settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), useSyntheticId); break; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java index 9c4ea4013bcd1..93962a23f7c4a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java @@ -86,7 +86,7 @@ protected void localClusterStateOperation( for (DataStream ds : dataStreams.values()) { // We choose to not count time series backing indices that do not belong to a time series data stream. - if (ds.getIndexMode() != IndexMode.TIME_SERIES) { + if (IndexMode.isTsdb(ds.getIndexMode()) == false) { continue; } tsDataStreamCount++; @@ -96,7 +96,7 @@ protected void localClusterStateOperation( for (Index backingIndex : ds.getIndices()) { IndexMetadata indexMetadata = projectMetadata.index(backingIndex); - if (indexMetadata.getIndexMode() != IndexMode.TIME_SERIES) { + if (IndexMode.isTsdb(indexMetadata.getIndexMode()) == false) { continue; } tsIndexCount++; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DownsampleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DownsampleAction.java index 38132b2873707..8f0c3cfcaa086 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DownsampleAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DownsampleAction.java @@ -20,7 +20,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.xcontent.ConstructingObjectParser; @@ -220,7 +219,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { (index, project) -> { IndexMetadata indexMetadata = project.index(index); assert indexMetadata != null : "invalid cluster metadata. index [" + index.getName() + "] metadata not found"; - if (IndexSettings.MODE.get(indexMetadata.getSettings()) != IndexMode.TIME_SERIES) { + if (IndexSettings.MODE.get(indexMetadata.getSettings()).isTsdb() == false) { return false; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStep.java index e50964773bf6a..2c742393b3899 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStep.java @@ -10,7 +10,6 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.xpack.core.ilm.step.info.EmptyInfo; import org.elasticsearch.xpack.core.ilm.step.info.SingleMessageFieldInfo; @@ -44,7 +43,7 @@ public boolean isRetryable() { @Override public void evaluateCondition(ProjectState state, IndexMetadata indexMetadata, Listener listener, TimeValue masterTimeout) { String indexName = indexMetadata.getIndex().getName(); - if (IndexSettings.MODE.get(indexMetadata.getSettings()) != IndexMode.TIME_SERIES) { + if (IndexSettings.MODE.get(indexMetadata.getSettings()).isTsdb() == false) { // this index is not a time series index so no need to wait listener.onResponse(true, EmptyInfo.INSTANCE); return; diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index a095c029e7912..6d29f690364d7 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -282,7 +282,7 @@ protected void masterOperation( } // Assert source index is a time_series index - if (IndexSettings.MODE.get(sourceIndexMetadata.getSettings()) != IndexMode.TIME_SERIES) { + if (IndexSettings.MODE.get(sourceIndexMetadata.getSettings()).isTsdb() == false) { recordInvalidConfigurationMetrics(startTime); listener.onFailure( new ElasticsearchException( diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index ac8f8f03a6745..42a504ba96274 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -886,7 +886,7 @@ protected static void createAllTypesIndex( config.startObject("settings"); config.startObject("index"); config.field("mode", mode); - if (mode == IndexMode.TIME_SERIES) { + if (mode.isTsdb()) { config.field("routing_path", "f_keyword"); } if (nodeId != null) { @@ -945,7 +945,7 @@ private static void typeMapping(IndexMode indexMode, XContentBuilder config, Dat case NULL -> config.field("type", "keyword"); case KEYWORD -> { config.field("type", type.esType()); - if (indexMode == IndexMode.TIME_SERIES) { + if (indexMode.isTsdb()) { config.field("time_series_dimension", true); } } @@ -1331,7 +1331,7 @@ private static Matcher expectedType( ) { return switch (type) { case COUNTER_DOUBLE, COUNTER_LONG, COUNTER_INTEGER -> { - if (indexMode == IndexMode.TIME_SERIES) { + if (indexMode.isTsdb()) { yield equalTo(type.esType()); } yield equalTo(type.esType().replace("counter_", "")); @@ -1406,7 +1406,7 @@ protected boolean preserveClusterUponCompletion() { private boolean syntheticSourceByDefault() { return switch (indexMode) { - case TIME_SERIES, LOGSDB, COLUMNAR, LOGSDB_COLUMNAR -> true; + case TIME_SERIES, TSDB, LOGSDB, COLUMNAR, LOGSDB_COLUMNAR -> true; case STANDARD, LOOKUP, VECTORDB_DOCUMENT -> false; }; } @@ -1518,7 +1518,7 @@ private void assertReadersBuilt(Map readersBuilt) { expected = expected.entry( "_id", - indexMode == IndexMode.TIME_SERIES ? matchesList().item("column_at_a_time:TsIdFieldReader") + indexMode.isTsdb() ? matchesList().item("column_at_a_time:TsIdFieldReader") : indexMode.isStrictColumnar() ? matchesList().item("column_at_a_time:IdDocValuesReader") : matchesList().item("column_at_a_time:null").item("row_stride:BlockStoredFieldsReader.Id") ) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index e58a84804dbdc..aa427385d252d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -2459,7 +2459,7 @@ private LogicalPlan injectTimestampSort(Limit limit) { return limit; } - boolean hasTimeSeries = child.collect(EsRelation.class, r -> r.indexMode() == IndexMode.TIME_SERIES).isEmpty() == false; + boolean hasTimeSeries = child.collect(EsRelation.class, r -> r.indexMode().isTsdb()).isEmpty() == false; if (hasTimeSeries == false) { return limit; } @@ -2468,7 +2468,7 @@ private LogicalPlan injectTimestampSort(Limit limit) { return limit.transformDown(Limit.class, l -> { if (l.child().collect(Limit.class).isEmpty()) { var localChild = l.child(); - var localTimestampAttr = localChild.collect(EsRelation.class, r -> r.indexMode() == IndexMode.TIME_SERIES) + var localTimestampAttr = localChild.collect(EsRelation.class, r -> r.indexMode().isTsdb()) .stream() .findFirst() .flatMap(r -> r.output().stream().filter(a -> MetadataAttribute.TIMESTAMP_FIELD.equals(a.name())).findFirst()) @@ -3386,7 +3386,7 @@ private static class ImplicitCastAggregateMetricDoubles extends ParameterizedRul public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { Holder indexMode = new Holder<>(IndexMode.STANDARD); plan.forEachUp(EsRelation.class, esRelation -> { indexMode.set(esRelation.indexMode()); }); - isTimeSeries = indexMode.get() == IndexMode.TIME_SERIES; + isTimeSeries = indexMode.get().isTsdb(); return plan.transformUp(LogicalPlan.class, p -> doRule(p, context)); } @@ -3624,7 +3624,7 @@ private Expression doRule(EsqlBinaryComparison comparison) { private LogicalPlan doRule(Aggregate plan) { Holder indexMode = new Holder<>(IndexMode.STANDARD); plan.forEachUp(EsRelation.class, esRelation -> { indexMode.set(esRelation.indexMode()); }); - final boolean isTimeSeries = indexMode.get() == IndexMode.TIME_SERIES; + final boolean isTimeSeries = indexMode.get().isTsdb(); return plan.transformExpressionsOnly(AggregateFunction.class, aggFunc -> { if (ImplicitCastAggregateMetricDoubles.hasNativeSupport(aggFunc, isTimeSeries)) { return aggFunc; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java index b01021274030b..80de0d4203e54 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java @@ -146,7 +146,7 @@ protected PreAnalysis doPreAnalyze(LogicalPlan plan) { Holder useAggregateMetricDoubleWhenNotSupported = new Holder<>(false); Holder useDenseVectorWhenNotSupported = new Holder<>(false); indexes.forEach((ip, mode) -> { - if (mode == IndexMode.TIME_SERIES) { + if (mode.isTsdb()) { useAggregateMetricDoubleWhenNotSupported.set(true); } }); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/rules/ResolveUnmapped.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/rules/ResolveUnmapped.java index 3c048f16e1a27..85503752c42cf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/rules/ResolveUnmapped.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/rules/ResolveUnmapped.java @@ -305,7 +305,7 @@ private static void assertSourceType(LogicalPlan source) { switch (source) { case EsRelation esRelation -> { IndexMode mode = esRelation.indexMode(); - if ((mode == IndexMode.STANDARD || mode == IndexMode.TIME_SERIES) == false) { + if ((mode == IndexMode.STANDARD || mode.isTsdb()) == false) { throw new EsqlIllegalArgumentException( "invalid source type [{}] for unmapped field resolution", esRelation.indexMode() diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java index fdcac6f7479e4..b2fa4f2977eb9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java @@ -260,7 +260,7 @@ private static LogicalPlan rewriteOne( } if (relation.indexMode() != null && relation.indexMode() != IndexMode.STANDARD) { String message = switch (relation.indexMode()) { - case TIME_SERIES -> "TS command is not supported for datasets; dataset(s) requested: " + datasetNames; + case TIME_SERIES, TSDB -> "TS command is not supported for datasets; dataset(s) requested: " + datasetNames; case LOOKUP -> "LOOKUP JOIN against a dataset is not supported; dataset(s) requested: " + datasetNames; case LOGSDB -> "LOGSDB index mode on FROM is not supported; dataset(s) requested: " + datasetNames; default -> "FROM with index mode [" diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneUnusedIndexMode.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneUnusedIndexMode.java index f6c8f5342ba51..5f937f242b416 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneUnusedIndexMode.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PruneUnusedIndexMode.java @@ -24,7 +24,7 @@ public PruneUnusedIndexMode() { @Override protected LogicalPlan rule(EsRelation r) { - if (r.indexMode() == IndexMode.TIME_SERIES) { + if (r.indexMode().isTsdb()) { if (Expressions.anyMatch(r.output(), a -> MetadataAttribute.TSID_FIELD.equals(((Attribute) a).name())) == false) { return r.withIndexMode(IndexMode.STANDARD); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/IgnoreNullMetrics.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/IgnoreNullMetrics.java index fedb633721676..c48b2172356fc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/IgnoreNullMetrics.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/IgnoreNullMetrics.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.optimizer.rules.logical.local; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; @@ -62,10 +61,10 @@ public LogicalPlan rule(TimeSeriesAggregate agg) { */ private static boolean isMetricsQuery(LogicalPlan logicalPlan) { if (logicalPlan instanceof EsRelation r) { - return r.indexMode() == IndexMode.TIME_SERIES; + return r.indexMode().isTsdb(); } if (logicalPlan instanceof UnresolvedRelation r) { - return r.indexMode() == IndexMode.TIME_SERIES; + return r.indexMode().isTsdb(); } return false; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoad.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoad.java index ec66b42b16aab..ba3e21fae97cb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoad.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoad.java @@ -208,7 +208,7 @@ private PhysicalPlan transformProject(ProjectExec project) { * not just {@code TS} command queries. */ private boolean hasTimeSeriesShards() { - return context.searchStats().targetShards().values().stream().anyMatch(imd -> imd.getIndexMode() == IndexMode.TIME_SERIES); + return context.searchStats().targetShards().values().stream().anyMatch(imd -> IndexMode.isTsdb(imd.getIndexMode())); } private Expression replaceFieldsForFieldTransformations(Expression e, BlockLoaderExpression.PushedBlockLoaderExpression fuse) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/ReplaceRoundToWithQueryAndTags.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/ReplaceRoundToWithQueryAndTags.java index 0b19ada1a074f..77df210a45daa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/ReplaceRoundToWithQueryAndTags.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/ReplaceRoundToWithQueryAndTags.java @@ -343,8 +343,8 @@ protected PhysicalPlan rule(EvalExec evalExec, LocalPhysicalOptimizerContext ctx // prefixes. When prefix partitioning is not available (old codec), we fall back to replacing round_to // with QueryAndTags. if (((FieldAttribute) roundTo.field()).name().equals(MetadataAttribute.TIMESTAMP_FIELD) - && ctx.searchStats().targetShards().values().stream().allMatch(imd -> imd.getIndexMode() == IndexMode.TIME_SERIES)) { - if (queryExec.indexMode() != IndexMode.TIME_SERIES) { + && ctx.searchStats().targetShards().values().stream().allMatch(imd -> IndexMode.isTsdb(imd.getIndexMode()))) { + if (queryExec.indexMode().isTsdb() == false) { return evalExec; } // prefer partitioning by tsid prefixes @@ -568,7 +568,7 @@ private int roundingPointsThreshold(LocalPhysicalOptimizerContext ctx) { */ static int adjustedRoundingPointsThreshold(SearchStats stats, int threshold, QueryBuilder query, IndexMode indexMode) { int clauses = estimateQueryClauses(stats, query) + 1; - if (indexMode == IndexMode.TIME_SERIES) { + if (indexMode.isTsdb()) { // No doc partitioning for time_series sources; increase the threshold to trade overhead for parallelism. threshold *= 2; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/ReplaceSourceAttributes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/ReplaceSourceAttributes.java index 525ff7129adf6..c31fd30f7a6ce 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/ReplaceSourceAttributes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/ReplaceSourceAttributes.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; @@ -53,7 +52,7 @@ private static PhysicalPlan replaceEsSource(EsSourceExec plan) { final List attributes = new ArrayList<>(); attributes.add(getDocAttribute(plan)); - if (plan.indexMode() == IndexMode.TIME_SERIES) { + if (plan.indexMode().isTsdb()) { for (EsField field : EsQueryExec.TIME_SERIES_SOURCE_FIELDS) { attributes.add(new FieldAttribute(plan.source(), null, null, field.getName(), field)); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index 1a0094456e5bb..3319ad720b674 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -729,7 +729,7 @@ public PlanFactory visitStatsCommand(EsqlBaseParser.StatsCommandContext ctx) { return input -> { boolean hasAggregate = input.anyMatch(p -> p instanceof Aggregate); boolean hasPromqlCommand = input.anyMatch(p -> p instanceof PromqlCommand); - boolean hasTimeSeries = input.anyMatch(p -> p instanceof UnresolvedRelation ur && ur.indexMode() == IndexMode.TIME_SERIES); + boolean hasTimeSeries = input.anyMatch(p -> p instanceof UnresolvedRelation ur && ur.indexMode().isTsdb()); boolean hasInfoCommand = input.anyMatch(p -> p instanceof MetricsInfo || p instanceof TsInfo); if (hasAggregate == false && hasPromqlCommand == false && hasTimeSeries && hasInfoCommand == false) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java index 92fcf2d20a761..23a13a92eaf0d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java @@ -10,7 +10,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.xpack.esql.capabilities.PostAnalysisVerificationAware; import org.elasticsearch.xpack.esql.capabilities.TelemetryAware; import org.elasticsearch.xpack.esql.common.Failures; @@ -280,7 +279,7 @@ private static void checkUnsupportedStatsGroupingType(Expression e, Failures fai protected void checkTimeSeriesAggregates(Failures failures) { Holder isTimeSeriesIndexMode = new Holder<>(false); child().forEachDown(p -> { - if (p instanceof EsRelation er && er.indexMode() == IndexMode.TIME_SERIES) { + if (p instanceof EsRelation er && er.indexMode().isTsdb()) { isTimeSeriesIndexMode.set(true); } }); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java index ed1338a892993..0c7fcae0a7b45 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java @@ -11,7 +11,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.xpack.esql.capabilities.PostAnalysisPlanVerificationAware; import org.elasticsearch.xpack.esql.capabilities.TelemetryAware; import org.elasticsearch.xpack.esql.common.Failures; @@ -186,7 +185,7 @@ public BiConsumer postAnalysisPlanVerification() { } else { foundPreviousStats.set(true); } - } else if (lp instanceof EsRelation er && er.indexMode() == IndexMode.TIME_SERIES) { + } else if (lp instanceof EsRelation er && er.indexMode().isTsdb()) { isTimeSeries.set(true); } }); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MetricsInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MetricsInfo.java index 8b6c9496e51c7..069f251cdfa2a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MetricsInfo.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MetricsInfo.java @@ -9,7 +9,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.xpack.esql.capabilities.PostAnalysisVerificationAware; import org.elasticsearch.xpack.esql.capabilities.TelemetryAware; import org.elasticsearch.xpack.esql.common.Failures; @@ -138,7 +137,7 @@ public void postAnalysisVerification(Failures failures) { if (failures.hasFailures()) { return; } - boolean hasTsSource = child().anyMatch(p -> p instanceof EsRelation er && er.indexMode() == IndexMode.TIME_SERIES); + boolean hasTsSource = child().anyMatch(p -> p instanceof EsRelation er && er.indexMode().isTsdb()); if (hasTsSource == false) { failures.add(fail(this, "METRICS_INFO can only be used with TS source command")); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/TsInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/TsInfo.java index d421510f0cbb3..3c285b169b074 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/TsInfo.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/TsInfo.java @@ -9,7 +9,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.xpack.esql.capabilities.PostAnalysisVerificationAware; import org.elasticsearch.xpack.esql.capabilities.TelemetryAware; import org.elasticsearch.xpack.esql.common.Failures; @@ -138,7 +137,7 @@ public void postAnalysisVerification(Failures failures) { if (failures.hasFailures()) { return; } - boolean hasTsSource = child().anyMatch(p -> p instanceof EsRelation er && er.indexMode() == IndexMode.TIME_SERIES); + boolean hasTsSource = child().anyMatch(p -> p instanceof EsRelation er && er.indexMode().isTsdb()); if (hasTsSource == false) { failures.add(fail(this, "TS_INFO can only be used with TS source command")); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java index 44d4460156648..e29f8d499e836 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java @@ -204,6 +204,6 @@ public String toString() { * which changes a number of behaviors in the planner. */ public boolean isTimeSeriesMode() { - return indexMode == IndexMode.TIME_SERIES; + return indexMode.isTsdb(); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java index d7a4e97bea924..9f0f1337af503 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java @@ -253,11 +253,11 @@ public EsQueryExec withLimit(Expression limit) { } public boolean canPushSorts() { - return indexMode != IndexMode.TIME_SERIES; + return indexMode.isTsdb() == false; } public EsQueryExec withSorts(List sorts) { - if (indexMode == IndexMode.TIME_SERIES) { + if (indexMode.isTsdb()) { assert false : "time-series index mode doesn't support sorts"; throw new UnsupportedOperationException("time-series index mode doesn't support sorts"); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 9a955b43715c0..4cb9b243a29a1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -41,7 +41,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.Releasable; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.mapper.BlockLoader; @@ -486,7 +485,7 @@ public final PhysicalOperation sourcePhysicalOperation(EsQueryExec esQueryExec, scoring, directoryBytesRead ); - } else if (esQueryExec.indexMode() == IndexMode.TIME_SERIES) { + } else if (esQueryExec.indexMode().isTsdb()) { luceneFactory = new TimeSeriesSourceOperator.Factory( shardContexts, querySupplier(esQueryExec.queryBuilderAndTags()), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index c18b953c73212..3af1d7afdf4c7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -1762,7 +1762,7 @@ private PhysicalOperation emptySourceForAttributes(List attributes) { private MetricsInfoOperator.MetricFieldLookup createMetricFieldLookup(IndexedByShardId shardContexts) { Map mappingsByIndex = new HashMap<>(); for (ShardContext shard : shardContexts.iterable()) { - if (shard.indexSettings().getMode() == IndexMode.TIME_SERIES) { + if (shard.indexSettings().getMode().isTsdb()) { mappingsByIndex.putIfAbsent(shard.indexSettings().getIndex().getName(), shard.mappingLookup()); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 7a1056e9fd808..e3ad26e719c03 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -33,7 +33,7 @@ import org.elasticsearch.index.mapper.IndexModeFieldMapper; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesExpressionGrouper; import org.elasticsearch.iplocation.api.IpDataLookupInfo; @@ -1866,7 +1866,7 @@ private void preAnalyzeMainIndices( indexPattern.indexPattern(), result.fieldNames, createQueryFilter(indexMode, requestFilter), - indexMode == IndexMode.TIME_SERIES, + indexMode.isTsdb(), // TODO: In case of subqueries, the different main index resolutions don't know about each other's minimum version. // This is bad because `FROM (FROM remote1:*) (FROM remote2:*)` can have different minimum versions // while resolving each subquery's main index pattern. We'll determine the correct overall minimum transport version @@ -1962,7 +1962,7 @@ private void preAnalyzeFlatMainIndices( projectRouting, result.fieldNames, createQueryFilter(indexMode, requestFilter), - indexMode == IndexMode.TIME_SERIES, + indexMode.isTsdb(), // TODO: Same problem with subqueries as preAnalyzeMainIndices, see above. result.minimumTransportVersion(), preAnalysis.useAggregateMetricDoubleWhenNotSupported(), @@ -1998,8 +1998,15 @@ private void preAnalyzeFlatMainIndices( private static QueryBuilder createQueryFilter(IndexMode indexMode, QueryBuilder requestFilter) { return switch (indexMode) { - case IndexMode.TIME_SERIES -> { - var indexModeFilter = new TermQueryBuilder(IndexModeFieldMapper.NAME, IndexMode.TIME_SERIES.getName()); + case IndexMode.TIME_SERIES, IndexMode.TSDB -> { + // Match either alias: the requested indexMode is a fixed sentinel (e.g. the TS command always + // declares IndexMode.TIME_SERIES), but the concrete backing indices may be configured with + // either [index.mode=time_series] or its alias [index.mode=tsdb]. + var indexModeFilter = new TermsQueryBuilder( + IndexModeFieldMapper.NAME, + IndexMode.TIME_SERIES.getName(), + IndexMode.TSDB.getName() + ); yield requestFilter != null ? new BoolQueryBuilder().filter(requestFilter).filter(indexModeFilter) : indexModeFilter; } default -> requestFilter; @@ -2008,7 +2015,7 @@ private static QueryBuilder createQueryFilter(IndexMode indexMode, QueryBuilder // visible for testing static boolean shouldRetryConcreteTimeSeriesResolution(IndexMode indexMode, IndexResolution resolution, IndexPattern indexPattern) { - return indexMode == IndexMode.TIME_SERIES + return indexMode.isTsdb() && resolution.isValid() && resolution.resolvedIndices().isEmpty() && EsqlCCSUtils.concreteIndexRequested(indexPattern.indexPattern()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java index e638372f37caf..f1a4dcf0bfab0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.telemetry; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.ChangePoint; @@ -99,8 +98,8 @@ public enum FeatureMetric { MV_EXPAND(MvExpand.class::isInstance), SHOW(ShowInfo.class::isInstance), ROW(Row.class::isInstance), - FROM(x -> x instanceof EsRelation relation && relation.indexMode() != IndexMode.TIME_SERIES), - TS(x -> x instanceof EsRelation relation && relation.indexMode() == IndexMode.TIME_SERIES), + FROM(x -> x instanceof EsRelation relation && relation.indexMode().isTsdb() == false), + TS(x -> x instanceof EsRelation relation && relation.indexMode().isTsdb()), EXTERNAL(plan -> plan instanceof org.elasticsearch.xpack.esql.plan.logical.ExternalRelation), DROP(Drop.class::isInstance), KEEP(Keep.class::isInstance), diff --git a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TimeSeriesToTsdbIndexModeRollingUpgradeIT.java b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TimeSeriesToTsdbIndexModeRollingUpgradeIT.java new file mode 100644 index 0000000000000..df3be41a515b1 --- /dev/null +++ b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TimeSeriesToTsdbIndexModeRollingUpgradeIT.java @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import org.elasticsearch.client.Request; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.test.rest.ObjectPath; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.xpack.logsdb.TsdbIT.getTemplate; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.notNullValue; + +/** + * Verifies that a data stream's backing indices can switch from {@link IndexMode#TIME_SERIES} to + * its preferred alias {@link IndexMode#TSDB} via a template update + rollover, once the whole + * cluster has been upgraded to a version that understands {@code tsdb} (old-version clusters in + * this suite never do, since {@code tsdb} did not exist as an {@code index.mode} value before). + * The pre-switch ({@code time_series}) and post-switch ({@code tsdb}) backing indices must both + * remain queryable, together, once the switch has happened. + */ +public class TimeSeriesToTsdbIndexModeRollingUpgradeIT extends AbstractLogsdbRollingUpgradeTestCase { + + private static final String DATA_STREAM_NAME = "k10s"; + private static final int DOCS_PER_ROUND = 4 * 128; + + // Kept short (and explicit, rather than relying on the plugin's defaults) so the test can + // deterministically place documents into the pre-switch vs. post-switch backing index by + // choosing their @timestamp, rather than depending on how much real wall-clock time the + // rolling upgrade itself takes. A backing index's [start_time, end_time) window is anchored + // to its real creation time, not to any document's @timestamp; on rollover the new index's + // start_time is exactly the old index's end_time (contiguous, no overlap/gap) - see + // DataStreamIndexSettingsProvider#provideAdditionalSettings. + private static final String LOOK_AHEAD_AND_BACK_TIME = "1m"; + + private int totalDocsIndexed = 0; + + public void testSwitchFromTimeSeriesToTsdb() throws Exception { + String templateId = getClass().getSimpleName().toLowerCase(Locale.ROOT); + String timeSeriesTemplate = withLookAheadAndBack(getTemplate(null, null)); + createTemplate(DATA_STREAM_NAME, templateId, timeSeriesTemplate); + + // Anchor for the whole test: the first backing index's window is [testStart - 1m, testStart + // + 1m). Documents timestamped near testStart land there regardless of real elapsed time. + Instant testStart = Instant.now(); + + // Old cluster: only "time_series" exists as an index.mode value. + indexRound(testStart); + verifyWriteIndexMode(IndexMode.TIME_SERIES); + assertSearchAndQuery(); + + // Mixed cluster: keep indexing/querying against the still-time_series write index. + clusterRollingUpgrade(index -> { + indexRound(testStart.plusSeconds(index + 1)); + verifyWriteIndexMode(IndexMode.TIME_SERIES); + assertSearchAndQuery(); + }); + + String firstBackingIndex = getDataStreamBackingIndexNames(DATA_STREAM_NAME).getFirst(); + + // Fully upgraded: switch the template to "tsdb" and roll over onto a new backing index. + String tsdbTemplate = withLookAheadAndBack(getTemplate(null, null)).replace("\"time_series\"", "\"tsdb\""); + createTemplate(DATA_STREAM_NAME, templateId, tsdbTemplate); + rolloverDataStream(DATA_STREAM_NAME); + + List backingIndices = getDataStreamBackingIndexNames(DATA_STREAM_NAME); + assertThat(backingIndices, hasSize(2)); + assertThat(backingIndices.get(0), equalTo(firstBackingIndex)); + String secondBackingIndex = backingIndices.get(1); + + verifyIndexMode(IndexMode.TIME_SERIES, firstBackingIndex); + verifyIndexMode(IndexMode.TSDB, secondBackingIndex); + + // The second backing index's window starts exactly where the first one's ends + // (testStart + 1m); land comfortably inside it, regardless of how long the upgrade took. + indexRound(testStart.plusSeconds(90)); + assertSearchAndQuery(); + + var forceMergeRequest = new Request("POST", "/" + DATA_STREAM_NAME + "/_forcemerge"); + forceMergeRequest.addParameter("max_num_segments", "1"); + assertOK(client().performRequest(forceMergeRequest)); + ensureGreen(DATA_STREAM_NAME); + assertSearchAndQuery(); + } + + private static String withLookAheadAndBack(String template) { + return template.replace( + "\"mode\": \"time_series\"", + "\"mode\": \"time_series\", \"look_ahead_time\": \"" + + LOOK_AHEAD_AND_BACK_TIME + + "\", \"look_back_time\": \"" + + LOOK_AHEAD_AND_BACK_TIME + + "\"" + ); + } + + private void indexRound(Instant timestamp) throws Exception { + bulkIndex(DATA_STREAM_NAME, 4, 128, timestamp, TsdbIndexingRollingUpgradeIT::docSupplier); + totalDocsIndexed += DOCS_PER_ROUND; + } + + private void rolloverDataStream(String dataStreamName) throws IOException { + var request = new Request("POST", "/" + dataStreamName + "/_rollover"); + assertOK(client().performRequest(request)); + } + + private void verifyWriteIndexMode(IndexMode indexMode) throws IOException { + verifyIndexMode(indexMode, getDataStreamBackingIndexNames(DATA_STREAM_NAME).getLast()); + } + + private void verifyIndexMode(IndexMode indexMode, String index) throws IOException { + var settings = (Map) getIndexSettings(index, true).get(index); + assertThat(((Map) settings.get("settings")).get(IndexSettings.MODE.getKey()), equalTo(indexMode.getName())); + } + + private void assertSearchAndQuery() throws Exception { + search(); + query(); + } + + private void search() throws Exception { + var searchRequest = new Request("POST", "/" + DATA_STREAM_NAME + "/_search"); + searchRequest.setJsonEntity("{\"size\": 0}"); + var response = client().performRequest(searchRequest); + assertOK(response); + var responseBody = entityAsMap(response); + Integer totalCount = ObjectPath.evaluate(responseBody, "hits.total.value"); + assertThat(totalCount, equalTo(totalDocsIndexed)); + } + + /** + * Groups by {@code _index} to prove the data stream query spans every backing index - + * including both the {@code time_series} and the {@code tsdb} one once the switch has + * happened - and that their per-index counts sum to every document ever indexed. + */ + private void query() throws Exception { + var queryRequest = new Request("POST", "/_query"); + queryRequest.setJsonEntity(""" + { + "query": "FROM $ds METADATA _index | STATS count = COUNT(*) BY _index | SORT _index | LIMIT 10" + } + """.replace("$ds", DATA_STREAM_NAME)); + var response = client().performRequest(queryRequest); + assertOK(response); + var responseBody = entityAsMap(response); + + List backingIndices = getDataStreamBackingIndexNames(DATA_STREAM_NAME); + List> values = ObjectPath.evaluate(responseBody, "values"); + assertThat("values=" + values, values, hasSize(backingIndices.size())); + + long countedTotal = 0; + List queriedIndices = new ArrayList<>(); + for (List row : values) { + Number count = (Number) row.get(0); + assertThat(count, notNullValue()); + countedTotal += count.longValue(); + queriedIndices.add((String) row.get(1)); + } + assertThat(countedTotal, equalTo((long) totalDocsIndexed)); + assertThat(queriedIndices, containsInAnyOrder(backingIndices.toArray())); + } +} diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index 118aac503848f..41f50634642bb 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -269,7 +269,7 @@ MappingHints getMappingHints( if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(tmpIndexMetadata.getSettings()) || indexMode == IndexMode.LOGSDB || indexMode == IndexMode.LOGSDB_COLUMNAR - || indexMode == IndexMode.TIME_SERIES) { + || IndexMode.isTsdb(indexMode)) { // In case when index mode is tsdb or logsdb and only _source.mode mapping attribute is specified, then the default // could be wrong. However, it doesn't really matter, because if the _source.mode mapping attribute is set to stored, // then configuring the index.mapping.source.mode setting to stored has no effect. Additionally _source.mode can't be set @@ -385,8 +385,8 @@ private IndexMetadata buildIndexMetadataForMapperService( .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) .put(IndexSettings.INDEX_FAST_REFRESH_SETTING.getKey(), false); // Avoid warnings for non-system indexes. - if (templateIndexMode == IndexMode.TIME_SERIES) { - finalResolvedSettings.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES); + if (IndexMode.isTsdb(templateIndexMode)) { + finalResolvedSettings.put(IndexSettings.MODE.getKey(), templateIndexMode); // Avoid failing because index.routing_path is missing (in case fields are marked as dimension) finalResolvedSettings.putList(INDEX_ROUTING_PATH.getKey(), List.of("path")); } @@ -413,7 +413,7 @@ private static SourceFieldMapper.Mode fallbackSourceMode(Settings settings, Inde * The GA-ed use cases in which synthetic source usage is allowed with gold or platinum license. */ private boolean isLegacyLicensedUsageOfSyntheticSourceAllowed(IndexMode templateIndexMode, String indexName, String dataStreamName) { - if (templateIndexMode == IndexMode.TIME_SERIES) { + if (IndexMode.isTsdb(templateIndexMode)) { return true; } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index fc17aaea04c7d..21a8a5222a200 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -145,7 +145,7 @@ public Builder(String name, IndexSettings indexSettings) { if (useTimeSeriesDocValuesSkippers(indexSettings, dimension.get())) { return false; } - if (indexSettings.getMode() == IndexMode.TIME_SERIES) { + if (indexSettings.getMode().isTsdb()) { var metricType = getMetric().getValue(); return metricType != MetricType.COUNTER && metricType != MetricType.GAUGE; } else { @@ -399,7 +399,7 @@ public Query rangeQuery( @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - if (indexMode == IndexMode.TIME_SERIES && metricType == TimeSeriesParams.MetricType.COUNTER) { + if (indexMode.isTsdb() && metricType == TimeSeriesParams.MetricType.COUNTER) { // Counters are not supported by ESQL so we load them in null return ConstantNull.INSTANCE; } @@ -508,7 +508,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext failIfNoDocValues(); } - ValuesSourceType valuesSourceType = indexMode == IndexMode.TIME_SERIES && metricType == TimeSeriesParams.MetricType.COUNTER + ValuesSourceType valuesSourceType = indexMode.isTsdb() && metricType == TimeSeriesParams.MetricType.COUNTER ? TimeSeriesValuesSourceType.COUNTER : IndexNumericFieldData.NumericType.LONG.getValuesSourceType(); diff --git a/x-pack/plugin/stateless/src/test/java/org/elasticsearch/xpack/stateless/commits/IndexCommitTimestampFieldRangeTests.java b/x-pack/plugin/stateless/src/test/java/org/elasticsearch/xpack/stateless/commits/IndexCommitTimestampFieldRangeTests.java index 4e63a9d5200b2..df4ab70030c38 100644 --- a/x-pack/plugin/stateless/src/test/java/org/elasticsearch/xpack/stateless/commits/IndexCommitTimestampFieldRangeTests.java +++ b/x-pack/plugin/stateless/src/test/java/org/elasticsearch/xpack/stateless/commits/IndexCommitTimestampFieldRangeTests.java @@ -120,10 +120,10 @@ public void testSoftDeletesAreAlmostAlwaysDisregardedForTimestampRange() throws IndexCommit previousCommit = null; try (Directory directory = newDirectory()) { try (IndexWriter indexWriter = new IndexWriter(directory, indexWriterConfig)) { - String docId1 = indexMode == IndexMode.TIME_SERIES ? TimeSeriesRoutingHashFieldMapper.encode(1) : "docId1"; + String docId1 = indexMode.isTsdb() ? TimeSeriesRoutingHashFieldMapper.encode(1) : "docId1"; long timestamp1 = randomLongBetween(10_000L, 1_000_000L); indexWriter.addDocument(mapper.parse(source(docId1, b -> { b.field("@timestamp", timestamp1); }, null)).rootDoc()); - String docId2 = indexMode == IndexMode.TIME_SERIES ? TimeSeriesRoutingHashFieldMapper.encode(2) : "docId2"; + String docId2 = indexMode.isTsdb() ? TimeSeriesRoutingHashFieldMapper.encode(2) : "docId2"; long timestamp2 = randomLongBetween(10_000L, 1_000_000L); indexWriter.addDocument(mapper.parse(source(docId2, b -> { b.field("@timestamp", timestamp2); }, null)).rootDoc()); boolean doc1Deleted = randomBoolean(); @@ -143,12 +143,12 @@ public void testSoftDeletesAreAlmostAlwaysDisregardedForTimestampRange() throws ); previousCommit = currentCommit; } - String docId3 = indexMode == IndexMode.TIME_SERIES ? TimeSeriesRoutingHashFieldMapper.encode(3) : "docId3"; + String docId3 = indexMode.isTsdb() ? TimeSeriesRoutingHashFieldMapper.encode(3) : "docId3"; long timestamp3 = randomLongBetween(10_000L, 1_000_000L); indexWriter.addDocument(mapper.parse(source(docId3, b -> { b.field("@timestamp", timestamp3); }, null)).rootDoc()); // create a 1-doc segment indexWriter.flush(); - String docId4 = indexMode == IndexMode.TIME_SERIES ? TimeSeriesRoutingHashFieldMapper.encode(4) : "docId4"; + String docId4 = indexMode.isTsdb() ? TimeSeriesRoutingHashFieldMapper.encode(4) : "docId4"; long timestamp4 = randomLongBetween(10_000L, 1_000_000L); indexWriter.addDocument(mapper.parse(source(docId4, b -> { b.field("@timestamp", timestamp4); }, null)).rootDoc()); deleteDoc(doc1Deleted ? docId2 : docId1, indexWriter, indexMode, mapper); @@ -197,7 +197,7 @@ public void testFieldValueRangeForBatchedCompoundCommit() throws Exception { return List.of( mapper.parse( source( - indexMode == IndexMode.TIME_SERIES + indexMode.isTsdb() ? TimeSeriesRoutingHashFieldMapper.encode(docId.incrementAndGet()) : String.valueOf(docId.incrementAndGet()), b -> { @@ -297,7 +297,7 @@ private void testFieldValueRange(boolean useCFS, IndexMode indexMode) throws Exc for (int i = 0; i < docCount; i++) { // make sure timestamp doesn't look like a year long timestamp = randomLongBetween(10_000L, 1_000_000L); - String docId = indexMode == IndexMode.TIME_SERIES ? TimeSeriesRoutingHashFieldMapper.encode(i) : randomUUID(); + String docId = indexMode.isTsdb() ? TimeSeriesRoutingHashFieldMapper.encode(i) : randomUUID(); indexWriter.addDocument(mapper.parse(source(docId, b -> { b.field("@timestamp", timestamp); }, null)).rootDoc()); flushEveryNDocCount--; commitEveryNDocCount--; @@ -400,12 +400,12 @@ private DocumentMapper getDocumentMapper(IndexMode indexMode) throws IOException b.field("doc_values", docValues).field("store", allowStore).endObject(); }), indexMode); } - } else if (indexMode == IndexMode.TIME_SERIES) { - // TIME_SERIES indexing modes use a fixed mapping for the @timestamp field + } else if (indexMode.isTsdb()) { + // TIME_SERIES/TSDB indexing modes use a fixed mapping for the @timestamp field // Synthetic id need cumbersome special care in test setup so we turn that // off in this test Settings settings = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexSettings.MODE.getKey(), indexMode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "uid") .put(IndexSettings.SYNTHETIC_ID.getKey(), false) .build(); From d963bb1a993d35f8c1cd2fcd8973937343a4969f Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:37:37 +0300 Subject: [PATCH 02/23] Update docs/changelog/152901.yaml --- docs/changelog/152901.yaml | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 docs/changelog/152901.yaml diff --git a/docs/changelog/152901.yaml b/docs/changelog/152901.yaml new file mode 100644 index 0000000000000..d1f0825ec7ea7 --- /dev/null +++ b/docs/changelog/152901.yaml @@ -0,0 +1,46 @@ +area: TSDB +highlight: + body: |- + ## Summary + + Add `IndexMode.TSDB` as a new, first-class enum value that behaves + identically to `TIME_SERIES` (delegates every overridden method), so + `index.mode: tsdb` is now accepted anywhere `index.mode: time_series` is + — without renaming or changing anything about existing `time_series` + indices, which keep emitting/persisting exactly as before (`time_series` + remains the canonical output string; `tsdb` is a preferred alias). + + - New helpers `IndexMode#isTsdb()`, `IndexMode.isTsdb(IndexMode)` (null-safe), and `IndexMode.isTsdbName(String)` (for raw, un-parsed `index.mode` setting strings) replace direct `== IndexMode.TIME_SERIES` comparisons across the codebase, so both spellings are treated identically everywhere. + - `Enum.valueOf(IndexMode.class, ...)` / `IndexMode.valueOf` parse sites were **deliberately left untouched** — since `TSDB` is a new constant (not a rename) whose name already uppercases to match `"tsdb"`, these already accept both spellings for free. + - New `TransportVersion` (`index_mode_tsdb_added`) gates `TSDB`'s wire serialization for mixed-version clusters. + - New `TsdbIndexModeTests` covering `fromString`, `getName`, `isTsdb`/`isTsdbName`, the `index.mode` setting parser, serialization round-trip + version-gate failure, and behavior parity with `TIME_SERIES`. + - New `TimeSeriesToTsdbIndexModeRollingUpgradeIT` — a real rolling-upgrade test that creates a data stream on `time_series`, upgrades the cluster, switches the template to `tsdb` and rolls over, and verifies the old backing index stays `time_series`, the new one is `tsdb`, and both remain queryable together (verified via `FROM ds METADATA _index | STATS count = COUNT(*) BY _index`). + + ## Notable bugs found and fixed along the way + + - `EsqlSession`'s `_index_mode` term filter (used to gate `TS`/`PROMQL` command queries) was built from the command's fixed sentinel mode, so it only ever matched `"time_series"` — it would have silently excluded any concrete `index.mode: tsdb` index from a `TS` query, and incorrectly rejected it as "not a time series index" on retry. Fixed with a `TermsQueryBuilder` matching both aliases. + - Two colon-style `switch`/`if` statements (`FollowingEngineTests`, `LogsdbIndexModeSettingsProvider`) with a `default`/hardcoded branch would have broken once `TSDB` became reachable via `randomFrom(IndexMode.availableModes())` — not caught by the compiler since these aren't exhaustive expression switches. + - A couple of `randomValueOtherThan(IndexMode.TIME_SERIES, ...)` test patterns would let a randomly-picked `TSDB` slip through as "not time series," causing spurious failures — fixed with `randomValueOtherThanMany(m -> m.isTsdb(), ...)`. + + ## Test plan + + - [x] `TsdbIndexModeTests` (new) — all pass + - [x] `IndexSettingsTests`, `MapperServiceTests`, `MappingLookupTests`, `IndexModeFieldTypeTests`, `SourceFieldMapperTests`, `PerFieldMapperCodecTests`, `TSDBDocValuesFormatSelectorTests`, `ResolveIndexTests` — all pass + - [x] ESQL `analysis`, `optimizer`, `datasources`, `session` test suites — all pass + - [x] `FollowingEngineTests` (ccr), `IndexCommitTimestampFieldRangeTests` (stateless) — all pass + - [x] `TimeSeriesToTsdbIndexModeRollingUpgradeIT` (new) — run against real rolling upgrades from both 9.5.0 and 8.19.0 + - [x] Full compile sweep across every touched module + - [x] `spotlessJavaCheck` clean across every touched module + + ## Follow-up (separate PR) + + Updating YAML REST tests and existing Java tests to prefer/cover `tsdb` + alongside `time_series` is intentionally deferred to a follow-up PR. + + Related to https://github.com/elastic/elasticsearch/issues/152902 + notable: true + title: Add `IndexMode.TSDB` as a preferred alias for TIME_SERIES +issues: [] +pr: 152901 +summary: Add `IndexMode.TSDB` as a preferred alias for TIME_SERIES +type: enhancement From 508dd8ae6d0946d5227345801a56ad50bcc251e0 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:38:35 +0300 Subject: [PATCH 03/23] Add `IndexMode.TSDB` alias for `TIME_SERIES` Introduced `IndexMode.TSDB` as an alias for `TIME_SERIES`, added new tests, and fixed related bugs. Deferred updates to YAML REST tests for a follow-up PR. --- docs/changelog/152901.yaml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/docs/changelog/152901.yaml b/docs/changelog/152901.yaml index d1f0825ec7ea7..01cbd0850da2a 100644 --- a/docs/changelog/152901.yaml +++ b/docs/changelog/152901.yaml @@ -10,34 +10,6 @@ highlight: indices, which keep emitting/persisting exactly as before (`time_series` remains the canonical output string; `tsdb` is a preferred alias). - - New helpers `IndexMode#isTsdb()`, `IndexMode.isTsdb(IndexMode)` (null-safe), and `IndexMode.isTsdbName(String)` (for raw, un-parsed `index.mode` setting strings) replace direct `== IndexMode.TIME_SERIES` comparisons across the codebase, so both spellings are treated identically everywhere. - - `Enum.valueOf(IndexMode.class, ...)` / `IndexMode.valueOf` parse sites were **deliberately left untouched** — since `TSDB` is a new constant (not a rename) whose name already uppercases to match `"tsdb"`, these already accept both spellings for free. - - New `TransportVersion` (`index_mode_tsdb_added`) gates `TSDB`'s wire serialization for mixed-version clusters. - - New `TsdbIndexModeTests` covering `fromString`, `getName`, `isTsdb`/`isTsdbName`, the `index.mode` setting parser, serialization round-trip + version-gate failure, and behavior parity with `TIME_SERIES`. - - New `TimeSeriesToTsdbIndexModeRollingUpgradeIT` — a real rolling-upgrade test that creates a data stream on `time_series`, upgrades the cluster, switches the template to `tsdb` and rolls over, and verifies the old backing index stays `time_series`, the new one is `tsdb`, and both remain queryable together (verified via `FROM ds METADATA _index | STATS count = COUNT(*) BY _index`). - - ## Notable bugs found and fixed along the way - - - `EsqlSession`'s `_index_mode` term filter (used to gate `TS`/`PROMQL` command queries) was built from the command's fixed sentinel mode, so it only ever matched `"time_series"` — it would have silently excluded any concrete `index.mode: tsdb` index from a `TS` query, and incorrectly rejected it as "not a time series index" on retry. Fixed with a `TermsQueryBuilder` matching both aliases. - - Two colon-style `switch`/`if` statements (`FollowingEngineTests`, `LogsdbIndexModeSettingsProvider`) with a `default`/hardcoded branch would have broken once `TSDB` became reachable via `randomFrom(IndexMode.availableModes())` — not caught by the compiler since these aren't exhaustive expression switches. - - A couple of `randomValueOtherThan(IndexMode.TIME_SERIES, ...)` test patterns would let a randomly-picked `TSDB` slip through as "not time series," causing spurious failures — fixed with `randomValueOtherThanMany(m -> m.isTsdb(), ...)`. - - ## Test plan - - - [x] `TsdbIndexModeTests` (new) — all pass - - [x] `IndexSettingsTests`, `MapperServiceTests`, `MappingLookupTests`, `IndexModeFieldTypeTests`, `SourceFieldMapperTests`, `PerFieldMapperCodecTests`, `TSDBDocValuesFormatSelectorTests`, `ResolveIndexTests` — all pass - - [x] ESQL `analysis`, `optimizer`, `datasources`, `session` test suites — all pass - - [x] `FollowingEngineTests` (ccr), `IndexCommitTimestampFieldRangeTests` (stateless) — all pass - - [x] `TimeSeriesToTsdbIndexModeRollingUpgradeIT` (new) — run against real rolling upgrades from both 9.5.0 and 8.19.0 - - [x] Full compile sweep across every touched module - - [x] `spotlessJavaCheck` clean across every touched module - - ## Follow-up (separate PR) - - Updating YAML REST tests and existing Java tests to prefer/cover `tsdb` - alongside `time_series` is intentionally deferred to a follow-up PR. - - Related to https://github.com/elastic/elasticsearch/issues/152902 notable: true title: Add `IndexMode.TSDB` as a preferred alias for TIME_SERIES issues: [] From 5ccdda9682983ad1cc98c23c81f8ef60d0d21cb3 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 3 Jul 2026 21:23:35 +0300 Subject: [PATCH 04/23] Fix NPEs and BWC gap surfaced by TSDB alias CI run NumberFieldType/GeoPointFieldType/ScaledFloatFieldType/ UnsignedLongFieldType all have a real (non-test-only) code path where indexMode is null, e.g. TokenCountFieldType passes null explicitly. The earlier == IndexMode.TIME_SERIES checks tolerated that; the isTsdb() instance-method replacements did not. Switch these four call sites to the null-safe IndexMode.isTsdb(IndexMode) static instead. Also gate AllSupportedFieldsTestCase's mode=tsdb parameterization on minVersion().supports(IndexMode.INDEX_MODE_TSDB_ADDED), matching the existing pattern for vectordb_document/columnar, so multi-cluster BWC runs skip creating a tsdb index against a remote that predates it. --- .../index/mapper/extras/ScaledFloatFieldMapper.java | 4 ++-- .../org/elasticsearch/index/mapper/GeoPointFieldMapper.java | 2 +- .../org/elasticsearch/index/mapper/NumberFieldMapper.java | 2 +- .../xpack/esql/qa/rest/AllSupportedFieldsTestCase.java | 3 +++ .../xpack/unsignedlong/UnsignedLongFieldMapper.java | 4 ++-- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 2d6d4c8b6d760..fbfb1ac4fbcb8 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -407,7 +407,7 @@ public Query rangeQuery( @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - if (indexMode.isTsdb() && metricType == TimeSeriesParams.MetricType.COUNTER) { + if (IndexMode.isTsdb(indexMode) && metricType == TimeSeriesParams.MetricType.COUNTER) { // Counters are not supported by ESQL so we load them in null return ConstantNull.INSTANCE; } @@ -495,7 +495,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext failIfNoDocValues(); } - ValuesSourceType valuesSourceType = indexMode.isTsdb() && metricType == TimeSeriesParams.MetricType.COUNTER + ValuesSourceType valuesSourceType = IndexMode.isTsdb(indexMode) && metricType == TimeSeriesParams.MetricType.COUNTER ? TimeSeriesValuesSourceType.COUNTER : IndexNumericFieldData.NumericType.LONG.getValuesSourceType(); if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 2dedda670b336..fef392b83eadd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -482,7 +482,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext failIfNoDocValues(); } - ValuesSourceType valuesSourceType = indexMode.isTsdb() && metricType == TimeSeriesParams.MetricType.POSITION + ValuesSourceType valuesSourceType = IndexMode.isTsdb(indexMode) && metricType == TimeSeriesParams.MetricType.POSITION ? TimeSeriesValuesSourceType.POSITION : CoreValuesSourceType.GEOPOINT; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index c7acdcf2f0e17..bc3cbdc5a623e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -2341,7 +2341,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext failIfNoDocValues(); } - ValuesSourceType valuesSourceType = indexMode.isTsdb() && metricType == TimeSeriesParams.MetricType.COUNTER + ValuesSourceType valuesSourceType = IndexMode.isTsdb(indexMode) && metricType == TimeSeriesParams.MetricType.COUNTER ? TimeSeriesValuesSourceType.COUNTER : type.numericType.getValuesSourceType(); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 42a504ba96274..1c3042058f214 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -276,6 +276,9 @@ public void createIndices() throws IOException { minVersion().supports(IndexMode.VECTORDB_DOCUMENT_INDEX_MODE) ); } + if (indexMode == IndexMode.TSDB) { + assumeTrue("Cluster has nodes that do not support index.mode=tsdb", minVersion().supports(IndexMode.INDEX_MODE_TSDB_ADDED)); + } if (indexMode == IndexMode.COLUMNAR || indexMode == IndexMode.LOGSDB_COLUMNAR) { // Gate on the cluster capability rather than the test runner build: columnar index modes are only available when the // feature flag is enabled on the nodes (e.g. they are disabled in upgrade clusters), in which case these tests skip. diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 21a8a5222a200..9c0282b519fd5 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -399,7 +399,7 @@ public Query rangeQuery( @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - if (indexMode.isTsdb() && metricType == TimeSeriesParams.MetricType.COUNTER) { + if (IndexMode.isTsdb(indexMode) && metricType == TimeSeriesParams.MetricType.COUNTER) { // Counters are not supported by ESQL so we load them in null return ConstantNull.INSTANCE; } @@ -508,7 +508,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext failIfNoDocValues(); } - ValuesSourceType valuesSourceType = indexMode.isTsdb() && metricType == TimeSeriesParams.MetricType.COUNTER + ValuesSourceType valuesSourceType = IndexMode.isTsdb(indexMode) && metricType == TimeSeriesParams.MetricType.COUNTER ? TimeSeriesValuesSourceType.COUNTER : IndexNumericFieldData.NumericType.LONG.getValuesSourceType(); From 57bf03298efb90625cc3dc0c9bdd9b2f1dde27ca Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Sun, 5 Jul 2026 11:45:09 +0300 Subject: [PATCH 05/23] Update 152901.yaml --- docs/changelog/152901.yaml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/changelog/152901.yaml b/docs/changelog/152901.yaml index 01cbd0850da2a..93cd5657f2529 100644 --- a/docs/changelog/152901.yaml +++ b/docs/changelog/152901.yaml @@ -3,12 +3,16 @@ highlight: body: |- ## Summary - Add `IndexMode.TSDB` as a new, first-class enum value that behaves - identically to `TIME_SERIES` (delegates every overridden method), so - `index.mode: tsdb` is now accepted anywhere `index.mode: time_series` is - — without renaming or changing anything about existing `time_series` - indices, which keep emitting/persisting exactly as before (`time_series` - remains the canonical output string; `tsdb` is a preferred alias). + Introduce `IndexMode.TSDB` as a new, first-class index mode for metrics. + It currently behaves identically to `TIME_SERIES`, so `index.mode: tsdb` + is now accepted anywhere `index.mode: time_series` is — without renaming + or changing anything about existing `time_series` indices. + + The new mode will be automatically selected and applied to new and + existing OTel and Prometheus data streams, with newer backing indices + using the `TSDB` mode. The querying experience and downsampling support + remains seamless, regardless of the mix of `TSDB` and `TIME_SERIES` + indices in a data stream. notable: true title: Add `IndexMode.TSDB` as a preferred alias for TIME_SERIES From 295243c3e7c8c20de84cf9c062a2948e56b12972 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Sun, 5 Jul 2026 12:15:46 +0300 Subject: [PATCH 06/23] Add TSDB alias unit test coverage (WIP checkpoint) Extend unit tests to assert IndexMode.TSDB behaves identically to IndexMode.TIME_SERIES wherever production code now uses isTsdb(). --- .../DataStreamIndexSettingsProviderTests.java | 293 +++++++++++++++++- ...astTimeSeriesIndexCreationActionTests.java | 97 +++++- .../TransportGetDataStreamsActionTests.java | 67 ++++ .../DataStreamLifecycleServiceTests.java | 47 +++ .../extras/ScaledFloatFieldMapperTests.java | 18 ++ .../cluster/metadata/DataStreamTests.java | 54 ++++ .../routing/DimensionsExtractorTests.java | 32 +- .../cluster/routing/IndexRoutingTests.java | 70 ++++- .../routing/RoutingPathExtractorTests.java | 34 +- ...SeriesEligibleWriteWindowLocatorTests.java | 20 +- .../index/IndexSettingsTests.java | 85 ++++- .../index/IndexSortSettingsTests.java | 28 ++ .../index/TimeSeriesModeTests.java | 57 +++- .../index/engine/InternalEngineTests.java | 12 +- .../mapper/GeoPointFieldMapperTests.java | 17 + .../index/mapper/MappingLookupTests.java | 30 ++ ...meSeriesMetadataFieldBlockLoaderTests.java | 32 ++ ...TimeSeriesRoutingHashFieldMapperTests.java | 21 ++ .../index/mapper/NumberFieldMapperTests.java | 17 + .../xpack/core/ilm/DownsampleActionTests.java | 43 +++ ...UntilTimeSeriesEndTimePassesStepTests.java | 93 ++++++ .../TransportDownsampleActionTests.java | 68 +++- .../xpack/esql/session/EsqlSession.java | 3 +- .../xpack/esql/analysis/AnalyzerTests.java | 63 ++++ .../datasources/DatasetRewriterTests.java | 6 +- .../PushExpressionsToFieldLoadTests.java | 37 +++ .../local/SubstituteRoundToTests.java | 108 +++++++ .../xpack/esql/session/EsqlSessionTests.java | 70 +++++ .../LogsdbIndexModeSettingsProviderTests.java | 97 ++++++ ...dexSettingsProviderLegacyLicenseTests.java | 70 +++++ .../UnsignedLongFieldMapperTests.java | 17 + 31 files changed, 1692 insertions(+), 14 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index a1f048fe1bbec..6a56e6364d156 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -158,6 +158,82 @@ public void testGetAdditionalIndexSettings() throws Exception { } } + /** + * The {@code tsdb} alias delegates {@link IndexMode#isTsdb()} to {@code true} just like + * {@link IndexMode#TIME_SERIES}, so creating a backing index with a template index mode of + * {@link IndexMode#TSDB} must inject the same start/end time, sequence-number, synthetic id + * and dimension settings as {@link IndexMode#TIME_SERIES} does. + */ + public void testGetAdditionalIndexSettingsWithTsdbAlias() throws Exception { + ProjectMetadata projectMetadata = emptyProject(); + String dataStreamName = "logs-app1"; + + Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); + Settings settings = Settings.builder() + .put("index.dimensions_tsid_strategy_enabled", indexDimensionsTsidStrategyEnabledSetting) + .build(); + String mapping = """ + { + "_doc": { + "properties": { + "field1": { + "type": "long" + }, + "field2": { + "type": "keyword" + }, + "field3": { + "type": "keyword", + "time_series_dimension": true + }, + "field4": { + "type": "long", + "time_series_dimension": true + }, + "field5": { + "type": "ip", + "time_series_dimension": true + }, + "field6": { + "type": "boolean", + "time_series_dimension": true + } + } + } + } + """; + Settings.Builder additionalSettings = builder(); + provider.provideAdditionalSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 1), + dataStreamName, + IndexMode.TSDB, + projectMetadata, + now, + settings, + List.of(new CompressedXContent(mapping)), + indexVersion, + additionalSettings + ); + Settings result = additionalSettings.build(); + // The index.time_series.end_time setting requires index.mode to be set to tsdb adding it here so that we read this setting: + // (in production the index.mode setting is usually provided in an index or component template) + result = builder().put(result).put("index.mode", "tsdb").build(); + assertThat(result.size(), equalTo(maybeAdjustIndexSettingCount(4))); + assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TSDB)); + assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); + assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); + if (expectedIndexDimensionsTsidOptimizationEnabled) { + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field3", "field4", "field5", "field6")); + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), empty()); + } else { + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field3", "field4", "field5", "field6")); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty()); + } + if (expectedDisabledSequenceNumbers) { + assertThat(IndexSettings.DISABLE_SEQUENCE_NUMBERS.get(result), equalTo(true)); + } + } + public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throws Exception { ProjectMetadata projectMetadata = emptyProject(); String dataStreamName = "logs-app1"; @@ -526,6 +602,48 @@ public void testGetAdditionalIndexSettingsMigrateToTsdb() { } } + /** + * The migration check at index-creation time relies on {@link IndexMode#isTsdb(IndexMode)}, so + * a data stream currently in {@code standard} mode must also migrate into time series mode when + * the matching index template specifies the {@link IndexMode#TSDB} alias, not just + * {@link IndexMode#TIME_SERIES}. + */ + public void testGetAdditionalIndexSettingsMigrateToTsdbAlias() { + Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); + String dataStreamName = "logs-app1"; + IndexMetadata idx = createFirstBackingIndex(dataStreamName).build(); + DataStream existingDataStream = newInstance(dataStreamName, List.of(idx.getIndex())); + ProjectMetadata projectMetadata = ProjectMetadata.builder(randomProjectIdOrDefault()) + .dataStreams(Map.of(dataStreamName, existingDataStream), Map.of()) + .build(); + + Settings settings = Settings.EMPTY; + Settings.Builder additionalSettings = builder(); + provider.provideAdditionalSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 2), + dataStreamName, + IndexMode.TSDB, + projectMetadata, + now, + settings, + List.of(), + indexVersion, + additionalSettings + ); + Settings result = additionalSettings.build(); + // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this + // setting: + // (in production the index.mode setting is usually provided in an index or component template) + result = builder().put(result).put("index.mode", "time_series").build(); + assertThat(result.size(), equalTo(maybeAdjustIndexSettingCount(3))); + assertThat(result.get(IndexSettings.MODE.getKey()), equalTo("time_series")); + assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); + assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); + if (expectedDisabledSequenceNumbers) { + assertThat(IndexSettings.DISABLE_SEQUENCE_NUMBERS.get(result), equalTo(true)); + } + } + public void testGetAdditionalIndexSettingsDowngradeFromTsdb() { String dataStreamName = "logs-app1"; Instant twoHoursAgo = Instant.now().minus(4, ChronoUnit.HOURS).truncatedTo(ChronoUnit.MILLIS); @@ -950,6 +1068,34 @@ public void testAddNewDimension() throws Exception { assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field2")); } + /** + * Same scenario as {@link #testAddNewDimension()}, but with an index in {@link IndexMode#TSDB} + * (the {@code tsdb} alias) rather than {@link IndexMode#TIME_SERIES}, so that the + * {@code assert IndexMode.isTsdb(...)} sanity check in + * {@link DataStreamIndexSettingsProvider#onUpdateMappings} is exercised with the alias too. + */ + public void testAddNewDimensionWithTsdbAlias() throws Exception { + String newMapping = """ + { + "_doc": { + "properties": { + "field1": { + "type": "keyword", + "time_series_dimension": true + }, + "field2": { + "type": "keyword", + "time_series_dimension": true + } + } + } + } + """; + Settings result = onUpdateMappings("field1", "field1", newMapping, IndexMode.TSDB); + assertThat(result.size(), equalTo(1)); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field2")); + } + public void testAddNewDimensionIndexDimensionsUnset() throws Exception { String newMapping = """ { @@ -1068,6 +1214,15 @@ private Settings generateTsdbSettings(String mapping, Instant now) throws IOExce } private Settings onUpdateMappings(String routingPath, String dimensions, String newMapping) throws IOException { + return onUpdateMappings(routingPath, dimensions, newMapping, IndexMode.TIME_SERIES); + } + + /** + * Same as {@link #onUpdateMappings(String, String, String)} but with an explicit {@code indexMode}, + * so that tests can assert {@link IndexMode#TSDB} satisfies the {@code assert IndexMode.isTsdb(...)} + * sanity check in {@link DataStreamIndexSettingsProvider#onUpdateMappings}. + */ + private Settings onUpdateMappings(String routingPath, String dimensions, String newMapping, IndexMode indexMode) throws IOException { String dataStreamName = "logs-app1"; Settings.Builder currentSettings = Settings.builder() .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) @@ -1076,7 +1231,7 @@ private Settings onUpdateMappings(String routingPath, String dimensions, String .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES); + .put(IndexSettings.MODE.getKey(), indexMode); IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1)) .settings(currentSettings) @@ -1164,6 +1319,74 @@ public void testClusterSettingsDefineSeqNoDisabledDefault() throws Exception { assertFalse(additionalSettings.build().hasValue(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey())); } + /** + * Same scenario as {@link #testClusterSettingsDefineSeqNoDisabledDefault()}, but with a + * template index mode of {@link IndexMode#TSDB} (the {@code tsdb} alias) rather than + * {@link IndexMode#TIME_SERIES}, since the sequence-number-disabling gate is keyed off + * {@link IndexMode#isTsdb()}. + */ + public void testClusterSettingsDefineSeqNoDisabledDefaultWithTsdbAlias() throws Exception { + Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); + IndexVersion version = IndexVersionUtils.randomVersionBetween( + IndexVersions.TIME_SERIES_DISABLE_SEQUENCE_NUMBERS_DEFAULT, + IndexVersion.current() + ); + String dataStreamName = "metrics-app1"; + + // With seq_no_disabled=false, the provider must NOT set index.disable_sequence_numbers + DataStreamIndexSettingsProvider providerWithSeqNoEnabled = new DataStreamIndexSettingsProvider( + im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), + Settings.builder().put(DataStreamIndexSettingsProvider.SUPPORT_SEQ_NO_DISABLED.getKey(), false).build() + ); + Settings.Builder additionalSettings = builder(); + providerWithSeqNoEnabled.provideAdditionalSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 1), + dataStreamName, + IndexMode.TSDB, + emptyProject(), + now, + Settings.EMPTY, + List.of(), + version, + additionalSettings + ); + assertFalse(additionalSettings.build().hasValue(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey())); + + // With seq_no_disabled=true (default), the provider must set index.disable_sequence_numbers=true + DataStreamIndexSettingsProvider providerWithSeqNoDisabled = new DataStreamIndexSettingsProvider( + im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), + Settings.builder().put(DataStreamIndexSettingsProvider.SUPPORT_SEQ_NO_DISABLED.getKey(), true).build() + ); + additionalSettings = builder(); + providerWithSeqNoDisabled.provideAdditionalSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 1), + dataStreamName, + IndexMode.TSDB, + emptyProject(), + now, + Settings.EMPTY, + List.of(), + version, + additionalSettings + ); + assertThat(IndexSettings.DISABLE_SEQUENCE_NUMBERS.get(additionalSettings.build()), equalTo(true)); + + // When index.disable_sequence_numbers is explicitly set in the template, the cluster setting must not override it + additionalSettings = builder(); + providerWithSeqNoDisabled.provideAdditionalSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 1), + dataStreamName, + IndexMode.TSDB, + emptyProject(), + now, + Settings.builder().put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), false).build(), + List.of(), + version, + additionalSettings + ); + assertFalse(additionalSettings.build().hasValue(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey())); + } + public void testClusterSettingsDefineSyntheticIdEnabledDefault() throws Exception { Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); IndexVersion version = IndexVersionUtils.randomVersionBetween( @@ -1226,4 +1449,72 @@ public void testClusterSettingsDefineSyntheticIdEnabledDefault() throws Exceptio assertFalse(additionalSettings.build().hasValue(IndexSettings.SYNTHETIC_ID.getKey())); } + /** + * Same scenario as {@link #testClusterSettingsDefineSyntheticIdEnabledDefault()}, but with a + * template index mode of {@link IndexMode#TSDB} (the {@code tsdb} alias) rather than + * {@link IndexMode#TIME_SERIES}, since the synthetic {@code _id} gate is keyed off + * {@link IndexMode#isTsdb()}. + */ + public void testClusterSettingsDefineSyntheticIdEnabledDefaultWithTsdbAlias() throws Exception { + Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); + IndexVersion version = IndexVersionUtils.randomVersionBetween( + IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD, + IndexVersion.current() + ); + String dataStreamName = "metrics-app1"; + + // With synthetic_id_enabled=false, the provider must set index.mapping.synthetic_id=false + DataStreamIndexSettingsProvider providerWithSyntheticIdDisabled = new DataStreamIndexSettingsProvider( + im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), + Settings.builder().put(DataStreamIndexSettingsProvider.SUPPORT_SYNTHETIC_ID.getKey(), false).build() + ); + Settings.Builder additionalSettings = builder(); + providerWithSyntheticIdDisabled.provideAdditionalSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 1), + dataStreamName, + IndexMode.TSDB, + emptyProject(), + now, + Settings.EMPTY, + List.of(), + version, + additionalSettings + ); + assertThat(additionalSettings.build().getAsBoolean(IndexSettings.SYNTHETIC_ID.getKey(), true), equalTo(false)); + + // With synthetic_id_enabled=true (default), the provider must set index.mapping.synthetic_id=true + DataStreamIndexSettingsProvider providerWithSyntheticIdEnabled = new DataStreamIndexSettingsProvider( + im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), + Settings.builder().put(DataStreamIndexSettingsProvider.SUPPORT_SYNTHETIC_ID.getKey(), true).build() + ); + additionalSettings = builder(); + providerWithSyntheticIdEnabled.provideAdditionalSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 1), + dataStreamName, + IndexMode.TSDB, + emptyProject(), + now, + Settings.EMPTY, + List.of(), + version, + additionalSettings + ); + assertThat(additionalSettings.build().getAsBoolean(IndexSettings.SYNTHETIC_ID.getKey(), false), equalTo(true)); + + // When index.mapping.synthetic_id is explicitly set in the template, the cluster setting must not override it + additionalSettings = builder(); + providerWithSyntheticIdEnabled.provideAdditionalSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 1), + dataStreamName, + IndexMode.TSDB, + emptyProject(), + now, + Settings.builder().put(IndexSettings.SYNTHETIC_ID.getKey(), false).build(), + List.of(), + version, + additionalSettings + ); + assertFalse(additionalSettings.build().hasValue(IndexSettings.SYNTHETIC_ID.getKey())); + } + } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java index cac3ecdba85db..dc73b65d90ce2 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java @@ -462,15 +462,110 @@ private ClusterState stateWithNoTsdbIndices() { } private static IndexMetadata createIndexMetadata(String indexName, Instant startTime, Instant endTime) { + return createIndexMetadata(indexName, startTime, endTime, IndexMode.TIME_SERIES); + } + + private static IndexMetadata createIndexMetadata(String indexName, Instant startTime, Instant endTime, IndexMode indexMode) { Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()); if (startTime != null && endTime != null) { - settings.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) + settings.put(IndexSettings.MODE.getKey(), indexMode.getName()) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime.toEpochMilli()) .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime.toEpochMilli()); } return IndexMetadata.builder(indexName).settings(settings).numberOfShards(1).numberOfReplicas(0).build(); } + /** + * Builds a ProjectMetadata with a TSDB data stream whose backing indices use the {@link IndexMode#TSDB} + * alias rather than {@link IndexMode#TIME_SERIES}, to verify that {@code IndexMode.isTsdb} treats the + * alias identically to {@code TIME_SERIES} in {@link TransportPastTimeSeriesIndexCreationAction}. + */ + private ProjectMetadata projectWithTsdbAliasDataStream(List> timeSlices) { + List backingIndices = new ArrayList<>(); + long generation = 1L; + for (Tuple slice : timeSlices) { + String indexName = DataStream.getDefaultBackingIndexName(DATA_STREAM, generation, slice.v1().toEpochMilli()); + backingIndices.add(createIndexMetadata(indexName, slice.v1(), slice.v2(), IndexMode.TSDB)); + generation++; + } + DataStream ds = DataStream.builder(DATA_STREAM, backingIndices.stream().map(IndexMetadata::getIndex).toList()) + .setGeneration(generation) + .setIndexMode(IndexMode.TSDB) + .build(); + ProjectMetadata.Builder builder = ProjectMetadata.builder(projectId); + for (IndexMetadata im : backingIndices) { + builder.put(im, false); + } + return builder.put(ds).build(); + } + + /** Builds a ClusterState with a {@link IndexMode#TSDB}-alias data stream whose backing indices cover the given time ranges. */ + private ClusterState stateWithExistingTsdbAlias(List> timeSlices, Instant now) { + List> allSlices = new ArrayList<>(timeSlices); + allSlices.add(Tuple.tuple(now, now.plus(randomIntBetween(1, 3), ChronoUnit.DAYS))); + return ClusterState.builder(ClusterName.DEFAULT).putProjectMetadata(projectWithTsdbAliasDataStream(allSlices)).build(); + } + + /** + * Equivalent of {@link #testSortAndRetrieve} that uses the {@link IndexMode#TSDB} alias instead of + * {@link IndexMode#TIME_SERIES} for the backing indices, to confirm {@code retrieveSortedTimeWindows} + * (gated by {@code IndexMode.isTsdb}) recognizes the alias mode the same way. + */ + public void testSortAndRetrieveWithTsdbAlias() { + Instant start1 = Instant.parse("2024-01-15T00:00:00Z"); + Instant start2 = Instant.parse("2024-01-16T00:00:00Z"); + Instant start3 = Instant.parse("2024-01-17T00:00:00Z"); + Instant end = Instant.parse("2024-01-18T00:00:00Z"); + ProjectMetadata project = projectWithTsdbAliasDataStream( + List.of(Tuple.tuple(start3, end), Tuple.tuple(start1, start2), Tuple.tuple(start2, start3)) + ); + // Add a non-TSDB index to the mixed data stream to confirm it's still excluded from the windows. + String nonTsdbName = randomIndexName(); + IndexMetadata nonTsdb = IndexMetadata.builder(nonTsdbName) + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + DataStream mixedDs = project.dataStreams().get(DATA_STREAM).unsafeAddBackingIndex(nonTsdb.getIndex()); + project = ProjectMetadata.builder(project).put(nonTsdb, false).put(mixedDs).build(); + + var result = PastTimeSeriesIndexCreationExecutor.retrieveSortedTimeWindows(project.dataStreams().get(DATA_STREAM), project); + assertThat(result, hasSize(1)); + TransportPastTimeSeriesIndexCreationAction.CoveredTimeWindow timeWindow = result.pop(); + assertThat(timeWindow.start(), is(start1.toEpochMilli())); + assertThat(timeWindow.end(), is(end.toEpochMilli())); + } + + /** + * Equivalent of the happy-path portion of {@link #testCreateIndicesWhenNeeded} that uses the + * {@link IndexMode#TSDB} alias instead of {@link IndexMode#TIME_SERIES}, to confirm + * {@code validateDataStream} and the time-range computation (both gated by {@code IndexMode.isTsdb}) + * behave identically for the alias. + */ + public void testCreateIndicesWhenNeededWithTsdbAlias() throws Exception { + Instant now = Instant.now(); + ClusterState clusterState = stateWithExistingTsdbAlias(List.of(), now); + List dayOffsets = List.of(5, 3, 2); + // Add two timestamps that fall within one index + { + Instant ts1 = getTimestampWithinDay(now, dayOffsets.getFirst(), randomIntBetween(2, 5)); + Instant ts2 = getTimestampWithinDay(now, dayOffsets.getFirst(), randomIntBetween(7, 12)); + TaskResult result = run(clusterState, ts1.toEpochMilli(), ts2.toEpochMilli()); + assertThat(result.covered, containsInAnyOrder(ts1, ts2)); + assertThat(result.createdNames.size(), is(1)); + clusterState = result.state(); + } + + // Add two timestamp that will create two indices + { + Instant ts1 = getTimestampWithinDay(now, dayOffsets.get(1), randomIntBetween(1, 5)); + Instant ts2 = getTimestampWithinDay(now, dayOffsets.get(2), randomIntBetween(13, 18)); + TaskResult result = run(clusterState, ts1.toEpochMilli(), ts2.toEpochMilli()); + assertThat(result.covered, containsInAnyOrder(ts1, ts2)); + assertThat(result.createdNames.size(), is(2)); + } + } + public void testOutsideEligibleWriteWindowFails() throws Exception { Instant now = Instant.now(); ClusterState clusterState = stateWithExisting(List.of(), now); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index 5b63d470d755f..c0a4dfe7cdec9 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.TestIndexNameExpressionResolver; @@ -322,6 +323,72 @@ public void testGetTimeSeriesDataStreamWithOutOfOrderIndices() { ); } + public void testGetTimeSeriesDataStreamUsingTsdbAlias() { + // Same scenario as testGetTimeSeriesDataStreamWithOutOfOrderIndices, but using the "tsdb" alias for + // index.mode instead of "time_series". DataStreamTestHelper#getClusterStateWithDataStream hardcodes + // IndexMode.TIME_SERIES, so the data stream and backing indices are built by hand here to exercise + // IndexMode.isTsdb(...) with IndexMode.TSDB at both call sites in + // TransportGetDataStreamsAction#innerOperation. + Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); + String dataStreamName = "ds-1"; + Instant sixHoursAgo = now.minus(6, ChronoUnit.HOURS); + Instant fourHoursAgo = now.minus(4, ChronoUnit.HOURS); + Instant twoHoursAgo = now.minus(2, ChronoUnit.HOURS); + Instant twoHoursAhead = now.plus(2, ChronoUnit.HOURS); + + List> timeSlices = List.of( + new Tuple<>(fourHoursAgo, twoHoursAgo), + new Tuple<>(sixHoursAgo, fourHoursAgo), + new Tuple<>(twoHoursAgo, twoHoursAhead) + ); + + ProjectMetadata.Builder mBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()); + List backingIndices = new ArrayList<>(); + long generation = 1L; + for (Tuple tuple : timeSlices) { + Instant start = tuple.v1(); + Instant end = tuple.v2(); + Settings settings = Settings.builder() + .put("index.mode", "tsdb") + .put("index.routing_path", "uid") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(start)) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(end)) + .build(); + var im = createIndexMetadata(getDefaultBackingIndexName(dataStreamName, generation, start.toEpochMilli()), true, settings, 0); + mBuilder.put(im, true); + backingIndices.add(im); + generation++; + } + DataStream dataStream = DataStream.builder( + dataStreamName, + backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()) + ).setGeneration(generation).setIndexMode(IndexMode.TSDB).build(); + mBuilder.put(dataStream); + + var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); + var response = TransportGetDataStreamsAction.innerOperation( + projectStateFromProject(mBuilder), + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings(), + dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), + null, + metadataDataStreamsService + ); + assertThat( + response.getDataStreams(), + contains( + allOf( + transformedMatch(d -> d.getDataStream().getName(), equalTo(dataStreamName)), + transformedMatch(d -> d.getTimeSeries().temporalRanges(), contains(new Tuple<>(sixHoursAgo, twoHoursAhead))) + ) + ) + ); + } + public void testGetTimeSeriesMixedDataStream() { Instant instant = Instant.parse("2023-06-06T14:00:00.000Z").truncatedTo(ChronoUnit.SECONDS); String dataStream1 = "ds-1"; diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index 0f0c803ae08eb..e0e19644407de 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -43,6 +43,8 @@ import org.elasticsearch.datastreams.lifecycle.health.DataStreamLifecycleHealthInfoPublisher; import org.elasticsearch.dlm.DataStreamLifecycleErrorStore; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.MergePolicyConfig; import org.elasticsearch.test.EqualsHashCodeTestUtils; @@ -1236,6 +1238,51 @@ public void testTimeSeriesIndicesStillWithinTimeBounds() { } } + /** + * Same coverage as {@link #testTimeSeriesIndicesStillWithinTimeBounds()} but using the {@link IndexMode#TSDB} alias + * instead of {@link IndexMode#TIME_SERIES}, to ensure {@code isTsdb()} gates this behaviour identically for both. + */ + public void testTimeSeriesIndicesStillWithinTimeBoundsWithTsdbIndexMode() { + Instant currentTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); + // These ranges are on the edge of each other's temporal boundaries. + Instant start1 = currentTime.minus(6, ChronoUnit.HOURS); + Instant end1 = currentTime.minus(4, ChronoUnit.HOURS); + Instant start2 = currentTime.minus(4, ChronoUnit.HOURS); + Instant end2 = currentTime.plus(2, ChronoUnit.HOURS); + + IndexMetadata outOfBoundsIndex = IndexMetadata.builder(randomAlphaOfLengthBetween(10, 30)) + .settings( + indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.TSDB) + .put("index.routing_path", "uid") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), start1.toEpochMilli()) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), end1.toEpochMilli()) + ) + .build(); + IndexMetadata withinBoundsIndex = IndexMetadata.builder(randomAlphaOfLengthBetween(10, 30)) + .settings( + indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.TSDB) + .put("index.routing_path", "uid") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), start2.toEpochMilli()) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), end2.toEpochMilli()) + ) + .build(); + + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put(outOfBoundsIndex, true) + .put(withinBoundsIndex, true) + .build(); + + Set indices = DataStreamLifecycleService.timeSeriesIndicesStillWithinTimeBounds( + project, + List.of(outOfBoundsIndex.getIndex(), withinBoundsIndex.getIndex()), + currentTime::toEpochMilli + ); + + assertThat(indices, containsInAnyOrder(withinBoundsIndex.getIndex())); + } + public void testTrackingTimeStats() { AtomicLong now = new AtomicLong(0); long delta = randomLongBetween(10, 10000); diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java index b5ca68e6c6698..743e726464acd 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java @@ -357,6 +357,24 @@ public void testTimeSeriesIndexDefault() throws Exception { assertFalse(ft.indexType().hasDenseIndex()); } + /** + * The {@code tsdb} index mode is an alias for {@code time_series} (see {@link IndexMode#isTsdb()}) + * so it must produce the same doc-values-only defaulting for metric fields. + */ + public void testTimeSeriesIndexDefaultTsdbAlias() throws Exception { + var randomMetricType = randomFrom(TimeSeriesParams.MetricType.scalar()); + var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); + var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_metric", randomMetricType.toString()); + })); + var ft = (ScaledFloatFieldMapper.ScaledFloatFieldType) mapperService.fieldType("field"); + assertThat(ft.getMetricType(), equalTo(randomMetricType)); + assertTrue(ft.hasDocValues()); + assertFalse(ft.indexType().hasDenseIndex()); + } + @Override protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { // Large floats are a terrible idea but the round trip should still work no matter how badly you configure the field diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 76d342ff1c69e..317fecd595a0d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -274,6 +274,24 @@ public void testRolloverUpgradeToTsdbDataStream() { assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TIME_SERIES)); } + public void testRolloverUpgradeToTsdbAliasDataStream() { + DataStream ds = DataStreamTestHelper.randomInstance() + .copy() + .setReplicated(false) + .setIndexMode(randomBoolean() ? IndexMode.STANDARD : null) + .build(); + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + var newCoordinates = ds.nextWriteIndexAndGeneration(project, ds.getDataComponent()); + + var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), IndexMode.TSDB, null); + assertThat(rolledDs.getName(), equalTo(ds.getName())); + assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); + assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1)); + assertTrue(rolledDs.getIndices().containsAll(ds.getIndices())); + assertTrue(rolledDs.getIndices().contains(rolledDs.getWriteIndex())); + assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TSDB)); + } + public void testRolloverUpgradeToLogsdbDataStream() { DataStream ds = DataStreamTestHelper.randomInstance() .copy() @@ -325,6 +343,20 @@ public void testRolloverFromLogsdbToTsdb() { assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TIME_SERIES)); } + public void testRolloverFromLogsdbToTsdbAlias() { + DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.LOGSDB).build(); + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + var newCoordinates = ds.nextWriteIndexAndGeneration(project, ds.getDataComponent()); + + var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), IndexMode.TSDB, null); + assertThat(rolledDs.getName(), equalTo(ds.getName())); + assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); + assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1)); + assertTrue(rolledDs.getIndices().containsAll(ds.getIndices())); + assertTrue(rolledDs.getIndices().contains(rolledDs.getWriteIndex())); + assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TSDB)); + } + public void testRolloverDowngradeFromTsdbToRegularDataStream() { DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.TIME_SERIES).build(); final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); @@ -460,6 +492,17 @@ public void testRolloverFromColumnarToTimeSeries() { assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TIME_SERIES)); } + public void testRolloverFromColumnarToTsdbAlias() { + DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.COLUMNAR).build(); + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + var newCoordinates = ds.nextWriteIndexAndGeneration(project, ds.getDataComponent()); + + var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), IndexMode.TSDB, null); + assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); + assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1)); + assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TSDB)); + } + public void testRolloverDowngradeFromTimeSeriesDataStreamToColumnar() { DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.TIME_SERIES).build(); final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); @@ -503,6 +546,17 @@ public void testRolloverFromColumnarLogsdbToTimeSeries() { assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TIME_SERIES)); } + public void testRolloverFromColumnarLogsdbToTsdbAlias() { + DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.LOGSDB_COLUMNAR).build(); + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + var newCoordinates = ds.nextWriteIndexAndGeneration(project, ds.getDataComponent()); + + var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), IndexMode.TSDB, null); + assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); + assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1)); + assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TSDB)); + } + public void testRolloverFromLogsdbToColumnarLogsdb() { DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.LOGSDB).build(); final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/DimensionsExtractorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/DimensionsExtractorTests.java index 3dc7f627a6dae..9b8765e10e515 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/DimensionsExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/DimensionsExtractorTests.java @@ -144,6 +144,32 @@ public void testTsidIsAttachedToIndexRequest() throws IOException { assertThat("tsid should be attached to the IndexRequest after extraction", req.tsid(), notNullValue()); } + /** + * {@code index.mode: tsdb} is a preferred alias for {@code time_series}; it must select the same + * {@link IndexRouting.ExtractFromSource.ForIndexDimensions} strategy and produce byte-identical tsids + * and shard assignments. + */ + public void testTsdbAliasProducesSameTsidAsTimeSeries() throws IOException { + Map doc = new LinkedHashMap<>(); + doc.put("dim.host", "node-7"); + doc.put("dim.region", "us-west-2"); + doc.put("metric", "cpu.user"); + BytesReference json = toJson(doc); + + IndexRouting.ExtractFromSource.ForIndexDimensions timeSeriesStrategy = forIndexDimensions("dim.*", IndexMode.TIME_SERIES); + IndexRequest timeSeriesReq = new IndexRequest("test").id("d1").source(json, XContentType.JSON); + timeSeriesStrategy.preProcess(timeSeriesReq); + int timeSeriesShardId = timeSeriesStrategy.indexShard(timeSeriesReq); + + IndexRouting.ExtractFromSource.ForIndexDimensions tsdbStrategy = forIndexDimensions("dim.*", IndexMode.TSDB); + IndexRequest tsdbReq = new IndexRequest("test").id("d1").source(json, XContentType.JSON); + tsdbStrategy.preProcess(tsdbReq); + int tsdbShardId = tsdbStrategy.indexShard(tsdbReq); + + assertThat("tsid must be identical for time_series and tsdb aliases", tsdbReq.tsid(), equalTo(timeSeriesReq.tsid())); + assertThat("shard id must be identical for time_series and tsdb aliases", tsdbShardId, equalTo(timeSeriesShardId)); + } + public void testResetClearsBuilderBetweenDocuments() throws IOException { IndexRouting.ExtractFromSource.ForIndexDimensions strategyA = forIndexDimensions("dim.*"); IndexRouting.ExtractFromSource.ForIndexDimensions strategyB = forIndexDimensions("dim.*"); @@ -200,10 +226,14 @@ private void assertExtractorMatchesSourceParser(Map doc, String } private static IndexRouting.ExtractFromSource.ForIndexDimensions forIndexDimensions(String dimensionPath) { + return forIndexDimensions(dimensionPath, IndexMode.TIME_SERIES); + } + + private static IndexRouting.ExtractFromSource.ForIndexDimensions forIndexDimensions(String dimensionPath, IndexMode indexMode) { Settings settings = Settings.builder() .put(SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) .put(IndexMetadata.INDEX_DIMENSIONS.getKey(), dimensionPath) - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(IndexSettings.MODE.getKey(), indexMode) .build(); IndexMetadata md = IndexMetadata.builder("test").settings(settings).numberOfShards(8).numberOfReplicas(0).build(); IndexRouting routing = IndexRouting.fromIndexMetadata(md); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index def3b94695712..38199ba65b84b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -752,6 +752,34 @@ public void testRoutingPathWithSingleBytePrefixTsid() throws IOException { assertIndexShard(fixture, Map.of("dim.a", "true"), 1); } + /** + * Same scenario as {@link #testRoutingPathWithSingleBytePrefixTsid()} but with {@code index.mode: tsdb}, + * the preferred alias for {@code time_series}. {@link IndexMode#TSDB} must select the same + * {@link IndexRouting.ExtractFromSource.ForIndexDimensions} strategy, track the routing hash the same + * way, and produce identical shard assignments as {@link IndexMode#TIME_SERIES}. + */ + public void testRoutingPathWithSingleBytePrefixTsidUsingTsdbAlias() throws IOException { + TimeSeriesRoutingFixture fixture = indexRoutingForTimeSeriesDimensions( + IndexVersionUtils.randomVersionOnOrAfter(IndexVersions.TSID_SINGLE_PREFIX_BYTE_FEATURE_FLAG), + 8, + "dim.*,other.*,top", + randomBoolean(), + IndexMode.TSDB + ); + assumeTrue("require single-byte-prefix tsid", TsidBuilder.useSingleBytePrefixLayout(fixture.routing.creationVersion)); + assertIndexShard(fixture, Map.of("dim", Map.of("a", "a")), 5); + assertIndexShard(fixture, Map.of("dim", Map.of("a", "b")), 3); + assertIndexShard(fixture, Map.of("dim", Map.of("c", "d")), 7); + assertIndexShard(fixture, Map.of("other", Map.of("a", "a")), 1); + assertIndexShard(fixture, Map.of("top", "a"), 6); + assertIndexShard(fixture, Map.of("dim", Map.of("c", "d"), "top", "b"), 0); + assertIndexShard(fixture, Map.of("dim.a", "a"), 5); + assertIndexShard(fixture, Map.of("dim.a", 1), 2); + assertIndexShard(fixture, Map.of("dim.a", "1"), 0); + assertIndexShard(fixture, Map.of("dim.a", true), 3); + assertIndexShard(fixture, Map.of("dim.a", "true"), 1); + } + public void testRoutingPathReadWithInvalidString() { int shards = between(2, 1000); IndexRouting indexRouting = indexRoutingForPath(shards, "foo").routing(); @@ -1243,7 +1271,29 @@ private TimeSeriesRoutingFixture indexRoutingForTimeSeriesDimensions( String path, boolean useSyntheticId ) { - return getIndexRoutingWithSetting(createdVersion, shards, path, IndexMetadata.INDEX_DIMENSIONS.getKey(), useSyntheticId); + return indexRoutingForTimeSeriesDimensions(createdVersion, shards, path, useSyntheticId, IndexMode.TIME_SERIES); + } + + /** + * Same as {@link #indexRoutingForTimeSeriesDimensions(IndexVersion, int, String, boolean)} but lets the + * caller pick the index mode, so callers can prove {@link IndexMode#TSDB} routes identically to + * {@link IndexMode#TIME_SERIES}. + */ + private TimeSeriesRoutingFixture indexRoutingForTimeSeriesDimensions( + IndexVersion createdVersion, + int shards, + String path, + boolean useSyntheticId, + IndexMode indexMode + ) { + return getIndexRoutingWithSetting( + createdVersion, + shards, + path, + IndexMetadata.INDEX_DIMENSIONS.getKey(), + useSyntheticId, + indexMode + ); } /** @@ -1255,12 +1305,26 @@ private static TimeSeriesRoutingFixture getIndexRoutingWithSetting( String path, String setting, boolean useSyntheticId + ) { + return getIndexRoutingWithSetting(indexVersion, shards, path, setting, useSyntheticId, IndexMode.TIME_SERIES); + } + + /** + * Same as {@link #getIndexRoutingWithSetting(IndexVersion, int, String, String, boolean)} but lets the + * caller pick the index mode. + */ + private static TimeSeriesRoutingFixture getIndexRoutingWithSetting( + IndexVersion indexVersion, + int shards, + String path, + String setting, + boolean useSyntheticId, + IndexMode indexMode ) { if (useSyntheticId) { assert indexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94) : "Can't use synthetic id with this index version"; } - Settings.Builder settingsBuilder = settings(indexVersion).put(setting, path) - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES); + Settings.Builder settingsBuilder = settings(indexVersion).put(setting, path).put(IndexSettings.MODE.getKey(), indexMode); if (indexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94)) { settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), useSyntheticId); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingPathExtractorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingPathExtractorTests.java index 95e50c92d2e88..8b2b98111ca68 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingPathExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingPathExtractorTests.java @@ -122,6 +122,34 @@ public void testNoThrowOnArrayAtNonRoutingField() throws IOException { } } + /** + * {@code index.mode: tsdb} is a preferred alias for {@code time_series}; it must flow through the same + * {@code trackTimeSeriesRoutingHash} logic in {@link IndexRouting.ExtractFromSource}, so the routing hash + * that {@link IndexRouting.ExtractFromSource#postProcess} attaches to the request must be identical + * regardless of which alias created the index. + */ + public void testTsdbAliasProducesSameRoutingHashAsTimeSeries() throws IOException { + Map doc = new LinkedHashMap<>(); + doc.put("dim.host", "node-7"); + doc.put("dim.region", "us-west-2"); + doc.put("metric", "cpu.user"); + BytesReference json = toJson(doc); + + IndexRouting.ExtractFromSource.ForRoutingPath timeSeriesStrategy = forRoutingPath("dim.*", IndexMode.TIME_SERIES); + IndexRequest timeSeriesReq = new IndexRequest("test").id("doc-1").source(json, XContentType.JSON); + timeSeriesStrategy.preProcess(timeSeriesReq); + timeSeriesStrategy.indexShard(timeSeriesReq); + timeSeriesStrategy.postProcess(timeSeriesReq); + + IndexRouting.ExtractFromSource.ForRoutingPath tsdbStrategy = forRoutingPath("dim.*", IndexMode.TSDB); + IndexRequest tsdbReq = new IndexRequest("test").id("doc-1").source(json, XContentType.JSON); + tsdbStrategy.preProcess(tsdbReq); + tsdbStrategy.indexShard(tsdbReq); + tsdbStrategy.postProcess(tsdbReq); + + assertThat("routing hash must be identical for time_series and tsdb aliases", tsdbReq.routing(), equalTo(timeSeriesReq.routing())); + } + public void testReuseAcrossDocumentsClearsBuilderState() throws IOException { // Same extractor, two different documents with different routing-path values should produce // the same shard ids as fresh extractors. @@ -182,10 +210,14 @@ private void assertExtractorMatchesSourceParser(Map doc, String } private static IndexRouting.ExtractFromSource.ForRoutingPath forRoutingPath(String routingPath) { + return forRoutingPath(routingPath, IndexMode.TIME_SERIES); + } + + private static IndexRouting.ExtractFromSource.ForRoutingPath forRoutingPath(String routingPath, IndexMode indexMode) { Settings settings = Settings.builder() .put(SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(IndexSettings.MODE.getKey(), indexMode) .build(); IndexMetadata md = IndexMetadata.builder("test").settings(settings).numberOfShards(8).numberOfReplicas(0).build(); IndexRouting routing = IndexRouting.fromIndexMetadata(md); diff --git a/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java b/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java index 4e608fb73e876..07b8d65644ea7 100644 --- a/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java +++ b/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java @@ -84,6 +84,20 @@ public void testWriteWindowDefinedByRetention() { } } + public void testWriteWindowDefinedByRetentionWithTsdbAlias() { + // IndexMode.TSDB is a preferred alias for IndexMode.TIME_SERIES, so a data stream configured + // with index.mode: tsdb must be treated identically by the eligible write window logic. + TimeValue retention = TimeValue.timeValueDays(30); + DataStreamLifecycle lifecycle = DataStreamLifecycle.dataLifecycleBuilder().dataRetention(retention).build(); + DataStream dataStream = dataStream("metrics-test", lifecycle, IndexMode.TSDB); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + long requestTimestamp = randomNonNegativeLong(); + assertThat( + DLM_ONLY.getEligibleWriteWindowStart(dataStream, project, null, requestTimestamp), + equalTo(requestTimestamp - retention.getMillis()) + ); + } + public void testWriteWindowDefinedByFrozenAfter() { TimeValue frozenAfter = TimeValue.timeValueDays(7); DataStreamLifecycle lifecycle = DataStreamLifecycle.dataLifecycleBuilder() @@ -136,7 +150,11 @@ public long getEligibleWriteWindowFromPolicy(String policy, ProjectMetadata proj } private static DataStream dataStream(String name, DataStreamLifecycle lifecycle) { + return dataStream(name, lifecycle, IndexMode.TIME_SERIES); + } + + private static DataStream dataStream(String name, DataStreamLifecycle lifecycle, IndexMode indexMode) { Index index = new Index(DataStream.getDefaultBackingIndexName(name, 1), randomAlphaOfLength(10)); - return DataStream.builder(name, List.of(index)).setLifecycle(lifecycle).setIndexMode(IndexMode.TIME_SERIES).build(); + return DataStream.builder(name, List.of(index)).setLifecycle(lifecycle).setIndexMode(indexMode).build(); } } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 796e496a87cf8..7831ac779ddd7 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -281,6 +281,27 @@ public void testSliceEnabledSettingRejectedForTimeSeriesMode() { assertThat(exception.getMessage(), containsString("time_series")); } + public void testSliceEnabledSettingRejectedForTsdbAlias() { + assumeTrue("slice indexing feature flag must be enabled", SliceIndexing.SLICE_FEATURE_FLAG.isEnabled()); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> new IndexSettings( + newIndexMeta( + "index", + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim") + .put(IndexSettings.SLICE_ENABLED.getKey(), true) + .build() + ), + Settings.EMPTY + ) + ); + assertThat(exception.getMessage(), containsString("index.slice.enabled")); + assertThat(exception.getMessage(), containsString("index.mode")); + assertThat(exception.getMessage(), containsString("tsdb")); + } + @TestLogging(reason = "testing warning logging", value = "org.elasticsearch.index.IndexSettings:WARN") public void testDenseVectorExperimentalFeaturesWarnsWhenExplicitlyEnabled() { MockLog.assertThatLogger(() -> { @@ -1016,6 +1037,29 @@ public void testSyntheticIdCorrectSettings() { assertTrue(indexMetadata.useTimeSeriesSyntheticId()); } + public void testSyntheticIdCorrectSettingsWithTsdbAlias() { + IndexVersion version = IndexVersionUtils.randomVersionBetween( + IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94, + IndexVersion.current() + ); + IndexMode mode = IndexMode.TSDB; + String codec = version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_BEST_COMPRESSION) + ? randomBoolean() ? CodecService.DEFAULT_CODEC : CodecService.BEST_COMPRESSION_CODEC + : CodecService.DEFAULT_CODEC; + + Settings settings = Settings.builder() + .put(IndexSettings.SYNTHETIC_ID.getKey(), true) + .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codec) + .put(IndexSettings.MODE.getKey(), mode) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some-routing") + .build(); + IndexMetadata indexMetadata = newIndexMeta("some-index", settings, version); + + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + assertTrue(indexSettings.useTimeSeriesSyntheticId()); + assertTrue(indexMetadata.useTimeSeriesSyntheticId()); + } + public void testSyntheticIdDefaultValueTrue() { IndexVersion version = IndexVersionUtils.randomVersionBetween( IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD, @@ -1038,6 +1082,28 @@ public void testSyntheticIdDefaultValueTrue() { assertTrue(indexMetadata.useTimeSeriesSyntheticId()); } + public void testSyntheticIdDefaultValueTrueWithTsdbAlias() { + IndexVersion version = IndexVersionUtils.randomVersionBetween( + IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD, + IndexVersion.current() + ); + IndexMode mode = IndexMode.TSDB; + String codec = version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_BEST_COMPRESSION) + ? randomBoolean() ? CodecService.DEFAULT_CODEC : CodecService.BEST_COMPRESSION_CODEC + : CodecService.DEFAULT_CODEC; + + Settings settings = Settings.builder() + .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codec) + .put(IndexSettings.MODE.getKey(), mode) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some-routing") + .build(); + IndexMetadata indexMetadata = newIndexMeta("some-index", settings, version); + + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + assertTrue(indexSettings.useTimeSeriesSyntheticId()); + assertTrue(indexMetadata.useTimeSeriesSyntheticId()); + } + public void testSyntheticIdDefaultValueFalse() { IndexVersion version = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD); IndexMode mode = IndexMode.TIME_SERIES; @@ -1055,6 +1121,23 @@ public void testSyntheticIdDefaultValueFalse() { assertFalse(indexMetadata.useTimeSeriesSyntheticId()); } + public void testSyntheticIdDefaultValueFalseWithTsdbAlias() { + IndexVersion version = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD); + IndexMode mode = IndexMode.TSDB; + String codec = CodecService.DEFAULT_CODEC; + + Settings settings = Settings.builder() + .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codec) + .put(IndexSettings.MODE.getKey(), mode) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some-routing") + .build(); + IndexMetadata indexMetadata = newIndexMeta("some-index", settings, version); + + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + assertFalse(indexSettings.useTimeSeriesSyntheticId()); + assertFalse(indexMetadata.useTimeSeriesSyntheticId()); + } + public void testSyntheticIdBadVersion() { IndexVersion badVersion = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94); IndexMode mode = IndexMode.TIME_SERIES; @@ -1205,7 +1288,7 @@ public void testDisableSequenceNumbersRequiresDocValuesOnly() { public void testDisableSequenceNumbersRequiresDocValuesOnlyForNonStandardModes() { IndexVersion indexVersion = IndexVersionUtils.randomVersionBetween(IndexVersions.DISABLE_SEQUENCE_NUMBERS, IndexVersion.current()); - List modes = new ArrayList<>(List.of(IndexMode.TIME_SERIES, IndexMode.LOGSDB)); + List modes = new ArrayList<>(List.of(IndexMode.TIME_SERIES, IndexMode.TSDB, IndexMode.LOGSDB)); if (IndexMode.COLUMNAR_FEATURE_FLAG.isEnabled()) { modes.addAll(List.of(IndexMode.LOGSDB_COLUMNAR, IndexMode.COLUMNAR)); } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index e381789f988d7..2c9d42b7376a2 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -375,6 +375,34 @@ public void testTimeSeriesModeNoTimestamp() { assertThat(e.getMessage(), equalTo("unknown index sort field:[@timestamp] required by [index.mode=time_series]")); } + public void testTimeSeriesModeWithTsdbAlias() { + IndexSettings indexSettings = indexSettings( + Settings.builder() + .put(IndexSettings.MODE.getKey(), "tsdb") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some_dimension") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") + .build() + ); + Sort sort = buildIndexSort(indexSettings, TimeSeriesIdFieldMapper.FIELD_TYPE, new DateFieldMapper.DateFieldType("@timestamp")); + assertThat(sort.getSort(), arrayWithSize(2)); + assertThat(sort.getSort()[0].getField(), equalTo("_tsid")); + assertThat(sort.getSort()[1].getField(), equalTo("@timestamp")); + } + + public void testTimeSeriesModeNoTimestampWithTsdbAlias() { + IndexSettings indexSettings = indexSettings( + Settings.builder() + .put(IndexSettings.MODE.getKey(), "tsdb") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some_dimension") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") + .build() + ); + Exception e = expectThrows(IllegalArgumentException.class, () -> buildIndexSort(indexSettings, TimeSeriesIdFieldMapper.FIELD_TYPE)); + assertThat(e.getMessage(), equalTo("unknown index sort field:[@timestamp] required by [index.mode=tsdb]")); + } + public void testLogsdbIndexSortWithArrays() { Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), "logsdb") diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index ade44b4333b48..cc68b1c398843 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -54,6 +54,22 @@ public void testPartitioned() { assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.routing_partition_size]")); } + /** + * The {@code tsdb} alias delegates {@link IndexMode#validateWithOtherSettings} to + * {@link IndexMode#TIME_SERIES}, so it must reject the same incompatible settings with the + * same (canonical {@code time_series}) error message. + */ + public void testPartitionedWithTsdbAlias() { + Settings s = Settings.builder() + .put(getSettingsWithMode("tsdb", randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) + .put(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.getKey(), 2) + .build(); + IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); + Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); + assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.routing_partition_size]")); + } + public void testSortField() { Settings s = Settings.builder().put(getSettings()).put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "a").build(); IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); @@ -84,6 +100,15 @@ public void testWithoutRoutingPath() { assertThat(e.getMessage(), containsString("[index.mode=time_series] requires a non-empty [index.routing_path]")); } + public void testWithoutRoutingPathWithTsdbAlias() { + Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "tsdb").build(); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> new IndexSettings(IndexSettingsTests.newIndexMeta("test", s), Settings.EMPTY) + ); + assertThat(e.getMessage(), containsString("[index.mode=time_series] requires a non-empty [index.routing_path]")); + } + public void testWithEmptyRoutingPath() { Settings s = getSettings(""); Exception e = expectThrows( @@ -141,6 +166,16 @@ public void testRequiredRouting() { assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); } + public void testRequiredRoutingWithTsdbAlias() { + Settings s = getSettingsWithMode("tsdb", randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z"); + var mapperService = new TestMapperServiceBuilder().settings(s).applyDefaultMapping(false).build(); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> withMapping(mapperService, topMapping(b -> b.startObject("_routing").field("required", true).endObject())) + ); + assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); + } + public void testValidateAlias() { Settings s = getSettings(); IndexSettings.MODE.get(s).validateAlias(null, null); // Doesn't throw exception @@ -158,6 +193,18 @@ public void testValidateAliasWithSearchRouting() { assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); } + public void testValidateAliasWithTsdbAlias() { + Settings s = getSettingsWithMode("tsdb", randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z"); + assertSame(IndexMode.TSDB, IndexSettings.MODE.get(s)); + IndexSettings.MODE.get(s).validateAlias(null, null); // Doesn't throw exception + + Exception e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s).validateAlias("r", null)); + assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); + + e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s).validateAlias(null, "r")); + assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); + } + public void testRoutingPathMatchesObject() throws IOException { Settings s = getSettings("dim.o*"); createMapperService(s, mapping(b -> { @@ -359,8 +406,16 @@ private Settings getSettings(String routingPath) { } private Settings getSettings(String routingPath, String startTime, String endTime) { + return getSettingsWithMode("time_series", routingPath, startTime, endTime); + } + + /** + * Same as {@link #getSettings(String, String, String)} but with an explicit {@code index.mode} + * value, so that tests can assert the {@code tsdb} alias behaves identically to {@code time_series}. + */ + private Settings getSettingsWithMode(String mode, String routingPath, String startTime, String endTime) { return Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexSettings.MODE.getKey(), mode) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 1980e9b5ff71e..380e7227537f8 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -8302,11 +8302,21 @@ public void testIndexBatchFastPathOnly() throws IOException { public void testIndexBatchTimeSeriesPhase2() throws IOException { // planPrimarySubBatch Phase 2 uses timeSeriesBatchLoadDocIdAndVersion for TIME_SERIES // indices, which does a single sorted segment scan with timestamp-based segment skipping. + runIndexBatchTimeSeriesPhase2(IndexMode.TIME_SERIES); + } + + public void testIndexBatchTimeSeriesPhase2WithTsdbAlias() throws IOException { + // IndexMode.TSDB is a preferred alias for TIME_SERIES (delegates isTsdb() etc. to + // TIME_SERIES), so it must exercise the exact same batch phase 2 behavior. + runIndexBatchTimeSeriesPhase2(IndexMode.TSDB); + } + + private void runIndexBatchTimeSeriesPhase2(IndexMode mode) throws IOException { IndexSettings tsSettings = IndexSettingsModule.newIndexSettings( "test", Settings.builder() .put(defaultSettings.getSettings()) - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) + .put(IndexSettings.MODE.getKey(), mode.getName()) .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") .build() ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index 214d7f2f7c9c5..81348f302c93d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -170,6 +170,23 @@ public void testTimeSeriesIndexDefault() throws Exception { assertTrue(ft.indexType().hasOnlyDocValues()); } + /** + * The {@code tsdb} index mode is an alias for {@code time_series} (see {@link IndexMode#isTsdb()}) + * so it must produce the same doc-values-only defaulting for metric fields. + */ + public void testTimeSeriesIndexDefaultTsdbAlias() throws Exception { + var positionMetricType = TimeSeriesParams.MetricType.POSITION; + var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); + var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_metric", positionMetricType.toString()); + })); + var ft = (GeoPointFieldMapper.GeoPointFieldType) mapperService.fieldType("field"); + assertThat(ft.getMetricType(), equalTo(positionMetricType)); + assertTrue(ft.indexType().hasOnlyDocValues()); + } + public void testMetricAndDocvalues() { Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { minimalMapping(b); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java index 2678d42f85c38..763a8fb111377 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java @@ -198,6 +198,21 @@ public MetricType getMetricType() { e = expectThrows(MapperParsingException.class, () -> tsMappingLookup.validateDoesNotShadow("metric")); assertThat(e.getMessage(), equalTo("Field [metric] attempted to shadow a time_series_metric")); tsMappingLookup.validateDoesNotShadow("plain"); + + // The tsdb index mode is an alias for time_series (see IndexMode#isTsdb()), so it must reject + // shadowing of dimension/metric fields identically. + MappingLookup tsdbMappingLookup = createMappingLookup( + List.of(dimMapper, metricMapper, plainMapper), + emptyList(), + emptyList(), + IndexMode.TSDB + ); + tsdbMappingLookup.validateDoesNotShadow("not_mapped"); + e = expectThrows(MapperParsingException.class, () -> tsdbMappingLookup.validateDoesNotShadow("dim")); + assertThat(e.getMessage(), equalTo("Field [dim] attempted to shadow a time_series_dimension")); + e = expectThrows(MapperParsingException.class, () -> tsdbMappingLookup.validateDoesNotShadow("metric")); + assertThat(e.getMessage(), equalTo("Field [metric] attempted to shadow a time_series_metric")); + tsdbMappingLookup.validateDoesNotShadow("plain"); } public void testShadowingOnConstruction() { @@ -241,6 +256,21 @@ public MetricType getMetricType() { : "Field [metric] attempted to shadow a time_series_metric" ) ); + + // The tsdb index mode is an alias for time_series (see IndexMode#isTsdb()), so it must reject + // the same shadowing on construction. + Exception tsdbException = expectThrows( + MapperParsingException.class, + () -> createMappingLookup(List.of(dimMapper, metricMapper), emptyList(), List.of(shadowing), IndexMode.TSDB) + ); + assertThat( + tsdbException.getMessage(), + equalTo( + shadowDim + ? "Field [dim] attempted to shadow a time_series_dimension" + : "Field [metric] attempted to shadow a time_series_metric" + ) + ); } private void assertAnalyzes(Analyzer analyzer, String field, String output) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java index 4552344f7a1cc..bd28d963115d5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java @@ -60,6 +60,12 @@ public class TimeSeriesMetadataFieldBlockLoaderTests extends MapperServiceTestCa */ private static final Settings TSDB_SYNTHETIC_SETTINGS = tsdbSettings(null, "host"); + /** + * Same as {@link #TSDB_SYNTHETIC_SETTINGS} but using the {@link IndexMode#TSDB} alias instead of + * {@link IndexMode#TIME_SERIES}; {@link IndexMode#isTsdb()} treats both identically. + */ + private static final Settings TSDB_ALIAS_SYNTHETIC_SETTINGS = tsdbAliasSettings(null, "host"); + /** * TSDB index with {@code index.mapping.source.mode: stored}. Stored source preserves the raw * indexed bytes (e.g. CBOR for Prometheus remote-write documents), so the {@code _timeseries} loader must explicitly normalize to JSON. @@ -165,6 +171,18 @@ private static Settings tsdbSettings(SourceFieldMapper.Mode sourceMode, String r return builder.build(); } + private static Settings tsdbAliasSettings(SourceFieldMapper.Mode sourceMode, String routingPath) { + Settings.Builder builder = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z"); + if (sourceMode != null) { + builder.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), sourceMode); + } + return builder.build(); + } + public void testDimensionsOnly() throws IOException { BlockLoader loader = createBlockLoader( TSDB_SYNTHETIC_SETTINGS, @@ -175,6 +193,20 @@ public void testDimensionsOnly() throws IOException { assertThat(sourcePaths(loader), equalTo(Set.of("host", "env", "region"))); } + /** + * The {@code tsdb} index mode is an alias for {@code time_series} (see {@link IndexMode#isTsdb()}) + * so {@link TimeSeriesMetadataFieldBlockLoader} must be selected and behave identically. + */ + public void testDimensionsOnlyTsdbAlias() throws IOException { + BlockLoader loader = createBlockLoader( + TSDB_ALIAS_SYNTHETIC_SETTINGS, + MAPPING, + new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of()) + ); + assertThat(loader, instanceOf(TimeSeriesMetadataFieldBlockLoader.class)); + assertThat(sourcePaths(loader), equalTo(Set.of("host", "env", "region"))); + } + public void testDimensionsAndMetrics() throws IOException { BlockLoader loader = createBlockLoader( TSDB_SYNTHETIC_SETTINGS, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java index 34b5b577bd3ff..a49f75a3674e9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java @@ -103,6 +103,27 @@ public void testEnabledInTimeSeriesMode() throws Exception { assertEquals(hash, getRoutingHash(doc)); } + /** + * The {@code tsdb} index mode is an alias for {@code time_series} (see {@link IndexMode#isTsdb()}) + * so {@link TimeSeriesRoutingHashFieldMapper} must be enabled identically. + */ + @SuppressWarnings("unchecked") + public void testEnabledInTsdbAliasMode() throws Exception { + Settings.Builder settingsBuilder = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.name()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing path is required") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-10-29T00:00:00Z"); + DocumentMapper docMapper = createMapperService( + settingsBuilder.build(), + mapping(b -> { b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); }) + ).documentMapper(); + + int hash = randomInt(); + ParsedDocument doc = parseDocument(hash, docMapper, b -> b.field("a", "value")); + assertThat(doc.rootDoc().getField("a").binaryValue(), equalTo(new BytesRef("value"))); + assertEquals(hash, getRoutingHash(doc)); + } + public void testRetrievedFromIdInTimeSeriesMode() throws Exception { boolean syntheticId = randomBoolean(); DocumentMapper docMapper = createMapper(mapping(b -> { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 5eebc6a967b67..7c875f4505916 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -289,6 +289,23 @@ public void testTimeSeriesIndexDefault() throws Exception { assertTrue(ft.indexType().hasOnlyDocValues()); } + /** + * The {@code tsdb} index mode is an alias for {@code time_series} (see {@link IndexMode#isTsdb()}) + * so it must produce the same doc-values-only defaulting for metric fields. + */ + public void testTimeSeriesIndexDefaultTsdbAlias() throws Exception { + var randomMetricType = randomFrom(TimeSeriesParams.MetricType.scalar()); + var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); + var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_metric", randomMetricType.toString()); + })); + var ft = (NumberFieldMapper.NumberFieldType) mapperService.fieldType("field"); + assertThat(ft.getMetricType(), equalTo(randomMetricType)); + assertTrue(ft.indexType().hasOnlyDocValues()); + } + public void testMetricAndDocvalues() { Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { minimalMapping(b); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java index 7d94e2fad5835..ac8b9b02a55d0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java @@ -312,6 +312,49 @@ public void testDownsamplingPrerequisitesStep() { } } + public void testDownsamplingPrerequisitesStepWithTsdbAlias() { + DateHistogramInterval fixedInterval = ConfigTestHelpers.randomInterval(); + boolean withForceMerge = randomBoolean(); + DownsampleAction action = new DownsampleAction(fixedInterval, WAIT_TIMEOUT, withForceMerge, randomSamplingMethod()); + String phase = randomAlphaOfLengthBetween(1, 10); + StepKey nextStepKey = new StepKey( + randomAlphaOfLengthBetween(1, 10), + randomAlphaOfLengthBetween(1, 10), + randomAlphaOfLengthBetween(1, 10) + ); + { + // indices using the tsdb alias of the time series index mode execute the action + BranchingStep branchingStep = getFirstBranchingStep(action, phase, nextStepKey, withForceMerge); + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.TSDB) + .put("index.routing_path", "uid") + .build(); + IndexMetadata indexMetadata = newIndexMeta("test", settings); + + ProjectState state = projectStateFromProject(ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true)); + + branchingStep.performAction(indexMetadata.getIndex(), state); + assertThat(branchingStep.getNextStepKey().name(), is(CheckNotDataStreamWriteIndexStep.NAME)); + } + { + // already downsampled indices for the interval skip the action, regardless of the index mode alias used + BranchingStep branchingStep = getFirstBranchingStep(action, phase, nextStepKey, withForceMerge); + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.TSDB) + .put("index.routing_path", "uid") + .put(IndexMetadata.INDEX_DOWNSAMPLE_STATUS_KEY, IndexMetadata.DownsampleTaskStatus.SUCCESS) + .put(IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_NAME.getKey(), "test") + .build(); + String indexName = DOWNSAMPLED_INDEX_PREFIX + fixedInterval + "-test"; + IndexMetadata indexMetadata = newIndexMeta(indexName, settings); + + ProjectState state = projectStateFromProject(ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true)); + + branchingStep.performAction(indexMetadata.getIndex(), state); + assertThat(branchingStep.getNextStepKey(), is(nextStepKey)); + } + } + private static BranchingStep getFirstBranchingStep(DownsampleAction action, String phase, StepKey nextStepKey, boolean withForceMerge) { List steps = action.toSteps(null, phase, nextStepKey); assertNotNull(steps); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java index db30c56f1d556..de79d48a7662f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java @@ -15,9 +15,13 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.xcontent.ToXContentObject; import java.time.Instant; @@ -145,4 +149,93 @@ public void onFailure(Exception e) { }, MASTER_TIMEOUT); } } + + /** + * {@link org.elasticsearch.index.IndexMode#TSDB} is a preferred alias for + * {@link org.elasticsearch.index.IndexMode#TIME_SERIES} and must gate {@link WaitUntilTimeSeriesEndTimePassesStep} + * identically. {@link #testEvaluateCondition()} only exercises indices whose {@code index.mode} is the literal + * {@code time_series}, so this repeats the time-series-specific assertions using {@code index.mode: tsdb} directly. + */ + public void testEvaluateConditionTsdbAlias() { + Instant currentTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); + Instant startTimeLapsed = currentTime.minus(6, ChronoUnit.HOURS); + Instant endTimeLapsed = currentTime.minus(2, ChronoUnit.HOURS); + Instant startTimeFuture = currentTime.minus(2, ChronoUnit.HOURS); + Instant endTimeFuture = currentTime.plus(2, ChronoUnit.HOURS); + + WaitUntilTimeSeriesEndTimePassesStep step = new WaitUntilTimeSeriesEndTimePassesStep( + randomStepKey(), + randomStepKey(), + () -> currentTime + ); + + { + // end_time has lapsed already so condition must be met + IndexMetadata indexMeta = createTsdbIndexMetadata("tsdb-index-lapsed", startTimeLapsed, endTimeLapsed); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMeta, true).build(); + ProjectState projectState = ClusterState.builder(ClusterName.DEFAULT) + .putProjectMetadata(project) + .build() + .projectState(project.id()); + + step.evaluateCondition(projectState, indexMeta, new AsyncWaitStep.Listener() { + + @Override + public void onResponse(boolean complete, ToXContentObject informationContext) { + assertThat(complete, is(true)); + } + + @Override + public void onFailure(Exception e) { + throw new AssertionError("Unexpected method call", e); + } + }, MASTER_TIMEOUT); + } + + { + // end_time is in the future + IndexMetadata indexMeta = createTsdbIndexMetadata("tsdb-index-future", startTimeFuture, endTimeFuture); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMeta, true).build(); + ProjectState projectState = ClusterState.builder(ClusterName.DEFAULT) + .putProjectMetadata(project) + .build() + .projectState(project.id()); + + step.evaluateCondition(projectState, indexMeta, new AsyncWaitStep.Listener() { + + @Override + public void onResponse(boolean complete, ToXContentObject informationContext) { + assertThat(complete, is(false)); + String information = Strings.toString(informationContext); + assertThat( + information, + containsString( + "The [index.time_series.end_time] setting for index [" + + indexMeta.getIndex().getName() + + "] is [" + + endTimeFuture.toEpochMilli() + + "]. Waiting until the index's time series end time lapses before proceeding with action [" + + step.getKey().action() + + "] as the index can still accept writes." + ) + ); + } + + @Override + public void onFailure(Exception e) { + throw new AssertionError("Unexpected method call", e); + } + }, MASTER_TIMEOUT); + } + } + + private static IndexMetadata createTsdbIndexMetadata(String indexName, Instant startTime, Instant endTime) { + Settings settings = indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "uid") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(startTime)) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(endTime)) + .build(); + return IndexMetadata.builder(indexName).settings(settings).build(); + } } diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java index 263661a23572e..9f4aefe831b97 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java @@ -266,6 +266,63 @@ public void testDownsampling() { verifyIndexFinalisation(); } + /** + * Same as {@link #testDownsampling()} but using {@link IndexMode#TSDB}, the preferred alias for + * {@link IndexMode#TIME_SERIES}, to configure the source index. This verifies that the + * {@code index.mode}-gated validation in {@link TransportDownsampleAction} relies on + * {@link IndexMode#isTsdb()} rather than an exact match against {@link IndexMode#TIME_SERIES}, + * so a source index configured with the {@code tsdb} alias is downsampled successfully as well. + */ + public void testDownsamplingWithTsdbIndexModeAlias() { + var projectMetadata = ProjectMetadata.builder(projectId) + .put(createSourceIndexMetadata(sourceIndex, primaryShards, replicaShards, IndexMode.TSDB)) + .build(); + + var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .putProjectMetadata(projectMetadata) + .blocks(ClusterBlocks.builder().addIndexBlock(projectId, sourceIndex, IndexMetadata.INDEX_WRITE_BLOCK)) + .build(); + + when(projectResolver.getProjectMetadata(any(ClusterState.class))).thenReturn(projectMetadata); + + Answer mockPersistentTask = invocation -> { + ActionListener> listener = invocation.getArgument(4); + PersistentTasksCustomMetadata.PersistentTask task1 = mock(PersistentTasksCustomMetadata.PersistentTask.class); + when(task1.getId()).thenReturn(randomAlphaOfLength(10)); + DownsampleShardPersistentTaskState runningTaskState = new DownsampleShardPersistentTaskState( + DownsampleShardIndexerStatus.COMPLETED, + null + ); + when(task1.getState()).thenReturn(runningTaskState); + listener.onResponse(task1); + return null; + }; + doAnswer(mockPersistentTask).when(persistentTaskService).sendStartRequest(anyString(), anyString(), any(), any(), any()); + doAnswer(mockPersistentTask).when(persistentTaskService).waitForPersistentTaskCondition(any(), anyString(), any(), any(), any()); + doAnswer(invocation -> { + var listener = invocation.getArgument(1, TransportDownsampleAction.UpdateDownsampleIndexSettingsActionListener.class); + listener.onResponse(AcknowledgedResponse.TRUE); + return null; + }).when(indicesAdminClient).updateSettings(any(), any()); + assertSuccessfulUpdateDownsampleStatus(clusterState); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.masterOperation( + task, + new DownsampleAction.Request( + ESTestCase.TEST_REQUEST_TIMEOUT, + sourceIndex, + targetIndex, + TimeValue.ONE_HOUR, + new DownsampleConfig(new DateHistogramInterval("5m"), randomSamplingMethod()) + ), + clusterState, + listener + ); + safeGet(listener); + verifyIndexFinalisation(); + } + public void testDownsamplingWithShortCircuitAfterCreation() { var projectMetadata = ProjectMetadata.builder(projectId) .put(createSourceIndexMetadata(sourceIndex, primaryShards, replicaShards)) @@ -495,11 +552,20 @@ private void verifyIndexFinalisation() { } private IndexMetadata.Builder createSourceIndexMetadata(String sourceIndex, int primaryShards, int replicaShards) { + return createSourceIndexMetadata(sourceIndex, primaryShards, replicaShards, IndexMode.TIME_SERIES); + } + + private IndexMetadata.Builder createSourceIndexMetadata( + String sourceIndex, + int primaryShards, + int replicaShards, + IndexMode indexMode + ) { return IndexMetadata.builder(sourceIndex) .settings( indexSettings(IndexVersion.current(), randomUUID(), primaryShards, replicaShards).put( IndexSettings.MODE.getKey(), - IndexMode.TIME_SERIES.getName() + indexMode.getName() ) .put("index.routing_path", "dimensions") .put(IndexMetadata.SETTING_BLOCKS_WRITE, true) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 35a555b344b35..3846ca477e2ff 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -1997,7 +1997,8 @@ private void preAnalyzeFlatMainIndices( ); } - private static QueryBuilder createQueryFilter(IndexMode indexMode, QueryBuilder requestFilter) { + // visible for testing + static QueryBuilder createQueryFilter(IndexMode indexMode, QueryBuilder requestFilter) { return switch (indexMode) { case IndexMode.TIME_SERIES, IndexMode.TSDB -> { // Match either alias: the requested indexMode is a fixed sentinel (e.g. the TS command always diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 7a08a0e6b0f37..12267d25e9430 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2734,6 +2734,24 @@ public void testImplicitTimestampSortForTsQuery() { assertThat(orderChild.name(), equalTo("@timestamp")); } + public void testImplicitTimestampSortForTsQueryWithTsdbAlias() { + // Same as testImplicitTimestampSortForTsQuery, but the index uses IndexMode.TSDB instead of + // IndexMode.TIME_SERIES: AddImplicitTimestampSort is gated on IndexMode#isTsdb(), so both + // aliases must be treated identically. + var plan = tsdbAlias().query("TS test"); + + var limit = as(plan, Limit.class); + var orderBy = as(limit.child(), OrderBy.class); + var orders = orderBy.order(); + assertThat(orders, hasSize(1)); + + var order = orders.get(0); + assertThat(order.direction(), equalTo(Order.OrderDirection.DESC)); + assertThat(order.nullsPosition(), equalTo(Order.NullsPosition.LAST)); + var orderChild = as(order.child(), FieldAttribute.class); + assertThat(orderChild.name(), equalTo("@timestamp")); + } + public void testImplicitTimestampSortWithKeep() { // TS query with KEEP should have implicit sort var plan = tsdb().query("TS test | KEEP @timestamp, host"); @@ -5034,6 +5052,42 @@ public void testImplicitCastingForAggregateMetricDouble() { assertProjection(plan2, "s1", "s2", "min", "count", "avg", "cluster", "time_bucket"); } + public void testImplicitCastingForAggregateMetricDoubleWithTsdbAlias() { + // Same as testImplicitCastingForAggregateMetricDouble's TS branch, but the indices use + // IndexMode.TSDB instead of IndexMode.TIME_SERIES: ImplicitCastAggregateMetricDoubles is + // gated on IndexMode#isTsdb(), so both aliases must be treated identically. + assumeTrue( + "aggregate metric double implicit casting must be available", + EsqlCapabilities.Cap.AGGREGATE_METRIC_DOUBLE_V0.isEnabled() + ); + Map mapping = Map.of( + "@timestamp", + new EsField("@timestamp", DATETIME, Map.of(), true, EsField.TimeSeriesFieldType.NONE), + "cluster", + new EsField("cluster", KEYWORD, Map.of(), true, EsField.TimeSeriesFieldType.DIMENSION), + "metric_field", + new InvalidMappedField("metric_field", Map.of("aggregate_metric_double", Set.of("k8s-downsampled"), "double", Set.of("k8s"))) + ); + + var esIndex = new EsIndex( + "k8s,k8s-downsampled", + mapping, + Map.of("k8s", IndexMode.TSDB, "k8s-downsampled", IndexMode.TSDB), + Map.of(), + Map.of() + ); + var testAnalyzer = analyzer().addIndex(esIndex); + var plan2 = testAnalyzer.query(""" + TS k8s,k8s-downsampled | stats s1 = sum(sum_over_time(metric_field)), + s2 = sum(avg_over_time(metric_field)), + min = min(max_over_time(metric_field)), + count = count(count_over_time(metric_field)), + avg = avg(min_over_time(metric_field)) + by cluster, time_bucket = bucket(@timestamp,1minute) + """); + assertProjection(plan2, "s1", "s2", "min", "count", "avg", "cluster", "time_bucket"); + } + public void testToGaugeStrippedOnAggregateMetricDoubleAndGaugeUnion() { assumeTrue("to_gauge must be available", EsqlCapabilities.Cap.TO_GAUGE.isEnabled()); assumeTrue( @@ -5533,6 +5587,15 @@ private static TestAnalyzer tsdb() { return analyzer().addIndex("test", "tsdb-mapping.json", IndexMode.TIME_SERIES); } + /** + * Same fixture as {@link #tsdb()}, but built with the {@link IndexMode#TSDB} alias instead of + * {@link IndexMode#TIME_SERIES}. Used to prove that {@code index.mode: tsdb} is treated identically + * to {@code index.mode: time_series} by rules gated on {@link IndexMode#isTsdb()}. + */ + private static TestAnalyzer tsdbAlias() { + return analyzer().addIndex("test", "tsdb-mapping.json", IndexMode.TSDB); + } + private static TestAnalyzer k8s() { return analyzer().addK8sDownsampled(); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java index 17f125f0a9511..4bda8ec272318 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java @@ -154,11 +154,13 @@ public void testMixedMultipleIndicesAndDatasetsProducesUnionAll() { } public void testIndexModeNonStandardRejected() { - // Note on coverage: only TIME_SERIES (via TS) and LOOKUP (via LOOKUP JOIN) are user-reachable + // Note on coverage: only TIME_SERIES/TSDB (via TS) and LOOKUP (via LOOKUP JOIN) are user-reachable // through ESQL syntax; LOGSDB has no user-syntax path that constructs an UnresolvedRelation // with IndexMode.LOGSDB pointing at a dataset name. The LOGSDB branch is defensive code for // any future path that might set it. There is no IT analogue for LOGSDB — this unit case // pins the rejection-message contract. + // TSDB is included alongside TIME_SERIES to prove the "index.mode: tsdb" alias is rejected + // with the exact same message, since IndexMode.TSDB behaves identically to TIME_SERIES. DataSource parent = dataSource("s3_parent", Map.of()); Dataset dataset = new Dataset("logs", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); ProjectMetadata project = projectWith(Map.of("s3_parent", parent), Map.of("logs", dataset)); @@ -166,6 +168,8 @@ public void testIndexModeNonStandardRejected() { Map expectedFragments = Map.of( IndexMode.TIME_SERIES, "TS command is not supported for datasets", + IndexMode.TSDB, + "TS command is not supported for datasets", IndexMode.LOOKUP, "LOOKUP JOIN against a dataset is not supported", IndexMode.LOGSDB, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java index 9bfe118d9f658..a0d6a093469a1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java @@ -355,6 +355,27 @@ public void testRoundToInTsEval() { assertThat(pushed, hasSize(0)); } + /** + * Verifies ROUND_TO on a long field is NOT pushed to the block loader when + * the shard's index metadata reports the {@link IndexMode#TSDB} alias + * rather than {@link IndexMode#TIME_SERIES}. {@code TSDB} behaves + * identically to {@code TIME_SERIES} for {@code hasTimeSeriesShards()}, + * so this mirrors {@link #testRoundToInTsEval()}. + */ + public void testRoundToInTsEvalWithTsdbIndexMode() { + assumeTrue("ROUND_TO block loader must be enabled", EsqlCapabilities.Cap.ROUND_TO_BLOCK_LOADER.isEnabled()); + PhysicalPlan plan = tsPlannerOptimizer.plan(""" + TS k8s + | EVAL r = ROUND_TO(events_received, 100, 200, 300) + | SORT @timestamp + | LIMIT 10 + | KEEP r + """, tsdbSearchStats()); + + List pushed = findPushedFields(plan, "events_received", BlockLoaderFunctionConfig.Function.ROUND_TO); + assertThat(pushed, hasSize(0)); + } + /** * Verifies ROUND_TO on a datetime field ({@code @timestamp}) is NOT pushed * to the block loader in TS mode. @@ -856,4 +877,20 @@ public Map targetShards() { } }; } + + /** + * Same as {@link #tsSearchStats()} but reports the shard's index metadata + * using the {@link IndexMode#TSDB} alias instead of {@link IndexMode#TIME_SERIES}. + */ + private static SearchStats tsdbSearchStats() { + return new EsqlTestUtils.TestSearchStats() { + @Override + public Map targetShards() { + IndexMetadata indexMetadata = IndexMetadata.builder("k8s") + .settings(indexSettings(IndexVersion.current(), 1, 1).put(IndexSettings.MODE.getKey(), IndexMode.TSDB.name())) + .build(); + return Map.of(new ShardId(new Index("k8s", "n/a"), 0), indexMetadata); + } + }; + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java index 5bf3f3c0d96e4..ee4f81e3a1a59 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; +import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.common.settings.Settings; @@ -27,11 +28,14 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.elasticsearch.xpack.esql.analysis.Analyzer; +import org.elasticsearch.xpack.esql.analysis.EnrichResolution; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; @@ -43,7 +47,11 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc; import org.elasticsearch.xpack.esql.expression.function.scalar.math.RoundTo; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.index.EsIndexGenerator; +import org.elasticsearch.xpack.esql.index.IndexResolution; import org.elasticsearch.xpack.esql.optimizer.AbstractLocalPhysicalPlanOptimizerTests; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; import org.elasticsearch.xpack.esql.optimizer.TestPlannerOptimizer; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; @@ -80,8 +88,14 @@ import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_FUNCTION_REGISTRY; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptyInferenceResolution; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.testAnalyzerContext; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.indexResolutions; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_NANOS_FORMATTER; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateNanosToLong; @@ -1040,6 +1054,100 @@ public Map targetShards() { } } + /** + * {@link org.elasticsearch.index.IndexMode#TSDB} is a preferred alias for + * {@link IndexMode#TIME_SERIES}: it behaves identically everywhere, including in the + * {@link ReplaceRoundToWithQueryAndTags} rule. This mirrors + * {@link #testRoundToWithTimeSeriesIndices} but resolves the source index (and the target + * shards reported by {@link SearchStats}) with {@link IndexMode#TSDB} instead, verifying the + * filter-by-filter rewrite still applies identically. + */ + public void testRoundToWithTimeSeriesIndicesUsingTsdbAlias() { + Map minValue = Map.of( + "@timestamp", + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-10-20T12:15:03.360Z") + ); + Map maxValue = Map.of( + "@timestamp", + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-10-20T14:55:01.543Z") + ); + SearchStats searchStats = new EsqlTestUtils.TestSearchStatsWithMinMax(minValue, maxValue) { + @Override + public Map targetShards() { + var indexMetadata = IndexMetadata.builder("test_index") + .settings( + ESTestCase.indexSettings(IndexVersion.current(), 1, 1).put(IndexSettings.MODE.getKey(), IndexMode.TSDB.name()) + ) + .build(); + return Map.of(new ShardId(new Index("id", "n/a"), 1), indexMetadata); + } + }; + TestPlannerOptimizer tsdbPlannerOptimizer = tsdbAliasPlannerOptimizer(); + // enable filter-by-filter for rate aggregations + { + String q = """ + TS k8s + | STATS max(rate(network.total_bytes_in)) BY cluster, BUCKET(@timestamp, 1 hour) + | LIMIT 10 + """; + PhysicalPlan plan = tsdbPlannerOptimizer.plan(q, searchStats); + int queryAndTags = plainQueryAndTags(plan); + assertThat(queryAndTags, equalTo(4)); + } + // disable filter-by-filter for non-rate aggregations + { + String q = """ + TS k8s + | STATS max(avg_over_time(network.bytes_in)) BY cluster, BUCKET(@timestamp, 1 hour) + | LIMIT 10 + """; + PhysicalPlan plan = tsdbPlannerOptimizer.plan(q, searchStats); + int queryAndTags = plainQueryAndTags(plan); + assertThat(queryAndTags, equalTo(1)); + } + } + + /** + * Directly exercises {@link ReplaceRoundToWithQueryAndTags#adjustedRoundingPointsThreshold}: since + * {@link IndexMode#TSDB} is a preferred alias for {@link IndexMode#TIME_SERIES}, both must double the + * rounding points threshold identically. + */ + public void testAdjustedRoundingPointsThresholdForTsdbAlias() { + int threshold = between(1, 1000); + int timeSeriesThreshold = ReplaceRoundToWithQueryAndTags.adjustedRoundingPointsThreshold( + searchStats, + threshold, + null, + IndexMode.TIME_SERIES + ); + int tsdbThreshold = ReplaceRoundToWithQueryAndTags.adjustedRoundingPointsThreshold(searchStats, threshold, null, IndexMode.TSDB); + assertThat(tsdbThreshold, equalTo(timeSeriesThreshold)); + assertThat(tsdbThreshold, equalTo(threshold * 2)); + } + + // Builds an analyzer/planner pair mirroring AbstractLocalPhysicalPlanOptimizerTests#init's + // plannerOptimizerTimeSeries/timeSeriesAnalyzer, but resolving the "k8s" index with the TSDB alias + // instead of TIME_SERIES, to verify IndexMode.TSDB is handled identically. + private TestPlannerOptimizer tsdbAliasPlannerOptimizer() { + var timeSeriesMapping = loadMapping("k8s-mappings.json"); + var tsdbIndex = IndexResolution.valid(EsIndexGenerator.esIndex("k8s", timeSeriesMapping, Map.of("k8s", IndexMode.TSDB))); + Analyzer tsdbAliasAnalyzer = new Analyzer( + testAnalyzerContext( + EsqlTestUtils.TEST_CFG, + TEST_FUNCTION_REGISTRY, + indexResolutions(tsdbIndex), + new EnrichResolution(), + emptyInferenceResolution() + ), + TEST_VERIFIER + ); + return new TestPlannerOptimizer( + config, + tsdbAliasAnalyzer, + new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small(), TransportVersion.current())) + ); + } + /** * Exercises the block loader fallback in {@link ReplaceRoundToWithQueryAndTags}: * when query-and-tags is disabled (threshold=0) but block loader is supported, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java index cf4e808d9e0f7..7d1e7e922309a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java @@ -13,6 +13,11 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.mapper.IndexModeFieldMapper; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; @@ -51,9 +56,13 @@ import static java.util.stream.Collectors.toMap; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_PARSER; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.sameInstance; public class EsqlSessionTests extends ESTestCase { @@ -78,6 +87,67 @@ public void testShouldNotRetryWildcardTimeSeriesResolution() { ); } + // IndexMode.TSDB is a preferred alias for IndexMode.TIME_SERIES (isTsdb() is true for both); this pins that + // shouldRetryConcreteTimeSeriesResolution treats the alias identically to the canonical constant. + public void testShouldRetryConcreteTsdbAliasResolution() { + assertTrue( + EsqlSession.shouldRetryConcreteTimeSeriesResolution( + IndexMode.TSDB, + IndexResolution.empty("logs"), + new IndexPattern(EMPTY, "logs") + ) + ); + } + + public void testShouldNotRetryWildcardTsdbAliasResolution() { + assertFalse( + EsqlSession.shouldRetryConcreteTimeSeriesResolution( + IndexMode.TSDB, + IndexResolution.empty("logs*"), + new IndexPattern(EMPTY, "logs*") + ) + ); + } + + // createQueryFilter builds the _index_mode filter used to resolve the concrete backing indices of a TS/METRICS + // source. The requested indexMode is a fixed sentinel (TS always declares IndexMode.TIME_SERIES) but the + // concrete backing indices may be configured with either [index.mode=time_series] or its alias + // [index.mode=tsdb], so the filter must match both terms regardless of which indexMode triggered it. + public void testCreateQueryFilterForTimeSeriesMatchesBothIndexModeAliases() { + QueryBuilder filter = EsqlSession.createQueryFilter(IndexMode.TIME_SERIES, null); + TermsQueryBuilder termsFilter = as(filter, TermsQueryBuilder.class); + assertThat(termsFilter.fieldName(), equalTo(IndexModeFieldMapper.NAME)); + assertThat(termsFilter.values(), containsInAnyOrder(IndexMode.TIME_SERIES.getName(), IndexMode.TSDB.getName())); + } + + public void testCreateQueryFilterForTsdbAliasMatchesBothIndexModeAliases() { + QueryBuilder filter = EsqlSession.createQueryFilter(IndexMode.TSDB, null); + TermsQueryBuilder termsFilter = as(filter, TermsQueryBuilder.class); + assertThat(termsFilter.fieldName(), equalTo(IndexModeFieldMapper.NAME)); + assertThat(termsFilter.values(), containsInAnyOrder(IndexMode.TIME_SERIES.getName(), IndexMode.TSDB.getName())); + } + + public void testCreateQueryFilterCombinesRequestFilterWithIndexModeFilter() { + QueryBuilder requestFilter = QueryBuilders.matchAllQuery(); + QueryBuilder filter = EsqlSession.createQueryFilter(IndexMode.TSDB, requestFilter); + BoolQueryBuilder boolFilter = as(filter, BoolQueryBuilder.class); + assertThat(boolFilter.filter(), hasSize(2)); + assertThat(boolFilter.filter(), hasItem(requestFilter)); + TermsQueryBuilder termsFilter = boolFilter.filter() + .stream() + .filter(TermsQueryBuilder.class::isInstance) + .map(TermsQueryBuilder.class::cast) + .findFirst() + .orElseThrow(); + assertThat(termsFilter.fieldName(), equalTo(IndexModeFieldMapper.NAME)); + assertThat(termsFilter.values(), containsInAnyOrder(IndexMode.TIME_SERIES.getName(), IndexMode.TSDB.getName())); + } + + public void testCreateQueryFilterForStandardModeReturnsRequestFilterUnchanged() { + QueryBuilder requestFilter = QueryBuilders.matchAllQuery(); + assertThat(EsqlSession.createQueryFilter(IndexMode.STANDARD, requestFilter), sameInstance(requestFilter)); + } + public void testRefineConcreteTimeSeriesResolutionReturnsHelpfulError() { IndexResolution resolution = EsqlSession.refineConcreteTimeSeriesResolution( new IndexPattern(EMPTY, "logs"), diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 414aa2e6ab987..c553c0cdcc158 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -277,6 +277,30 @@ public void testOnExplicitTimeSeriesIndex() throws IOException { assertTrue(additionalIndexSettings.isEmpty()); } + public void testOnExplicitTsdbIndex() throws IOException { + // Same scenario as testOnExplicitTimeSeriesIndex, but using the "tsdb" alias for index.mode. + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + logsdbLicenseService, + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + Settings.Builder settingsBuilder = builder(); + provider.provideAdditionalSettings( + null, + "logs-apache-production", + null, + emptyProject(), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()).build(), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), + IndexVersion.current(), + settingsBuilder + ); + final Settings additionalIndexSettings = settingsBuilder.build(); + + assertTrue(additionalIndexSettings.isEmpty()); + } + public void testNonLogsDataStream() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( logsdbLicenseService, @@ -669,6 +693,44 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { } } + public void testNewIndexHasSyntheticSourceUsageTsdb() throws IOException { + // Same scenario as testNewIndexHasSyntheticSourceUsageTimeSeries, but exercised through the "tsdb" index.mode + // alias to make sure IndexMode.isTsdb(...) treats it identically to "time_series". + String dataStreamName = DATA_STREAM_NAME; + String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); + String mapping = """ + { + "properties": { + "my_field": { + "type": "keyword", + "time_series_dimension": true + } + } + } + """; + LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); + { + Settings settings = Settings.builder().put("index.mode", "tsdb").put("index.routing_path", "my_field").build(); + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) + .hasSyntheticSourceUsage(); + assertTrue(result); + } + { + Settings settings = Settings.builder().put("index.mode", "tsdb").put("index.routing_path", "my_field").build(); + boolean result = provider.getMappingHints(indexName, null, settings, List.of()).hasSyntheticSourceUsage(); + assertTrue(result); + } + { + boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of()).hasSyntheticSourceUsage(); + assertFalse(result); + } + { + boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(getMapping(mapping)))) + .hasSyntheticSourceUsage(); + assertFalse(result); + } + } + public void testNewIndexHasSyntheticSourceUsageInvalidSettings() throws IOException { String dataStreamName = DATA_STREAM_NAME; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); @@ -793,6 +855,41 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { assertThat(newMapperServiceCounter.get(), equalTo(4)); } + public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceTsdbAlias() { + // Same scenario as the IndexMode.TIME_SERIES case in testGetAdditionalIndexSettingsDowngradeFromSyntheticSource, + // but using the IndexMode.TSDB alias to make sure the template mode injection in + // LogsdbIndexModeSettingsProvider#buildIndexMetadataForMapperService treats it identically. + String dataStreamName = DATA_STREAM_NAME; + ProjectMetadata project = DataStreamTestHelper.getProjectWithDataStreams( + List.of(Tuple.tuple(dataStreamName, 1)), + List.of(), + Instant.now().toEpochMilli(), + builder().build(), + 1 + ); + LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); + Settings settings = builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .build(); + logsdbLicenseService.setSyntheticSourceFallback(true); + + Settings.Builder settingsBuilder = builder(); + provider.provideAdditionalSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 2), + dataStreamName, + IndexMode.TSDB, + project, + Instant.ofEpochMilli(1L), + settings, + List.of(), + IndexVersion.current(), + settingsBuilder + ); + Settings result = settingsBuilder.build(); + assertThat(result.size(), equalTo(1)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertThat(newMapperServiceCounter.get(), equalTo(1)); + } + public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceOldNode() { logsdbLicenseService.setSyntheticSourceFallback(true); LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(true, Version.V_8_16_0); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java index 19b30822e4df6..08333ffc64287 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java @@ -148,6 +148,29 @@ public void testGetAdditionalIndexSettingsTsdb() throws IOException { assertEquals(Settings.EMPTY, result); } + public void testGetAdditionalIndexSettingsTsdbAlias() throws IOException { + // Same scenario as testGetAdditionalIndexSettingsTsdb, but using the IndexMode.TSDB alias to make sure the + // legacy license check in LogsdbIndexModeSettingsProvider#isLegacyLicensedUsageOfSyntheticSourceAllowed + // treats it identically to IndexMode.TIME_SERIES. + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + String dataStreamName = "metrics-my-app"; + String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); + Settings.Builder builder = Settings.builder(); + provider.provideAdditionalSettings( + indexName, + dataStreamName, + IndexMode.TSDB, + null, + null, + settings, + List.of(), + IndexVersion.current(), + builder + ); + var result = builder.build(); + assertEquals(Settings.EMPTY, result); + } + public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception { long start = LocalDateTime.of(2025, 2, 5, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); License license = createGoldOrPlatinumLicense(start); @@ -192,4 +215,51 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception var expected = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED").build(); assertEquals(expected, result); } + + public void testGetAdditionalIndexSettingsTsdbAfterCutoffDateAlias() throws Exception { + // Same scenario as testGetAdditionalIndexSettingsTsdbAfterCutoffDate, but using the IndexMode.TSDB alias to + // make sure the legacy license check treats it identically to IndexMode.TIME_SERIES. + long start = LocalDateTime.of(2025, 2, 5, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); + License license = createGoldOrPlatinumLicense(start); + long time = LocalDateTime.of(2024, 12, 31, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); + var licenseState = new XPackLicenseState(() -> time, new XPackLicenseStatus(license.operationMode(), true, null)); + + var licenseService = new LogsdbLicenseService(Settings.EMPTY); + licenseService.setLicenseState(licenseState); + var mockLicenseService = mock(LicenseService.class); + when(mockLicenseService.getLicense()).thenReturn(license); + + LogsdbLicenseService syntheticSourceLicenseService = new LogsdbLicenseService(Settings.EMPTY); + syntheticSourceLicenseService.setLicenseState(licenseState); + syntheticSourceLicenseService.setLicenseService(mockLicenseService); + + provider = new LogsdbIndexModeSettingsProvider(syntheticSourceLicenseService, Settings.EMPTY); + provider.init( + im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), + IndexVersion::current, + () -> Version.CURRENT, + true, + true + ); + + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + String dataStreamName = "metrics-my-app"; + String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); + Settings.Builder builder = Settings.builder(); + provider.provideAdditionalSettings( + indexName, + dataStreamName, + IndexMode.TSDB, + null, + null, + settings, + List.of(), + IndexVersion.current(), + builder + ); + + var result = builder.build(); + var expected = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED").build(); + assertEquals(expected, result); + } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index 18801ab2e3446..b77788d18023a 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -352,6 +352,23 @@ public void testTimeSeriesIndexDefault() throws Exception { assertTrue(ft.indexType().hasOnlyDocValues()); } + /** + * The {@code tsdb} index mode is an alias for {@code time_series} (see {@link IndexMode#isTsdb()}) + * so it must produce the same doc-values-only defaulting for metric fields. + */ + public void testTimeSeriesIndexDefaultTsdbAlias() throws Exception { + var randomMetricType = randomFrom(TimeSeriesParams.MetricType.scalar()); + var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); + var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_metric", randomMetricType.toString()); + })); + var ft = (UnsignedLongFieldMapper.UnsignedLongFieldType) mapperService.fieldType("field"); + assertThat(ft.getMetricType(), equalTo(randomMetricType)); + assertTrue(ft.indexType().hasOnlyDocValues()); + } + @Override protected Object generateRandomInputValue(MappedFieldType ft) { Number n = randomNumericValue(); From 032444b8999e33437de5c3420a5cc1024327045a Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Sun, 5 Jul 2026 14:21:31 +0300 Subject: [PATCH 07/23] Add remaining TSDB alias unit test coverage Fix IndexSettingsTests assertion to expect the canonical time_series message text, and add FeatureMetric TSDB coverage. --- .../index/IndexSettingsTests.java | 9 ++++++- .../esql/telemetry/FeatureMetricTests.java | 26 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 7831ac779ddd7..5b01993241f67 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -281,6 +281,13 @@ public void testSliceEnabledSettingRejectedForTimeSeriesMode() { assertThat(exception.getMessage(), containsString("time_series")); } + /** + * {@code index.mode: tsdb} must be rejected exactly like {@code time_series}. Note that the error + * message itself always names {@code time_series}: {@link IndexMode#TSDB} delegates + * {@code validateWithOtherSettings} straight to {@link IndexMode#TIME_SERIES}, whose message uses the + * canonical {@code time_series} name regardless of which alias was actually configured (mirroring + * {@code tsdbMode()}'s behavior elsewhere in {@link IndexMode}). + */ public void testSliceEnabledSettingRejectedForTsdbAlias() { assumeTrue("slice indexing feature flag must be enabled", SliceIndexing.SLICE_FEATURE_FLAG.isEnabled()); IllegalArgumentException exception = expectThrows( @@ -299,7 +306,7 @@ public void testSliceEnabledSettingRejectedForTsdbAlias() { ); assertThat(exception.getMessage(), containsString("index.slice.enabled")); assertThat(exception.getMessage(), containsString("index.mode")); - assertThat(exception.getMessage(), containsString("tsdb")); + assertThat(exception.getMessage(), containsString("time_series")); } @TestLogging(reason = "testing warning logging", value = "org.elasticsearch.index.IndexSettings:WARN") diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetricTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetricTests.java index f47097519e71e..6b518b696886f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetricTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetricTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.plan.IndexPattern; +import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedExternalRelation; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; @@ -18,6 +19,8 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.relation; + /** * {@link FeatureMetric#set} throws for any plan node it cannot map and that is not on the exclusion list. Since the * exclusion list matches the {@code UnresolvedSourceRelation} marker, both {@code FROM}-style leaf shapes must be @@ -53,4 +56,27 @@ public void testExternalRelationIsExcluded() { FeatureMetric.set(external, bitset); assertTrue("excluded plans must not set any telemetry bit", bitset.isEmpty()); } + + /** + * {@link FeatureMetric#TS} and {@link FeatureMetric#FROM} branch on {@code EsRelation#indexMode()#isTsdb()}. + * {@code IndexMode.TSDB} is a preferred alias for {@code IndexMode.TIME_SERIES} (isTsdb() is true for both), so + * both constants must flip the TS bit - never the FROM bit - exactly like the canonical TIME_SERIES constant. + */ + public void testTimeSeriesRelationSetsTsMetricForBothIndexModeAliases() { + for (IndexMode mode : List.of(IndexMode.TIME_SERIES, IndexMode.TSDB)) { + EsRelation relation = relation(mode); + BitSet bitset = new BitSet(); + FeatureMetric.set(relation, bitset); + assertTrue("indexMode [" + mode + "] must set the TS telemetry bit", bitset.get(FeatureMetric.TS.ordinal())); + assertFalse("indexMode [" + mode + "] must not set the FROM telemetry bit", bitset.get(FeatureMetric.FROM.ordinal())); + } + } + + public void testStandardRelationSetsFromMetricNotTs() { + EsRelation relation = relation(IndexMode.STANDARD); + BitSet bitset = new BitSet(); + FeatureMetric.set(relation, bitset); + assertTrue("STANDARD indexMode must set the FROM telemetry bit", bitset.get(FeatureMetric.FROM.ordinal())); + assertFalse("STANDARD indexMode must not set the TS telemetry bit", bitset.get(FeatureMetric.TS.ordinal())); + } } From be5bb04b756084014beeec4a8739c387ea7d0286 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Sun, 5 Jul 2026 11:27:51 +0000 Subject: [PATCH 08/23] [CI] Auto commit changes from spotless --- .../elasticsearch/cluster/routing/IndexRoutingTests.java | 9 +-------- .../mapper/TimeSeriesRoutingHashFieldMapperTests.java | 7 +++---- .../xpack/downsample/TransportDownsampleActionTests.java | 7 +------ 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 38199ba65b84b..e3b7d7e8be01d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -1286,14 +1286,7 @@ private TimeSeriesRoutingFixture indexRoutingForTimeSeriesDimensions( boolean useSyntheticId, IndexMode indexMode ) { - return getIndexRoutingWithSetting( - createdVersion, - shards, - path, - IndexMetadata.INDEX_DIMENSIONS.getKey(), - useSyntheticId, - indexMode - ); + return getIndexRoutingWithSetting(createdVersion, shards, path, IndexMetadata.INDEX_DIMENSIONS.getKey(), useSyntheticId, indexMode); } /** diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java index a49f75a3674e9..812ea25b35ce2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java @@ -113,10 +113,9 @@ public void testEnabledInTsdbAliasMode() throws Exception { .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing path is required") .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-10-29T00:00:00Z"); - DocumentMapper docMapper = createMapperService( - settingsBuilder.build(), - mapping(b -> { b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); }) - ).documentMapper(); + DocumentMapper docMapper = createMapperService(settingsBuilder.build(), mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + })).documentMapper(); int hash = randomInt(); ParsedDocument doc = parseDocument(hash, docMapper, b -> b.field("a", "value")); diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java index 9f4aefe831b97..94dd73d806910 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java @@ -555,12 +555,7 @@ private IndexMetadata.Builder createSourceIndexMetadata(String sourceIndex, int return createSourceIndexMetadata(sourceIndex, primaryShards, replicaShards, IndexMode.TIME_SERIES); } - private IndexMetadata.Builder createSourceIndexMetadata( - String sourceIndex, - int primaryShards, - int replicaShards, - IndexMode indexMode - ) { + private IndexMetadata.Builder createSourceIndexMetadata(String sourceIndex, int primaryShards, int replicaShards, IndexMode indexMode) { return IndexMetadata.builder(sourceIndex) .settings( indexSettings(IndexVersion.current(), randomUUID(), primaryShards, replicaShards).put( From 90efebebc60cafabf8c38ffe9d3d45c7d73b5ee1 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Sun, 5 Jul 2026 14:40:36 +0300 Subject: [PATCH 09/23] Update 152901.yaml --- docs/changelog/152901.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/changelog/152901.yaml b/docs/changelog/152901.yaml index 93cd5657f2529..94a1fd63ed78a 100644 --- a/docs/changelog/152901.yaml +++ b/docs/changelog/152901.yaml @@ -1,8 +1,6 @@ area: TSDB highlight: body: |- - ## Summary - Introduce `IndexMode.TSDB` as a new, first-class index mode for metrics. It currently behaves identically to `TIME_SERIES`, so `index.mode: tsdb` is now accepted anywhere `index.mode: time_series` is — without renaming From 1eb0b65d159badc754740692eebed8371cf74b8f Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Sun, 5 Jul 2026 15:12:33 +0300 Subject: [PATCH 10/23] Stop calling TSDB an alias in test names and comments Rename TsdbAlias-style identifiers to Tsdb and reword comments, since alias has a specific meaning (index aliases) in Elasticsearch. Also add TS-command TSDB coverage in TimeSeriesIT after confirming no production bug: concrete, wildcard, and mixed-mode alias lookups all resolve correctly today. --- .../DataStreamIndexSettingsProviderTests.java | 20 ++--- ...astTimeSeriesIndexCreationActionTests.java | 30 +++---- .../TransportGetDataStreamsActionTests.java | 4 +- .../extras/ScaledFloatFieldMapperTests.java | 4 +- .../cluster/metadata/DataStreamTests.java | 8 +- .../routing/DimensionsExtractorTests.java | 8 +- .../cluster/routing/IndexRoutingTests.java | 4 +- .../routing/RoutingPathExtractorTests.java | 6 +- ...SeriesEligibleWriteWindowLocatorTests.java | 4 +- .../index/IndexSettingsTests.java | 8 +- .../index/IndexSortSettingsTests.java | 4 +- .../index/TimeSeriesModeTests.java | 12 +-- .../index/engine/InternalEngineTests.java | 4 +- .../mapper/GeoPointFieldMapperTests.java | 4 +- ...meSeriesMetadataFieldBlockLoaderTests.java | 12 +-- ...TimeSeriesRoutingHashFieldMapperTests.java | 4 +- .../index/mapper/NumberFieldMapperTests.java | 4 +- .../xpack/core/ilm/DownsampleActionTests.java | 4 +- ...UntilTimeSeriesEndTimePassesStepTests.java | 4 +- .../xpack/esql/action/TimeSeriesIT.java | 86 +++++++++++++++++++ .../xpack/esql/analysis/AnalyzerTests.java | 14 +-- .../local/SubstituteRoundToTests.java | 18 ++-- .../xpack/esql/session/EsqlSessionTests.java | 16 ++-- .../LogsdbIndexModeSettingsProviderTests.java | 8 +- ...dexSettingsProviderLegacyLicenseTests.java | 8 +- .../UnsignedLongFieldMapperTests.java | 4 +- 26 files changed, 194 insertions(+), 108 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index 6a56e6364d156..f180b90151098 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -159,12 +159,12 @@ public void testGetAdditionalIndexSettings() throws Exception { } /** - * The {@code tsdb} alias delegates {@link IndexMode#isTsdb()} to {@code true} just like + * {@link IndexMode#TSDB} resolves {@link IndexMode#isTsdb()} to {@code true} just like * {@link IndexMode#TIME_SERIES}, so creating a backing index with a template index mode of * {@link IndexMode#TSDB} must inject the same start/end time, sequence-number, synthetic id * and dimension settings as {@link IndexMode#TIME_SERIES} does. */ - public void testGetAdditionalIndexSettingsWithTsdbAlias() throws Exception { + public void testGetAdditionalIndexSettingsWithTsdb() throws Exception { ProjectMetadata projectMetadata = emptyProject(); String dataStreamName = "logs-app1"; @@ -605,10 +605,10 @@ public void testGetAdditionalIndexSettingsMigrateToTsdb() { /** * The migration check at index-creation time relies on {@link IndexMode#isTsdb(IndexMode)}, so * a data stream currently in {@code standard} mode must also migrate into time series mode when - * the matching index template specifies the {@link IndexMode#TSDB} alias, not just + * the matching index template specifies {@link IndexMode#TSDB} directly, not just * {@link IndexMode#TIME_SERIES}. */ - public void testGetAdditionalIndexSettingsMigrateToTsdbAlias() { + public void testGetAdditionalIndexSettingsMigrateToTsdbMode() { Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); String dataStreamName = "logs-app1"; IndexMetadata idx = createFirstBackingIndex(dataStreamName).build(); @@ -1070,11 +1070,11 @@ public void testAddNewDimension() throws Exception { /** * Same scenario as {@link #testAddNewDimension()}, but with an index in {@link IndexMode#TSDB} - * (the {@code tsdb} alias) rather than {@link IndexMode#TIME_SERIES}, so that the + * (the {@code tsdb} value) rather than {@link IndexMode#TIME_SERIES}, so that the * {@code assert IndexMode.isTsdb(...)} sanity check in * {@link DataStreamIndexSettingsProvider#onUpdateMappings} is exercised with the alias too. */ - public void testAddNewDimensionWithTsdbAlias() throws Exception { + public void testAddNewDimensionWithTsdb() throws Exception { String newMapping = """ { "_doc": { @@ -1321,11 +1321,11 @@ public void testClusterSettingsDefineSeqNoDisabledDefault() throws Exception { /** * Same scenario as {@link #testClusterSettingsDefineSeqNoDisabledDefault()}, but with a - * template index mode of {@link IndexMode#TSDB} (the {@code tsdb} alias) rather than + * template index mode of {@link IndexMode#TSDB} (the {@code tsdb} value) rather than * {@link IndexMode#TIME_SERIES}, since the sequence-number-disabling gate is keyed off * {@link IndexMode#isTsdb()}. */ - public void testClusterSettingsDefineSeqNoDisabledDefaultWithTsdbAlias() throws Exception { + public void testClusterSettingsDefineSeqNoDisabledDefaultWithTsdb() throws Exception { Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); IndexVersion version = IndexVersionUtils.randomVersionBetween( IndexVersions.TIME_SERIES_DISABLE_SEQUENCE_NUMBERS_DEFAULT, @@ -1451,11 +1451,11 @@ public void testClusterSettingsDefineSyntheticIdEnabledDefault() throws Exceptio /** * Same scenario as {@link #testClusterSettingsDefineSyntheticIdEnabledDefault()}, but with a - * template index mode of {@link IndexMode#TSDB} (the {@code tsdb} alias) rather than + * template index mode of {@link IndexMode#TSDB} (the {@code tsdb} value) rather than * {@link IndexMode#TIME_SERIES}, since the synthetic {@code _id} gate is keyed off * {@link IndexMode#isTsdb()}. */ - public void testClusterSettingsDefineSyntheticIdEnabledDefaultWithTsdbAlias() throws Exception { + public void testClusterSettingsDefineSyntheticIdEnabledDefaultWithTsdb() throws Exception { Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); IndexVersion version = IndexVersionUtils.randomVersionBetween( IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD, diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java index dc73b65d90ce2..710088398f81c 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java @@ -476,11 +476,11 @@ private static IndexMetadata createIndexMetadata(String indexName, Instant start } /** - * Builds a ProjectMetadata with a TSDB data stream whose backing indices use the {@link IndexMode#TSDB} - * alias rather than {@link IndexMode#TIME_SERIES}, to verify that {@code IndexMode.isTsdb} treats the - * alias identically to {@code TIME_SERIES} in {@link TransportPastTimeSeriesIndexCreationAction}. + * Builds a ProjectMetadata with a TSDB data stream whose backing indices use {@link IndexMode#TSDB} + * rather than {@link IndexMode#TIME_SERIES}, to verify that {@code IndexMode.isTsdb} treats the two + * identically in {@link TransportPastTimeSeriesIndexCreationAction}. */ - private ProjectMetadata projectWithTsdbAliasDataStream(List> timeSlices) { + private ProjectMetadata projectWithTsdbDataStream(List> timeSlices) { List backingIndices = new ArrayList<>(); long generation = 1L; for (Tuple slice : timeSlices) { @@ -499,24 +499,24 @@ private ProjectMetadata projectWithTsdbAliasDataStream(List> timeSlices, Instant now) { + /** Builds a ClusterState with an {@link IndexMode#TSDB} data stream whose backing indices cover the given time ranges. */ + private ClusterState stateWithExistingTsdb(List> timeSlices, Instant now) { List> allSlices = new ArrayList<>(timeSlices); allSlices.add(Tuple.tuple(now, now.plus(randomIntBetween(1, 3), ChronoUnit.DAYS))); - return ClusterState.builder(ClusterName.DEFAULT).putProjectMetadata(projectWithTsdbAliasDataStream(allSlices)).build(); + return ClusterState.builder(ClusterName.DEFAULT).putProjectMetadata(projectWithTsdbDataStream(allSlices)).build(); } /** - * Equivalent of {@link #testSortAndRetrieve} that uses the {@link IndexMode#TSDB} alias instead of + * Equivalent of {@link #testSortAndRetrieve} that uses {@link IndexMode#TSDB} directly instead of * {@link IndexMode#TIME_SERIES} for the backing indices, to confirm {@code retrieveSortedTimeWindows} - * (gated by {@code IndexMode.isTsdb}) recognizes the alias mode the same way. + * (gated by {@code IndexMode.isTsdb}) recognizes it the same way. */ - public void testSortAndRetrieveWithTsdbAlias() { + public void testSortAndRetrieveWithTsdb() { Instant start1 = Instant.parse("2024-01-15T00:00:00Z"); Instant start2 = Instant.parse("2024-01-16T00:00:00Z"); Instant start3 = Instant.parse("2024-01-17T00:00:00Z"); Instant end = Instant.parse("2024-01-18T00:00:00Z"); - ProjectMetadata project = projectWithTsdbAliasDataStream( + ProjectMetadata project = projectWithTsdbDataStream( List.of(Tuple.tuple(start3, end), Tuple.tuple(start1, start2), Tuple.tuple(start2, start3)) ); // Add a non-TSDB index to the mixed data stream to confirm it's still excluded from the windows. @@ -538,13 +538,13 @@ public void testSortAndRetrieveWithTsdbAlias() { /** * Equivalent of the happy-path portion of {@link #testCreateIndicesWhenNeeded} that uses the - * {@link IndexMode#TSDB} alias instead of {@link IndexMode#TIME_SERIES}, to confirm + * {@link IndexMode#TSDB} instead of {@link IndexMode#TIME_SERIES}, to confirm * {@code validateDataStream} and the time-range computation (both gated by {@code IndexMode.isTsdb}) - * behave identically for the alias. + * behave identically for it. */ - public void testCreateIndicesWhenNeededWithTsdbAlias() throws Exception { + public void testCreateIndicesWhenNeededWithTsdb() throws Exception { Instant now = Instant.now(); - ClusterState clusterState = stateWithExistingTsdbAlias(List.of(), now); + ClusterState clusterState = stateWithExistingTsdb(List.of(), now); List dayOffsets = List.of(5, 3, 2); // Add two timestamps that fall within one index { diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index c0a4dfe7cdec9..ee0bdde92ffa6 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -323,8 +323,8 @@ public void testGetTimeSeriesDataStreamWithOutOfOrderIndices() { ); } - public void testGetTimeSeriesDataStreamUsingTsdbAlias() { - // Same scenario as testGetTimeSeriesDataStreamWithOutOfOrderIndices, but using the "tsdb" alias for + public void testGetTimeSeriesDataStreamUsingTsdb() { + // Same scenario as testGetTimeSeriesDataStreamWithOutOfOrderIndices, but using the "tsdb" value for // index.mode instead of "time_series". DataStreamTestHelper#getClusterStateWithDataStream hardcodes // IndexMode.TIME_SERIES, so the data stream and backing indices are built by hand here to exercise // IndexMode.isTsdb(...) with IndexMode.TSDB at both call sites in diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java index 743e726464acd..279106d5b370e 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java @@ -358,10 +358,10 @@ public void testTimeSeriesIndexDefault() throws Exception { } /** - * The {@code tsdb} index mode is an alias for {@code time_series} (see {@link IndexMode#isTsdb()}) + * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) * so it must produce the same doc-values-only defaulting for metric fields. */ - public void testTimeSeriesIndexDefaultTsdbAlias() throws Exception { + public void testTimeSeriesIndexDefaultTsdb() throws Exception { var randomMetricType = randomFrom(TimeSeriesParams.MetricType.scalar()); var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 317fecd595a0d..916e4be8d738a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -274,7 +274,7 @@ public void testRolloverUpgradeToTsdbDataStream() { assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TIME_SERIES)); } - public void testRolloverUpgradeToTsdbAliasDataStream() { + public void testRolloverUpgradeToTsdbDataStreamMode() { DataStream ds = DataStreamTestHelper.randomInstance() .copy() .setReplicated(false) @@ -343,7 +343,7 @@ public void testRolloverFromLogsdbToTsdb() { assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TIME_SERIES)); } - public void testRolloverFromLogsdbToTsdbAlias() { + public void testRolloverFromLogsdbToTsdbMode() { DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.LOGSDB).build(); final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); var newCoordinates = ds.nextWriteIndexAndGeneration(project, ds.getDataComponent()); @@ -492,7 +492,7 @@ public void testRolloverFromColumnarToTimeSeries() { assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TIME_SERIES)); } - public void testRolloverFromColumnarToTsdbAlias() { + public void testRolloverFromColumnarToTsdb() { DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.COLUMNAR).build(); final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); var newCoordinates = ds.nextWriteIndexAndGeneration(project, ds.getDataComponent()); @@ -546,7 +546,7 @@ public void testRolloverFromColumnarLogsdbToTimeSeries() { assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TIME_SERIES)); } - public void testRolloverFromColumnarLogsdbToTsdbAlias() { + public void testRolloverFromColumnarLogsdbToTsdb() { DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.LOGSDB_COLUMNAR).build(); final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); var newCoordinates = ds.nextWriteIndexAndGeneration(project, ds.getDataComponent()); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/DimensionsExtractorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/DimensionsExtractorTests.java index 9b8765e10e515..b332e67e7bfbb 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/DimensionsExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/DimensionsExtractorTests.java @@ -145,11 +145,11 @@ public void testTsidIsAttachedToIndexRequest() throws IOException { } /** - * {@code index.mode: tsdb} is a preferred alias for {@code time_series}; it must select the same + * {@code index.mode: tsdb} is a preferred alternative to {@code time_series}; it must select the same * {@link IndexRouting.ExtractFromSource.ForIndexDimensions} strategy and produce byte-identical tsids * and shard assignments. */ - public void testTsdbAliasProducesSameTsidAsTimeSeries() throws IOException { + public void testTsdbProducesSameTsidAsTimeSeries() throws IOException { Map doc = new LinkedHashMap<>(); doc.put("dim.host", "node-7"); doc.put("dim.region", "us-west-2"); @@ -166,8 +166,8 @@ public void testTsdbAliasProducesSameTsidAsTimeSeries() throws IOException { tsdbStrategy.preProcess(tsdbReq); int tsdbShardId = tsdbStrategy.indexShard(tsdbReq); - assertThat("tsid must be identical for time_series and tsdb aliases", tsdbReq.tsid(), equalTo(timeSeriesReq.tsid())); - assertThat("shard id must be identical for time_series and tsdb aliases", tsdbShardId, equalTo(timeSeriesShardId)); + assertThat("tsid must be identical for time_series and tsdb", tsdbReq.tsid(), equalTo(timeSeriesReq.tsid())); + assertThat("shard id must be identical for time_series and tsdb", tsdbShardId, equalTo(timeSeriesShardId)); } public void testResetClearsBuilderBetweenDocuments() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index e3b7d7e8be01d..795e5fb72a83f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -754,11 +754,11 @@ public void testRoutingPathWithSingleBytePrefixTsid() throws IOException { /** * Same scenario as {@link #testRoutingPathWithSingleBytePrefixTsid()} but with {@code index.mode: tsdb}, - * the preferred alias for {@code time_series}. {@link IndexMode#TSDB} must select the same + * the preferred alternative to {@code time_series}. {@link IndexMode#TSDB} must select the same * {@link IndexRouting.ExtractFromSource.ForIndexDimensions} strategy, track the routing hash the same * way, and produce identical shard assignments as {@link IndexMode#TIME_SERIES}. */ - public void testRoutingPathWithSingleBytePrefixTsidUsingTsdbAlias() throws IOException { + public void testRoutingPathWithSingleBytePrefixTsidUsingTsdb() throws IOException { TimeSeriesRoutingFixture fixture = indexRoutingForTimeSeriesDimensions( IndexVersionUtils.randomVersionOnOrAfter(IndexVersions.TSID_SINGLE_PREFIX_BYTE_FEATURE_FLAG), 8, diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingPathExtractorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingPathExtractorTests.java index 8b2b98111ca68..e9322be5817c5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingPathExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingPathExtractorTests.java @@ -123,12 +123,12 @@ public void testNoThrowOnArrayAtNonRoutingField() throws IOException { } /** - * {@code index.mode: tsdb} is a preferred alias for {@code time_series}; it must flow through the same + * {@code index.mode: tsdb} is a preferred alternative to {@code time_series}; it must flow through the same * {@code trackTimeSeriesRoutingHash} logic in {@link IndexRouting.ExtractFromSource}, so the routing hash * that {@link IndexRouting.ExtractFromSource#postProcess} attaches to the request must be identical * regardless of which alias created the index. */ - public void testTsdbAliasProducesSameRoutingHashAsTimeSeries() throws IOException { + public void testTsdbProducesSameRoutingHashAsTimeSeries() throws IOException { Map doc = new LinkedHashMap<>(); doc.put("dim.host", "node-7"); doc.put("dim.region", "us-west-2"); @@ -147,7 +147,7 @@ public void testTsdbAliasProducesSameRoutingHashAsTimeSeries() throws IOExceptio tsdbStrategy.indexShard(tsdbReq); tsdbStrategy.postProcess(tsdbReq); - assertThat("routing hash must be identical for time_series and tsdb aliases", tsdbReq.routing(), equalTo(timeSeriesReq.routing())); + assertThat("routing hash must be identical for time_series and tsdb", tsdbReq.routing(), equalTo(timeSeriesReq.routing())); } public void testReuseAcrossDocumentsClearsBuilderState() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java b/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java index 07b8d65644ea7..ced206ef6b61f 100644 --- a/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java +++ b/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java @@ -84,8 +84,8 @@ public void testWriteWindowDefinedByRetention() { } } - public void testWriteWindowDefinedByRetentionWithTsdbAlias() { - // IndexMode.TSDB is a preferred alias for IndexMode.TIME_SERIES, so a data stream configured + public void testWriteWindowDefinedByRetentionWithTsdb() { + // IndexMode.TSDB is a preferred alternative to IndexMode.TIME_SERIES, so a data stream configured // with index.mode: tsdb must be treated identically by the eligible write window logic. TimeValue retention = TimeValue.timeValueDays(30); DataStreamLifecycle lifecycle = DataStreamLifecycle.dataLifecycleBuilder().dataRetention(retention).build(); diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 5b01993241f67..8087837308b19 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -288,7 +288,7 @@ public void testSliceEnabledSettingRejectedForTimeSeriesMode() { * canonical {@code time_series} name regardless of which alias was actually configured (mirroring * {@code tsdbMode()}'s behavior elsewhere in {@link IndexMode}). */ - public void testSliceEnabledSettingRejectedForTsdbAlias() { + public void testSliceEnabledSettingRejectedForTsdb() { assumeTrue("slice indexing feature flag must be enabled", SliceIndexing.SLICE_FEATURE_FLAG.isEnabled()); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, @@ -1044,7 +1044,7 @@ public void testSyntheticIdCorrectSettings() { assertTrue(indexMetadata.useTimeSeriesSyntheticId()); } - public void testSyntheticIdCorrectSettingsWithTsdbAlias() { + public void testSyntheticIdCorrectSettingsWithTsdb() { IndexVersion version = IndexVersionUtils.randomVersionBetween( IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94, IndexVersion.current() @@ -1089,7 +1089,7 @@ public void testSyntheticIdDefaultValueTrue() { assertTrue(indexMetadata.useTimeSeriesSyntheticId()); } - public void testSyntheticIdDefaultValueTrueWithTsdbAlias() { + public void testSyntheticIdDefaultValueTrueWithTsdb() { IndexVersion version = IndexVersionUtils.randomVersionBetween( IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD, IndexVersion.current() @@ -1128,7 +1128,7 @@ public void testSyntheticIdDefaultValueFalse() { assertFalse(indexMetadata.useTimeSeriesSyntheticId()); } - public void testSyntheticIdDefaultValueFalseWithTsdbAlias() { + public void testSyntheticIdDefaultValueFalseWithTsdb() { IndexVersion version = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD); IndexMode mode = IndexMode.TSDB; String codec = CodecService.DEFAULT_CODEC; diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index 2c9d42b7376a2..c037328715b35 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -375,7 +375,7 @@ public void testTimeSeriesModeNoTimestamp() { assertThat(e.getMessage(), equalTo("unknown index sort field:[@timestamp] required by [index.mode=time_series]")); } - public void testTimeSeriesModeWithTsdbAlias() { + public void testTimeSeriesModeWithTsdb() { IndexSettings indexSettings = indexSettings( Settings.builder() .put(IndexSettings.MODE.getKey(), "tsdb") @@ -390,7 +390,7 @@ public void testTimeSeriesModeWithTsdbAlias() { assertThat(sort.getSort()[1].getField(), equalTo("@timestamp")); } - public void testTimeSeriesModeNoTimestampWithTsdbAlias() { + public void testTimeSeriesModeNoTimestampWithTsdb() { IndexSettings indexSettings = indexSettings( Settings.builder() .put(IndexSettings.MODE.getKey(), "tsdb") diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index cc68b1c398843..3043a11432f51 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -55,11 +55,11 @@ public void testPartitioned() { } /** - * The {@code tsdb} alias delegates {@link IndexMode#validateWithOtherSettings} to + * {@link IndexMode#TSDB} delegates {@link IndexMode#validateWithOtherSettings} to * {@link IndexMode#TIME_SERIES}, so it must reject the same incompatible settings with the * same (canonical {@code time_series}) error message. */ - public void testPartitionedWithTsdbAlias() { + public void testPartitionedWithTsdb() { Settings s = Settings.builder() .put(getSettingsWithMode("tsdb", randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) @@ -100,7 +100,7 @@ public void testWithoutRoutingPath() { assertThat(e.getMessage(), containsString("[index.mode=time_series] requires a non-empty [index.routing_path]")); } - public void testWithoutRoutingPathWithTsdbAlias() { + public void testWithoutRoutingPathWithTsdb() { Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "tsdb").build(); Exception e = expectThrows( IllegalArgumentException.class, @@ -166,7 +166,7 @@ public void testRequiredRouting() { assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); } - public void testRequiredRoutingWithTsdbAlias() { + public void testRequiredRoutingWithTsdb() { Settings s = getSettingsWithMode("tsdb", randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z"); var mapperService = new TestMapperServiceBuilder().settings(s).applyDefaultMapping(false).build(); Exception e = expectThrows( @@ -193,7 +193,7 @@ public void testValidateAliasWithSearchRouting() { assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); } - public void testValidateAliasWithTsdbAlias() { + public void testValidateAliasWithTsdb() { Settings s = getSettingsWithMode("tsdb", randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z"); assertSame(IndexMode.TSDB, IndexSettings.MODE.get(s)); IndexSettings.MODE.get(s).validateAlias(null, null); // Doesn't throw exception @@ -411,7 +411,7 @@ private Settings getSettings(String routingPath, String startTime, String endTim /** * Same as {@link #getSettings(String, String, String)} but with an explicit {@code index.mode} - * value, so that tests can assert the {@code tsdb} alias behaves identically to {@code time_series}. + * value, so that tests can assert the {@code tsdb} value behaves identically to {@code time_series}. */ private Settings getSettingsWithMode(String mode, String routingPath, String startTime, String endTime) { return Settings.builder() diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 380e7227537f8..0d481c33f325e 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -8305,8 +8305,8 @@ public void testIndexBatchTimeSeriesPhase2() throws IOException { runIndexBatchTimeSeriesPhase2(IndexMode.TIME_SERIES); } - public void testIndexBatchTimeSeriesPhase2WithTsdbAlias() throws IOException { - // IndexMode.TSDB is a preferred alias for TIME_SERIES (delegates isTsdb() etc. to + public void testIndexBatchTimeSeriesPhase2WithTsdb() throws IOException { + // IndexMode.TSDB is a preferred alternative to TIME_SERIES (delegates isTsdb() etc. to // TIME_SERIES), so it must exercise the exact same batch phase 2 behavior. runIndexBatchTimeSeriesPhase2(IndexMode.TSDB); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index 81348f302c93d..af9120e071cf2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -171,10 +171,10 @@ public void testTimeSeriesIndexDefault() throws Exception { } /** - * The {@code tsdb} index mode is an alias for {@code time_series} (see {@link IndexMode#isTsdb()}) + * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) * so it must produce the same doc-values-only defaulting for metric fields. */ - public void testTimeSeriesIndexDefaultTsdbAlias() throws Exception { + public void testTimeSeriesIndexDefaultTsdb() throws Exception { var positionMetricType = TimeSeriesParams.MetricType.POSITION; var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java index bd28d963115d5..5ab874317f918 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java @@ -61,10 +61,10 @@ public class TimeSeriesMetadataFieldBlockLoaderTests extends MapperServiceTestCa private static final Settings TSDB_SYNTHETIC_SETTINGS = tsdbSettings(null, "host"); /** - * Same as {@link #TSDB_SYNTHETIC_SETTINGS} but using the {@link IndexMode#TSDB} alias instead of + * Same as {@link #TSDB_SYNTHETIC_SETTINGS} but using {@link IndexMode#TSDB} directly instead of * {@link IndexMode#TIME_SERIES}; {@link IndexMode#isTsdb()} treats both identically. */ - private static final Settings TSDB_ALIAS_SYNTHETIC_SETTINGS = tsdbAliasSettings(null, "host"); + private static final Settings TSDB_MODE_SYNTHETIC_SETTINGS = tsdbModeSettings(null, "host"); /** * TSDB index with {@code index.mapping.source.mode: stored}. Stored source preserves the raw @@ -171,7 +171,7 @@ private static Settings tsdbSettings(SourceFieldMapper.Mode sourceMode, String r return builder.build(); } - private static Settings tsdbAliasSettings(SourceFieldMapper.Mode sourceMode, String routingPath) { + private static Settings tsdbModeSettings(SourceFieldMapper.Mode sourceMode, String routingPath) { Settings.Builder builder = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) @@ -194,12 +194,12 @@ public void testDimensionsOnly() throws IOException { } /** - * The {@code tsdb} index mode is an alias for {@code time_series} (see {@link IndexMode#isTsdb()}) + * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) * so {@link TimeSeriesMetadataFieldBlockLoader} must be selected and behave identically. */ - public void testDimensionsOnlyTsdbAlias() throws IOException { + public void testDimensionsOnlyTsdb() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_ALIAS_SYNTHETIC_SETTINGS, + TSDB_MODE_SYNTHETIC_SETTINGS, MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of()) ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java index 812ea25b35ce2..a9be59f22fd70 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java @@ -104,11 +104,11 @@ public void testEnabledInTimeSeriesMode() throws Exception { } /** - * The {@code tsdb} index mode is an alias for {@code time_series} (see {@link IndexMode#isTsdb()}) + * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) * so {@link TimeSeriesRoutingHashFieldMapper} must be enabled identically. */ @SuppressWarnings("unchecked") - public void testEnabledInTsdbAliasMode() throws Exception { + public void testEnabledInTsdbMode() throws Exception { Settings.Builder settingsBuilder = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.name()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing path is required") .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 7c875f4505916..020d67afa7919 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -290,10 +290,10 @@ public void testTimeSeriesIndexDefault() throws Exception { } /** - * The {@code tsdb} index mode is an alias for {@code time_series} (see {@link IndexMode#isTsdb()}) + * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) * so it must produce the same doc-values-only defaulting for metric fields. */ - public void testTimeSeriesIndexDefaultTsdbAlias() throws Exception { + public void testTimeSeriesIndexDefaultTsdb() throws Exception { var randomMetricType = randomFrom(TimeSeriesParams.MetricType.scalar()); var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java index ac8b9b02a55d0..31bb54897b460 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java @@ -312,7 +312,7 @@ public void testDownsamplingPrerequisitesStep() { } } - public void testDownsamplingPrerequisitesStepWithTsdbAlias() { + public void testDownsamplingPrerequisitesStepWithTsdb() { DateHistogramInterval fixedInterval = ConfigTestHelpers.randomInterval(); boolean withForceMerge = randomBoolean(); DownsampleAction action = new DownsampleAction(fixedInterval, WAIT_TIMEOUT, withForceMerge, randomSamplingMethod()); @@ -323,7 +323,7 @@ public void testDownsamplingPrerequisitesStepWithTsdbAlias() { randomAlphaOfLengthBetween(1, 10) ); { - // indices using the tsdb alias of the time series index mode execute the action + // indices using the tsdb value of the time series index mode execute the action BranchingStep branchingStep = getFirstBranchingStep(action, phase, nextStepKey, withForceMerge); Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TSDB) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java index de79d48a7662f..c128aa30ddb34 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java @@ -151,12 +151,12 @@ public void onFailure(Exception e) { } /** - * {@link org.elasticsearch.index.IndexMode#TSDB} is a preferred alias for + * {@link org.elasticsearch.index.IndexMode#TSDB} is a preferred alternative to * {@link org.elasticsearch.index.IndexMode#TIME_SERIES} and must gate {@link WaitUntilTimeSeriesEndTimePassesStep} * identically. {@link #testEvaluateCondition()} only exercises indices whose {@code index.mode} is the literal * {@code time_series}, so this repeats the time-series-specific assertions using {@code index.mode: tsdb} directly. */ - public void testEvaluateConditionTsdbAlias() { + public void testEvaluateConditionTsdb() { Instant currentTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); Instant startTimeLapsed = currentTime.minus(6, ChronoUnit.HOURS); Instant endTimeLapsed = currentTime.minus(2, ChronoUnit.HOURS); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java index 8b9f8033f6668..568bea3bf1ec8 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.action; +import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.TsidBuilder; import org.elasticsearch.common.Randomness; @@ -78,6 +79,91 @@ public void testEmpty() { run("TS empty_index | LIMIT 1").close(); } + public void testTsdbModeProbe() throws Exception { + Settings settings = Settings.builder().put("mode", "tsdb").putList("routing_path", List.of("host")).build(); + client().admin() + .indices() + .prepareCreate("hosts_tsdb_probe") + .setSettings(settings) + .setMapping( + "@timestamp", + "type=date", + "host", + "type=keyword,time_series_dimension=true", + "cpu", + "type=double,time_series_metric=gauge" + ) + .get(); + long timestamp = DEFAULT_DATE_TIME_FORMATTER.parseMillis("2024-04-15T00:00:00Z"); + for (int i = 0; i < 10; i++) { + client().prepareIndex("hosts_tsdb_probe") + .setSource("@timestamp", timestamp + i * 1000, "host", "h" + (i % 2), "cpu", (double) i) + .get(); + } + client().admin().indices().prepareRefresh("hosts_tsdb_probe").get(); + try (EsqlQueryResponse resp = run("TS hosts_tsdb_probe METADATA _index_mode | STATS load=avg(cpu) BY host | SORT host")) { + List> rows = EsqlTestUtils.getValuesList(resp); + assertThat(rows, hasSize(2)); + } + try (EsqlQueryResponse resp = run("TS hosts_tsdb_probe METADATA _index_mode, _index | STATS BY _index, _index_mode")) { + List> rows = EsqlTestUtils.getValuesList(resp); + assertThat(rows, equalTo(List.of(List.of("hosts_tsdb_probe", "tsdb")))); + } + // mixed wildcard: "hosts" fixture is mode=time_series, "hosts_tsdb_probe" is mode=tsdb + try (EsqlQueryResponse resp = run("TS hosts* METADATA _index_mode, _index | STATS BY _index, _index_mode | SORT _index")) { + List> rows = EsqlTestUtils.getValuesList(resp); + assertThat(rows, equalTo(List.of(List.of("hosts", "time_series"), List.of("hosts_tsdb_probe", "tsdb")))); + } + } + + /** + * Mirrors how a real data stream ends up with backing indices in both spellings after migrating + * to the new preferred {@code tsdb} spelling (e.g. OTel/Prometheus streams, see docs/changelog/152901.yaml): + * a single logical name (here, an alias) fronting one {@code time_series}-mode index and one + * {@code tsdb}-mode index. {@code TS name}, a concrete (non-wildcard) request, must resolve both. + */ + public void testTsdbModeProbeMixedAlias() throws Exception { + String mapping = """ + { + "properties": { + "@timestamp": { "type": "date" }, + "host": { "type": "keyword", "time_series_dimension": true }, + "cpu": { "type": "double", "time_series_metric": "gauge" } + } + } + """; + client().admin() + .indices() + .prepareCreate("hosts_mixed_alias_v1") + .setSettings( + Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").putList("routing_path", List.of("host")).build() + ) + .setMapping(mapping) + .addAlias(new Alias("hosts_mixed_alias")) + .get(); + client().admin() + .indices() + .prepareCreate("hosts_mixed_alias_v2") + .setSettings(Settings.builder().put(IndexSettings.MODE.getKey(), "tsdb").putList("routing_path", List.of("host")).build()) + .setMapping(mapping) + .addAlias(new Alias("hosts_mixed_alias")) + .get(); + client().prepareIndex("hosts_mixed_alias_v1").setSource("@timestamp", "2024-04-15T00:00:00Z", "host", "h0", "cpu", 1.0).get(); + client().prepareIndex("hosts_mixed_alias_v2").setSource("@timestamp", "2024-04-15T01:00:00Z", "host", "h1", "cpu", 2.0).get(); + client().admin().indices().prepareRefresh("hosts_mixed_alias").get(); + + try ( + EsqlQueryResponse resp = run("TS hosts_mixed_alias METADATA _index_mode, _index | STATS BY _index, _index_mode | SORT _index") + ) { + List> rows = EsqlTestUtils.getValuesList(resp); + assertThat(rows, equalTo(List.of(List.of("hosts_mixed_alias_v1", "time_series"), List.of("hosts_mixed_alias_v2", "tsdb")))); + } + try (EsqlQueryResponse resp = run("TS hosts_mixed_alias | STATS total=sum(cpu)")) { + List> rows = EsqlTestUtils.getValuesList(resp); + assertThat(rows, equalTo(List.of(List.of(3.0)))); + } + } + record Doc( Collection project, String host, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 12267d25e9430..6b3d7a6e3fb2b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2734,11 +2734,11 @@ public void testImplicitTimestampSortForTsQuery() { assertThat(orderChild.name(), equalTo("@timestamp")); } - public void testImplicitTimestampSortForTsQueryWithTsdbAlias() { + public void testImplicitTimestampSortForTsQueryWithTsdb() { // Same as testImplicitTimestampSortForTsQuery, but the index uses IndexMode.TSDB instead of // IndexMode.TIME_SERIES: AddImplicitTimestampSort is gated on IndexMode#isTsdb(), so both - // aliases must be treated identically. - var plan = tsdbAlias().query("TS test"); + // must be treated identically. + var plan = tsdbMode().query("TS test"); var limit = as(plan, Limit.class); var orderBy = as(limit.child(), OrderBy.class); @@ -5052,10 +5052,10 @@ public void testImplicitCastingForAggregateMetricDouble() { assertProjection(plan2, "s1", "s2", "min", "count", "avg", "cluster", "time_bucket"); } - public void testImplicitCastingForAggregateMetricDoubleWithTsdbAlias() { + public void testImplicitCastingForAggregateMetricDoubleWithTsdb() { // Same as testImplicitCastingForAggregateMetricDouble's TS branch, but the indices use // IndexMode.TSDB instead of IndexMode.TIME_SERIES: ImplicitCastAggregateMetricDoubles is - // gated on IndexMode#isTsdb(), so both aliases must be treated identically. + // gated on IndexMode#isTsdb(), so both must be treated identically. assumeTrue( "aggregate metric double implicit casting must be available", EsqlCapabilities.Cap.AGGREGATE_METRIC_DOUBLE_V0.isEnabled() @@ -5588,11 +5588,11 @@ private static TestAnalyzer tsdb() { } /** - * Same fixture as {@link #tsdb()}, but built with the {@link IndexMode#TSDB} alias instead of + * Same fixture as {@link #tsdb()}, but built with {@link IndexMode#TSDB} directly instead of * {@link IndexMode#TIME_SERIES}. Used to prove that {@code index.mode: tsdb} is treated identically * to {@code index.mode: time_series} by rules gated on {@link IndexMode#isTsdb()}. */ - private static TestAnalyzer tsdbAlias() { + private static TestAnalyzer tsdbMode() { return analyzer().addIndex("test", "tsdb-mapping.json", IndexMode.TSDB); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java index ee4f81e3a1a59..df6bb6c112417 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java @@ -1055,14 +1055,14 @@ public Map targetShards() { } /** - * {@link org.elasticsearch.index.IndexMode#TSDB} is a preferred alias for + * {@link org.elasticsearch.index.IndexMode#TSDB} is a preferred alternative to * {@link IndexMode#TIME_SERIES}: it behaves identically everywhere, including in the * {@link ReplaceRoundToWithQueryAndTags} rule. This mirrors * {@link #testRoundToWithTimeSeriesIndices} but resolves the source index (and the target * shards reported by {@link SearchStats}) with {@link IndexMode#TSDB} instead, verifying the * filter-by-filter rewrite still applies identically. */ - public void testRoundToWithTimeSeriesIndicesUsingTsdbAlias() { + public void testRoundToWithTimeSeriesIndicesUsingTsdb() { Map minValue = Map.of( "@timestamp", DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-10-20T12:15:03.360Z") @@ -1082,7 +1082,7 @@ public Map targetShards() { return Map.of(new ShardId(new Index("id", "n/a"), 1), indexMetadata); } }; - TestPlannerOptimizer tsdbPlannerOptimizer = tsdbAliasPlannerOptimizer(); + TestPlannerOptimizer tsdbPlannerOptimizer = tsdbPlannerOptimizer(); // enable filter-by-filter for rate aggregations { String q = """ @@ -1109,10 +1109,10 @@ public Map targetShards() { /** * Directly exercises {@link ReplaceRoundToWithQueryAndTags#adjustedRoundingPointsThreshold}: since - * {@link IndexMode#TSDB} is a preferred alias for {@link IndexMode#TIME_SERIES}, both must double the + * {@link IndexMode#TSDB} is a preferred alternative to {@link IndexMode#TIME_SERIES}, both must double the * rounding points threshold identically. */ - public void testAdjustedRoundingPointsThresholdForTsdbAlias() { + public void testAdjustedRoundingPointsThresholdForTsdb() { int threshold = between(1, 1000); int timeSeriesThreshold = ReplaceRoundToWithQueryAndTags.adjustedRoundingPointsThreshold( searchStats, @@ -1126,12 +1126,12 @@ public void testAdjustedRoundingPointsThresholdForTsdbAlias() { } // Builds an analyzer/planner pair mirroring AbstractLocalPhysicalPlanOptimizerTests#init's - // plannerOptimizerTimeSeries/timeSeriesAnalyzer, but resolving the "k8s" index with the TSDB alias + // plannerOptimizerTimeSeries/timeSeriesAnalyzer, but resolving the "k8s" index with IndexMode.TSDB // instead of TIME_SERIES, to verify IndexMode.TSDB is handled identically. - private TestPlannerOptimizer tsdbAliasPlannerOptimizer() { + private TestPlannerOptimizer tsdbPlannerOptimizer() { var timeSeriesMapping = loadMapping("k8s-mappings.json"); var tsdbIndex = IndexResolution.valid(EsIndexGenerator.esIndex("k8s", timeSeriesMapping, Map.of("k8s", IndexMode.TSDB))); - Analyzer tsdbAliasAnalyzer = new Analyzer( + Analyzer tsdbAnalyzer = new Analyzer( testAnalyzerContext( EsqlTestUtils.TEST_CFG, TEST_FUNCTION_REGISTRY, @@ -1143,7 +1143,7 @@ private TestPlannerOptimizer tsdbAliasPlannerOptimizer() { ); return new TestPlannerOptimizer( config, - tsdbAliasAnalyzer, + tsdbAnalyzer, new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small(), TransportVersion.current())) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java index 7d1e7e922309a..a32598ee0cca9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java @@ -87,9 +87,9 @@ public void testShouldNotRetryWildcardTimeSeriesResolution() { ); } - // IndexMode.TSDB is a preferred alias for IndexMode.TIME_SERIES (isTsdb() is true for both); this pins that - // shouldRetryConcreteTimeSeriesResolution treats the alias identically to the canonical constant. - public void testShouldRetryConcreteTsdbAliasResolution() { + // IndexMode.TSDB is a preferred alternative to IndexMode.TIME_SERIES (isTsdb() is true for both); this pins + // that shouldRetryConcreteTimeSeriesResolution treats it identically to the canonical constant. + public void testShouldRetryConcreteTsdbResolution() { assertTrue( EsqlSession.shouldRetryConcreteTimeSeriesResolution( IndexMode.TSDB, @@ -99,7 +99,7 @@ public void testShouldRetryConcreteTsdbAliasResolution() { ); } - public void testShouldNotRetryWildcardTsdbAliasResolution() { + public void testShouldNotRetryWildcardTsdbResolution() { assertFalse( EsqlSession.shouldRetryConcreteTimeSeriesResolution( IndexMode.TSDB, @@ -111,16 +111,16 @@ public void testShouldNotRetryWildcardTsdbAliasResolution() { // createQueryFilter builds the _index_mode filter used to resolve the concrete backing indices of a TS/METRICS // source. The requested indexMode is a fixed sentinel (TS always declares IndexMode.TIME_SERIES) but the - // concrete backing indices may be configured with either [index.mode=time_series] or its alias - // [index.mode=tsdb], so the filter must match both terms regardless of which indexMode triggered it. - public void testCreateQueryFilterForTimeSeriesMatchesBothIndexModeAliases() { + // concrete backing indices may be configured with either [index.mode=time_series] or [index.mode=tsdb], + // so the filter must match both terms regardless of which indexMode triggered it. + public void testCreateQueryFilterForTimeSeriesMatchesBothIndexModeValues() { QueryBuilder filter = EsqlSession.createQueryFilter(IndexMode.TIME_SERIES, null); TermsQueryBuilder termsFilter = as(filter, TermsQueryBuilder.class); assertThat(termsFilter.fieldName(), equalTo(IndexModeFieldMapper.NAME)); assertThat(termsFilter.values(), containsInAnyOrder(IndexMode.TIME_SERIES.getName(), IndexMode.TSDB.getName())); } - public void testCreateQueryFilterForTsdbAliasMatchesBothIndexModeAliases() { + public void testCreateQueryFilterForTsdbMatchesBothIndexModeValues() { QueryBuilder filter = EsqlSession.createQueryFilter(IndexMode.TSDB, null); TermsQueryBuilder termsFilter = as(filter, TermsQueryBuilder.class); assertThat(termsFilter.fieldName(), equalTo(IndexModeFieldMapper.NAME)); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index c553c0cdcc158..dfbff3eb8367a 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -278,7 +278,7 @@ public void testOnExplicitTimeSeriesIndex() throws IOException { } public void testOnExplicitTsdbIndex() throws IOException { - // Same scenario as testOnExplicitTimeSeriesIndex, but using the "tsdb" alias for index.mode. + // Same scenario as testOnExplicitTimeSeriesIndex, but using the "tsdb" value for index.mode. final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( logsdbLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() @@ -695,7 +695,7 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { public void testNewIndexHasSyntheticSourceUsageTsdb() throws IOException { // Same scenario as testNewIndexHasSyntheticSourceUsageTimeSeries, but exercised through the "tsdb" index.mode - // alias to make sure IndexMode.isTsdb(...) treats it identically to "time_series". + // value to make sure IndexMode.isTsdb(...) treats it identically to "time_series". String dataStreamName = DATA_STREAM_NAME; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); String mapping = """ @@ -855,9 +855,9 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { assertThat(newMapperServiceCounter.get(), equalTo(4)); } - public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceTsdbAlias() { + public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceTsdb() { // Same scenario as the IndexMode.TIME_SERIES case in testGetAdditionalIndexSettingsDowngradeFromSyntheticSource, - // but using the IndexMode.TSDB alias to make sure the template mode injection in + // but using IndexMode.TSDB directly to make sure the template mode injection in // LogsdbIndexModeSettingsProvider#buildIndexMetadataForMapperService treats it identically. String dataStreamName = DATA_STREAM_NAME; ProjectMetadata project = DataStreamTestHelper.getProjectWithDataStreams( diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java index 08333ffc64287..d6bf9e90a74de 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java @@ -148,8 +148,8 @@ public void testGetAdditionalIndexSettingsTsdb() throws IOException { assertEquals(Settings.EMPTY, result); } - public void testGetAdditionalIndexSettingsTsdbAlias() throws IOException { - // Same scenario as testGetAdditionalIndexSettingsTsdb, but using the IndexMode.TSDB alias to make sure the + public void testGetAdditionalIndexSettingsTsdbMode() throws IOException { + // Same scenario as testGetAdditionalIndexSettingsTsdb, but using IndexMode.TSDB directly to make sure the // legacy license check in LogsdbIndexModeSettingsProvider#isLegacyLicensedUsageOfSyntheticSourceAllowed // treats it identically to IndexMode.TIME_SERIES. Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); @@ -216,8 +216,8 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception assertEquals(expected, result); } - public void testGetAdditionalIndexSettingsTsdbAfterCutoffDateAlias() throws Exception { - // Same scenario as testGetAdditionalIndexSettingsTsdbAfterCutoffDate, but using the IndexMode.TSDB alias to + public void testGetAdditionalIndexSettingsTsdbAfterCutoffDateMode() throws Exception { + // Same scenario as testGetAdditionalIndexSettingsTsdbAfterCutoffDate, but using IndexMode.TSDB directly to // make sure the legacy license check treats it identically to IndexMode.TIME_SERIES. long start = LocalDateTime.of(2025, 2, 5, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); License license = createGoldOrPlatinumLicense(start); diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index b77788d18023a..f8c347487739f 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -353,10 +353,10 @@ public void testTimeSeriesIndexDefault() throws Exception { } /** - * The {@code tsdb} index mode is an alias for {@code time_series} (see {@link IndexMode#isTsdb()}) + * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) * so it must produce the same doc-values-only defaulting for metric fields. */ - public void testTimeSeriesIndexDefaultTsdbAlias() throws Exception { + public void testTimeSeriesIndexDefaultTsdb() throws Exception { var randomMetricType = randomFrom(TimeSeriesParams.MetricType.scalar()); var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); From e9679bec1a3494ca005748465883ac209500ba53 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Sun, 5 Jul 2026 15:24:32 +0300 Subject: [PATCH 11/23] Finish removing alias wording for TSDB from remaining files Covers IndexMode.java's own javadoc plus test files touched by other concurrent agents that the earlier sweep missed. --- .../lifecycle/DataStreamLifecycleServiceTests.java | 2 +- .../src/main/java/org/elasticsearch/index/IndexMode.java | 6 +++--- .../main/java/org/elasticsearch/index/IndexSettings.java | 2 +- .../java/org/elasticsearch/index/TsdbIndexModeTests.java | 2 +- .../org/elasticsearch/index/mapper/MappingLookupTests.java | 4 ++-- .../xpack/downsample/TransportDownsampleActionTests.java | 6 +++--- .../org/elasticsearch/xpack/esql/session/EsqlSession.java | 4 ++-- .../xpack/esql/datasources/DatasetRewriterTests.java | 2 +- .../physical/local/PushExpressionsToFieldLoadTests.java | 4 ++-- .../xpack/esql/telemetry/FeatureMetricTests.java | 7 ++++--- 10 files changed, 20 insertions(+), 19 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index e0e19644407de..e413627581257 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -1239,7 +1239,7 @@ public void testTimeSeriesIndicesStillWithinTimeBounds() { } /** - * Same coverage as {@link #testTimeSeriesIndicesStillWithinTimeBounds()} but using the {@link IndexMode#TSDB} alias + * Same coverage as {@link #testTimeSeriesIndicesStillWithinTimeBounds()} but using {@link IndexMode#TSDB} * instead of {@link IndexMode#TIME_SERIES}, to ensure {@code isTsdb()} gates this behaviour identically for both. */ public void testTimeSeriesIndicesStillWithinTimeBoundsWithTsdbIndexMode() { diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 1293f63795913..1c701711b7538 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -308,7 +308,7 @@ public boolean isColumnar() { } }, /** - * Alias of {@link #TIME_SERIES} with the preferred name {@code tsdb}. Behaves identically + * Preferred alternative to {@link #TIME_SERIES} with the name {@code tsdb}. Behaves identically * to {@link #TIME_SERIES} in every respect other than {@link #getName()}; existing * {@code time_series} indices are unaffected and {@link #fromString} accepts both spellings. */ @@ -1051,7 +1051,7 @@ public boolean isStrictColumnar() { /** * Whether this index mode represents a time series (tsdb) index, regardless of whether it - * was declared as {@code time_series} or its preferred alias {@code tsdb}. + * was declared as {@code time_series} or {@code tsdb}. */ public boolean isTsdb() { return this == TIME_SERIES || this == TSDB; @@ -1068,7 +1068,7 @@ public static boolean isTsdb(@Nullable IndexMode mode) { /** * Whether the given raw {@code index.mode} setting value names a time series (tsdb) index - * mode, accepting both {@code time_series} and its preferred alias {@code tsdb}, + * mode, accepting both {@code time_series} and {@code tsdb}, * case-insensitively. Use this instead of comparing against {@link #TIME_SERIES}'s or * {@link #TSDB}'s {@link #getName()} directly when the value hasn't been parsed with * {@link #fromString} yet. diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 82a6278a26bca..c51a82630bc43 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -778,7 +778,7 @@ public void validate(Boolean enabled) {} @Override public void validate(Boolean enabled, Map, Object> settings) { if (enabled) { - // Verify if index mode is a time series mode (TIME_SERIES or its alias TSDB) + // Verify if index mode is a time series mode (TIME_SERIES or TSDB) var indexMode = (IndexMode) settings.get(MODE); if (indexMode.isTsdb() == false) { throw new IllegalArgumentException( diff --git a/server/src/test/java/org/elasticsearch/index/TsdbIndexModeTests.java b/server/src/test/java/org/elasticsearch/index/TsdbIndexModeTests.java index 6d8d338912829..3a5ed25454df9 100644 --- a/server/src/test/java/org/elasticsearch/index/TsdbIndexModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TsdbIndexModeTests.java @@ -22,7 +22,7 @@ import static org.hamcrest.Matchers.equalTo; /** - * {@link IndexMode#TSDB} is a preferred alias for {@link IndexMode#TIME_SERIES}: both parse from + * {@link IndexMode#TSDB} is a preferred alternative to {@link IndexMode#TIME_SERIES}: both parse from * (and behave identically for) the same {@code index.mode} settings, but {@code time_series} * remains the canonical emitted/persisted string so existing indices are unaffected. */ diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java index 763a8fb111377..4742c49ebabb6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java @@ -199,7 +199,7 @@ public MetricType getMetricType() { assertThat(e.getMessage(), equalTo("Field [metric] attempted to shadow a time_series_metric")); tsMappingLookup.validateDoesNotShadow("plain"); - // The tsdb index mode is an alias for time_series (see IndexMode#isTsdb()), so it must reject + // IndexMode.TSDB is equivalent to time_series (see IndexMode#isTsdb()), so it must reject // shadowing of dimension/metric fields identically. MappingLookup tsdbMappingLookup = createMappingLookup( List.of(dimMapper, metricMapper, plainMapper), @@ -257,7 +257,7 @@ public MetricType getMetricType() { ) ); - // The tsdb index mode is an alias for time_series (see IndexMode#isTsdb()), so it must reject + // IndexMode.TSDB is equivalent to time_series (see IndexMode#isTsdb()), so it must reject // the same shadowing on construction. Exception tsdbException = expectThrows( MapperParsingException.class, diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java index 94dd73d806910..2569659217939 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java @@ -267,13 +267,13 @@ public void testDownsampling() { } /** - * Same as {@link #testDownsampling()} but using {@link IndexMode#TSDB}, the preferred alias for + * Same as {@link #testDownsampling()} but using {@link IndexMode#TSDB}, the preferred alternative to * {@link IndexMode#TIME_SERIES}, to configure the source index. This verifies that the * {@code index.mode}-gated validation in {@link TransportDownsampleAction} relies on * {@link IndexMode#isTsdb()} rather than an exact match against {@link IndexMode#TIME_SERIES}, - * so a source index configured with the {@code tsdb} alias is downsampled successfully as well. + * so a source index configured with {@code index.mode: tsdb} is downsampled successfully as well. */ - public void testDownsamplingWithTsdbIndexModeAlias() { + public void testDownsamplingWithTsdbIndexMode() { var projectMetadata = ProjectMetadata.builder(projectId) .put(createSourceIndexMetadata(sourceIndex, primaryShards, replicaShards, IndexMode.TSDB)) .build(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 3846ca477e2ff..e5182ceff64ec 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -2001,9 +2001,9 @@ private void preAnalyzeFlatMainIndices( static QueryBuilder createQueryFilter(IndexMode indexMode, QueryBuilder requestFilter) { return switch (indexMode) { case IndexMode.TIME_SERIES, IndexMode.TSDB -> { - // Match either alias: the requested indexMode is a fixed sentinel (e.g. the TS command always + // Match both values: the requested indexMode is a fixed sentinel (e.g. the TS command always // declares IndexMode.TIME_SERIES), but the concrete backing indices may be configured with - // either [index.mode=time_series] or its alias [index.mode=tsdb]. + // either [index.mode=time_series] or [index.mode=tsdb]. var indexModeFilter = new TermsQueryBuilder( IndexModeFieldMapper.NAME, IndexMode.TIME_SERIES.getName(), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java index 4bda8ec272318..c18027158eb8b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java @@ -159,7 +159,7 @@ public void testIndexModeNonStandardRejected() { // with IndexMode.LOGSDB pointing at a dataset name. The LOGSDB branch is defensive code for // any future path that might set it. There is no IT analogue for LOGSDB — this unit case // pins the rejection-message contract. - // TSDB is included alongside TIME_SERIES to prove the "index.mode: tsdb" alias is rejected + // TSDB is included alongside TIME_SERIES to prove "index.mode: tsdb" is rejected // with the exact same message, since IndexMode.TSDB behaves identically to TIME_SERIES. DataSource parent = dataSource("s3_parent", Map.of()); Dataset dataset = new Dataset("logs", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java index a0d6a093469a1..e38bd9dc47e94 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java @@ -357,7 +357,7 @@ public void testRoundToInTsEval() { /** * Verifies ROUND_TO on a long field is NOT pushed to the block loader when - * the shard's index metadata reports the {@link IndexMode#TSDB} alias + * the shard's index metadata reports {@link IndexMode#TSDB} * rather than {@link IndexMode#TIME_SERIES}. {@code TSDB} behaves * identically to {@code TIME_SERIES} for {@code hasTimeSeriesShards()}, * so this mirrors {@link #testRoundToInTsEval()}. @@ -880,7 +880,7 @@ public Map targetShards() { /** * Same as {@link #tsSearchStats()} but reports the shard's index metadata - * using the {@link IndexMode#TSDB} alias instead of {@link IndexMode#TIME_SERIES}. + * using {@link IndexMode#TSDB} instead of {@link IndexMode#TIME_SERIES}. */ private static SearchStats tsdbSearchStats() { return new EsqlTestUtils.TestSearchStats() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetricTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetricTests.java index 6b518b696886f..37adab33869e9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetricTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetricTests.java @@ -59,10 +59,11 @@ public void testExternalRelationIsExcluded() { /** * {@link FeatureMetric#TS} and {@link FeatureMetric#FROM} branch on {@code EsRelation#indexMode()#isTsdb()}. - * {@code IndexMode.TSDB} is a preferred alias for {@code IndexMode.TIME_SERIES} (isTsdb() is true for both), so - * both constants must flip the TS bit - never the FROM bit - exactly like the canonical TIME_SERIES constant. + * {@code IndexMode.TSDB} is a preferred alternative to {@code IndexMode.TIME_SERIES} (isTsdb() is true for + * both), so both constants must flip the TS bit - never the FROM bit - exactly like the canonical TIME_SERIES + * constant. */ - public void testTimeSeriesRelationSetsTsMetricForBothIndexModeAliases() { + public void testTimeSeriesRelationSetsTsMetricForBothIndexModes() { for (IndexMode mode : List.of(IndexMode.TIME_SERIES, IndexMode.TSDB)) { EsRelation relation = relation(mode); BitSet bitset = new BitSet(); From 4a153094bbf2b378f250721ffb2813e134518bea Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Sun, 5 Jul 2026 18:10:44 +0300 Subject: [PATCH 12/23] Make TSDB validation errors name the actual mode IndexMode.TSDB delegated validateWithOtherSettings/validateMapping/ validateAlias straight to TIME_SERIES's method bodies, where unqualified getName()/this always resolved to TIME_SERIES. Extract shared static utilities that take the calling mode explicitly, so a tsdb-mode index's validation errors correctly say index.mode=tsdb instead of always canonicalizing to time_series. Add TSDB coverage for the sort-setting and routing-partition-size rejection paths. --- .../org/elasticsearch/index/IndexMode.java | 208 ++++++++++-------- .../index/IndexSettingsTests.java | 10 +- .../index/TimeSeriesModeTests.java | 54 ++++- 3 files changed, 172 insertions(+), 100 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 1c701711b7538..56b4278445fd5 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -130,93 +130,17 @@ public SourceFieldMapper.Mode defaultSourceMode() { TIME_SERIES("time_series") { @Override void validateWithOtherSettings(Map, Object> settings) { - if (settings.get(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING) != Integer.valueOf(1)) { - throw new IllegalArgumentException(error(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING)); - } - - var settingsWithIndexMode = Settings.builder().put(IndexSettings.MODE.getKey(), getName()).build(); - - for (Setting unsupported : TIME_SERIES_UNSUPPORTED) { - if (false == Objects.equals(unsupported.getDefault(settingsWithIndexMode), settings.get(unsupported))) { - throw new IllegalArgumentException(error(unsupported)); - } - } - if (Boolean.TRUE.equals(settings.get(IndexSettings.SLICE_ENABLED))) { - throw new IllegalArgumentException( - "The setting [" - + IndexSettings.SLICE_ENABLED.getKey() - + "] cannot be used with [" - + IndexSettings.MODE.getKey() - + "=" - + IndexMode.TIME_SERIES.getName() - + "]." - ); - } - Setting> routingPath = IndexMetadata.INDEX_ROUTING_PATH; - if (isEmpty(settings, routingPath) && isEmpty(settings, IndexMetadata.INDEX_DIMENSIONS)) { - // index.dimensions is a private setting that only gets populated for data streams. - // We don't include it in the error message to not confuse users that are manually creating time series indices - // which is the only case where this error can occur. - throw new IllegalArgumentException(tsdbMode() + " requires a non-empty [" + routingPath.getKey() + "]"); - } - } - - private static boolean isEmpty(Map, Object> settings, Setting> setting) { - return Objects.equals(setting.getDefault(Settings.EMPTY), settings.get(setting)); - } - - private static String error(Setting unsupported) { - return tsdbMode() + " is incompatible with [" + unsupported.getKey() + "]"; + validateTimeSeriesSettings(this, settings); } @Override public void validateMapping(MappingLookup lookup, Settings settings) { - if (((RoutingFieldMapper) lookup.getMapper(RoutingFieldMapper.NAME)).required()) { - throw new IllegalArgumentException(routingRequiredBad()); - } - validateTemporalityField(lookup, settings); - } - - private static void validateTemporalityField(MappingLookup lookup, Settings settings) { - String temporalityFieldName = IndexSettings.TIME_SERIES_TEMPORALITY_FIELD.get(settings); - if (temporalityFieldName == null || temporalityFieldName.isEmpty()) { - return; - } - MappedFieldType fieldType = lookup.getFieldType(temporalityFieldName); - if (fieldType == null) { - // We silently ignore this case because of composable templates: - // composable templates are verified without being merged with the default mapping - // This means the temporality field might not exist during verification, - // but we'll add it automatically when the index is created through the default mapping - return; - } - if (fieldType instanceof KeywordFieldMapper.KeywordFieldType == false) { - throw new IllegalArgumentException( - "[" - + IndexSettings.TIME_SERIES_TEMPORALITY_FIELD.getKey() - + "] field [" - + temporalityFieldName - + "] must be of type [keyword] but is [" - + fieldType.typeName() - + "]" - ); - } - if (fieldType.isDimension() == false) { - throw new IllegalArgumentException( - "[" - + IndexSettings.TIME_SERIES_TEMPORALITY_FIELD.getKey() - + "] field [" - + temporalityFieldName - + "] must be a [time_series_dimension]" - ); - } + validateTimeSeriesMapping(this, lookup, settings); } @Override public void validateAlias(@Nullable String indexRouting, @Nullable String searchRouting) { - if (indexRouting != null || searchRouting != null) { - throw new IllegalArgumentException(routingRequiredBad()); - } + validateTimeSeriesAlias(this, indexRouting, searchRouting); } @Override @@ -254,10 +178,6 @@ public TimestampBounds getTimestampBound(IndexMetadata indexMetadata) { return new TimestampBounds(indexMetadata.getTimeSeriesStart(), indexMetadata.getTimeSeriesEnd()); } - private static String routingRequiredBad() { - return "routing is forbidden on CRUD operations that target indices in " + tsdbMode(); - } - @Override public MetadataFieldMapper timeSeriesIdFieldMapper(MappingParserContext c) { return TimeSeriesIdFieldMapper.getInstance(c); @@ -315,17 +235,17 @@ public boolean isColumnar() { TSDB("tsdb") { @Override void validateWithOtherSettings(Map, Object> settings) { - TIME_SERIES.validateWithOtherSettings(settings); + validateTimeSeriesSettings(this, settings); } @Override public void validateMapping(MappingLookup lookup, Settings settings) { - TIME_SERIES.validateMapping(lookup, settings); + validateTimeSeriesMapping(this, lookup, settings); } @Override public void validateAlias(@Nullable String indexRouting, @Nullable String searchRouting) { - TIME_SERIES.validateAlias(indexRouting, searchRouting); + validateTimeSeriesAlias(this, indexRouting, searchRouting); } @Override @@ -898,6 +818,122 @@ private static CompressedXContent createDefaultMapping(CheckedConsumer, Object> settings) { + if (settings.get(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING) != Integer.valueOf(1)) { + throw new IllegalArgumentException(unsupportedTimeSeriesSettingError(mode, IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING)); + } + + var settingsWithIndexMode = Settings.builder().put(IndexSettings.MODE.getKey(), mode.getName()).build(); + + for (Setting unsupported : TIME_SERIES_UNSUPPORTED) { + if (false == Objects.equals(unsupported.getDefault(settingsWithIndexMode), settings.get(unsupported))) { + throw new IllegalArgumentException(unsupportedTimeSeriesSettingError(mode, unsupported)); + } + } + if (Boolean.TRUE.equals(settings.get(IndexSettings.SLICE_ENABLED))) { + throw new IllegalArgumentException( + "The setting [" + + IndexSettings.SLICE_ENABLED.getKey() + + "] cannot be used with [" + + IndexSettings.MODE.getKey() + + "=" + + mode.getName() + + "]." + ); + } + Setting> routingPath = IndexMetadata.INDEX_ROUTING_PATH; + if (isDefaultRoutingPathSetting(settings, routingPath) && isDefaultRoutingPathSetting(settings, IndexMetadata.INDEX_DIMENSIONS)) { + // index.dimensions is a private setting that only gets populated for data streams. + // We don't include it in the error message to not confuse users that are manually creating time series indices + // which is the only case where this error can occur. + throw new IllegalArgumentException( + "[" + IndexSettings.MODE.getKey() + "=" + mode.getName() + "] requires a non-empty [" + routingPath.getKey() + "]" + ); + } + } + + private static boolean isDefaultRoutingPathSetting(Map, Object> settings, Setting> setting) { + return Objects.equals(setting.getDefault(Settings.EMPTY), settings.get(setting)); + } + + private static String unsupportedTimeSeriesSettingError(IndexMode mode, Setting unsupported) { + return "[" + IndexSettings.MODE.getKey() + "=" + mode.getName() + "] is incompatible with [" + unsupported.getKey() + "]"; + } + + /** + * Shared {@code validateMapping} implementation for {@link #TIME_SERIES} and {@link #TSDB}. Takes the + * calling {@code mode} explicitly (see {@link #validateTimeSeriesSettings}) so the routing-required + * error names the mode that was actually configured. + */ + private static void validateTimeSeriesMapping(IndexMode mode, MappingLookup lookup, Settings settings) { + if (((RoutingFieldMapper) lookup.getMapper(RoutingFieldMapper.NAME)).required()) { + throw new IllegalArgumentException(routingRequiredBad(mode)); + } + validateTemporalityField(lookup, settings); + } + + private static void validateTemporalityField(MappingLookup lookup, Settings settings) { + String temporalityFieldName = IndexSettings.TIME_SERIES_TEMPORALITY_FIELD.get(settings); + if (temporalityFieldName == null || temporalityFieldName.isEmpty()) { + return; + } + MappedFieldType fieldType = lookup.getFieldType(temporalityFieldName); + if (fieldType == null) { + // We silently ignore this case because of composable templates: + // composable templates are verified without being merged with the default mapping + // This means the temporality field might not exist during verification, + // but we'll add it automatically when the index is created through the default mapping + return; + } + if (fieldType instanceof KeywordFieldMapper.KeywordFieldType == false) { + throw new IllegalArgumentException( + "[" + + IndexSettings.TIME_SERIES_TEMPORALITY_FIELD.getKey() + + "] field [" + + temporalityFieldName + + "] must be of type [keyword] but is [" + + fieldType.typeName() + + "]" + ); + } + if (fieldType.isDimension() == false) { + throw new IllegalArgumentException( + "[" + + IndexSettings.TIME_SERIES_TEMPORALITY_FIELD.getKey() + + "] field [" + + temporalityFieldName + + "] must be a [time_series_dimension]" + ); + } + } + + /** + * Shared {@code validateAlias} implementation for {@link #TIME_SERIES} and {@link #TSDB}. See + * {@link #validateTimeSeriesSettings} for why {@code mode} is passed explicitly. + */ + private static void validateTimeSeriesAlias(IndexMode mode, @Nullable String indexRouting, @Nullable String searchRouting) { + if (indexRouting != null || searchRouting != null) { + throw new IllegalArgumentException(routingRequiredBad(mode)); + } + } + + private static String routingRequiredBad(IndexMode mode) { + return "routing is forbidden on CRUD operations that target indices in [" + + IndexSettings.MODE.getKey() + + "=" + + mode.getName() + + "]"; + } + static final List> VALIDATE_WITH_SETTINGS = List.copyOf( Stream.concat( Stream.of( diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 8087837308b19..2aa4cec6b529a 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -282,11 +282,9 @@ public void testSliceEnabledSettingRejectedForTimeSeriesMode() { } /** - * {@code index.mode: tsdb} must be rejected exactly like {@code time_series}. Note that the error - * message itself always names {@code time_series}: {@link IndexMode#TSDB} delegates - * {@code validateWithOtherSettings} straight to {@link IndexMode#TIME_SERIES}, whose message uses the - * canonical {@code time_series} name regardless of which alias was actually configured (mirroring - * {@code tsdbMode()}'s behavior elsewhere in {@link IndexMode}). + * {@code index.mode: tsdb} must be rejected exactly like {@code time_series}, and the rejection + * message must name the mode that was actually configured ({@code tsdb}), not the canonical + * {@code time_series} name. */ public void testSliceEnabledSettingRejectedForTsdb() { assumeTrue("slice indexing feature flag must be enabled", SliceIndexing.SLICE_FEATURE_FLAG.isEnabled()); @@ -306,7 +304,7 @@ public void testSliceEnabledSettingRejectedForTsdb() { ); assertThat(exception.getMessage(), containsString("index.slice.enabled")); assertThat(exception.getMessage(), containsString("index.mode")); - assertThat(exception.getMessage(), containsString("time_series")); + assertThat(exception.getMessage(), containsString("tsdb")); } @TestLogging(reason = "testing warning logging", value = "org.elasticsearch.index.IndexSettings:WARN") diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index 3043a11432f51..1a4a5d644f513 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -55,9 +55,9 @@ public void testPartitioned() { } /** - * {@link IndexMode#TSDB} delegates {@link IndexMode#validateWithOtherSettings} to - * {@link IndexMode#TIME_SERIES}, so it must reject the same incompatible settings with the - * same (canonical {@code time_series}) error message. + * {@link IndexMode#TSDB} shares its {@link IndexMode#validateWithOtherSettings} implementation with + * {@link IndexMode#TIME_SERIES}, so it must reject the same incompatible settings - naming the mode + * that was actually configured ({@code tsdb}) in the error message. */ public void testPartitionedWithTsdb() { Settings s = Settings.builder() @@ -67,7 +67,7 @@ public void testPartitionedWithTsdb() { .build(); IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.routing_partition_size]")); + assertThat(e.getMessage(), equalTo("[index.mode=tsdb] is incompatible with [index.routing_partition_size]")); } public void testSortField() { @@ -77,6 +77,22 @@ public void testSortField() { assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.field]")); } + /** + * Same as {@link #testSortField()} but using {@link IndexMode#TSDB}. This exercises the + * {@code validateTimeSeriesSettings} path in {@link IndexMode} that builds a synthetic + * {@code Settings} from the calling mode's own name to compute this setting's default value for + * comparison, so it must reject the same way and name {@code tsdb} in the error message. + */ + public void testSortFieldWithTsdb() { + Settings s = Settings.builder() + .put(getSettingsWithMode("tsdb", randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "a") + .build(); + IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); + Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); + assertThat(e.getMessage(), equalTo("[index.mode=tsdb] is incompatible with [index.sort.field]")); + } + public void testSortMode() { Settings s = Settings.builder().put(getSettings()).put(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last").build(); IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); @@ -84,6 +100,17 @@ public void testSortMode() { assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.missing]")); } + /** Same as {@link #testSortMode()} but using {@link IndexMode#TSDB}. */ + public void testSortModeWithTsdb() { + Settings s = Settings.builder() + .put(getSettingsWithMode("tsdb", randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) + .put(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last") + .build(); + IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); + Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); + assertThat(e.getMessage(), equalTo("[index.mode=tsdb] is incompatible with [index.sort.missing]")); + } + public void testSortOrder() { Settings s = Settings.builder().put(getSettings()).put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "desc").build(); IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); @@ -91,6 +118,17 @@ public void testSortOrder() { assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.order]")); } + /** Same as {@link #testSortOrder()} but using {@link IndexMode#TSDB}. */ + public void testSortOrderWithTsdb() { + Settings s = Settings.builder() + .put(getSettingsWithMode("tsdb", randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) + .put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "desc") + .build(); + IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); + Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); + assertThat(e.getMessage(), equalTo("[index.mode=tsdb] is incompatible with [index.sort.order]")); + } + public void testWithoutRoutingPath() { Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); Exception e = expectThrows( @@ -106,7 +144,7 @@ public void testWithoutRoutingPathWithTsdb() { IllegalArgumentException.class, () -> new IndexSettings(IndexSettingsTests.newIndexMeta("test", s), Settings.EMPTY) ); - assertThat(e.getMessage(), containsString("[index.mode=time_series] requires a non-empty [index.routing_path]")); + assertThat(e.getMessage(), containsString("[index.mode=tsdb] requires a non-empty [index.routing_path]")); } public void testWithEmptyRoutingPath() { @@ -173,7 +211,7 @@ public void testRequiredRoutingWithTsdb() { IllegalArgumentException.class, () -> withMapping(mapperService, topMapping(b -> b.startObject("_routing").field("required", true).endObject())) ); - assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); + assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=tsdb]")); } public void testValidateAlias() { @@ -199,10 +237,10 @@ public void testValidateAliasWithTsdb() { IndexSettings.MODE.get(s).validateAlias(null, null); // Doesn't throw exception Exception e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s).validateAlias("r", null)); - assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); + assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=tsdb]")); e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s).validateAlias(null, "r")); - assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); + assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=tsdb]")); } public void testRoutingPathMatchesObject() throws IOException { From c61ab62627527d31a4e1793e7c6689b3564057c2 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 6 Jul 2026 10:33:14 +0300 Subject: [PATCH 13/23] Add tsdb_index_mode REST capability Mirrors the existing logsdb/vectordb_document/columnar index mode capabilities so YAML tests can gate on cluster support for tsdb. --- .../rest/action/admin/indices/CreateIndexCapabilities.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java index 411aa505f7705..1282bdda2d025 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java @@ -39,6 +39,11 @@ public class CreateIndexCapabilities { */ private static final String VECTORDB_DOCUMENT_INDEX_MODE_CAPABILITY = "vectordb_document_index_mode"; + /** + * Support for the 'tsdb' index mode + */ + private static final String TSDB_INDEX_MODE_CAPABILITY = "tsdb_index_mode"; + private static final String NESTED_DENSE_VECTOR_SYNTHETIC_TEST = "nested_dense_vector_synthetic_test"; private static final String POORLY_FORMATTED_BAD_REQUEST = "poorly_formatted_bad_request"; @@ -67,6 +72,7 @@ public class CreateIndexCapabilities { caps.add(COLUMNAR_INDEX_MODES_CAPABILITY); } caps.add(VECTORDB_DOCUMENT_INDEX_MODE_CAPABILITY); + caps.add(TSDB_INDEX_MODE_CAPABILITY); CAPABILITIES = Set.copyOf(caps); } } From 0e540d0d2748363778bce4674186083de66a94ad Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 6 Jul 2026 16:46:12 +0300 Subject: [PATCH 14/23] Address review feedback on TSDB test coverage - Fix testGetAdditionalIndexSettingsMigrateToTsdbMode: it exercised IndexMode.TSDB but asserted on hardcoded "time_series", a copy-paste leftover from its TIME_SERIES sibling test. - Add testRolloverBetweenTimeSeriesAndTsdbDataStreamModes covering rollover directly between TIME_SERIES and TSDB in both directions, not just upgrading from standard. --- .../DataStreamIndexSettingsProviderTests.java | 6 ++--- .../cluster/metadata/DataStreamTests.java | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index f180b90151098..1c4f1872d6fc9 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -631,12 +631,12 @@ public void testGetAdditionalIndexSettingsMigrateToTsdbMode() { additionalSettings ); Settings result = additionalSettings.build(); - // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this + // The index.time_series.end_time setting requires index.mode to be set to tsdb adding it here so that we read this // setting: // (in production the index.mode setting is usually provided in an index or component template) - result = builder().put(result).put("index.mode", "time_series").build(); + result = builder().put(result).put("index.mode", "tsdb").build(); assertThat(result.size(), equalTo(maybeAdjustIndexSettingCount(3))); - assertThat(result.get(IndexSettings.MODE.getKey()), equalTo("time_series")); + assertThat(result.get(IndexSettings.MODE.getKey()), equalTo("tsdb")); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); if (expectedDisabledSequenceNumbers) { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 916e4be8d738a..2fea95f323360 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -292,6 +292,28 @@ public void testRolloverUpgradeToTsdbDataStreamMode() { assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TSDB)); } + /** + * {@link IndexMode#TIME_SERIES} and {@link IndexMode#TSDB} behave identically, so a data stream + * already using one of them must also be able to roll over into the other - not just upgrade from + * {@code standard} - covering both directions of the switch. + */ + public void testRolloverBetweenTimeSeriesAndTsdbDataStreamModes() { + for (IndexMode fromMode : List.of(IndexMode.TIME_SERIES, IndexMode.TSDB)) { + IndexMode toMode = fromMode == IndexMode.TIME_SERIES ? IndexMode.TSDB : IndexMode.TIME_SERIES; + DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(fromMode).build(); + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + var newCoordinates = ds.nextWriteIndexAndGeneration(project, ds.getDataComponent()); + + var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), toMode, null); + assertThat(rolledDs.getName(), equalTo(ds.getName())); + assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); + assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1)); + assertTrue(rolledDs.getIndices().containsAll(ds.getIndices())); + assertTrue(rolledDs.getIndices().contains(rolledDs.getWriteIndex())); + assertThat(rolledDs.getIndexMode(), equalTo(toMode)); + } + } + public void testRolloverUpgradeToLogsdbDataStream() { DataStream ds = DataStreamTestHelper.randomInstance() .copy() From cb824e2b9f0c7decade3e81a3a06bc6e827ebbc9 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:27:47 +0300 Subject: [PATCH 15/23] Add `IndexMode.TSDB` as a preferred alias for TIME_SERIES The new `IndexMode.TSDB` will be automatically selected for new and existing OTel and Prometheus data streams, ensuring a seamless querying experience. --- docs/changelog/152901.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/docs/changelog/152901.yaml b/docs/changelog/152901.yaml index 94a1fd63ed78a..fbc05f23155bb 100644 --- a/docs/changelog/152901.yaml +++ b/docs/changelog/152901.yaml @@ -4,13 +4,9 @@ highlight: Introduce `IndexMode.TSDB` as a new, first-class index mode for metrics. It currently behaves identically to `TIME_SERIES`, so `index.mode: tsdb` is now accepted anywhere `index.mode: time_series` is — without renaming - or changing anything about existing `time_series` indices. - - The new mode will be automatically selected and applied to new and - existing OTel and Prometheus data streams, with newer backing indices - using the `TSDB` mode. The querying experience and downsampling support - remains seamless, regardless of the mix of `TSDB` and `TIME_SERIES` - indices in a data stream. + or changing anything about existing `time_series` indices. The querying + experience and downsampling support remains seamless, regardless of the + mix of `TSDB` and `TIME_SERIES` indices in a data stream. notable: true title: Add `IndexMode.TSDB` as a preferred alias for TIME_SERIES From 260d1e66bb5a03e61e515b1606255e22d70c8dd0 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Mon, 6 Jul 2026 20:45:26 +0300 Subject: [PATCH 16/23] Apply suggestion from @felixbarny Co-authored-by: Felix Barnsteiner --- docs/changelog/152901.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/152901.yaml b/docs/changelog/152901.yaml index fbc05f23155bb..db632847d3c5d 100644 --- a/docs/changelog/152901.yaml +++ b/docs/changelog/152901.yaml @@ -2,7 +2,7 @@ area: TSDB highlight: body: |- Introduce `IndexMode.TSDB` as a new, first-class index mode for metrics. - It currently behaves identically to `TIME_SERIES`, so `index.mode: tsdb` + It behaves identically to `TIME_SERIES`, so `index.mode: tsdb` is now accepted anywhere `index.mode: time_series` is — without renaming or changing anything about existing `time_series` indices. The querying experience and downsampling support remains seamless, regardless of the From 57594f9e672a60ede66e9a0ec8f75bd185aea311 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 6 Jul 2026 21:27:02 +0300 Subject: [PATCH 17/23] Merge duplicated TSDB/TIME_SERIES unit tests Collapse TIME_SERIES/TSDB test method pairs into single tests using randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB), matching the existing LOGSDB/LOGSDB_COLUMNAR randomization pattern. Also fixes a ClassCastException in NumberFieldMapperTests's shared testTimeSeriesIndexDefault, uncovered by this merge for subclasses whose field type isn't a NumberFieldMapper.NumberFieldType. --- .../DataStreamIndexSettingsProviderTests.java | 315 ++---------------- ...astTimeSeriesIndexCreationActionTests.java | 102 ++---- .../TransportGetDataStreamsActionTests.java | 65 +--- .../DataStreamLifecycleServiceTests.java | 79 +++-- .../extras/ScaledFloatFieldMapperTests.java | 19 +- .../cluster/routing/IndexRoutingTests.java | 32 +- ...SeriesEligibleWriteWindowLocatorTests.java | 23 +- .../index/IndexSettingsTests.java | 98 +----- .../index/IndexSortSettingsTests.java | 36 +- .../index/TimeSeriesModeTests.java | 127 +++---- .../index/engine/InternalEngineTests.java | 6 +- .../mapper/GeoPointFieldMapperTests.java | 18 +- .../index/mapper/MappingLookupTests.java | 39 +-- ...meSeriesMetadataFieldBlockLoaderTests.java | 52 +-- ...TimeSeriesRoutingHashFieldMapperTests.java | 19 +- .../index/mapper/NumberFieldMapperTests.java | 20 +- .../xpack/core/ilm/DownsampleActionTests.java | 56 +--- ...UntilTimeSeriesEndTimePassesStepTests.java | 130 ++------ .../TransportDownsampleActionTests.java | 61 +--- .../xpack/esql/session/EsqlSession.java | 26 +- .../xpack/esql/analysis/AnalyzerTests.java | 81 +---- .../PushExpressionsToFieldLoadTests.java | 42 +-- .../local/SubstituteRoundToTests.java | 90 +---- .../xpack/esql/session/EsqlSessionTests.java | 48 +-- .../LogsdbIndexModeSettingsProviderTests.java | 111 +----- ...dexSettingsProviderLegacyLicenseTests.java | 80 +---- .../UnsignedLongFieldMapperTests.java | 18 +- 27 files changed, 312 insertions(+), 1481 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index 1c4f1872d6fc9..e3cb6873d162c 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -88,83 +88,14 @@ int maybeAdjustIndexSettingCount(int baseCount) { return count; } - public void testGetAdditionalIndexSettings() throws Exception { - ProjectMetadata projectMetadata = emptyProject(); - String dataStreamName = "logs-app1"; - - Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); - Settings settings = Settings.builder() - .put("index.dimensions_tsid_strategy_enabled", indexDimensionsTsidStrategyEnabledSetting) - .build(); - String mapping = """ - { - "_doc": { - "properties": { - "field1": { - "type": "long" - }, - "field2": { - "type": "keyword" - }, - "field3": { - "type": "keyword", - "time_series_dimension": true - }, - "field4": { - "type": "long", - "time_series_dimension": true - }, - "field5": { - "type": "ip", - "time_series_dimension": true - }, - "field6": { - "type": "boolean", - "time_series_dimension": true - } - } - } - } - """; - Settings.Builder additionalSettings = builder(); - provider.provideAdditionalSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 1), - dataStreamName, - IndexMode.TIME_SERIES, - projectMetadata, - now, - settings, - List.of(new CompressedXContent(mapping)), - indexVersion, - additionalSettings - ); - Settings result = additionalSettings.build(); - // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: - // (in production the index.mode setting is usually provided in an index or component template) - result = builder().put(result).put("index.mode", "time_series").build(); - assertThat(result.size(), equalTo(maybeAdjustIndexSettingCount(4))); - assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); - assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); - assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - if (expectedIndexDimensionsTsidOptimizationEnabled) { - assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field3", "field4", "field5", "field6")); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), empty()); - } else { - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field3", "field4", "field5", "field6")); - assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty()); - } - if (expectedDisabledSequenceNumbers) { - assertThat(IndexSettings.DISABLE_SEQUENCE_NUMBERS.get(result), equalTo(true)); - } - } - /** * {@link IndexMode#TSDB} resolves {@link IndexMode#isTsdb()} to {@code true} just like * {@link IndexMode#TIME_SERIES}, so creating a backing index with a template index mode of * {@link IndexMode#TSDB} must inject the same start/end time, sequence-number, synthetic id * and dimension settings as {@link IndexMode#TIME_SERIES} does. */ - public void testGetAdditionalIndexSettingsWithTsdb() throws Exception { + public void testGetAdditionalIndexSettings() throws Exception { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); ProjectMetadata projectMetadata = emptyProject(); String dataStreamName = "logs-app1"; @@ -206,7 +137,7 @@ public void testGetAdditionalIndexSettingsWithTsdb() throws Exception { provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TSDB, + mode, projectMetadata, now, settings, @@ -215,11 +146,11 @@ public void testGetAdditionalIndexSettingsWithTsdb() throws Exception { additionalSettings ); Settings result = additionalSettings.build(); - // The index.time_series.end_time setting requires index.mode to be set to tsdb adding it here so that we read this setting: + // The index.time_series.end_time setting requires index.mode to be set adding it here so that we read this setting: // (in production the index.mode setting is usually provided in an index or component template) - result = builder().put(result).put("index.mode", "tsdb").build(); + result = builder().put(result).put("index.mode", mode.getName()).build(); assertThat(result.size(), equalTo(maybeAdjustIndexSettingCount(4))); - assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TSDB)); + assertThat(IndexSettings.MODE.get(result), equalTo(mode)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); if (expectedIndexDimensionsTsidOptimizationEnabled) { @@ -567,48 +498,14 @@ public void testGetAdditionalIndexSettingsNonTsdbTemplate() { assertThat(result.size(), equalTo(0)); } - public void testGetAdditionalIndexSettingsMigrateToTsdb() { - Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); - String dataStreamName = "logs-app1"; - IndexMetadata idx = createFirstBackingIndex(dataStreamName).build(); - DataStream existingDataStream = newInstance(dataStreamName, List.of(idx.getIndex())); - ProjectMetadata projectMetadata = ProjectMetadata.builder(randomProjectIdOrDefault()) - .dataStreams(Map.of(dataStreamName, existingDataStream), Map.of()) - .build(); - - Settings settings = Settings.EMPTY; - Settings.Builder additionalSettings = builder(); - provider.provideAdditionalSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 2), - dataStreamName, - IndexMode.TIME_SERIES, - projectMetadata, - now, - settings, - List.of(), - indexVersion, - additionalSettings - ); - Settings result = additionalSettings.build(); - // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: - // (in production the index.mode setting is usually provided in an index or component template) - result = builder().put(result).put("index.mode", "time_series").build(); - assertThat(result.size(), equalTo(maybeAdjustIndexSettingCount(3))); - assertThat(result.get(IndexSettings.MODE.getKey()), equalTo("time_series")); - assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); - assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - if (expectedDisabledSequenceNumbers) { - assertThat(IndexSettings.DISABLE_SEQUENCE_NUMBERS.get(result), equalTo(true)); - } - } - /** * The migration check at index-creation time relies on {@link IndexMode#isTsdb(IndexMode)}, so * a data stream currently in {@code standard} mode must also migrate into time series mode when * the matching index template specifies {@link IndexMode#TSDB} directly, not just * {@link IndexMode#TIME_SERIES}. */ - public void testGetAdditionalIndexSettingsMigrateToTsdbMode() { + public void testGetAdditionalIndexSettingsMigrateToTsdb() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); String dataStreamName = "logs-app1"; IndexMetadata idx = createFirstBackingIndex(dataStreamName).build(); @@ -622,7 +519,7 @@ public void testGetAdditionalIndexSettingsMigrateToTsdbMode() { provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, - IndexMode.TSDB, + mode, projectMetadata, now, settings, @@ -631,12 +528,11 @@ public void testGetAdditionalIndexSettingsMigrateToTsdbMode() { additionalSettings ); Settings result = additionalSettings.build(); - // The index.time_series.end_time setting requires index.mode to be set to tsdb adding it here so that we read this - // setting: + // The index.time_series.end_time setting requires index.mode to be set adding it here so that we read this setting: // (in production the index.mode setting is usually provided in an index or component template) - result = builder().put(result).put("index.mode", "tsdb").build(); + result = builder().put(result).put("index.mode", mode.getName()).build(); assertThat(result.size(), equalTo(maybeAdjustIndexSettingCount(3))); - assertThat(result.get(IndexSettings.MODE.getKey()), equalTo("tsdb")); + assertThat(result.get(IndexSettings.MODE.getKey()), equalTo(mode.getName())); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); if (expectedDisabledSequenceNumbers) { @@ -1046,35 +942,12 @@ public void testDynamicTemplatePrecedence() throws Exception { } } - public void testAddNewDimension() throws Exception { - String newMapping = """ - { - "_doc": { - "properties": { - "field1": { - "type": "keyword", - "time_series_dimension": true - }, - "field2": { - "type": "keyword", - "time_series_dimension": true - } - } - } - } - """; - Settings result = onUpdateMappings("field1", "field1", newMapping); - assertThat(result.size(), equalTo(1)); - assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field2")); - } - /** - * Same scenario as {@link #testAddNewDimension()}, but with an index in {@link IndexMode#TSDB} - * (the {@code tsdb} value) rather than {@link IndexMode#TIME_SERIES}, so that the - * {@code assert IndexMode.isTsdb(...)} sanity check in - * {@link DataStreamIndexSettingsProvider#onUpdateMappings} is exercised with the alias too. + * {@link IndexMode#TSDB} (the {@code tsdb} value) must be handled the same as + * {@link IndexMode#TIME_SERIES}, so that the {@code assert IndexMode.isTsdb(...)} sanity check + * in {@link DataStreamIndexSettingsProvider#onUpdateMappings} is exercised with the alias too. */ - public void testAddNewDimensionWithTsdb() throws Exception { + public void testAddNewDimension() throws Exception { String newMapping = """ { "_doc": { @@ -1091,7 +964,7 @@ public void testAddNewDimensionWithTsdb() throws Exception { } } """; - Settings result = onUpdateMappings("field1", "field1", newMapping, IndexMode.TSDB); + Settings result = onUpdateMappings("field1", "field1", newMapping, randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB)); assertThat(result.size(), equalTo(1)); assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field2")); } @@ -1257,75 +1130,12 @@ private Settings onUpdateMappings(String routingPath, String dimensions, String return additionalSettings.build(); } - public void testClusterSettingsDefineSeqNoDisabledDefault() throws Exception { - Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); - IndexVersion version = IndexVersionUtils.randomVersionBetween( - IndexVersions.TIME_SERIES_DISABLE_SEQUENCE_NUMBERS_DEFAULT, - IndexVersion.current() - ); - String dataStreamName = "metrics-app1"; - - // With seq_no_disabled=false, the provider must NOT set index.disable_sequence_numbers - DataStreamIndexSettingsProvider providerWithSeqNoEnabled = new DataStreamIndexSettingsProvider( - im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), - Settings.builder().put(DataStreamIndexSettingsProvider.SUPPORT_SEQ_NO_DISABLED.getKey(), false).build() - ); - Settings.Builder additionalSettings = builder(); - providerWithSeqNoEnabled.provideAdditionalSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 1), - dataStreamName, - IndexMode.TIME_SERIES, - emptyProject(), - now, - Settings.EMPTY, - List.of(), - version, - additionalSettings - ); - assertFalse(additionalSettings.build().hasValue(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey())); - - // With seq_no_disabled=true (default), the provider must set index.disable_sequence_numbers=true - DataStreamIndexSettingsProvider providerWithSeqNoDisabled = new DataStreamIndexSettingsProvider( - im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), - Settings.builder().put(DataStreamIndexSettingsProvider.SUPPORT_SEQ_NO_DISABLED.getKey(), true).build() - ); - additionalSettings = builder(); - providerWithSeqNoDisabled.provideAdditionalSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 1), - dataStreamName, - IndexMode.TIME_SERIES, - emptyProject(), - now, - Settings.EMPTY, - List.of(), - version, - additionalSettings - ); - assertThat(IndexSettings.DISABLE_SEQUENCE_NUMBERS.get(additionalSettings.build()), equalTo(true)); - - // When index.disable_sequence_numbers is explicitly set in the template, the cluster setting must not override it - additionalSettings = builder(); - providerWithSeqNoDisabled.provideAdditionalSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 1), - dataStreamName, - IndexMode.TIME_SERIES, - emptyProject(), - now, - Settings.builder().put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), false).build(), - List.of(), - version, - additionalSettings - ); - assertFalse(additionalSettings.build().hasValue(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey())); - } - /** - * Same scenario as {@link #testClusterSettingsDefineSeqNoDisabledDefault()}, but with a - * template index mode of {@link IndexMode#TSDB} (the {@code tsdb} value) rather than - * {@link IndexMode#TIME_SERIES}, since the sequence-number-disabling gate is keyed off - * {@link IndexMode#isTsdb()}. + * The sequence-number-disabling gate is keyed off {@link IndexMode#isTsdb()}, so it must behave + * identically for a template index mode of {@link IndexMode#TIME_SERIES} and {@link IndexMode#TSDB}. */ - public void testClusterSettingsDefineSeqNoDisabledDefaultWithTsdb() throws Exception { + public void testClusterSettingsDefineSeqNoDisabledDefault() throws Exception { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); IndexVersion version = IndexVersionUtils.randomVersionBetween( IndexVersions.TIME_SERIES_DISABLE_SEQUENCE_NUMBERS_DEFAULT, @@ -1342,7 +1152,7 @@ public void testClusterSettingsDefineSeqNoDisabledDefaultWithTsdb() throws Excep providerWithSeqNoEnabled.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TSDB, + mode, emptyProject(), now, Settings.EMPTY, @@ -1361,7 +1171,7 @@ public void testClusterSettingsDefineSeqNoDisabledDefaultWithTsdb() throws Excep providerWithSeqNoDisabled.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TSDB, + mode, emptyProject(), now, Settings.EMPTY, @@ -1376,7 +1186,7 @@ public void testClusterSettingsDefineSeqNoDisabledDefaultWithTsdb() throws Excep providerWithSeqNoDisabled.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TSDB, + mode, emptyProject(), now, Settings.builder().put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), false).build(), @@ -1387,75 +1197,12 @@ public void testClusterSettingsDefineSeqNoDisabledDefaultWithTsdb() throws Excep assertFalse(additionalSettings.build().hasValue(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey())); } - public void testClusterSettingsDefineSyntheticIdEnabledDefault() throws Exception { - Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); - IndexVersion version = IndexVersionUtils.randomVersionBetween( - IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD, - IndexVersion.current() - ); - String dataStreamName = "metrics-app1"; - - // With synthetic_id_enabled=false, the provider must set index.mapping.synthetic_id=false - DataStreamIndexSettingsProvider providerWithSyntheticIdDisabled = new DataStreamIndexSettingsProvider( - im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), - Settings.builder().put(DataStreamIndexSettingsProvider.SUPPORT_SYNTHETIC_ID.getKey(), false).build() - ); - Settings.Builder additionalSettings = builder(); - providerWithSyntheticIdDisabled.provideAdditionalSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 1), - dataStreamName, - IndexMode.TIME_SERIES, - emptyProject(), - now, - Settings.EMPTY, - List.of(), - version, - additionalSettings - ); - assertThat(additionalSettings.build().getAsBoolean(IndexSettings.SYNTHETIC_ID.getKey(), true), equalTo(false)); - - // With synthetic_id_enabled=true (default), the provider must set index.mapping.synthetic_id=true - DataStreamIndexSettingsProvider providerWithSyntheticIdEnabled = new DataStreamIndexSettingsProvider( - im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), - Settings.builder().put(DataStreamIndexSettingsProvider.SUPPORT_SYNTHETIC_ID.getKey(), true).build() - ); - additionalSettings = builder(); - providerWithSyntheticIdEnabled.provideAdditionalSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 1), - dataStreamName, - IndexMode.TIME_SERIES, - emptyProject(), - now, - Settings.EMPTY, - List.of(), - version, - additionalSettings - ); - assertThat(additionalSettings.build().getAsBoolean(IndexSettings.SYNTHETIC_ID.getKey(), false), equalTo(true)); - - // When index.mapping.synthetic_id is explicitly set in the template, the cluster setting must not override it - additionalSettings = builder(); - providerWithSyntheticIdEnabled.provideAdditionalSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 1), - dataStreamName, - IndexMode.TIME_SERIES, - emptyProject(), - now, - Settings.builder().put(IndexSettings.SYNTHETIC_ID.getKey(), false).build(), - List.of(), - version, - additionalSettings - ); - assertFalse(additionalSettings.build().hasValue(IndexSettings.SYNTHETIC_ID.getKey())); - } - /** - * Same scenario as {@link #testClusterSettingsDefineSyntheticIdEnabledDefault()}, but with a - * template index mode of {@link IndexMode#TSDB} (the {@code tsdb} value) rather than - * {@link IndexMode#TIME_SERIES}, since the synthetic {@code _id} gate is keyed off - * {@link IndexMode#isTsdb()}. + * The synthetic {@code _id} gate is keyed off {@link IndexMode#isTsdb()}, so it must behave + * identically for a template index mode of {@link IndexMode#TIME_SERIES} and {@link IndexMode#TSDB}. */ - public void testClusterSettingsDefineSyntheticIdEnabledDefaultWithTsdb() throws Exception { + public void testClusterSettingsDefineSyntheticIdEnabledDefault() throws Exception { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); IndexVersion version = IndexVersionUtils.randomVersionBetween( IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD, @@ -1472,7 +1219,7 @@ public void testClusterSettingsDefineSyntheticIdEnabledDefaultWithTsdb() throws providerWithSyntheticIdDisabled.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TSDB, + mode, emptyProject(), now, Settings.EMPTY, @@ -1491,7 +1238,7 @@ public void testClusterSettingsDefineSyntheticIdEnabledDefaultWithTsdb() throws providerWithSyntheticIdEnabled.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TSDB, + mode, emptyProject(), now, Settings.EMPTY, @@ -1506,7 +1253,7 @@ public void testClusterSettingsDefineSyntheticIdEnabledDefaultWithTsdb() throws providerWithSyntheticIdEnabled.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TSDB, + mode, emptyProject(), now, Settings.builder().put(IndexSettings.SYNTHETIC_ID.getKey(), false).build(), diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java index 710088398f81c..dc72e4360244a 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java @@ -92,13 +92,13 @@ public void testSortAndRetrieve() { } // Retrieve all continuous TSDB indices in a single window { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Instant start1 = Instant.parse("2024-01-15T00:00:00Z"); Instant start2 = Instant.parse("2024-01-16T00:00:00Z"); Instant start3 = Instant.parse("2024-01-17T00:00:00Z"); Instant end = Instant.parse("2024-01-18T00:00:00Z"); - ProjectMetadata project = DataStreamTestHelper.getProjectWithDataStream( - projectId, - DATA_STREAM, + ProjectMetadata project = projectWithDataStream( + mode, List.of(Tuple.tuple(start3, end), Tuple.tuple(start1, start2), Tuple.tuple(start2, start3)) ); // Updated project metadata with a consequent and unsored ts backing indices in a mixed data stream @@ -160,8 +160,9 @@ public void testSortAndRetrieve() { } public void testCreateIndicesWhenNeeded() throws Exception { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Instant now = Instant.now(); - ClusterState clusterState = stateWithExisting(List.of(), now); + ClusterState clusterState = stateWithExisting(List.of(), now, mode); List dayOffsets = List.of(5, 3, 2); List createdNames = new ArrayList<>(); // Add two timestamps that fall within one index @@ -452,6 +453,20 @@ private ClusterState stateWithExisting(List> timeSlices, .build(); } + /** + * Equivalent of {@link #stateWithExisting(List, Instant)} that builds the data stream in the given + * {@link IndexMode}, to verify that {@code IndexMode.isTsdb} treats {@link IndexMode#TSDB} and + * {@link IndexMode#TIME_SERIES} identically. + */ + private ClusterState stateWithExisting(List> timeSlices, Instant now, IndexMode mode) { + if (mode == IndexMode.TIME_SERIES) { + return stateWithExisting(timeSlices, now); + } + List> allSlices = new ArrayList<>(timeSlices); + allSlices.add(Tuple.tuple(now, now.plus(randomIntBetween(1, 3), ChronoUnit.DAYS))); + return ClusterState.builder(ClusterName.DEFAULT).putProjectMetadata(projectWithDataStream(mode, allSlices)).build(); + } + /** Builds a ClusterState with a data stream that has one non-TSDB (standard) backing index. */ private ClusterState stateWithNoTsdbIndices() { String indexName = DataStream.getDefaultBackingIndexName(DATA_STREAM, 1); @@ -476,21 +491,21 @@ private static IndexMetadata createIndexMetadata(String indexName, Instant start } /** - * Builds a ProjectMetadata with a TSDB data stream whose backing indices use {@link IndexMode#TSDB} - * rather than {@link IndexMode#TIME_SERIES}, to verify that {@code IndexMode.isTsdb} treats the two - * identically in {@link TransportPastTimeSeriesIndexCreationAction}. + * Builds a ProjectMetadata with a data stream in the given {@link IndexMode} whose backing indices + * cover the given time ranges, to verify that {@code IndexMode.isTsdb} treats {@link IndexMode#TSDB} + * and {@link IndexMode#TIME_SERIES} identically in {@link TransportPastTimeSeriesIndexCreationAction}. */ - private ProjectMetadata projectWithTsdbDataStream(List> timeSlices) { + private ProjectMetadata projectWithDataStream(IndexMode mode, List> timeSlices) { List backingIndices = new ArrayList<>(); long generation = 1L; for (Tuple slice : timeSlices) { String indexName = DataStream.getDefaultBackingIndexName(DATA_STREAM, generation, slice.v1().toEpochMilli()); - backingIndices.add(createIndexMetadata(indexName, slice.v1(), slice.v2(), IndexMode.TSDB)); + backingIndices.add(createIndexMetadata(indexName, slice.v1(), slice.v2(), mode)); generation++; } DataStream ds = DataStream.builder(DATA_STREAM, backingIndices.stream().map(IndexMetadata::getIndex).toList()) .setGeneration(generation) - .setIndexMode(IndexMode.TSDB) + .setIndexMode(mode) .build(); ProjectMetadata.Builder builder = ProjectMetadata.builder(projectId); for (IndexMetadata im : backingIndices) { @@ -499,73 +514,6 @@ private ProjectMetadata projectWithTsdbDataStream(List> return builder.put(ds).build(); } - /** Builds a ClusterState with an {@link IndexMode#TSDB} data stream whose backing indices cover the given time ranges. */ - private ClusterState stateWithExistingTsdb(List> timeSlices, Instant now) { - List> allSlices = new ArrayList<>(timeSlices); - allSlices.add(Tuple.tuple(now, now.plus(randomIntBetween(1, 3), ChronoUnit.DAYS))); - return ClusterState.builder(ClusterName.DEFAULT).putProjectMetadata(projectWithTsdbDataStream(allSlices)).build(); - } - - /** - * Equivalent of {@link #testSortAndRetrieve} that uses {@link IndexMode#TSDB} directly instead of - * {@link IndexMode#TIME_SERIES} for the backing indices, to confirm {@code retrieveSortedTimeWindows} - * (gated by {@code IndexMode.isTsdb}) recognizes it the same way. - */ - public void testSortAndRetrieveWithTsdb() { - Instant start1 = Instant.parse("2024-01-15T00:00:00Z"); - Instant start2 = Instant.parse("2024-01-16T00:00:00Z"); - Instant start3 = Instant.parse("2024-01-17T00:00:00Z"); - Instant end = Instant.parse("2024-01-18T00:00:00Z"); - ProjectMetadata project = projectWithTsdbDataStream( - List.of(Tuple.tuple(start3, end), Tuple.tuple(start1, start2), Tuple.tuple(start2, start3)) - ); - // Add a non-TSDB index to the mixed data stream to confirm it's still excluded from the windows. - String nonTsdbName = randomIndexName(); - IndexMetadata nonTsdb = IndexMetadata.builder(nonTsdbName) - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); - DataStream mixedDs = project.dataStreams().get(DATA_STREAM).unsafeAddBackingIndex(nonTsdb.getIndex()); - project = ProjectMetadata.builder(project).put(nonTsdb, false).put(mixedDs).build(); - - var result = PastTimeSeriesIndexCreationExecutor.retrieveSortedTimeWindows(project.dataStreams().get(DATA_STREAM), project); - assertThat(result, hasSize(1)); - TransportPastTimeSeriesIndexCreationAction.CoveredTimeWindow timeWindow = result.pop(); - assertThat(timeWindow.start(), is(start1.toEpochMilli())); - assertThat(timeWindow.end(), is(end.toEpochMilli())); - } - - /** - * Equivalent of the happy-path portion of {@link #testCreateIndicesWhenNeeded} that uses the - * {@link IndexMode#TSDB} instead of {@link IndexMode#TIME_SERIES}, to confirm - * {@code validateDataStream} and the time-range computation (both gated by {@code IndexMode.isTsdb}) - * behave identically for it. - */ - public void testCreateIndicesWhenNeededWithTsdb() throws Exception { - Instant now = Instant.now(); - ClusterState clusterState = stateWithExistingTsdb(List.of(), now); - List dayOffsets = List.of(5, 3, 2); - // Add two timestamps that fall within one index - { - Instant ts1 = getTimestampWithinDay(now, dayOffsets.getFirst(), randomIntBetween(2, 5)); - Instant ts2 = getTimestampWithinDay(now, dayOffsets.getFirst(), randomIntBetween(7, 12)); - TaskResult result = run(clusterState, ts1.toEpochMilli(), ts2.toEpochMilli()); - assertThat(result.covered, containsInAnyOrder(ts1, ts2)); - assertThat(result.createdNames.size(), is(1)); - clusterState = result.state(); - } - - // Add two timestamp that will create two indices - { - Instant ts1 = getTimestampWithinDay(now, dayOffsets.get(1), randomIntBetween(1, 5)); - Instant ts2 = getTimestampWithinDay(now, dayOffsets.get(2), randomIntBetween(13, 18)); - TaskResult result = run(clusterState, ts1.toEpochMilli(), ts2.toEpochMilli()); - assertThat(result.covered, containsInAnyOrder(ts1, ts2)); - assertThat(result.createdNames.size(), is(2)); - } - } - public void testOutsideEligibleWriteWindowFails() throws Exception { Instant now = Instant.now(); ClusterState clusterState = stateWithExisting(List.of(), now); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index ee0bdde92ffa6..4cfa9bf558452 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -275,60 +275,15 @@ public void testGetTimeSeriesDataStream() { ); } + /** + * {@link IndexMode#TSDB} must be handled identically to {@link IndexMode#TIME_SERIES} at both + * {@code IndexMode.isTsdb(...)} call sites in {@link TransportGetDataStreamsAction#innerOperation}. + * {@link org.elasticsearch.cluster.metadata.DataStreamTestHelper#getClusterStateWithDataStream} + * hardcodes {@link IndexMode#TIME_SERIES}, so the data stream and backing indices are built by + * hand here so the mode can be randomized. + */ public void testGetTimeSeriesDataStreamWithOutOfOrderIndices() { - Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); - String dataStream = "ds-1"; - Instant sixHoursAgo = now.minus(6, ChronoUnit.HOURS); - Instant fourHoursAgo = now.minus(4, ChronoUnit.HOURS); - Instant twoHoursAgo = now.minus(2, ChronoUnit.HOURS); - Instant twoHoursAhead = now.plus(2, ChronoUnit.HOURS); - - var projectId = randomProjectIdOrDefault(); - ClusterState state; - { - var mBuilder = ProjectMetadata.builder(projectId); - DataStreamTestHelper.getClusterStateWithDataStream( - mBuilder, - dataStream, - List.of( - new Tuple<>(fourHoursAgo, twoHoursAgo), - new Tuple<>(sixHoursAgo, fourHoursAgo), - new Tuple<>(twoHoursAgo, twoHoursAhead) - ) - ); - state = ClusterState.builder(new ClusterName("_name")).putProjectMetadata(mBuilder.build()).build(); - } - - var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); - var response = TransportGetDataStreamsAction.innerOperation( - state.projectState(projectId), - req, - resolver, - systemIndices, - ClusterSettings.createBuiltInClusterSettings(), - dataStreamGlobalRetentionSettings, - emptyDataStreamFailureStoreSettings, - new IndexSettingProviders(Set.of()), - null, - metadataDataStreamsService - ); - assertThat( - response.getDataStreams(), - contains( - allOf( - transformedMatch(d -> d.getDataStream().getName(), equalTo(dataStream)), - transformedMatch(d -> d.getTimeSeries().temporalRanges(), contains(new Tuple<>(sixHoursAgo, twoHoursAhead))) - ) - ) - ); - } - - public void testGetTimeSeriesDataStreamUsingTsdb() { - // Same scenario as testGetTimeSeriesDataStreamWithOutOfOrderIndices, but using the "tsdb" value for - // index.mode instead of "time_series". DataStreamTestHelper#getClusterStateWithDataStream hardcodes - // IndexMode.TIME_SERIES, so the data stream and backing indices are built by hand here to exercise - // IndexMode.isTsdb(...) with IndexMode.TSDB at both call sites in - // TransportGetDataStreamsAction#innerOperation. + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); String dataStreamName = "ds-1"; Instant sixHoursAgo = now.minus(6, ChronoUnit.HOURS); @@ -349,7 +304,7 @@ public void testGetTimeSeriesDataStreamUsingTsdb() { Instant start = tuple.v1(); Instant end = tuple.v2(); Settings settings = Settings.builder() - .put("index.mode", "tsdb") + .put("index.mode", mode.getName()) .put("index.routing_path", "uid") .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(start)) .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(end)) @@ -362,7 +317,7 @@ public void testGetTimeSeriesDataStreamUsingTsdb() { DataStream dataStream = DataStream.builder( dataStreamName, backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()) - ).setGeneration(generation).setIndexMode(IndexMode.TSDB).build(); + ).setGeneration(generation).setIndexMode(mode).build(); mBuilder.put(dataStream); var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index bdfffc096afbe..60753b3a9da78 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -1184,6 +1184,11 @@ public void testWithTinyRetentions() { assertThat(failureStoreConditions.getMaxAge(), equalTo(TimeValue.timeValueHours(1))); // 12h retention -> 1h max_age } + /** + * Covers both {@link IndexMode#TIME_SERIES} and {@link IndexMode#TSDB}, the latter sharing the + * former's {@code isTsdb()}-gated behaviour, so {@link DataStreamLifecycleService#timeSeriesIndicesStillWithinTimeBounds} + * must treat both identically. + */ public void testTimeSeriesIndicesStillWithinTimeBounds() { Instant currentTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); // These ranges are on the edge of each other temporal boundaries. @@ -1242,51 +1247,43 @@ public void testTimeSeriesIndicesStillWithinTimeBounds() { ); assertThat(indices.size(), is(0)); } - } - - /** - * Same coverage as {@link #testTimeSeriesIndicesStillWithinTimeBounds()} but using {@link IndexMode#TSDB} - * instead of {@link IndexMode#TIME_SERIES}, to ensure {@code isTsdb()} gates this behaviour identically for both. - */ - public void testTimeSeriesIndicesStillWithinTimeBoundsWithTsdbIndexMode() { - Instant currentTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); - // These ranges are on the edge of each other's temporal boundaries. - Instant start1 = currentTime.minus(6, ChronoUnit.HOURS); - Instant end1 = currentTime.minus(4, ChronoUnit.HOURS); - Instant start2 = currentTime.minus(4, ChronoUnit.HOURS); - Instant end2 = currentTime.plus(2, ChronoUnit.HOURS); - IndexMetadata outOfBoundsIndex = IndexMetadata.builder(randomAlphaOfLengthBetween(10, 30)) - .settings( - indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) - .put(IndexSettings.MODE.getKey(), IndexMode.TSDB) - .put("index.routing_path", "uid") - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), start1.toEpochMilli()) - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), end1.toEpochMilli()) - ) - .build(); - IndexMetadata withinBoundsIndex = IndexMetadata.builder(randomAlphaOfLengthBetween(10, 30)) - .settings( - indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) - .put(IndexSettings.MODE.getKey(), IndexMode.TSDB) - .put("index.routing_path", "uid") - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), start2.toEpochMilli()) - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), end2.toEpochMilli()) - ) - .build(); + { + // same coverage as above, but building the indices directly with an explicitly configured + // index mode, randomized between IndexMode.TIME_SERIES and IndexMode.TSDB + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + IndexMetadata outOfBoundsIndex = IndexMetadata.builder(randomAlphaOfLengthBetween(10, 30)) + .settings( + indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), mode.getName()) + .put("index.routing_path", "uid") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), start1.toEpochMilli()) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), end1.toEpochMilli()) + ) + .build(); + IndexMetadata withinBoundsIndex = IndexMetadata.builder(randomAlphaOfLengthBetween(10, 30)) + .settings( + indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), mode.getName()) + .put("index.routing_path", "uid") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), start2.toEpochMilli()) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), end2.toEpochMilli()) + ) + .build(); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) - .put(outOfBoundsIndex, true) - .put(withinBoundsIndex, true) - .build(); + ProjectMetadata modeProject = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put(outOfBoundsIndex, true) + .put(withinBoundsIndex, true) + .build(); - Set indices = DataStreamLifecycleService.timeSeriesIndicesStillWithinTimeBounds( - project, - List.of(outOfBoundsIndex.getIndex(), withinBoundsIndex.getIndex()), - currentTime::toEpochMilli - ); + Set indices = DataStreamLifecycleService.timeSeriesIndicesStillWithinTimeBounds( + modeProject, + List.of(outOfBoundsIndex.getIndex(), withinBoundsIndex.getIndex()), + currentTime::toEpochMilli + ); - assertThat(indices, containsInAnyOrder(withinBoundsIndex.getIndex())); + assertThat(indices, containsInAnyOrder(withinBoundsIndex.getIndex())); + } } public void testTrackingTimeStats() { diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java index 279106d5b370e..e6a566cf8b2c8 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java @@ -343,27 +343,14 @@ public void testMetricAndDocvalues() { assertThat(e.getCause().getMessage(), containsString("Field [time_series_metric] requires that [doc_values] is true")); } - public void testTimeSeriesIndexDefault() throws Exception { - var randomMetricType = randomFrom(TimeSeriesParams.MetricType.scalar()); - var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); - var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> { - minimalMapping(b); - b.field("time_series_metric", randomMetricType.toString()); - })); - var ft = (ScaledFloatFieldMapper.ScaledFloatFieldType) mapperService.fieldType("field"); - assertThat(ft.getMetricType(), equalTo(randomMetricType)); - assertTrue(ft.hasDocValues()); - assertFalse(ft.indexType().hasDenseIndex()); - } - /** * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) * so it must produce the same doc-values-only defaulting for metric fields. */ - public void testTimeSeriesIndexDefaultTsdb() throws Exception { + public void testTimeSeriesIndexDefault() throws Exception { var randomMetricType = randomFrom(TimeSeriesParams.MetricType.scalar()); - var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) + var mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> { minimalMapping(b); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 795e5fb72a83f..1926042889a37 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -731,40 +731,18 @@ public void testRoutingPathBwcAfterTsidBasedRouting() throws IOException { assertIndexShard(fixture, Map.of("dim.a", "true"), 6); } - public void testRoutingPathWithSingleBytePrefixTsid() throws IOException { - TimeSeriesRoutingFixture fixture = indexRoutingForTimeSeriesDimensions( - IndexVersionUtils.randomVersionOnOrAfter(IndexVersions.TSID_SINGLE_PREFIX_BYTE_FEATURE_FLAG), - 8, - "dim.*,other.*,top", - randomBoolean() - ); - assumeTrue("require single-byte-prefix tsid", TsidBuilder.useSingleBytePrefixLayout(fixture.routing.creationVersion)); - assertIndexShard(fixture, Map.of("dim", Map.of("a", "a")), 5); - assertIndexShard(fixture, Map.of("dim", Map.of("a", "b")), 3); - assertIndexShard(fixture, Map.of("dim", Map.of("c", "d")), 7); - assertIndexShard(fixture, Map.of("other", Map.of("a", "a")), 1); - assertIndexShard(fixture, Map.of("top", "a"), 6); - assertIndexShard(fixture, Map.of("dim", Map.of("c", "d"), "top", "b"), 0); - assertIndexShard(fixture, Map.of("dim.a", "a"), 5); - assertIndexShard(fixture, Map.of("dim.a", 1), 2); - assertIndexShard(fixture, Map.of("dim.a", "1"), 0); - assertIndexShard(fixture, Map.of("dim.a", true), 3); - assertIndexShard(fixture, Map.of("dim.a", "true"), 1); - } - /** - * Same scenario as {@link #testRoutingPathWithSingleBytePrefixTsid()} but with {@code index.mode: tsdb}, - * the preferred alternative to {@code time_series}. {@link IndexMode#TSDB} must select the same - * {@link IndexRouting.ExtractFromSource.ForIndexDimensions} strategy, track the routing hash the same - * way, and produce identical shard assignments as {@link IndexMode#TIME_SERIES}. + * {@link IndexMode#TSDB} must select the same {@link IndexRouting.ExtractFromSource.ForIndexDimensions} + * strategy, track the routing hash the same way, and produce identical shard assignments as + * {@link IndexMode#TIME_SERIES}. */ - public void testRoutingPathWithSingleBytePrefixTsidUsingTsdb() throws IOException { + public void testRoutingPathWithSingleBytePrefixTsid() throws IOException { TimeSeriesRoutingFixture fixture = indexRoutingForTimeSeriesDimensions( IndexVersionUtils.randomVersionOnOrAfter(IndexVersions.TSID_SINGLE_PREFIX_BYTE_FEATURE_FLAG), 8, "dim.*,other.*,top", randomBoolean(), - IndexMode.TSDB + randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB) ); assumeTrue("require single-byte-prefix tsid", TsidBuilder.useSingleBytePrefixLayout(fixture.routing.creationVersion)); assertIndexShard(fixture, Map.of("dim", Map.of("a", "a")), 5); diff --git a/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java b/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java index ced206ef6b61f..f958101f06bd7 100644 --- a/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java +++ b/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java @@ -51,12 +51,17 @@ public void testInfiniteWriteWindow() { assertThat(DLM_ONLY.getEligibleWriteWindowStart(dataStream, project, null, randomNonNegativeLong()), equalTo(-1L)); } + /** + * IndexMode.TSDB is a preferred alternative to IndexMode.TIME_SERIES, so a data stream configured + * with either mode must be treated identically by the eligible write window logic. + */ public void testWriteWindowDefinedByRetention() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); // Configured retention { TimeValue retention = TimeValue.timeValueDays(30); DataStreamLifecycle lifecycle = DataStreamLifecycle.dataLifecycleBuilder().dataRetention(retention).build(); - DataStream dataStream = dataStream("metrics-test", lifecycle); + DataStream dataStream = dataStream("metrics-test", lifecycle, mode); ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); long requestTimestamp = randomNonNegativeLong(); assertThat( @@ -73,7 +78,7 @@ public void testWriteWindowDefinedByRetention() { lifecycleBuilder.dataRetention(retention); } DataStreamLifecycle lifecycle = lifecycleBuilder.build(); - DataStream dataStream = dataStream("metrics-test", lifecycle); + DataStream dataStream = dataStream("metrics-test", lifecycle, mode); ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(null, globalMax); long requestTimestamp = randomNonNegativeLong(); @@ -84,20 +89,6 @@ public void testWriteWindowDefinedByRetention() { } } - public void testWriteWindowDefinedByRetentionWithTsdb() { - // IndexMode.TSDB is a preferred alternative to IndexMode.TIME_SERIES, so a data stream configured - // with index.mode: tsdb must be treated identically by the eligible write window logic. - TimeValue retention = TimeValue.timeValueDays(30); - DataStreamLifecycle lifecycle = DataStreamLifecycle.dataLifecycleBuilder().dataRetention(retention).build(); - DataStream dataStream = dataStream("metrics-test", lifecycle, IndexMode.TSDB); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); - long requestTimestamp = randomNonNegativeLong(); - assertThat( - DLM_ONLY.getEligibleWriteWindowStart(dataStream, project, null, requestTimestamp), - equalTo(requestTimestamp - retention.getMillis()) - ); - } - public void testWriteWindowDefinedByFrozenAfter() { TimeValue frozenAfter = TimeValue.timeValueDays(7); DataStreamLifecycle lifecycle = DataStreamLifecycle.dataLifecycleBuilder() diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 2aa4cec6b529a..75c6a4c4d4037 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -260,41 +260,21 @@ public void testSliceEnabledSettingRequiresFeatureFlag() { assertThat(exception.getMessage(), containsString("unknown setting [index.slice.enabled]")); } - public void testSliceEnabledSettingRejectedForTimeSeriesMode() { - assumeTrue("slice indexing feature flag must be enabled", SliceIndexing.SLICE_FEATURE_FLAG.isEnabled()); - IllegalArgumentException exception = expectThrows( - IllegalArgumentException.class, - () -> new IndexSettings( - newIndexMeta( - "index", - Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim") - .put(IndexSettings.SLICE_ENABLED.getKey(), true) - .build() - ), - Settings.EMPTY - ) - ); - assertThat(exception.getMessage(), containsString("index.slice.enabled")); - assertThat(exception.getMessage(), containsString("index.mode")); - assertThat(exception.getMessage(), containsString("time_series")); - } - /** * {@code index.mode: tsdb} must be rejected exactly like {@code time_series}, and the rejection - * message must name the mode that was actually configured ({@code tsdb}), not the canonical + * message must name the mode that was actually configured, not always the canonical * {@code time_series} name. */ - public void testSliceEnabledSettingRejectedForTsdb() { + public void testSliceEnabledSettingRejectedForTimeSeriesMode() { assumeTrue("slice indexing feature flag must be enabled", SliceIndexing.SLICE_FEATURE_FLAG.isEnabled()); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> new IndexSettings( newIndexMeta( "index", Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) + .put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim") .put(IndexSettings.SLICE_ENABLED.getKey(), true) .build() @@ -304,7 +284,7 @@ public void testSliceEnabledSettingRejectedForTsdb() { ); assertThat(exception.getMessage(), containsString("index.slice.enabled")); assertThat(exception.getMessage(), containsString("index.mode")); - assertThat(exception.getMessage(), containsString("tsdb")); + assertThat(exception.getMessage(), containsString(mode.getName())); } @TestLogging(reason = "testing warning logging", value = "org.elasticsearch.index.IndexSettings:WARN") @@ -1024,30 +1004,7 @@ public void testSyntheticIdCorrectSettings() { IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94, IndexVersion.current() ); - IndexMode mode = IndexMode.TIME_SERIES; - String codec = version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_BEST_COMPRESSION) - ? randomBoolean() ? CodecService.DEFAULT_CODEC : CodecService.BEST_COMPRESSION_CODEC - : CodecService.DEFAULT_CODEC; - - Settings settings = Settings.builder() - .put(IndexSettings.SYNTHETIC_ID.getKey(), true) - .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codec) - .put(IndexSettings.MODE.getKey(), mode) - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some-routing") - .build(); - IndexMetadata indexMetadata = newIndexMeta("some-index", settings, version); - - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); - assertTrue(indexSettings.useTimeSeriesSyntheticId()); - assertTrue(indexMetadata.useTimeSeriesSyntheticId()); - } - - public void testSyntheticIdCorrectSettingsWithTsdb() { - IndexVersion version = IndexVersionUtils.randomVersionBetween( - IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94, - IndexVersion.current() - ); - IndexMode mode = IndexMode.TSDB; + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); String codec = version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_BEST_COMPRESSION) ? randomBoolean() ? CodecService.DEFAULT_CODEC : CodecService.BEST_COMPRESSION_CODEC : CodecService.DEFAULT_CODEC; @@ -1070,29 +1027,7 @@ public void testSyntheticIdDefaultValueTrue() { IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD, IndexVersion.current() ); - IndexMode mode = IndexMode.TIME_SERIES; - String codec = version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_BEST_COMPRESSION) - ? randomBoolean() ? CodecService.DEFAULT_CODEC : CodecService.BEST_COMPRESSION_CODEC - : CodecService.DEFAULT_CODEC; - - Settings settings = Settings.builder() - .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codec) - .put(IndexSettings.MODE.getKey(), mode) - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some-routing") - .build(); - IndexMetadata indexMetadata = newIndexMeta("some-index", settings, version); - - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); - assertTrue(indexSettings.useTimeSeriesSyntheticId()); - assertTrue(indexMetadata.useTimeSeriesSyntheticId()); - } - - public void testSyntheticIdDefaultValueTrueWithTsdb() { - IndexVersion version = IndexVersionUtils.randomVersionBetween( - IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD, - IndexVersion.current() - ); - IndexMode mode = IndexMode.TSDB; + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); String codec = version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_BEST_COMPRESSION) ? randomBoolean() ? CodecService.DEFAULT_CODEC : CodecService.BEST_COMPRESSION_CODEC : CodecService.DEFAULT_CODEC; @@ -1111,24 +1046,7 @@ public void testSyntheticIdDefaultValueTrueWithTsdb() { public void testSyntheticIdDefaultValueFalse() { IndexVersion version = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD); - IndexMode mode = IndexMode.TIME_SERIES; - String codec = CodecService.DEFAULT_CODEC; - - Settings settings = Settings.builder() - .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codec) - .put(IndexSettings.MODE.getKey(), mode) - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some-routing") - .build(); - IndexMetadata indexMetadata = newIndexMeta("some-index", settings, version); - - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); - assertFalse(indexSettings.useTimeSeriesSyntheticId()); - assertFalse(indexMetadata.useTimeSeriesSyntheticId()); - } - - public void testSyntheticIdDefaultValueFalseWithTsdb() { - IndexVersion version = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD); - IndexMode mode = IndexMode.TSDB; + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); String codec = CodecService.DEFAULT_CODEC; Settings settings = Settings.builder() diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index c037328715b35..e3480d9994cda 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -348,9 +348,10 @@ public void testSortMissingValueDateNanoFieldPre714() { } public void testTimeSeriesMode() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); IndexSettings indexSettings = indexSettings( Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some_dimension") .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") @@ -363,44 +364,17 @@ public void testTimeSeriesMode() { } public void testTimeSeriesModeNoTimestamp() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); IndexSettings indexSettings = indexSettings( Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some_dimension") .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") .build() ); Exception e = expectThrows(IllegalArgumentException.class, () -> buildIndexSort(indexSettings, TimeSeriesIdFieldMapper.FIELD_TYPE)); - assertThat(e.getMessage(), equalTo("unknown index sort field:[@timestamp] required by [index.mode=time_series]")); - } - - public void testTimeSeriesModeWithTsdb() { - IndexSettings indexSettings = indexSettings( - Settings.builder() - .put(IndexSettings.MODE.getKey(), "tsdb") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some_dimension") - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") - .build() - ); - Sort sort = buildIndexSort(indexSettings, TimeSeriesIdFieldMapper.FIELD_TYPE, new DateFieldMapper.DateFieldType("@timestamp")); - assertThat(sort.getSort(), arrayWithSize(2)); - assertThat(sort.getSort()[0].getField(), equalTo("_tsid")); - assertThat(sort.getSort()[1].getField(), equalTo("@timestamp")); - } - - public void testTimeSeriesModeNoTimestampWithTsdb() { - IndexSettings indexSettings = indexSettings( - Settings.builder() - .put(IndexSettings.MODE.getKey(), "tsdb") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some_dimension") - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") - .build() - ); - Exception e = expectThrows(IllegalArgumentException.class, () -> buildIndexSort(indexSettings, TimeSeriesIdFieldMapper.FIELD_TYPE)); - assertThat(e.getMessage(), equalTo("unknown index sort field:[@timestamp] required by [index.mode=tsdb]")); + assertThat(e.getMessage(), equalTo("unknown index sort field:[@timestamp] required by [index.mode=" + mode.getName() + "]")); } public void testLogsdbIndexSortWithArrays() { diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index 1a4a5d644f513..49c5bb9785acb 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -43,108 +43,69 @@ public void testConfigureIndex() { assertSame(IndexMode.TIME_SERIES, IndexSettings.MODE.get(s)); } - public void testPartitioned() { - Settings s = Settings.builder() - .put(getSettings()) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) - .put(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.getKey(), 2) - .build(); - IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); - Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.routing_partition_size]")); - } - /** * {@link IndexMode#TSDB} shares its {@link IndexMode#validateWithOtherSettings} implementation with * {@link IndexMode#TIME_SERIES}, so it must reject the same incompatible settings - naming the mode - * that was actually configured ({@code tsdb}) in the error message. + * that was actually configured in the error message. */ - public void testPartitionedWithTsdb() { + public void testPartitioned() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Settings s = Settings.builder() - .put(getSettingsWithMode("tsdb", randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) + .put(getSettingsWithMode(mode.getName(), randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) .put(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.getKey(), 2) .build(); IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), equalTo("[index.mode=tsdb] is incompatible with [index.routing_partition_size]")); - } - - public void testSortField() { - Settings s = Settings.builder().put(getSettings()).put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "a").build(); - IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); - Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.field]")); + assertThat(e.getMessage(), equalTo("[index.mode=" + mode.getName() + "] is incompatible with [index.routing_partition_size]")); } /** - * Same as {@link #testSortField()} but using {@link IndexMode#TSDB}. This exercises the - * {@code validateTimeSeriesSettings} path in {@link IndexMode} that builds a synthetic + * Exercises the {@code validateTimeSeriesSettings} path in {@link IndexMode} that builds a synthetic * {@code Settings} from the calling mode's own name to compute this setting's default value for - * comparison, so it must reject the same way and name {@code tsdb} in the error message. + * comparison, so it must reject the same way for both {@link IndexMode#TIME_SERIES} and {@link IndexMode#TSDB}. */ - public void testSortFieldWithTsdb() { + public void testSortField() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Settings s = Settings.builder() - .put(getSettingsWithMode("tsdb", randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) + .put(getSettingsWithMode(mode.getName(), randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "a") .build(); IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), equalTo("[index.mode=tsdb] is incompatible with [index.sort.field]")); + assertThat(e.getMessage(), equalTo("[index.mode=" + mode.getName() + "] is incompatible with [index.sort.field]")); } public void testSortMode() { - Settings s = Settings.builder().put(getSettings()).put(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last").build(); - IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); - Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.missing]")); - } - - /** Same as {@link #testSortMode()} but using {@link IndexMode#TSDB}. */ - public void testSortModeWithTsdb() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Settings s = Settings.builder() - .put(getSettingsWithMode("tsdb", randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) + .put(getSettingsWithMode(mode.getName(), randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) .put(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last") .build(); IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), equalTo("[index.mode=tsdb] is incompatible with [index.sort.missing]")); + assertThat(e.getMessage(), equalTo("[index.mode=" + mode.getName() + "] is incompatible with [index.sort.missing]")); } public void testSortOrder() { - Settings s = Settings.builder().put(getSettings()).put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "desc").build(); - IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); - Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.order]")); - } - - /** Same as {@link #testSortOrder()} but using {@link IndexMode#TSDB}. */ - public void testSortOrderWithTsdb() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Settings s = Settings.builder() - .put(getSettingsWithMode("tsdb", randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) + .put(getSettingsWithMode(mode.getName(), randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) .put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "desc") .build(); IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), equalTo("[index.mode=tsdb] is incompatible with [index.sort.order]")); + assertThat(e.getMessage(), equalTo("[index.mode=" + mode.getName() + "] is incompatible with [index.sort.order]")); } public void testWithoutRoutingPath() { - Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), mode.getName()).build(); Exception e = expectThrows( IllegalArgumentException.class, () -> new IndexSettings(IndexSettingsTests.newIndexMeta("test", s), Settings.EMPTY) ); - assertThat(e.getMessage(), containsString("[index.mode=time_series] requires a non-empty [index.routing_path]")); - } - - public void testWithoutRoutingPathWithTsdb() { - Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "tsdb").build(); - Exception e = expectThrows( - IllegalArgumentException.class, - () -> new IndexSettings(IndexSettingsTests.newIndexMeta("test", s), Settings.EMPTY) - ); - assertThat(e.getMessage(), containsString("[index.mode=tsdb] requires a non-empty [index.routing_path]")); + assertThat(e.getMessage(), containsString("[index.mode=" + mode.getName() + "] requires a non-empty [index.routing_path]")); } public void testWithEmptyRoutingPath() { @@ -195,52 +156,44 @@ public void testSetDefaultTimeRangeValue() { } public void testRequiredRouting() { - Settings s = getSettings(); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings s = getSettingsWithMode(mode.getName(), randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z"); var mapperService = new TestMapperServiceBuilder().settings(s).applyDefaultMapping(false).build(); Exception e = expectThrows( IllegalArgumentException.class, () -> withMapping(mapperService, topMapping(b -> b.startObject("_routing").field("required", true).endObject())) ); - assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); - } - - public void testRequiredRoutingWithTsdb() { - Settings s = getSettingsWithMode("tsdb", randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z"); - var mapperService = new TestMapperServiceBuilder().settings(s).applyDefaultMapping(false).build(); - Exception e = expectThrows( - IllegalArgumentException.class, - () -> withMapping(mapperService, topMapping(b -> b.startObject("_routing").field("required", true).endObject())) + assertThat( + e.getMessage(), + equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=" + mode.getName() + "]") ); - assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=tsdb]")); } public void testValidateAlias() { - Settings s = getSettings(); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings s = getSettingsWithMode(mode.getName(), randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z"); + assertSame(mode, IndexSettings.MODE.get(s)); IndexSettings.MODE.get(s).validateAlias(null, null); // Doesn't throw exception } public void testValidateAliasWithIndexRouting() { - Settings s = getSettings(); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings s = getSettingsWithMode(mode.getName(), randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z"); Exception e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s).validateAlias("r", null)); - assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); + assertThat( + e.getMessage(), + equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=" + mode.getName() + "]") + ); } public void testValidateAliasWithSearchRouting() { - Settings s = getSettings(); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings s = getSettingsWithMode(mode.getName(), randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z"); Exception e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s).validateAlias(null, "r")); - assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); - } - - public void testValidateAliasWithTsdb() { - Settings s = getSettingsWithMode("tsdb", randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z"); - assertSame(IndexMode.TSDB, IndexSettings.MODE.get(s)); - IndexSettings.MODE.get(s).validateAlias(null, null); // Doesn't throw exception - - Exception e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s).validateAlias("r", null)); - assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=tsdb]")); - - e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s).validateAlias(null, "r")); - assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=tsdb]")); + assertThat( + e.getMessage(), + equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=" + mode.getName() + "]") + ); } public void testRoutingPathMatchesObject() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 0d481c33f325e..3a71c61392622 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -8302,13 +8302,9 @@ public void testIndexBatchFastPathOnly() throws IOException { public void testIndexBatchTimeSeriesPhase2() throws IOException { // planPrimarySubBatch Phase 2 uses timeSeriesBatchLoadDocIdAndVersion for TIME_SERIES // indices, which does a single sorted segment scan with timestamp-based segment skipping. - runIndexBatchTimeSeriesPhase2(IndexMode.TIME_SERIES); - } - - public void testIndexBatchTimeSeriesPhase2WithTsdb() throws IOException { // IndexMode.TSDB is a preferred alternative to TIME_SERIES (delegates isTsdb() etc. to // TIME_SERIES), so it must exercise the exact same batch phase 2 behavior. - runIndexBatchTimeSeriesPhase2(IndexMode.TSDB); + runIndexBatchTimeSeriesPhase2(randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB)); } private void runIndexBatchTimeSeriesPhase2(IndexMode mode) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index af9120e071cf2..dc759191960be 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -157,26 +157,14 @@ public final void testPositionMetricType() throws IOException { assertParseMinimalWarnings(); } - public void testTimeSeriesIndexDefault() throws Exception { - var positionMetricType = TimeSeriesParams.MetricType.POSITION; - var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); - var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> { - minimalMapping(b); - b.field("time_series_metric", positionMetricType.toString()); - })); - var ft = (GeoPointFieldMapper.GeoPointFieldType) mapperService.fieldType("field"); - assertThat(ft.getMetricType(), equalTo(positionMetricType)); - assertTrue(ft.indexType().hasOnlyDocValues()); - } - /** * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) * so it must produce the same doc-values-only defaulting for metric fields. */ - public void testTimeSeriesIndexDefaultTsdb() throws Exception { + public void testTimeSeriesIndexDefault() throws Exception { var positionMetricType = TimeSeriesParams.MetricType.POSITION; - var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) + var mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> { minimalMapping(b); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java index 4742c49ebabb6..61134cbca6e5a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java @@ -190,7 +190,7 @@ public MetricType getMetricType() { List.of(dimMapper, metricMapper, plainMapper), emptyList(), emptyList(), - IndexMode.TIME_SERIES + randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB) ); tsMappingLookup.validateDoesNotShadow("not_mapped"); Exception e = expectThrows(MapperParsingException.class, () -> tsMappingLookup.validateDoesNotShadow("dim")); @@ -198,21 +198,6 @@ public MetricType getMetricType() { e = expectThrows(MapperParsingException.class, () -> tsMappingLookup.validateDoesNotShadow("metric")); assertThat(e.getMessage(), equalTo("Field [metric] attempted to shadow a time_series_metric")); tsMappingLookup.validateDoesNotShadow("plain"); - - // IndexMode.TSDB is equivalent to time_series (see IndexMode#isTsdb()), so it must reject - // shadowing of dimension/metric fields identically. - MappingLookup tsdbMappingLookup = createMappingLookup( - List.of(dimMapper, metricMapper, plainMapper), - emptyList(), - emptyList(), - IndexMode.TSDB - ); - tsdbMappingLookup.validateDoesNotShadow("not_mapped"); - e = expectThrows(MapperParsingException.class, () -> tsdbMappingLookup.validateDoesNotShadow("dim")); - assertThat(e.getMessage(), equalTo("Field [dim] attempted to shadow a time_series_dimension")); - e = expectThrows(MapperParsingException.class, () -> tsdbMappingLookup.validateDoesNotShadow("metric")); - assertThat(e.getMessage(), equalTo("Field [metric] attempted to shadow a time_series_metric")); - tsdbMappingLookup.validateDoesNotShadow("plain"); } public void testShadowingOnConstruction() { @@ -246,25 +231,15 @@ public MetricType getMetricType() { Exception e = expectThrows( MapperParsingException.class, - () -> createMappingLookup(List.of(dimMapper, metricMapper), emptyList(), List.of(shadowing), IndexMode.TIME_SERIES) - ); - assertThat( - e.getMessage(), - equalTo( - shadowDim - ? "Field [dim] attempted to shadow a time_series_dimension" - : "Field [metric] attempted to shadow a time_series_metric" + () -> createMappingLookup( + List.of(dimMapper, metricMapper), + emptyList(), + List.of(shadowing), + randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB) ) ); - - // IndexMode.TSDB is equivalent to time_series (see IndexMode#isTsdb()), so it must reject - // the same shadowing on construction. - Exception tsdbException = expectThrows( - MapperParsingException.class, - () -> createMappingLookup(List.of(dimMapper, metricMapper), emptyList(), List.of(shadowing), IndexMode.TSDB) - ); assertThat( - tsdbException.getMessage(), + e.getMessage(), equalTo( shadowDim ? "Field [dim] attempted to shadow a time_series_dimension" diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java index 5ab874317f918..77cf4994bbc82 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java @@ -58,31 +58,29 @@ public class TimeSeriesMetadataFieldBlockLoaderTests extends MapperServiceTestCa * Plain TSDB index. {@link IndexMode#TIME_SERIES} defaults {@code _source.mode} to * {@link SourceFieldMapper.Mode#SYNTHETIC}, so synthetic source reconstructs {@code _source} as JSON. */ - private static final Settings TSDB_SYNTHETIC_SETTINGS = tsdbSettings(null, "host"); - - /** - * Same as {@link #TSDB_SYNTHETIC_SETTINGS} but using {@link IndexMode#TSDB} directly instead of - * {@link IndexMode#TIME_SERIES}; {@link IndexMode#isTsdb()} treats both identically. - */ - private static final Settings TSDB_MODE_SYNTHETIC_SETTINGS = tsdbModeSettings(null, "host"); + private static final Settings TSDB_SYNTHETIC_SETTINGS = tsdbSettings(IndexMode.TIME_SERIES, null, "host"); /** * TSDB index with {@code index.mapping.source.mode: stored}. Stored source preserves the raw * indexed bytes (e.g. CBOR for Prometheus remote-write documents), so the {@code _timeseries} loader must explicitly normalize to JSON. */ - private static final Settings TSDB_STORED_SETTINGS = tsdbSettings(SourceFieldMapper.Mode.STORED, "host"); + private static final Settings TSDB_STORED_SETTINGS = tsdbSettings(IndexMode.TIME_SERIES, SourceFieldMapper.Mode.STORED, "host"); /** * TSDB index that mirrors the Prometheus mapping: a {@code labels} {@code passthrough} object marked as * {@code time_series_dimension: true}. Dimension fields show up under {@code labels.*} and the index uses stored source. */ - private static final Settings TSDB_PROMETHEUS_LIKE_SETTINGS = tsdbSettings(SourceFieldMapper.Mode.STORED, "labels.*"); + private static final Settings TSDB_PROMETHEUS_LIKE_SETTINGS = tsdbSettings( + IndexMode.TIME_SERIES, + SourceFieldMapper.Mode.STORED, + "labels.*" + ); /** * OTel-like TSDB index: an {@code attributes} passthrough object. Dimension fields show up under * {@code attributes.*} and also as root-level aliases. Synthetic source (JSON) is used. */ - private static final Settings TSDB_OTEL_LIKE_SETTINGS = tsdbSettings(null, "attributes.*"); + private static final Settings TSDB_OTEL_LIKE_SETTINGS = tsdbSettings(IndexMode.TIME_SERIES, null, "attributes.*"); private static final String MAPPING = """ { @@ -159,21 +157,9 @@ public class TimeSeriesMetadataFieldBlockLoaderTests extends MapperServiceTestCa } """; - private static Settings tsdbSettings(SourceFieldMapper.Mode sourceMode, String routingPath) { - Settings.Builder builder = Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z"); - if (sourceMode != null) { - builder.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), sourceMode); - } - return builder.build(); - } - - private static Settings tsdbModeSettings(SourceFieldMapper.Mode sourceMode, String routingPath) { + private static Settings tsdbSettings(IndexMode mode, SourceFieldMapper.Mode sourceMode, String routingPath) { Settings.Builder builder = Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) + .put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z"); @@ -183,23 +169,15 @@ private static Settings tsdbModeSettings(SourceFieldMapper.Mode sourceMode, Stri return builder.build(); } - public void testDimensionsOnly() throws IOException { - BlockLoader loader = createBlockLoader( - TSDB_SYNTHETIC_SETTINGS, - MAPPING, - new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of()) - ); - assertThat(loader, instanceOf(TimeSeriesMetadataFieldBlockLoader.class)); - assertThat(sourcePaths(loader), equalTo(Set.of("host", "env", "region"))); - } - /** * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) - * so {@link TimeSeriesMetadataFieldBlockLoader} must be selected and behave identically. + * so {@link TimeSeriesMetadataFieldBlockLoader} must be selected and behave identically regardless of + * which of the two mode names configured the index. */ - public void testDimensionsOnlyTsdb() throws IOException { + public void testDimensionsOnly() throws IOException { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); BlockLoader loader = createBlockLoader( - TSDB_MODE_SYNTHETIC_SETTINGS, + tsdbSettings(mode, null, "host"), MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of()) ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java index a9be59f22fd70..edccad46072de 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java @@ -91,25 +91,14 @@ private static int getRoutingHash(ParsedDocument document) { return TimeSeriesRoutingHashFieldMapper.decode(Uid.decodeId(value.bytes)); } - @SuppressWarnings("unchecked") - public void testEnabledInTimeSeriesMode() throws Exception { - DocumentMapper docMapper = createMapper(mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - })); - - int hash = randomInt(); - ParsedDocument doc = parseDocument(hash, docMapper, b -> b.field("a", "value")); - assertThat(doc.rootDoc().getField("a").binaryValue(), equalTo(new BytesRef("value"))); - assertEquals(hash, getRoutingHash(doc)); - } - /** * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) - * so {@link TimeSeriesRoutingHashFieldMapper} must be enabled identically. + * so {@link TimeSeriesRoutingHashFieldMapper} must be enabled identically for both. */ @SuppressWarnings("unchecked") - public void testEnabledInTsdbMode() throws Exception { - Settings.Builder settingsBuilder = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.name()) + public void testEnabledInTimeSeriesMode() throws Exception { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings.Builder settingsBuilder = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), mode.name()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing path is required") .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-10-29T00:00:00Z"); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 020d67afa7919..719703fd98d64 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -278,24 +278,8 @@ public void testMetricType() throws IOException { public void testTimeSeriesIndexDefault() throws Exception { var randomMetricType = randomFrom(TimeSeriesParams.MetricType.scalar()); - var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); - var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> { - minimalMapping(b); - b.field("time_series_metric", randomMetricType.toString()); - })); - var ft = (NumberFieldMapper.NumberFieldType) mapperService.fieldType("field"); - assertThat(ft.getMetricType(), equalTo(randomMetricType)); - assertTrue(ft.indexType().hasOnlyDocValues()); - } - - /** - * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) - * so it must produce the same doc-values-only defaulting for metric fields. - */ - public void testTimeSeriesIndexDefaultTsdb() throws Exception { - var randomMetricType = randomFrom(TimeSeriesParams.MetricType.scalar()); - var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> { minimalMapping(b); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java index 31bb54897b460..bc0320529268a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java @@ -269,12 +269,10 @@ public void testDownsamplingPrerequisitesStep() { assertThat(branchingStep.getNextStepKey(), is(nextStepKey)); } { - // time series indices execute the action + // time series indices execute the action, regardless of the index mode alias used BranchingStep branchingStep = getFirstBranchingStep(action, phase, nextStepKey, withForceMerge); - Settings settings = Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) - .put("index.routing_path", "uid") - .build(); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), mode).put("index.routing_path", "uid").build(); IndexMetadata indexMetadata = newIndexMeta("test", settings); ProjectState state = projectStateFromProject(ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true)); @@ -283,10 +281,11 @@ public void testDownsamplingPrerequisitesStep() { assertThat(branchingStep.getNextStepKey().name(), is(CheckNotDataStreamWriteIndexStep.NAME)); } { - // already downsampled indices for the interval skip the action + // already downsampled indices for the interval skip the action, regardless of the index mode alias used BranchingStep branchingStep = getFirstBranchingStep(action, phase, nextStepKey, withForceMerge); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Settings settings = Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(IndexSettings.MODE.getKey(), mode) .put("index.routing_path", "uid") .put(IndexMetadata.INDEX_DOWNSAMPLE_STATUS_KEY, IndexMetadata.DownsampleTaskStatus.SUCCESS) .put(IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_NAME.getKey(), "test") @@ -312,49 +311,6 @@ public void testDownsamplingPrerequisitesStep() { } } - public void testDownsamplingPrerequisitesStepWithTsdb() { - DateHistogramInterval fixedInterval = ConfigTestHelpers.randomInterval(); - boolean withForceMerge = randomBoolean(); - DownsampleAction action = new DownsampleAction(fixedInterval, WAIT_TIMEOUT, withForceMerge, randomSamplingMethod()); - String phase = randomAlphaOfLengthBetween(1, 10); - StepKey nextStepKey = new StepKey( - randomAlphaOfLengthBetween(1, 10), - randomAlphaOfLengthBetween(1, 10), - randomAlphaOfLengthBetween(1, 10) - ); - { - // indices using the tsdb value of the time series index mode execute the action - BranchingStep branchingStep = getFirstBranchingStep(action, phase, nextStepKey, withForceMerge); - Settings settings = Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.TSDB) - .put("index.routing_path", "uid") - .build(); - IndexMetadata indexMetadata = newIndexMeta("test", settings); - - ProjectState state = projectStateFromProject(ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true)); - - branchingStep.performAction(indexMetadata.getIndex(), state); - assertThat(branchingStep.getNextStepKey().name(), is(CheckNotDataStreamWriteIndexStep.NAME)); - } - { - // already downsampled indices for the interval skip the action, regardless of the index mode alias used - BranchingStep branchingStep = getFirstBranchingStep(action, phase, nextStepKey, withForceMerge); - Settings settings = Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.TSDB) - .put("index.routing_path", "uid") - .put(IndexMetadata.INDEX_DOWNSAMPLE_STATUS_KEY, IndexMetadata.DownsampleTaskStatus.SUCCESS) - .put(IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_NAME.getKey(), "test") - .build(); - String indexName = DOWNSAMPLED_INDEX_PREFIX + fixedInterval + "-test"; - IndexMetadata indexMetadata = newIndexMeta(indexName, settings); - - ProjectState state = projectStateFromProject(ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true)); - - branchingStep.performAction(indexMetadata.getIndex(), state); - assertThat(branchingStep.getNextStepKey(), is(nextStepKey)); - } - } - private static BranchingStep getFirstBranchingStep(DownsampleAction action, String phase, StepKey nextStepKey, boolean withForceMerge) { List steps = action.toSteps(null, phase, nextStepKey); assertNotNull(steps); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java index c128aa30ddb34..fc8abc1b543b7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java @@ -10,14 +10,10 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ProjectState; -import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.Tuple; -import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; @@ -26,7 +22,6 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.List; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; @@ -57,33 +52,36 @@ protected WaitUntilTimeSeriesEndTimePassesStep copyInstance(WaitUntilTimeSeriesE return new WaitUntilTimeSeriesEndTimePassesStep(instance.getKey(), instance.getNextStepKey(), Instant::now); } + /** + * {@link org.elasticsearch.index.IndexMode#TSDB} is a preferred alternative to + * {@link org.elasticsearch.index.IndexMode#TIME_SERIES} and must gate {@link WaitUntilTimeSeriesEndTimePassesStep} + * identically, so the {@code index.mode} used for the time-series-specific assertions is randomized between the two. + */ public void testEvaluateCondition() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Instant currentTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); // These ranges are on the edge of each other temporal boundaries. - Instant start1 = currentTime.minus(6, ChronoUnit.HOURS); - Instant end1 = currentTime.minus(2, ChronoUnit.HOURS); - Instant start2 = currentTime.minus(2, ChronoUnit.HOURS); - Instant end2 = currentTime.plus(2, ChronoUnit.HOURS); - - String dataStreamName = "logs_my-app_prod"; - final var project = DataStreamTestHelper.getProjectWithDataStream( - randomProjectIdOrDefault(), - dataStreamName, - List.of(Tuple.tuple(start1, end1), Tuple.tuple(start2, end2)) - ); - var state = ClusterState.builder(ClusterName.DEFAULT).putProjectMetadata(project).build().projectState(project.id()); - DataStream dataStream = project.dataStreams().get(dataStreamName); + Instant startTimeLapsed = currentTime.minus(6, ChronoUnit.HOURS); + Instant endTimeLapsed = currentTime.minus(2, ChronoUnit.HOURS); + Instant startTimeFuture = currentTime.minus(2, ChronoUnit.HOURS); + Instant endTimeFuture = currentTime.plus(2, ChronoUnit.HOURS); WaitUntilTimeSeriesEndTimePassesStep step = new WaitUntilTimeSeriesEndTimePassesStep( randomStepKey(), randomStepKey(), () -> currentTime ); + { // end_time has lapsed already so condition must be met - Index previousGeneration = dataStream.getIndices().get(0); + IndexMetadata indexMeta = createTimeSeriesIndexMetadata(mode, "ts-index-lapsed", startTimeLapsed, endTimeLapsed); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMeta, true).build(); + ProjectState projectState = ClusterState.builder(ClusterName.DEFAULT) + .putProjectMetadata(project) + .build() + .projectState(project.id()); - step.evaluateCondition(state, project.index(previousGeneration), new AsyncWaitStep.Listener() { + step.evaluateCondition(projectState, indexMeta, new AsyncWaitStep.Listener() { @Override public void onResponse(boolean complete, ToXContentObject informationContext) { @@ -99,9 +97,14 @@ public void onFailure(Exception e) { { // end_time is in the future - Index writeIndex = dataStream.getIndices().get(1); + IndexMetadata indexMeta = createTimeSeriesIndexMetadata(mode, "ts-index-future", startTimeFuture, endTimeFuture); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMeta, true).build(); + ProjectState projectState = ClusterState.builder(ClusterName.DEFAULT) + .putProjectMetadata(project) + .build() + .projectState(project.id()); - step.evaluateCondition(state, project.index(writeIndex), new AsyncWaitStep.Listener() { + step.evaluateCondition(projectState, indexMeta, new AsyncWaitStep.Listener() { @Override public void onResponse(boolean complete, ToXContentObject informationContext) { @@ -111,9 +114,9 @@ public void onResponse(boolean complete, ToXContentObject informationContext) { information, containsString( "The [index.time_series.end_time] setting for index [" - + writeIndex.getName() + + indexMeta.getIndex().getName() + "] is [" - + end2.toEpochMilli() + + endTimeFuture.toEpochMilli() + "]. Waiting until the index's time series end time lapses before proceeding with action [" + step.getKey().action() + "] as the index can still accept writes." @@ -134,50 +137,11 @@ public void onFailure(Exception e) { .settings(indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()).build()) .build(); - ProjectState newState = state.updateProject(ProjectMetadata.builder(project).put(indexMeta, true).build()); - step.evaluateCondition(newState, indexMeta, new AsyncWaitStep.Listener() { - - @Override - public void onResponse(boolean complete, ToXContentObject informationContext) { - assertThat(complete, is(true)); - } - - @Override - public void onFailure(Exception e) { - throw new AssertionError("Unexpected method call", e); - } - }, MASTER_TIMEOUT); - } - } - - /** - * {@link org.elasticsearch.index.IndexMode#TSDB} is a preferred alternative to - * {@link org.elasticsearch.index.IndexMode#TIME_SERIES} and must gate {@link WaitUntilTimeSeriesEndTimePassesStep} - * identically. {@link #testEvaluateCondition()} only exercises indices whose {@code index.mode} is the literal - * {@code time_series}, so this repeats the time-series-specific assertions using {@code index.mode: tsdb} directly. - */ - public void testEvaluateConditionTsdb() { - Instant currentTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); - Instant startTimeLapsed = currentTime.minus(6, ChronoUnit.HOURS); - Instant endTimeLapsed = currentTime.minus(2, ChronoUnit.HOURS); - Instant startTimeFuture = currentTime.minus(2, ChronoUnit.HOURS); - Instant endTimeFuture = currentTime.plus(2, ChronoUnit.HOURS); - - WaitUntilTimeSeriesEndTimePassesStep step = new WaitUntilTimeSeriesEndTimePassesStep( - randomStepKey(), - randomStepKey(), - () -> currentTime - ); - - { - // end_time has lapsed already so condition must be met - IndexMetadata indexMeta = createTsdbIndexMetadata("tsdb-index-lapsed", startTimeLapsed, endTimeLapsed); ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMeta, true).build(); ProjectState projectState = ClusterState.builder(ClusterName.DEFAULT) .putProjectMetadata(project) .build() .projectState(project.id()); - step.evaluateCondition(projectState, indexMeta, new AsyncWaitStep.Listener() { @Override @@ -191,47 +155,11 @@ public void onFailure(Exception e) { } }, MASTER_TIMEOUT); } - - { - // end_time is in the future - IndexMetadata indexMeta = createTsdbIndexMetadata("tsdb-index-future", startTimeFuture, endTimeFuture); - ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMeta, true).build(); - ProjectState projectState = ClusterState.builder(ClusterName.DEFAULT) - .putProjectMetadata(project) - .build() - .projectState(project.id()); - - step.evaluateCondition(projectState, indexMeta, new AsyncWaitStep.Listener() { - - @Override - public void onResponse(boolean complete, ToXContentObject informationContext) { - assertThat(complete, is(false)); - String information = Strings.toString(informationContext); - assertThat( - information, - containsString( - "The [index.time_series.end_time] setting for index [" - + indexMeta.getIndex().getName() - + "] is [" - + endTimeFuture.toEpochMilli() - + "]. Waiting until the index's time series end time lapses before proceeding with action [" - + step.getKey().action() - + "] as the index can still accept writes." - ) - ); - } - - @Override - public void onFailure(Exception e) { - throw new AssertionError("Unexpected method call", e); - } - }, MASTER_TIMEOUT); - } } - private static IndexMetadata createTsdbIndexMetadata(String indexName, Instant startTime, Instant endTime) { + private static IndexMetadata createTimeSeriesIndexMetadata(IndexMode mode, String indexName, Instant startTime, Instant endTime) { Settings settings = indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) - .put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) + .put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "uid") .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(startTime)) .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(endTime)) diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java index 2569659217939..939d522125a7e 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java @@ -217,65 +217,10 @@ public void tearDown() throws Exception { } public void testDownsampling() { + // index.mode may be either TIME_SERIES or its preferred alias TSDB; both must be handled identically. + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); var projectMetadata = ProjectMetadata.builder(projectId) - .put(createSourceIndexMetadata(sourceIndex, primaryShards, replicaShards)) - .build(); - - var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) - .putProjectMetadata(projectMetadata) - .blocks(ClusterBlocks.builder().addIndexBlock(projectId, sourceIndex, IndexMetadata.INDEX_WRITE_BLOCK)) - .build(); - - when(projectResolver.getProjectMetadata(any(ClusterState.class))).thenReturn(projectMetadata); - - Answer mockPersistentTask = invocation -> { - ActionListener> listener = invocation.getArgument(4); - PersistentTasksCustomMetadata.PersistentTask task1 = mock(PersistentTasksCustomMetadata.PersistentTask.class); - when(task1.getId()).thenReturn(randomAlphaOfLength(10)); - DownsampleShardPersistentTaskState runningTaskState = new DownsampleShardPersistentTaskState( - DownsampleShardIndexerStatus.COMPLETED, - null - ); - when(task1.getState()).thenReturn(runningTaskState); - listener.onResponse(task1); - return null; - }; - doAnswer(mockPersistentTask).when(persistentTaskService).sendStartRequest(anyString(), anyString(), any(), any(), any()); - doAnswer(mockPersistentTask).when(persistentTaskService).waitForPersistentTaskCondition(any(), anyString(), any(), any(), any()); - doAnswer(invocation -> { - var listener = invocation.getArgument(1, TransportDownsampleAction.UpdateDownsampleIndexSettingsActionListener.class); - listener.onResponse(AcknowledgedResponse.TRUE); - return null; - }).when(indicesAdminClient).updateSettings(any(), any()); - assertSuccessfulUpdateDownsampleStatus(clusterState); - - PlainActionFuture listener = new PlainActionFuture<>(); - action.masterOperation( - task, - new DownsampleAction.Request( - ESTestCase.TEST_REQUEST_TIMEOUT, - sourceIndex, - targetIndex, - TimeValue.ONE_HOUR, - new DownsampleConfig(new DateHistogramInterval("5m"), randomSamplingMethod()) - ), - clusterState, - listener - ); - safeGet(listener); - verifyIndexFinalisation(); - } - - /** - * Same as {@link #testDownsampling()} but using {@link IndexMode#TSDB}, the preferred alternative to - * {@link IndexMode#TIME_SERIES}, to configure the source index. This verifies that the - * {@code index.mode}-gated validation in {@link TransportDownsampleAction} relies on - * {@link IndexMode#isTsdb()} rather than an exact match against {@link IndexMode#TIME_SERIES}, - * so a source index configured with {@code index.mode: tsdb} is downsampled successfully as well. - */ - public void testDownsamplingWithTsdbIndexMode() { - var projectMetadata = ProjectMetadata.builder(projectId) - .put(createSourceIndexMetadata(sourceIndex, primaryShards, replicaShards, IndexMode.TSDB)) + .put(createSourceIndexMetadata(sourceIndex, primaryShards, replicaShards, mode)) .build(); var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 4496d9baefe87..c4e2387dd826d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -2007,20 +2007,18 @@ private void preAnalyzeFlatMainIndices( // visible for testing static QueryBuilder createQueryFilter(IndexMode indexMode, QueryBuilder requestFilter) { - return switch (indexMode) { - case IndexMode.TIME_SERIES, IndexMode.TSDB -> { - // Match both values: the requested indexMode is a fixed sentinel (e.g. the TS command always - // declares IndexMode.TIME_SERIES), but the concrete backing indices may be configured with - // either [index.mode=time_series] or [index.mode=tsdb]. - var indexModeFilter = new TermsQueryBuilder( - IndexModeFieldMapper.NAME, - IndexMode.TIME_SERIES.getName(), - IndexMode.TSDB.getName() - ); - yield requestFilter != null ? new BoolQueryBuilder().filter(requestFilter).filter(indexModeFilter) : indexModeFilter; - } - default -> requestFilter; - }; + if (IndexMode.isTsdb(indexMode)) { + // Match both values: the requested indexMode is a fixed sentinel (e.g. the TS command always + // declares IndexMode.TIME_SERIES), but the concrete backing indices may be configured with + // either [index.mode=time_series] or [index.mode=tsdb]. + var indexModeFilter = new TermsQueryBuilder( + IndexModeFieldMapper.NAME, + IndexMode.TIME_SERIES.getName(), + IndexMode.TSDB.getName() + ); + return requestFilter != null ? new BoolQueryBuilder().filter(requestFilter).filter(indexModeFilter) : indexModeFilter; + } + return requestFilter; } // visible for testing diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 6b3d7a6e3fb2b..14e9883f9eeac 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2719,26 +2719,11 @@ private static void assertDefaultLimitForQuery(String query, int expectedLimit) } public void testImplicitTimestampSortForTsQuery() { - // TS query without STATS or SORT should have implicit sort - var plan = tsdb().query("TS test"); - - var limit = as(plan, Limit.class); - var orderBy = as(limit.child(), OrderBy.class); - var orders = orderBy.order(); - assertThat(orders, hasSize(1)); - - var order = orders.get(0); - assertThat(order.direction(), equalTo(Order.OrderDirection.DESC)); - assertThat(order.nullsPosition(), equalTo(Order.NullsPosition.LAST)); - var orderChild = as(order.child(), FieldAttribute.class); - assertThat(orderChild.name(), equalTo("@timestamp")); - } - - public void testImplicitTimestampSortForTsQueryWithTsdb() { - // Same as testImplicitTimestampSortForTsQuery, but the index uses IndexMode.TSDB instead of - // IndexMode.TIME_SERIES: AddImplicitTimestampSort is gated on IndexMode#isTsdb(), so both - // must be treated identically. - var plan = tsdbMode().query("TS test"); + // TS query without STATS or SORT should have implicit sort. AddImplicitTimestampSort is + // gated on IndexMode#isTsdb(), so this exercises both IndexMode.TIME_SERIES and + // IndexMode.TSDB, which must be treated identically. + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + var plan = analyzer().addIndex("test", "tsdb-mapping.json", mode).query("TS test"); var limit = as(plan, Limit.class); var orderBy = as(limit.child(), OrderBy.class); @@ -5006,10 +4991,13 @@ private void verifyNameAndType(String actualName, DataType actualType, String ex } public void testImplicitCastingForAggregateMetricDouble() { + // ImplicitCastAggregateMetricDoubles is gated on IndexMode#isTsdb(), so the TS branch below + // exercises both IndexMode.TIME_SERIES and IndexMode.TSDB, which must be treated identically. assumeTrue( "aggregate metric double implicit casting must be available", EsqlCapabilities.Cap.AGGREGATE_METRIC_DOUBLE_V0.isEnabled() ); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Map mapping = Map.of( "@timestamp", new EsField("@timestamp", DATETIME, Map.of(), true, EsField.TimeSeriesFieldType.NONE), @@ -5019,13 +5007,7 @@ public void testImplicitCastingForAggregateMetricDouble() { new InvalidMappedField("metric_field", Map.of("aggregate_metric_double", Set.of("k8s-downsampled"), "double", Set.of("k8s"))) ); - var esIndex = new EsIndex( - "k8s,k8s-downsampled", - mapping, - Map.of("k8s", IndexMode.TIME_SERIES, "k8s-downsampled", IndexMode.TIME_SERIES), - Map.of(), - Map.of() - ); + var esIndex = new EsIndex("k8s,k8s-downsampled", mapping, Map.of("k8s", mode, "k8s-downsampled", mode), Map.of(), Map.of()); var testAnalyzer = analyzer().addIndex(esIndex); var stddevPlan = testAnalyzer.query(""" from k8s,k8s-downsampled | stats std_dev = std_dev(metric_field) @@ -5052,42 +5034,6 @@ public void testImplicitCastingForAggregateMetricDouble() { assertProjection(plan2, "s1", "s2", "min", "count", "avg", "cluster", "time_bucket"); } - public void testImplicitCastingForAggregateMetricDoubleWithTsdb() { - // Same as testImplicitCastingForAggregateMetricDouble's TS branch, but the indices use - // IndexMode.TSDB instead of IndexMode.TIME_SERIES: ImplicitCastAggregateMetricDoubles is - // gated on IndexMode#isTsdb(), so both must be treated identically. - assumeTrue( - "aggregate metric double implicit casting must be available", - EsqlCapabilities.Cap.AGGREGATE_METRIC_DOUBLE_V0.isEnabled() - ); - Map mapping = Map.of( - "@timestamp", - new EsField("@timestamp", DATETIME, Map.of(), true, EsField.TimeSeriesFieldType.NONE), - "cluster", - new EsField("cluster", KEYWORD, Map.of(), true, EsField.TimeSeriesFieldType.DIMENSION), - "metric_field", - new InvalidMappedField("metric_field", Map.of("aggregate_metric_double", Set.of("k8s-downsampled"), "double", Set.of("k8s"))) - ); - - var esIndex = new EsIndex( - "k8s,k8s-downsampled", - mapping, - Map.of("k8s", IndexMode.TSDB, "k8s-downsampled", IndexMode.TSDB), - Map.of(), - Map.of() - ); - var testAnalyzer = analyzer().addIndex(esIndex); - var plan2 = testAnalyzer.query(""" - TS k8s,k8s-downsampled | stats s1 = sum(sum_over_time(metric_field)), - s2 = sum(avg_over_time(metric_field)), - min = min(max_over_time(metric_field)), - count = count(count_over_time(metric_field)), - avg = avg(min_over_time(metric_field)) - by cluster, time_bucket = bucket(@timestamp,1minute) - """); - assertProjection(plan2, "s1", "s2", "min", "count", "avg", "cluster", "time_bucket"); - } - public void testToGaugeStrippedOnAggregateMetricDoubleAndGaugeUnion() { assumeTrue("to_gauge must be available", EsqlCapabilities.Cap.TO_GAUGE.isEnabled()); assumeTrue( @@ -5587,15 +5533,6 @@ private static TestAnalyzer tsdb() { return analyzer().addIndex("test", "tsdb-mapping.json", IndexMode.TIME_SERIES); } - /** - * Same fixture as {@link #tsdb()}, but built with {@link IndexMode#TSDB} directly instead of - * {@link IndexMode#TIME_SERIES}. Used to prove that {@code index.mode: tsdb} is treated identically - * to {@code index.mode: time_series} by rules gated on {@link IndexMode#isTsdb()}. - */ - private static TestAnalyzer tsdbMode() { - return analyzer().addIndex("test", "tsdb-mapping.json", IndexMode.TSDB); - } - private static TestAnalyzer k8s() { return analyzer().addK8sDownsampled(); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java index e38bd9dc47e94..ef66e701fd095 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java @@ -343,34 +343,14 @@ public void testRoundToTwoFields() { */ public void testRoundToInTsEval() { assumeTrue("ROUND_TO block loader must be enabled", EsqlCapabilities.Cap.ROUND_TO_BLOCK_LOADER.isEnabled()); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); PhysicalPlan plan = tsPlannerOptimizer.plan(""" TS k8s | EVAL r = ROUND_TO(events_received, 100, 200, 300) | SORT @timestamp | LIMIT 10 | KEEP r - """, tsSearchStats()); - - List pushed = findPushedFields(plan, "events_received", BlockLoaderFunctionConfig.Function.ROUND_TO); - assertThat(pushed, hasSize(0)); - } - - /** - * Verifies ROUND_TO on a long field is NOT pushed to the block loader when - * the shard's index metadata reports {@link IndexMode#TSDB} - * rather than {@link IndexMode#TIME_SERIES}. {@code TSDB} behaves - * identically to {@code TIME_SERIES} for {@code hasTimeSeriesShards()}, - * so this mirrors {@link #testRoundToInTsEval()}. - */ - public void testRoundToInTsEvalWithTsdbIndexMode() { - assumeTrue("ROUND_TO block loader must be enabled", EsqlCapabilities.Cap.ROUND_TO_BLOCK_LOADER.isEnabled()); - PhysicalPlan plan = tsPlannerOptimizer.plan(""" - TS k8s - | EVAL r = ROUND_TO(events_received, 100, 200, 300) - | SORT @timestamp - | LIMIT 10 - | KEEP r - """, tsdbSearchStats()); + """, tsSearchStats(mode)); List pushed = findPushedFields(plan, "events_received", BlockLoaderFunctionConfig.Function.ROUND_TO); assertThat(pushed, hasSize(0)); @@ -867,27 +847,15 @@ private List collectNodes(PhysicalPlan plan, Class targetShards() { - IndexMetadata indexMetadata = IndexMetadata.builder("k8s") - .settings(indexSettings(IndexVersion.current(), 1, 1).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())) - .build(); - return Map.of(new ShardId(new Index("k8s", "n/a"), 0), indexMetadata); - } - }; + return tsSearchStats(IndexMode.TIME_SERIES); } - /** - * Same as {@link #tsSearchStats()} but reports the shard's index metadata - * using {@link IndexMode#TSDB} instead of {@link IndexMode#TIME_SERIES}. - */ - private static SearchStats tsdbSearchStats() { + private static SearchStats tsSearchStats(IndexMode mode) { return new EsqlTestUtils.TestSearchStats() { @Override public Map targetShards() { IndexMetadata indexMetadata = IndexMetadata.builder("k8s") - .settings(indexSettings(IndexVersion.current(), 1, 1).put(IndexSettings.MODE.getKey(), IndexMode.TSDB.name())) + .settings(indexSettings(IndexVersion.current(), 1, 1).put(IndexSettings.MODE.getKey(), mode.name())) .build(); return Map.of(new ShardId(new Index("k8s", "n/a"), 0), indexMetadata); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java index df6bb6c112417..13388bdaf761a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java @@ -1010,6 +1010,7 @@ public void testSubqueryWithCountStarAndDateTrunc() { * {@link PushExpressionsToFieldLoadTests#testRoundToInTsEval} and friends. */ public void testRoundToWithTimeSeriesIndices() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Map minValue = Map.of( "@timestamp", DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-10-20T12:15:03.360Z") @@ -1023,13 +1024,13 @@ public void testRoundToWithTimeSeriesIndices() { public Map targetShards() { var indexMetadata = IndexMetadata.builder("test_index") .settings( - ESTestCase.indexSettings(IndexVersion.current(), 1, 1) - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) + ESTestCase.indexSettings(IndexVersion.current(), 1, 1).put(IndexSettings.MODE.getKey(), mode.getName()) ) .build(); return Map.of(new ShardId(new Index("id", "n/a"), 1), indexMetadata); } }; + TestPlannerOptimizer optimizer = plannerOptimizerForMode(mode); // enable filter-by-filter for rate aggregations { String q = """ @@ -1037,7 +1038,7 @@ public Map targetShards() { | STATS max(rate(network.total_bytes_in)) BY cluster, BUCKET(@timestamp, 1 hour) | LIMIT 10 """; - PhysicalPlan plan = plannerOptimizerTimeSeries.plan(q, searchStats, timeSeriesAnalyzer); + PhysicalPlan plan = optimizer.plan(q, searchStats); int queryAndTags = plainQueryAndTags(plan); assertThat(queryAndTags, equalTo(4)); } @@ -1048,60 +1049,7 @@ public Map targetShards() { | STATS max(avg_over_time(network.bytes_in)) BY cluster, BUCKET(@timestamp, 1 hour) | LIMIT 10 """; - PhysicalPlan plan = plannerOptimizerTimeSeries.plan(q, searchStats, timeSeriesAnalyzer); - int queryAndTags = plainQueryAndTags(plan); - assertThat(queryAndTags, equalTo(1)); - } - } - - /** - * {@link org.elasticsearch.index.IndexMode#TSDB} is a preferred alternative to - * {@link IndexMode#TIME_SERIES}: it behaves identically everywhere, including in the - * {@link ReplaceRoundToWithQueryAndTags} rule. This mirrors - * {@link #testRoundToWithTimeSeriesIndices} but resolves the source index (and the target - * shards reported by {@link SearchStats}) with {@link IndexMode#TSDB} instead, verifying the - * filter-by-filter rewrite still applies identically. - */ - public void testRoundToWithTimeSeriesIndicesUsingTsdb() { - Map minValue = Map.of( - "@timestamp", - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-10-20T12:15:03.360Z") - ); - Map maxValue = Map.of( - "@timestamp", - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-10-20T14:55:01.543Z") - ); - SearchStats searchStats = new EsqlTestUtils.TestSearchStatsWithMinMax(minValue, maxValue) { - @Override - public Map targetShards() { - var indexMetadata = IndexMetadata.builder("test_index") - .settings( - ESTestCase.indexSettings(IndexVersion.current(), 1, 1).put(IndexSettings.MODE.getKey(), IndexMode.TSDB.name()) - ) - .build(); - return Map.of(new ShardId(new Index("id", "n/a"), 1), indexMetadata); - } - }; - TestPlannerOptimizer tsdbPlannerOptimizer = tsdbPlannerOptimizer(); - // enable filter-by-filter for rate aggregations - { - String q = """ - TS k8s - | STATS max(rate(network.total_bytes_in)) BY cluster, BUCKET(@timestamp, 1 hour) - | LIMIT 10 - """; - PhysicalPlan plan = tsdbPlannerOptimizer.plan(q, searchStats); - int queryAndTags = plainQueryAndTags(plan); - assertThat(queryAndTags, equalTo(4)); - } - // disable filter-by-filter for non-rate aggregations - { - String q = """ - TS k8s - | STATS max(avg_over_time(network.bytes_in)) BY cluster, BUCKET(@timestamp, 1 hour) - | LIMIT 10 - """; - PhysicalPlan plan = tsdbPlannerOptimizer.plan(q, searchStats); + PhysicalPlan plan = optimizer.plan(q, searchStats); int queryAndTags = plainQueryAndTags(plan); assertThat(queryAndTags, equalTo(1)); } @@ -1112,30 +1060,24 @@ public Map targetShards() { * {@link IndexMode#TSDB} is a preferred alternative to {@link IndexMode#TIME_SERIES}, both must double the * rounding points threshold identically. */ - public void testAdjustedRoundingPointsThresholdForTsdb() { + public void testAdjustedRoundingPointsThreshold() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); int threshold = between(1, 1000); - int timeSeriesThreshold = ReplaceRoundToWithQueryAndTags.adjustedRoundingPointsThreshold( - searchStats, - threshold, - null, - IndexMode.TIME_SERIES - ); - int tsdbThreshold = ReplaceRoundToWithQueryAndTags.adjustedRoundingPointsThreshold(searchStats, threshold, null, IndexMode.TSDB); - assertThat(tsdbThreshold, equalTo(timeSeriesThreshold)); - assertThat(tsdbThreshold, equalTo(threshold * 2)); + int adjustedThreshold = ReplaceRoundToWithQueryAndTags.adjustedRoundingPointsThreshold(searchStats, threshold, null, mode); + assertThat(adjustedThreshold, equalTo(threshold * 2)); } // Builds an analyzer/planner pair mirroring AbstractLocalPhysicalPlanOptimizerTests#init's - // plannerOptimizerTimeSeries/timeSeriesAnalyzer, but resolving the "k8s" index with IndexMode.TSDB - // instead of TIME_SERIES, to verify IndexMode.TSDB is handled identically. - private TestPlannerOptimizer tsdbPlannerOptimizer() { + // plannerOptimizerTimeSeries/timeSeriesAnalyzer, but resolving the "k8s" index with the given + // IndexMode, so tests can verify IndexMode.TSDB is handled identically to IndexMode.TIME_SERIES. + private TestPlannerOptimizer plannerOptimizerForMode(IndexMode mode) { var timeSeriesMapping = loadMapping("k8s-mappings.json"); - var tsdbIndex = IndexResolution.valid(EsIndexGenerator.esIndex("k8s", timeSeriesMapping, Map.of("k8s", IndexMode.TSDB))); - Analyzer tsdbAnalyzer = new Analyzer( + var timeSeriesIndex = IndexResolution.valid(EsIndexGenerator.esIndex("k8s", timeSeriesMapping, Map.of("k8s", mode))); + Analyzer analyzer = new Analyzer( testAnalyzerContext( EsqlTestUtils.TEST_CFG, TEST_FUNCTION_REGISTRY, - indexResolutions(tsdbIndex), + indexResolutions(timeSeriesIndex), new EnrichResolution(), emptyInferenceResolution() ), @@ -1143,7 +1085,7 @@ private TestPlannerOptimizer tsdbPlannerOptimizer() { ); return new TestPlannerOptimizer( config, - tsdbAnalyzer, + analyzer, new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small(), TransportVersion.current())) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java index a32598ee0cca9..3e2ff2060954f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java @@ -68,44 +68,16 @@ public class EsqlSessionTests extends ESTestCase { public void testShouldRetryConcreteTimeSeriesResolution() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); assertTrue( - EsqlSession.shouldRetryConcreteTimeSeriesResolution( - IndexMode.TIME_SERIES, - IndexResolution.empty("logs"), - new IndexPattern(EMPTY, "logs") - ) + EsqlSession.shouldRetryConcreteTimeSeriesResolution(mode, IndexResolution.empty("logs"), new IndexPattern(EMPTY, "logs")) ); } public void testShouldNotRetryWildcardTimeSeriesResolution() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); assertFalse( - EsqlSession.shouldRetryConcreteTimeSeriesResolution( - IndexMode.TIME_SERIES, - IndexResolution.empty("logs*"), - new IndexPattern(EMPTY, "logs*") - ) - ); - } - - // IndexMode.TSDB is a preferred alternative to IndexMode.TIME_SERIES (isTsdb() is true for both); this pins - // that shouldRetryConcreteTimeSeriesResolution treats it identically to the canonical constant. - public void testShouldRetryConcreteTsdbResolution() { - assertTrue( - EsqlSession.shouldRetryConcreteTimeSeriesResolution( - IndexMode.TSDB, - IndexResolution.empty("logs"), - new IndexPattern(EMPTY, "logs") - ) - ); - } - - public void testShouldNotRetryWildcardTsdbResolution() { - assertFalse( - EsqlSession.shouldRetryConcreteTimeSeriesResolution( - IndexMode.TSDB, - IndexResolution.empty("logs*"), - new IndexPattern(EMPTY, "logs*") - ) + EsqlSession.shouldRetryConcreteTimeSeriesResolution(mode, IndexResolution.empty("logs*"), new IndexPattern(EMPTY, "logs*")) ); } @@ -113,15 +85,9 @@ public void testShouldNotRetryWildcardTsdbResolution() { // source. The requested indexMode is a fixed sentinel (TS always declares IndexMode.TIME_SERIES) but the // concrete backing indices may be configured with either [index.mode=time_series] or [index.mode=tsdb], // so the filter must match both terms regardless of which indexMode triggered it. - public void testCreateQueryFilterForTimeSeriesMatchesBothIndexModeValues() { - QueryBuilder filter = EsqlSession.createQueryFilter(IndexMode.TIME_SERIES, null); - TermsQueryBuilder termsFilter = as(filter, TermsQueryBuilder.class); - assertThat(termsFilter.fieldName(), equalTo(IndexModeFieldMapper.NAME)); - assertThat(termsFilter.values(), containsInAnyOrder(IndexMode.TIME_SERIES.getName(), IndexMode.TSDB.getName())); - } - - public void testCreateQueryFilterForTsdbMatchesBothIndexModeValues() { - QueryBuilder filter = EsqlSession.createQueryFilter(IndexMode.TSDB, null); + public void testCreateQueryFilterMatchesBothIndexModeValues() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + QueryBuilder filter = EsqlSession.createQueryFilter(mode, null); TermsQueryBuilder termsFilter = as(filter, TermsQueryBuilder.class); assertThat(termsFilter.fieldName(), equalTo(IndexModeFieldMapper.NAME)); assertThat(termsFilter.values(), containsInAnyOrder(IndexMode.TIME_SERIES.getName(), IndexMode.TSDB.getName())); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index dfbff3eb8367a..7b74ee83852f9 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -255,6 +255,8 @@ public void testOnExplicitStandardIndex() throws IOException { } public void testOnExplicitTimeSeriesIndex() throws IOException { + // Randomly exercise both IndexMode.TIME_SERIES and IndexMode.TSDB to make sure they behave identically. + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( logsdbLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() @@ -267,31 +269,7 @@ public void testOnExplicitTimeSeriesIndex() throws IOException { null, emptyProject(), Instant.now().truncatedTo(ChronoUnit.SECONDS), - Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()).build(), - List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), - IndexVersion.current(), - settingsBuilder - ); - final Settings additionalIndexSettings = settingsBuilder.build(); - - assertTrue(additionalIndexSettings.isEmpty()); - } - - public void testOnExplicitTsdbIndex() throws IOException { - // Same scenario as testOnExplicitTimeSeriesIndex, but using the "tsdb" value for index.mode. - final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( - logsdbLicenseService, - Settings.builder().put("cluster.logsdb.enabled", true).build() - ); - - Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalSettings( - null, - "logs-apache-production", - null, - emptyProject(), - Instant.now().truncatedTo(ChronoUnit.SECONDS), - Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()).build(), + Settings.builder().put(IndexSettings.MODE.getKey(), mode.getName()).build(), List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder @@ -658,6 +636,8 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException } public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { + // Randomly exercise both IndexMode.TIME_SERIES and IndexMode.TSDB to make sure they behave identically. + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); String dataStreamName = DATA_STREAM_NAME; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); String mapping = """ @@ -672,51 +652,13 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { """; LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); { - Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build(); - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) - .hasSyntheticSourceUsage(); - assertTrue(result); - } - { - Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build(); - boolean result = provider.getMappingHints(indexName, null, settings, List.of()).hasSyntheticSourceUsage(); - assertTrue(result); - } - { - boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of()).hasSyntheticSourceUsage(); - assertFalse(result); - } - { - boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(getMapping(mapping)))) - .hasSyntheticSourceUsage(); - assertFalse(result); - } - } - - public void testNewIndexHasSyntheticSourceUsageTsdb() throws IOException { - // Same scenario as testNewIndexHasSyntheticSourceUsageTimeSeries, but exercised through the "tsdb" index.mode - // value to make sure IndexMode.isTsdb(...) treats it identically to "time_series". - String dataStreamName = DATA_STREAM_NAME; - String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); - String mapping = """ - { - "properties": { - "my_field": { - "type": "keyword", - "time_series_dimension": true - } - } - } - """; - LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); - { - Settings settings = Settings.builder().put("index.mode", "tsdb").put("index.routing_path", "my_field").build(); + Settings settings = Settings.builder().put("index.mode", mode.getName()).put("index.routing_path", "my_field").build(); boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertTrue(result); } { - Settings settings = Settings.builder().put("index.mode", "tsdb").put("index.routing_path", "my_field").build(); + Settings settings = Settings.builder().put("index.mode", mode.getName()).put("index.routing_path", "my_field").build(); boolean result = provider.getMappingHints(indexName, null, settings, List.of()).hasSyntheticSourceUsage(); assertTrue(result); } @@ -772,6 +714,8 @@ public void testNewIndexHasSyntheticSourceUsageInvalidSettings() throws IOExcept } public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { + // Randomly exercise both IndexMode.TIME_SERIES and IndexMode.TSDB to make sure they behave identically. + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); String dataStreamName = DATA_STREAM_NAME; ProjectMetadata project = DataStreamTestHelper.getProjectWithDataStreams( List.of(Tuple.tuple(dataStreamName, 1)), @@ -822,7 +766,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, - IndexMode.TIME_SERIES, + mode, project, Instant.ofEpochMilli(1L), settings, @@ -855,41 +799,6 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { assertThat(newMapperServiceCounter.get(), equalTo(4)); } - public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceTsdb() { - // Same scenario as the IndexMode.TIME_SERIES case in testGetAdditionalIndexSettingsDowngradeFromSyntheticSource, - // but using IndexMode.TSDB directly to make sure the template mode injection in - // LogsdbIndexModeSettingsProvider#buildIndexMetadataForMapperService treats it identically. - String dataStreamName = DATA_STREAM_NAME; - ProjectMetadata project = DataStreamTestHelper.getProjectWithDataStreams( - List.of(Tuple.tuple(dataStreamName, 1)), - List.of(), - Instant.now().toEpochMilli(), - builder().build(), - 1 - ); - LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); - Settings settings = builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) - .build(); - logsdbLicenseService.setSyntheticSourceFallback(true); - - Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 2), - dataStreamName, - IndexMode.TSDB, - project, - Instant.ofEpochMilli(1L), - settings, - List.of(), - IndexVersion.current(), - settingsBuilder - ); - Settings result = settingsBuilder.build(); - assertThat(result.size(), equalTo(1)); - assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); - assertThat(newMapperServiceCounter.get(), equalTo(1)); - } - public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceOldNode() { logsdbLicenseService.setSyntheticSourceFallback(true); LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(true, Version.V_8_16_0); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java index d6bf9e90a74de..04d37e985efcb 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java @@ -129,6 +129,9 @@ public void testGetAdditionalIndexSettingsProfiling() throws IOException { } public void testGetAdditionalIndexSettingsTsdb() throws IOException { + // IndexMode.TSDB is a preferred alternative to IndexMode.TIME_SERIES and must be treated identically by the + // legacy license check in LogsdbIndexModeSettingsProvider#isLegacyLicensedUsageOfSyntheticSourceAllowed. + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); @@ -136,30 +139,7 @@ public void testGetAdditionalIndexSettingsTsdb() throws IOException { provider.provideAdditionalSettings( indexName, dataStreamName, - IndexMode.TIME_SERIES, - null, - null, - settings, - List.of(), - IndexVersion.current(), - builder - ); - var result = builder.build(); - assertEquals(Settings.EMPTY, result); - } - - public void testGetAdditionalIndexSettingsTsdbMode() throws IOException { - // Same scenario as testGetAdditionalIndexSettingsTsdb, but using IndexMode.TSDB directly to make sure the - // legacy license check in LogsdbIndexModeSettingsProvider#isLegacyLicensedUsageOfSyntheticSourceAllowed - // treats it identically to IndexMode.TIME_SERIES. - Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); - String dataStreamName = "metrics-my-app"; - String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); - Settings.Builder builder = Settings.builder(); - provider.provideAdditionalSettings( - indexName, - dataStreamName, - IndexMode.TSDB, + mode, null, null, settings, @@ -195,6 +175,9 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception true ); + // IndexMode.TSDB is a preferred alternative to IndexMode.TIME_SERIES and must be treated identically by the + // legacy license check. + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); @@ -202,54 +185,7 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception provider.provideAdditionalSettings( indexName, dataStreamName, - IndexMode.TIME_SERIES, - null, - null, - settings, - List.of(), - IndexVersion.current(), - builder - ); - - var result = builder.build(); - var expected = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED").build(); - assertEquals(expected, result); - } - - public void testGetAdditionalIndexSettingsTsdbAfterCutoffDateMode() throws Exception { - // Same scenario as testGetAdditionalIndexSettingsTsdbAfterCutoffDate, but using IndexMode.TSDB directly to - // make sure the legacy license check treats it identically to IndexMode.TIME_SERIES. - long start = LocalDateTime.of(2025, 2, 5, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); - License license = createGoldOrPlatinumLicense(start); - long time = LocalDateTime.of(2024, 12, 31, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli(); - var licenseState = new XPackLicenseState(() -> time, new XPackLicenseStatus(license.operationMode(), true, null)); - - var licenseService = new LogsdbLicenseService(Settings.EMPTY); - licenseService.setLicenseState(licenseState); - var mockLicenseService = mock(LicenseService.class); - when(mockLicenseService.getLicense()).thenReturn(license); - - LogsdbLicenseService syntheticSourceLicenseService = new LogsdbLicenseService(Settings.EMPTY); - syntheticSourceLicenseService.setLicenseState(licenseState); - syntheticSourceLicenseService.setLicenseService(mockLicenseService); - - provider = new LogsdbIndexModeSettingsProvider(syntheticSourceLicenseService, Settings.EMPTY); - provider.init( - im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), - IndexVersion::current, - () -> Version.CURRENT, - true, - true - ); - - Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); - String dataStreamName = "metrics-my-app"; - String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); - Settings.Builder builder = Settings.builder(); - provider.provideAdditionalSettings( - indexName, - dataStreamName, - IndexMode.TSDB, + mode, null, null, settings, diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index f8c347487739f..c5fbbd6fc35e5 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -339,26 +339,14 @@ public void testMetricAndDimension() { ); } - public void testTimeSeriesIndexDefault() throws Exception { - var randomMetricType = randomFrom(TimeSeriesParams.MetricType.scalar()); - var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); - var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> { - minimalMapping(b); - b.field("time_series_metric", randomMetricType.toString()); - })); - var ft = (UnsignedLongFieldMapper.UnsignedLongFieldType) mapperService.fieldType("field"); - assertThat(ft.getMetricType(), equalTo(randomMetricType)); - assertTrue(ft.indexType().hasOnlyDocValues()); - } - /** * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) * so it must produce the same doc-values-only defaulting for metric fields. */ - public void testTimeSeriesIndexDefaultTsdb() throws Exception { + public void testTimeSeriesIndexDefault() throws Exception { var randomMetricType = randomFrom(TimeSeriesParams.MetricType.scalar()); - var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TSDB.getName()) + var mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> { minimalMapping(b); From fed7b9401b97402a602066c73ae80cd7b2c3c297 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 6 Jul 2026 18:35:42 +0000 Subject: [PATCH 18/23] [CI] Auto commit changes from spotless --- .../rules/physical/local/SubstituteRoundToTests.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java index 13388bdaf761a..c15ad1f9b9d5a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java @@ -1023,9 +1023,7 @@ public void testRoundToWithTimeSeriesIndices() { @Override public Map targetShards() { var indexMetadata = IndexMetadata.builder("test_index") - .settings( - ESTestCase.indexSettings(IndexVersion.current(), 1, 1).put(IndexSettings.MODE.getKey(), mode.getName()) - ) + .settings(ESTestCase.indexSettings(IndexVersion.current(), 1, 1).put(IndexSettings.MODE.getKey(), mode.getName())) .build(); return Map.of(new ShardId(new Index("id", "n/a"), 1), indexMetadata); } From 9980edc2c8b16cb81433c7f62eb6711b65124055 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 6 Jul 2026 22:07:25 +0300 Subject: [PATCH 19/23] Simplify TransportGetDataStreamsActionTests TSDB merge Randomize the index mode inside the shared DataStreamTestHelper#getClusterStateWithDataStream instead of hand-building indices, so the merged test stays as close to the original helper-based version as possible. --- .../TransportGetDataStreamsActionTests.java | 56 ++++++------------- .../metadata/DataStreamTestHelper.java | 5 +- 2 files changed, 20 insertions(+), 41 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index 4cfa9bf558452..5b63d470d755f 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -33,7 +33,6 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.TestIndexNameExpressionResolver; @@ -275,54 +274,33 @@ public void testGetTimeSeriesDataStream() { ); } - /** - * {@link IndexMode#TSDB} must be handled identically to {@link IndexMode#TIME_SERIES} at both - * {@code IndexMode.isTsdb(...)} call sites in {@link TransportGetDataStreamsAction#innerOperation}. - * {@link org.elasticsearch.cluster.metadata.DataStreamTestHelper#getClusterStateWithDataStream} - * hardcodes {@link IndexMode#TIME_SERIES}, so the data stream and backing indices are built by - * hand here so the mode can be randomized. - */ public void testGetTimeSeriesDataStreamWithOutOfOrderIndices() { - IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); - String dataStreamName = "ds-1"; + String dataStream = "ds-1"; Instant sixHoursAgo = now.minus(6, ChronoUnit.HOURS); Instant fourHoursAgo = now.minus(4, ChronoUnit.HOURS); Instant twoHoursAgo = now.minus(2, ChronoUnit.HOURS); Instant twoHoursAhead = now.plus(2, ChronoUnit.HOURS); - List> timeSlices = List.of( - new Tuple<>(fourHoursAgo, twoHoursAgo), - new Tuple<>(sixHoursAgo, fourHoursAgo), - new Tuple<>(twoHoursAgo, twoHoursAhead) - ); - - ProjectMetadata.Builder mBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()); - List backingIndices = new ArrayList<>(); - long generation = 1L; - for (Tuple tuple : timeSlices) { - Instant start = tuple.v1(); - Instant end = tuple.v2(); - Settings settings = Settings.builder() - .put("index.mode", mode.getName()) - .put("index.routing_path", "uid") - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(start)) - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(end)) - .build(); - var im = createIndexMetadata(getDefaultBackingIndexName(dataStreamName, generation, start.toEpochMilli()), true, settings, 0); - mBuilder.put(im, true); - backingIndices.add(im); - generation++; + var projectId = randomProjectIdOrDefault(); + ClusterState state; + { + var mBuilder = ProjectMetadata.builder(projectId); + DataStreamTestHelper.getClusterStateWithDataStream( + mBuilder, + dataStream, + List.of( + new Tuple<>(fourHoursAgo, twoHoursAgo), + new Tuple<>(sixHoursAgo, fourHoursAgo), + new Tuple<>(twoHoursAgo, twoHoursAhead) + ) + ); + state = ClusterState.builder(new ClusterName("_name")).putProjectMetadata(mBuilder.build()).build(); } - DataStream dataStream = DataStream.builder( - dataStreamName, - backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()) - ).setGeneration(generation).setIndexMode(mode).build(); - mBuilder.put(dataStream); var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); var response = TransportGetDataStreamsAction.innerOperation( - projectStateFromProject(mBuilder), + state.projectState(projectId), req, resolver, systemIndices, @@ -337,7 +315,7 @@ public void testGetTimeSeriesDataStreamWithOutOfOrderIndices() { response.getDataStreams(), contains( allOf( - transformedMatch(d -> d.getDataStream().getName(), equalTo(dataStreamName)), + transformedMatch(d -> d.getDataStream().getName(), equalTo(dataStream)), transformedMatch(d -> d.getTimeSeries().temporalRanges(), contains(new Tuple<>(sixHoursAgo, twoHoursAhead))) ) ) diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index 2d1930edfe849..6cc1dc46f709a 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -575,6 +575,7 @@ public static void getClusterStateWithDataStream( String dataStreamName, List> timeSlices ) { + IndexMode indexMode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); List backingIndices = new ArrayList<>(); DataStream existing = builder.dataStream(dataStreamName); if (existing != null) { @@ -587,7 +588,7 @@ public static void getClusterStateWithDataStream( Instant start = tuple.v1(); Instant end = tuple.v2(); Settings settings = Settings.builder() - .put("index.mode", "time_series") + .put("index.mode", indexMode.getName()) .put("index.routing_path", "uid") .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(start)) .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(end)) @@ -600,7 +601,7 @@ public static void getClusterStateWithDataStream( var dataStreamBuilder = DataStream.builder( dataStreamName, backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()) - ).setGeneration(generation).setIndexMode(IndexMode.TIME_SERIES); + ).setGeneration(generation).setIndexMode(indexMode); if (existing != null) { dataStreamBuilder.setMetadata(existing.getMetadata()) .setHidden(existing.isHidden()) From d3ef602043ea1e4b7528f3f058c5d16c4b819e98 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 6 Jul 2026 22:10:58 +0300 Subject: [PATCH 20/23] Drop redundant TSDB block in lifecycle bounds test The data stream built at the top of the test already gets a randomized TIME_SERIES/TSDB mode via the shared DataStreamTestHelper#getClusterStateWithDataStream, so the extra hand-built-index block added for TSDB coverage was retesting the same behavior a second time. --- .../DataStreamLifecycleServiceTests.java | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index 60753b3a9da78..d37b0d37fed11 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -49,7 +49,6 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.MergePolicyConfig; import org.elasticsearch.snapshots.SearchableSnapshotsSettings; @@ -1247,43 +1246,6 @@ public void testTimeSeriesIndicesStillWithinTimeBounds() { ); assertThat(indices.size(), is(0)); } - - { - // same coverage as above, but building the indices directly with an explicitly configured - // index mode, randomized between IndexMode.TIME_SERIES and IndexMode.TSDB - IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); - IndexMetadata outOfBoundsIndex = IndexMetadata.builder(randomAlphaOfLengthBetween(10, 30)) - .settings( - indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) - .put(IndexSettings.MODE.getKey(), mode.getName()) - .put("index.routing_path", "uid") - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), start1.toEpochMilli()) - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), end1.toEpochMilli()) - ) - .build(); - IndexMetadata withinBoundsIndex = IndexMetadata.builder(randomAlphaOfLengthBetween(10, 30)) - .settings( - indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) - .put(IndexSettings.MODE.getKey(), mode.getName()) - .put("index.routing_path", "uid") - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), start2.toEpochMilli()) - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), end2.toEpochMilli()) - ) - .build(); - - ProjectMetadata modeProject = ProjectMetadata.builder(randomProjectIdOrDefault()) - .put(outOfBoundsIndex, true) - .put(withinBoundsIndex, true) - .build(); - - Set indices = DataStreamLifecycleService.timeSeriesIndicesStillWithinTimeBounds( - modeProject, - List.of(outOfBoundsIndex.getIndex(), withinBoundsIndex.getIndex()), - currentTime::toEpochMilli - ); - - assertThat(indices, containsInAnyOrder(withinBoundsIndex.getIndex())); - } } public void testTrackingTimeStats() { From 8d32e11cb26b1cd834433da7ae3f5ff4fd0f3005 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 6 Jul 2026 22:44:13 +0300 Subject: [PATCH 21/23] Collapse remaining TIME_SERIES-only test util overloads Several test helpers still had a no-arg/fixed-mode flavor that delegated to a sibling taking an explicit IndexMode, left over from the initial TSDB alias work. Fold each pair into a single function that randomizes between TIME_SERIES and TSDB internally, or pass the mode through explicitly where a fixed value is still needed. --- .../DataStreamIndexSettingsProviderTests.java | 17 ++---------- ...astTimeSeriesIndexCreationActionTests.java | 22 +++------------- .../routing/DimensionsExtractorTests.java | 2 +- .../cluster/routing/IndexRoutingTests.java | 26 +++++++------------ .../routing/RoutingPathExtractorTests.java | 2 +- ...SeriesEligibleWriteWindowLocatorTests.java | 14 +++++----- .../TransportDownsampleActionTests.java | 10 ++----- .../PushExpressionsToFieldLoadTests.java | 14 +++++----- 8 files changed, 32 insertions(+), 75 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index e3cb6873d162c..dce5307f14fe3 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -942,11 +942,6 @@ public void testDynamicTemplatePrecedence() throws Exception { } } - /** - * {@link IndexMode#TSDB} (the {@code tsdb} value) must be handled the same as - * {@link IndexMode#TIME_SERIES}, so that the {@code assert IndexMode.isTsdb(...)} sanity check - * in {@link DataStreamIndexSettingsProvider#onUpdateMappings} is exercised with the alias too. - */ public void testAddNewDimension() throws Exception { String newMapping = """ { @@ -964,7 +959,7 @@ public void testAddNewDimension() throws Exception { } } """; - Settings result = onUpdateMappings("field1", "field1", newMapping, randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB)); + Settings result = onUpdateMappings("field1", "field1", newMapping); assertThat(result.size(), equalTo(1)); assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field2")); } @@ -1087,15 +1082,7 @@ private Settings generateTsdbSettings(String mapping, Instant now) throws IOExce } private Settings onUpdateMappings(String routingPath, String dimensions, String newMapping) throws IOException { - return onUpdateMappings(routingPath, dimensions, newMapping, IndexMode.TIME_SERIES); - } - - /** - * Same as {@link #onUpdateMappings(String, String, String)} but with an explicit {@code indexMode}, - * so that tests can assert {@link IndexMode#TSDB} satisfies the {@code assert IndexMode.isTsdb(...)} - * sanity check in {@link DataStreamIndexSettingsProvider#onUpdateMappings}. - */ - private Settings onUpdateMappings(String routingPath, String dimensions, String newMapping, IndexMode indexMode) throws IOException { + IndexMode indexMode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); String dataStreamName = "logs-app1"; Settings.Builder currentSettings = Settings.builder() .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java index dc72e4360244a..4ff3c629e61a4 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java @@ -160,9 +160,8 @@ public void testSortAndRetrieve() { } public void testCreateIndicesWhenNeeded() throws Exception { - IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Instant now = Instant.now(); - ClusterState clusterState = stateWithExisting(List.of(), now, mode); + ClusterState clusterState = stateWithExisting(List.of(), now); List dayOffsets = List.of(5, 3, 2); List createdNames = new ArrayList<>(); // Add two timestamps that fall within one index @@ -453,20 +452,6 @@ private ClusterState stateWithExisting(List> timeSlices, .build(); } - /** - * Equivalent of {@link #stateWithExisting(List, Instant)} that builds the data stream in the given - * {@link IndexMode}, to verify that {@code IndexMode.isTsdb} treats {@link IndexMode#TSDB} and - * {@link IndexMode#TIME_SERIES} identically. - */ - private ClusterState stateWithExisting(List> timeSlices, Instant now, IndexMode mode) { - if (mode == IndexMode.TIME_SERIES) { - return stateWithExisting(timeSlices, now); - } - List> allSlices = new ArrayList<>(timeSlices); - allSlices.add(Tuple.tuple(now, now.plus(randomIntBetween(1, 3), ChronoUnit.DAYS))); - return ClusterState.builder(ClusterName.DEFAULT).putProjectMetadata(projectWithDataStream(mode, allSlices)).build(); - } - /** Builds a ClusterState with a data stream that has one non-TSDB (standard) backing index. */ private ClusterState stateWithNoTsdbIndices() { String indexName = DataStream.getDefaultBackingIndexName(DATA_STREAM, 1); @@ -477,7 +462,7 @@ private ClusterState stateWithNoTsdbIndices() { } private static IndexMetadata createIndexMetadata(String indexName, Instant startTime, Instant endTime) { - return createIndexMetadata(indexName, startTime, endTime, IndexMode.TIME_SERIES); + return createIndexMetadata(indexName, startTime, endTime, randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB)); } private static IndexMetadata createIndexMetadata(String indexName, Instant startTime, Instant endTime, IndexMode indexMode) { @@ -492,8 +477,7 @@ private static IndexMetadata createIndexMetadata(String indexName, Instant start /** * Builds a ProjectMetadata with a data stream in the given {@link IndexMode} whose backing indices - * cover the given time ranges, to verify that {@code IndexMode.isTsdb} treats {@link IndexMode#TSDB} - * and {@link IndexMode#TIME_SERIES} identically in {@link TransportPastTimeSeriesIndexCreationAction}. + * cover the given time ranges. */ private ProjectMetadata projectWithDataStream(IndexMode mode, List> timeSlices) { List backingIndices = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/DimensionsExtractorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/DimensionsExtractorTests.java index b332e67e7bfbb..3911c0e80657e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/DimensionsExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/DimensionsExtractorTests.java @@ -226,7 +226,7 @@ private void assertExtractorMatchesSourceParser(Map doc, String } private static IndexRouting.ExtractFromSource.ForIndexDimensions forIndexDimensions(String dimensionPath) { - return forIndexDimensions(dimensionPath, IndexMode.TIME_SERIES); + return forIndexDimensions(dimensionPath, randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB)); } private static IndexRouting.ExtractFromSource.ForIndexDimensions forIndexDimensions(String dimensionPath, IndexMode indexMode) { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 1926042889a37..3aadf8e0cb327 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -707,7 +707,8 @@ public void testRoutingPathBwcAfterTsidBasedRouting() throws IOException { ), 8, "dim.*,other.*,top", - randomBoolean() + randomBoolean(), + IndexMode.TIME_SERIES ); assertFalse(TsidBuilder.useSingleBytePrefixLayout(fixture.routing.creationVersion)); /* @@ -1243,20 +1244,6 @@ private TimeSeriesRoutingFixture indexRoutingForRoutingPath( return getIndexRoutingWithSetting(createdVersion, shards, path, IndexMetadata.INDEX_ROUTING_PATH.getKey(), useSyntheticId); } - private TimeSeriesRoutingFixture indexRoutingForTimeSeriesDimensions( - IndexVersion createdVersion, - int shards, - String path, - boolean useSyntheticId - ) { - return indexRoutingForTimeSeriesDimensions(createdVersion, shards, path, useSyntheticId, IndexMode.TIME_SERIES); - } - - /** - * Same as {@link #indexRoutingForTimeSeriesDimensions(IndexVersion, int, String, boolean)} but lets the - * caller pick the index mode, so callers can prove {@link IndexMode#TSDB} routes identically to - * {@link IndexMode#TIME_SERIES}. - */ private TimeSeriesRoutingFixture indexRoutingForTimeSeriesDimensions( IndexVersion createdVersion, int shards, @@ -1277,7 +1264,14 @@ private static TimeSeriesRoutingFixture getIndexRoutingWithSetting( String setting, boolean useSyntheticId ) { - return getIndexRoutingWithSetting(indexVersion, shards, path, setting, useSyntheticId, IndexMode.TIME_SERIES); + return getIndexRoutingWithSetting( + indexVersion, + shards, + path, + setting, + useSyntheticId, + randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB) + ); } /** diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingPathExtractorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingPathExtractorTests.java index e9322be5817c5..808ab5a945d63 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingPathExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingPathExtractorTests.java @@ -210,7 +210,7 @@ private void assertExtractorMatchesSourceParser(Map doc, String } private static IndexRouting.ExtractFromSource.ForRoutingPath forRoutingPath(String routingPath) { - return forRoutingPath(routingPath, IndexMode.TIME_SERIES); + return forRoutingPath(routingPath, randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB)); } private static IndexRouting.ExtractFromSource.ForRoutingPath forRoutingPath(String routingPath, IndexMode indexMode) { diff --git a/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java b/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java index f958101f06bd7..514547c37b477 100644 --- a/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java +++ b/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java @@ -56,12 +56,11 @@ public void testInfiniteWriteWindow() { * with either mode must be treated identically by the eligible write window logic. */ public void testWriteWindowDefinedByRetention() { - IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); // Configured retention { TimeValue retention = TimeValue.timeValueDays(30); DataStreamLifecycle lifecycle = DataStreamLifecycle.dataLifecycleBuilder().dataRetention(retention).build(); - DataStream dataStream = dataStream("metrics-test", lifecycle, mode); + DataStream dataStream = dataStream("metrics-test", lifecycle); ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); long requestTimestamp = randomNonNegativeLong(); assertThat( @@ -78,7 +77,7 @@ public void testWriteWindowDefinedByRetention() { lifecycleBuilder.dataRetention(retention); } DataStreamLifecycle lifecycle = lifecycleBuilder.build(); - DataStream dataStream = dataStream("metrics-test", lifecycle, mode); + DataStream dataStream = dataStream("metrics-test", lifecycle); ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(null, globalMax); long requestTimestamp = randomNonNegativeLong(); @@ -141,11 +140,10 @@ public long getEligibleWriteWindowFromPolicy(String policy, ProjectMetadata proj } private static DataStream dataStream(String name, DataStreamLifecycle lifecycle) { - return dataStream(name, lifecycle, IndexMode.TIME_SERIES); - } - - private static DataStream dataStream(String name, DataStreamLifecycle lifecycle, IndexMode indexMode) { Index index = new Index(DataStream.getDefaultBackingIndexName(name, 1), randomAlphaOfLength(10)); - return DataStream.builder(name, List.of(index)).setLifecycle(lifecycle).setIndexMode(indexMode).build(); + return DataStream.builder(name, List.of(index)) + .setLifecycle(lifecycle) + .setIndexMode(randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB)) + .build(); } } diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java index 939d522125a7e..690fd15f058d0 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java @@ -217,10 +217,8 @@ public void tearDown() throws Exception { } public void testDownsampling() { - // index.mode may be either TIME_SERIES or its preferred alias TSDB; both must be handled identically. - IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); var projectMetadata = ProjectMetadata.builder(projectId) - .put(createSourceIndexMetadata(sourceIndex, primaryShards, replicaShards, mode)) + .put(createSourceIndexMetadata(sourceIndex, primaryShards, replicaShards)) .build(); var clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) @@ -497,15 +495,11 @@ private void verifyIndexFinalisation() { } private IndexMetadata.Builder createSourceIndexMetadata(String sourceIndex, int primaryShards, int replicaShards) { - return createSourceIndexMetadata(sourceIndex, primaryShards, replicaShards, IndexMode.TIME_SERIES); - } - - private IndexMetadata.Builder createSourceIndexMetadata(String sourceIndex, int primaryShards, int replicaShards, IndexMode indexMode) { return IndexMetadata.builder(sourceIndex) .settings( indexSettings(IndexVersion.current(), randomUUID(), primaryShards, replicaShards).put( IndexSettings.MODE.getKey(), - indexMode.getName() + randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB).getName() ) .put("index.routing_path", "dimensions") .put(IndexMetadata.SETTING_BLOCKS_WRITE, true) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java index ef66e701fd095..7c577d93412ef 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java @@ -343,14 +343,13 @@ public void testRoundToTwoFields() { */ public void testRoundToInTsEval() { assumeTrue("ROUND_TO block loader must be enabled", EsqlCapabilities.Cap.ROUND_TO_BLOCK_LOADER.isEnabled()); - IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); PhysicalPlan plan = tsPlannerOptimizer.plan(""" TS k8s | EVAL r = ROUND_TO(events_received, 100, 200, 300) | SORT @timestamp | LIMIT 10 | KEEP r - """, tsSearchStats(mode)); + """, tsSearchStats()); List pushed = findPushedFields(plan, "events_received", BlockLoaderFunctionConfig.Function.ROUND_TO); assertThat(pushed, hasSize(0)); @@ -847,15 +846,16 @@ private List collectNodes(PhysicalPlan plan, Class targetShards() { IndexMetadata indexMetadata = IndexMetadata.builder("k8s") - .settings(indexSettings(IndexVersion.current(), 1, 1).put(IndexSettings.MODE.getKey(), mode.name())) + .settings( + indexSettings(IndexVersion.current(), 1, 1).put( + IndexSettings.MODE.getKey(), + randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB).name() + ) + ) .build(); return Map.of(new ShardId(new Index("k8s", "n/a"), 0), indexMetadata); } From ee9b58aa7278b69267d14a2331a2fb2fe90534d8 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 6 Jul 2026 22:58:52 +0300 Subject: [PATCH 22/23] Randomize tsdbSettings mode internally tsdbSettings took an explicit IndexMode arg with every static fixture hardcoding TIME_SERIES. Turn the fixtures into instance methods and randomize the mode inside tsdbSettings itself, since static fields can't use randomFrom before a test's Random exists. --- ...meSeriesMetadataFieldBlockLoaderTests.java | 72 +++++++++---------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java index 77cf4994bbc82..89ed3f6d8d84d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java @@ -58,29 +58,33 @@ public class TimeSeriesMetadataFieldBlockLoaderTests extends MapperServiceTestCa * Plain TSDB index. {@link IndexMode#TIME_SERIES} defaults {@code _source.mode} to * {@link SourceFieldMapper.Mode#SYNTHETIC}, so synthetic source reconstructs {@code _source} as JSON. */ - private static final Settings TSDB_SYNTHETIC_SETTINGS = tsdbSettings(IndexMode.TIME_SERIES, null, "host"); + private Settings tsdbSyntheticSettings() { + return tsdbSettings(null, "host"); + } /** * TSDB index with {@code index.mapping.source.mode: stored}. Stored source preserves the raw * indexed bytes (e.g. CBOR for Prometheus remote-write documents), so the {@code _timeseries} loader must explicitly normalize to JSON. */ - private static final Settings TSDB_STORED_SETTINGS = tsdbSettings(IndexMode.TIME_SERIES, SourceFieldMapper.Mode.STORED, "host"); + private Settings tsdbStoredSettings() { + return tsdbSettings(SourceFieldMapper.Mode.STORED, "host"); + } /** * TSDB index that mirrors the Prometheus mapping: a {@code labels} {@code passthrough} object marked as * {@code time_series_dimension: true}. Dimension fields show up under {@code labels.*} and the index uses stored source. */ - private static final Settings TSDB_PROMETHEUS_LIKE_SETTINGS = tsdbSettings( - IndexMode.TIME_SERIES, - SourceFieldMapper.Mode.STORED, - "labels.*" - ); + private Settings tsdbPrometheusLikeSettings() { + return tsdbSettings(SourceFieldMapper.Mode.STORED, "labels.*"); + } /** * OTel-like TSDB index: an {@code attributes} passthrough object. Dimension fields show up under * {@code attributes.*} and also as root-level aliases. Synthetic source (JSON) is used. */ - private static final Settings TSDB_OTEL_LIKE_SETTINGS = tsdbSettings(IndexMode.TIME_SERIES, null, "attributes.*"); + private Settings tsdbOtelLikeSettings() { + return tsdbSettings(null, "attributes.*"); + } private static final String MAPPING = """ { @@ -157,9 +161,13 @@ public class TimeSeriesMetadataFieldBlockLoaderTests extends MapperServiceTestCa } """; - private static Settings tsdbSettings(IndexMode mode, SourceFieldMapper.Mode sourceMode, String routingPath) { + /** + * Randomizes between {@link IndexMode#TIME_SERIES} and {@link IndexMode#TSDB} so callers exercise both, + * since {@link TimeSeriesMetadataFieldBlockLoader} must be selected and behave identically for either. + */ + private Settings tsdbSettings(SourceFieldMapper.Mode sourceMode, String routingPath) { Settings.Builder builder = Settings.builder() - .put(IndexSettings.MODE.getKey(), mode.getName()) + .put(IndexSettings.MODE.getKey(), randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB).getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z"); @@ -169,15 +177,9 @@ private static Settings tsdbSettings(IndexMode mode, SourceFieldMapper.Mode sour return builder.build(); } - /** - * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) - * so {@link TimeSeriesMetadataFieldBlockLoader} must be selected and behave identically regardless of - * which of the two mode names configured the index. - */ public void testDimensionsOnly() throws IOException { - IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); BlockLoader loader = createBlockLoader( - tsdbSettings(mode, null, "host"), + tsdbSettings(null, "host"), MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of()) ); @@ -187,7 +189,7 @@ public void testDimensionsOnly() throws IOException { public void testDimensionsAndMetrics() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_SYNTHETIC_SETTINGS, + tsdbSyntheticSettings(), MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(true, Set.of()) ); @@ -197,7 +199,7 @@ public void testDimensionsAndMetrics() throws IOException { public void testExcludedDimensions() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_SYNTHETIC_SETTINGS, + tsdbSyntheticSettings(), MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of("host", "region")) ); @@ -207,7 +209,7 @@ public void testExcludedDimensions() throws IOException { public void testExcludedDimensionsWithMetrics() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_SYNTHETIC_SETTINGS, + tsdbSyntheticSettings(), MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(true, Set.of("env")) ); @@ -224,7 +226,7 @@ public void testExcludedDimensionsWithMetrics() throws IOException { */ public void testExcludedDimensionsWithWildcardDimensionSetting() throws IOException { Settings settings = Settings.builder() - .put(TSDB_PROMETHEUS_LIKE_SETTINGS) + .put(tsdbPrometheusLikeSettings()) .putList(IndexMetadata.INDEX_DIMENSIONS.getKey(), "labels.*") .build(); BlockLoader loader = createBlockLoader( @@ -237,7 +239,7 @@ public void testExcludedDimensionsWithWildcardDimensionSetting() throws IOExcept } public void testNoConfigReturnSourceBlockLoader() throws IOException { - MapperService mapperService = createMapperService(TSDB_SYNTHETIC_SETTINGS, MAPPING); + MapperService mapperService = createMapperService(tsdbSyntheticSettings(), MAPPING); BlockLoader loader = mapperService.documentMapper() .sourceMapper() .fieldType() @@ -252,7 +254,7 @@ public void testNoConfigReturnSourceBlockLoader() throws IOException { */ public void testReadEmitsJsonForSyntheticSourceIndex() throws IOException { BytesReference json = bytes(XContentType.JSON, b -> writeTimestampAndDimensions(b, "host-1", "prod", "us-east-1")); - BytesRef value = readTimeSeriesValue(TSDB_SYNTHETIC_SETTINGS, MAPPING, sourceToParse(json, XContentType.JSON)); + BytesRef value = readTimeSeriesValue(tsdbSyntheticSettings(), MAPPING, sourceToParse(json, XContentType.JSON)); assertThat(parseJsonObject(value), equalTo(Map.of("host", "host-1", "env", "prod", "region", "us-east-1"))); } @@ -264,7 +266,7 @@ public void testReadEmitsJsonForSyntheticSourceIndex() throws IOException { */ public void testReadConvertsCborStoredSourceToJson() throws IOException { BytesReference cbor = bytes(XContentType.CBOR, b -> writeTimestampAndDimensions(b, "host-1", "prod", "us-east-1")); - BytesRef value = readTimeSeriesValue(TSDB_STORED_SETTINGS, MAPPING, sourceToParse(cbor, XContentType.CBOR)); + BytesRef value = readTimeSeriesValue(tsdbStoredSettings(), MAPPING, sourceToParse(cbor, XContentType.CBOR)); assertThat(parseJsonObject(value), equalTo(Map.of("host", "host-1", "env", "prod", "region", "us-east-1"))); } @@ -274,7 +276,7 @@ public void testReadConvertsCborStoredSourceToJson() throws IOException { */ public void testReadPassesThroughJsonStoredSource() throws IOException { BytesReference json = bytes(XContentType.JSON, b -> writeTimestampAndDimensions(b, "host-1", "prod", "us-east-1")); - BytesRef value = readTimeSeriesValue(TSDB_STORED_SETTINGS, MAPPING, sourceToParse(json, XContentType.JSON)); + BytesRef value = readTimeSeriesValue(tsdbStoredSettings(), MAPPING, sourceToParse(json, XContentType.JSON)); assertThat(parseJsonObject(value), equalTo(Map.of("host", "host-1", "env", "prod", "region", "us-east-1"))); } @@ -295,11 +297,7 @@ public void testReadJsonContainsDimensionKeyValuePairs() throws IOException { b.field("go_gc_cleanups_executed_cleanups_total", 1.0); b.endObject(); }); - BytesRef value = readTimeSeriesValue( - TSDB_PROMETHEUS_LIKE_SETTINGS, - PROMETHEUS_LIKE_MAPPING, - sourceToParse(cbor, XContentType.CBOR) - ); + BytesRef value = readTimeSeriesValue(tsdbPrometheusLikeSettings(), PROMETHEUS_LIKE_MAPPING, sourceToParse(cbor, XContentType.CBOR)); Map parsed = parseJsonObject(value); assertThat(parsed.keySet(), equalTo(Set.of("labels"))); @SuppressWarnings("unchecked") @@ -328,7 +326,7 @@ public void testWithoutFieldsExcludesLabelsFromOutput() throws IOException { b.endObject(); }); BytesRef value = readTimeSeriesValue( - TSDB_PROMETHEUS_LIKE_SETTINGS, + tsdbPrometheusLikeSettings(), PROMETHEUS_LIKE_MAPPING, sourceToParse(cbor, XContentType.CBOR), Set.of("labels.instance") @@ -350,7 +348,7 @@ public void testWithoutFieldsExcludesLabelsFromOutput() throws IOException { */ public void testOtelPassthroughAliasExcludedByShortName() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_OTEL_LIKE_SETTINGS, + tsdbOtelLikeSettings(), OTEL_LIKE_MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of("cpu")) ); @@ -365,7 +363,7 @@ public void testOtelPassthroughAliasExcludedByShortName() throws IOException { */ public void testPrometheusPassthroughAliasExcludedByShortName() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_PROMETHEUS_LIKE_SETTINGS, + tsdbPrometheusLikeSettings(), PROMETHEUS_LIKE_MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of("job")) ); @@ -378,7 +376,7 @@ public void testPrometheusPassthroughAliasExcludedByShortName() throws IOExcepti */ public void testWithoutAllDimensionsYieldsEmptySourcePaths() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_SYNTHETIC_SETTINGS, + tsdbSyntheticSettings(), MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of("host", "env", "region")) ); @@ -392,7 +390,7 @@ public void testWithoutAllDimensionsYieldsEmptySourcePaths() throws IOException */ public void testWithoutUnknownFieldIsIgnored() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_SYNTHETIC_SETTINGS, + tsdbSyntheticSettings(), MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of("does_not_exist")) ); @@ -406,7 +404,7 @@ public void testWithoutUnknownFieldIsIgnored() throws IOException { */ public void testExcludedDimensionsWithMetricsAndExclusion() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_SYNTHETIC_SETTINGS, + tsdbSyntheticSettings(), MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(true, Set.of("host")) ); @@ -420,7 +418,7 @@ public void testExcludedDimensionsWithMetricsAndExclusion() throws IOException { */ public void testOtelPassthroughConcreteNameExcludesCorrectly() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_OTEL_LIKE_SETTINGS, + tsdbOtelLikeSettings(), OTEL_LIKE_MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of("attributes.cpu")) ); From 57305a038f6f95dd465d065072a88a70475a6ca2 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 7 Jul 2026 07:09:19 +0300 Subject: [PATCH 23/23] Fix IndexRoutingTests message assertions for TSDB mode Several tests hardcoded "time_series mode" in their expected error message despite building the fixture through the now-randomized getIndexRoutingWithSetting, so a randomly chosen TSDB mode failed the assertion. Expose the actual IndexMode on TimeSeriesRoutingFixture and build the expected message from it. --- .../cluster/routing/IndexRoutingTests.java | 45 ++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 3aadf8e0cb327..cd140f0093416 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -60,9 +60,11 @@ public class IndexRoutingTests extends ESTestCase { /** - * Result of creating time_series index routing; carries the routing and the explicit synthetic-id setting used. + * Result of creating time_series index routing; carries the routing, the explicit synthetic-id setting + * used, and the {@link IndexMode} the fixture was built with (either {@link IndexMode#TIME_SERIES} or + * {@link IndexMode#TSDB}). */ - private record TimeSeriesRoutingFixture(IndexRouting routing, boolean useSyntheticId) {} + private record TimeSeriesRoutingFixture(IndexRouting routing, boolean useSyntheticId, IndexMode indexMode) {} public void testSimpleRoutingRejectsEmptyId() { IndexRouting indexRouting = IndexRouting.fromIndexMetadata( @@ -562,16 +564,21 @@ public void testRoutingPathMismatchSource() { } public void testRoutingPathUpdate() { - IndexRouting routing = indexRoutingForPath(between(1, 5), "foo").routing(); + TimeSeriesRoutingFixture fixture = indexRoutingForPath(between(1, 5), "foo"); + IndexRouting routing = fixture.routing(); Exception e = expectThrows( IllegalArgumentException.class, () -> routing.updateShard(randomAlphaOfLength(5), randomBoolean() ? null : randomAlphaOfLength(5)) ); - assertThat(e.getMessage(), equalTo("update is not supported because the destination index [test] is in time_series mode")); + assertThat( + e.getMessage(), + equalTo("update is not supported because the destination index [test] is in " + fixture.indexMode().getName() + " mode") + ); } public void testRoutingIndexWithRouting() throws IOException { - IndexRouting indexRouting = indexRoutingForPath(5, "foo").routing(); + TimeSeriesRoutingFixture fixture = indexRoutingForPath(5, "foo"); + IndexRouting indexRouting = fixture.routing(); String value = randomAlphaOfLength(5); BytesReference source = source(Map.of("foo", value)); String docRouting = randomAlphaOfLength(5); @@ -583,16 +590,23 @@ public void testRoutingIndexWithRouting() throws IOException { ); assertThat( e.getMessage(), - equalTo("specifying routing is not supported because the destination index [test] is in time_series mode") + equalTo( + "specifying routing is not supported because the destination index [test] is in " + fixture.indexMode().getName() + " mode" + ) ); } public void testRoutingPathCollectSearchWithRouting() { - IndexRouting routing = indexRoutingForPath(between(1, 5), "foo").routing(); + TimeSeriesRoutingFixture fixture = indexRoutingForPath(between(1, 5), "foo"); + IndexRouting routing = fixture.routing(); Exception e = expectThrows(IllegalArgumentException.class, () -> routing.collectSearchShards(randomAlphaOfLength(5), null)); assertThat( e.getMessage(), - equalTo("searching with a specified routing is not supported because the destination index [test] is in time_series mode") + equalTo( + "searching with a specified routing is not supported because the destination index [test] is in " + + fixture.indexMode().getName() + + " mode" + ) ); } @@ -761,16 +775,16 @@ public void testRoutingPathWithSingleBytePrefixTsid() throws IOException { public void testRoutingPathReadWithInvalidString() { int shards = between(2, 1000); - IndexRouting indexRouting = indexRoutingForPath(shards, "foo").routing(); - Exception e = expectThrows(ResourceNotFoundException.class, () -> shardIdForReadFromSourceExtracting(indexRouting, "!@#")); - assertThat(e.getMessage(), equalTo("invalid id [!@#] for index [test] in time_series mode")); + TimeSeriesRoutingFixture fixture = indexRoutingForPath(shards, "foo"); + Exception e = expectThrows(ResourceNotFoundException.class, () -> shardIdForReadFromSourceExtracting(fixture.routing(), "!@#")); + assertThat(e.getMessage(), equalTo("invalid id [!@#] for index [test] in " + fixture.indexMode().getName() + " mode")); } public void testRoutingPathReadWithShortString() { int shards = between(2, 1000); - IndexRouting indexRouting = indexRoutingForPath(shards, "foo").routing(); - Exception e = expectThrows(ResourceNotFoundException.class, () -> shardIdForReadFromSourceExtracting(indexRouting, "")); - assertThat(e.getMessage(), equalTo("invalid id [] for index [test] in time_series mode")); + TimeSeriesRoutingFixture fixture = indexRoutingForPath(shards, "foo"); + Exception e = expectThrows(ResourceNotFoundException.class, () -> shardIdForReadFromSourceExtracting(fixture.routing(), "")); + assertThat(e.getMessage(), equalTo("invalid id [] for index [test] in " + fixture.indexMode().getName() + " mode")); } public void testRoutingPathLogsdb() { @@ -1297,7 +1311,8 @@ private static TimeSeriesRoutingFixture getIndexRoutingWithSetting( IndexRouting.fromIndexMetadata( IndexMetadata.builder("test").settings(settingsBuilder).numberOfShards(shards).numberOfReplicas(1).build() ), - useSyntheticId + useSyntheticId, + indexMode ); }