Add IndexMode.TSDB as the preferred metrics mode

docs/changelog/152901.yaml

@@ -0,0 +1,16 @@
+area: TSDB
+highlight:
+  body: |-
+    Introduce `IndexMode.TSDB` as a new, first-class index mode for metrics.
+    It behaves identically to `TIME_SERIES`, so `index.mode: tsdb` 
+    is now accepted anywhere `index.mode: time_series` is — without renaming 
+    or changing anything about existing `time_series` indices. The querying 
+    experience and downsampling support remains seamless, regardless of the 
+    mix of `TSDB` and `TIME_SERIES` indices in a data stream.
+
+  notable: true
+  title: Add `IndexMode.TSDB` as a preferred alias for TIME_SERIES
+issues: []
+pr: 152901
+summary: Add `IndexMode.TSDB` as a preferred alias for TIME_SERIES
+type: enhancement

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java

@@ -152,7 +152,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
 
     public void testInvalidIndexMode() {
         final var indexName = randomIdentifier();
-        var randomNonTsdbIndexMode = randomValueOtherThan(IndexMode.TIME_SERIES, () -> randomFrom(IndexMode.availableModes()));
+        var randomNonTsdbIndexMode = randomValueOtherThanMany(m -> m.isTsdb(), () -> randomFrom(IndexMode.availableModes()));
 
         var exception = expectThrows(
             IllegalArgumentException.class,
@@ -168,7 +168,11 @@ public void testInvalidIndexMode() {
             containsString(
                 "The setting ["
                     + IndexSettings.SYNTHETIC_ID.getKey()
-                    + "] is only permitted when [index.mode] is set to [TIME_SERIES]. Current mode: ["
+                    + "] is only permitted when [index.mode] is set to ["
+                    + IndexMode.TIME_SERIES.name()
+                    + "] or ["
+                    + IndexMode.TSDB.name()
+                    + "]. Current mode: ["
                     + randomNonTsdbIndexMode.getName().toUpperCase(Locale.ROOT)
                     + "]."
             )

modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java

@@ -88,7 +88,14 @@ int maybeAdjustIndexSettingCount(int baseCount) {
         return count;
     }
 
+    /**
+     * {@link IndexMode#TSDB} resolves {@link IndexMode#isTsdb()} to {@code true} just like
+     * {@link IndexMode#TIME_SERIES}, so creating a backing index with a template index mode of
+     * {@link IndexMode#TSDB} must inject the same start/end time, sequence-number, synthetic id
+     * and dimension settings as {@link IndexMode#TIME_SERIES} does.
+     */
     public void testGetAdditionalIndexSettings() throws Exception {
+        IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB);
         ProjectMetadata projectMetadata = emptyProject();
         String dataStreamName = "logs-app1";
 
@@ -130,7 +137,7 @@ public void testGetAdditionalIndexSettings() throws Exception {
         provider.provideAdditionalSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            IndexMode.TIME_SERIES,
+            mode,
             projectMetadata,
             now,
             settings,
@@ -139,11 +146,11 @@ public void testGetAdditionalIndexSettings() throws Exception {
             additionalSettings
         );
         Settings result = additionalSettings.build();
-        // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
+        // The index.time_series.end_time setting requires index.mode to be set adding it here so that we read this setting:
         // (in production the index.mode setting is usually provided in an index or component template)
-        result = builder().put(result).put("index.mode", "time_series").build();
+        result = builder().put(result).put("index.mode", mode.getName()).build();
         assertThat(result.size(), equalTo(maybeAdjustIndexSettingCount(4)));
-        assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES));
+        assertThat(IndexSettings.MODE.get(result), equalTo(mode));
         assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
         assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
         if (expectedIndexDimensionsTsidOptimizationEnabled) {
@@ -491,7 +498,14 @@ public void testGetAdditionalIndexSettingsNonTsdbTemplate() {
         assertThat(result.size(), equalTo(0));
     }
 
+    /**
+     * The migration check at index-creation time relies on {@link IndexMode#isTsdb(IndexMode)}, so
+     * a data stream currently in {@code standard} mode must also migrate into time series mode when
+     * the matching index template specifies {@link IndexMode#TSDB} directly, not just
+     * {@link IndexMode#TIME_SERIES}.
+     */
     public void testGetAdditionalIndexSettingsMigrateToTsdb() {
+        IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB);
         Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
         String dataStreamName = "logs-app1";
         IndexMetadata idx = createFirstBackingIndex(dataStreamName).build();
@@ -505,7 +519,7 @@ public void testGetAdditionalIndexSettingsMigrateToTsdb() {
         provider.provideAdditionalSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 2),
             dataStreamName,
-            IndexMode.TIME_SERIES,
+            mode,
             projectMetadata,
             now,
             settings,
@@ -514,11 +528,11 @@ public void testGetAdditionalIndexSettingsMigrateToTsdb() {
             additionalSettings
         );
         Settings result = additionalSettings.build();
-        // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting:
+        // The index.time_series.end_time setting requires index.mode to be set adding it here so that we read this setting:
         // (in production the index.mode setting is usually provided in an index or component template)
-        result = builder().put(result).put("index.mode", "time_series").build();
+        result = builder().put(result).put("index.mode", mode.getName()).build();
         assertThat(result.size(), equalTo(maybeAdjustIndexSettingCount(3)));
-        assertThat(result.get(IndexSettings.MODE.getKey()), equalTo("time_series"));
+        assertThat(result.get(IndexSettings.MODE.getKey()), equalTo(mode.getName()));
         assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis())));
         assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
         if (expectedDisabledSequenceNumbers) {
@@ -1068,6 +1082,7 @@ private Settings generateTsdbSettings(String mapping, Instant now) throws IOExce
     }
 
     private Settings onUpdateMappings(String routingPath, String dimensions, String newMapping) throws IOException {
+        IndexMode indexMode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB);
         String dataStreamName = "logs-app1";
         Settings.Builder currentSettings = Settings.builder()
             .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath)
@@ -1076,7 +1091,7 @@ private Settings onUpdateMappings(String routingPath, String dimensions, String
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
             .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
-            .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES);
+            .put(IndexSettings.MODE.getKey(), indexMode);
 
         IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1))
             .settings(currentSettings)
@@ -1102,7 +1117,12 @@ private Settings onUpdateMappings(String routingPath, String dimensions, String
         return additionalSettings.build();
     }
 
+    /**
+     * The sequence-number-disabling gate is keyed off {@link IndexMode#isTsdb()}, so it must behave
+     * identically for a template index mode of {@link IndexMode#TIME_SERIES} and {@link IndexMode#TSDB}.
+     */
     public void testClusterSettingsDefineSeqNoDisabledDefault() throws Exception {
+        IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB);
         Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
         IndexVersion version = IndexVersionUtils.randomVersionBetween(
             IndexVersions.TIME_SERIES_DISABLE_SEQUENCE_NUMBERS_DEFAULT,
@@ -1119,7 +1139,7 @@ public void testClusterSettingsDefineSeqNoDisabledDefault() throws Exception {
         providerWithSeqNoEnabled.provideAdditionalSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            IndexMode.TIME_SERIES,
+            mode,
             emptyProject(),
             now,
             Settings.EMPTY,
@@ -1138,7 +1158,7 @@ public void testClusterSettingsDefineSeqNoDisabledDefault() throws Exception {
         providerWithSeqNoDisabled.provideAdditionalSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            IndexMode.TIME_SERIES,
+            mode,
             emptyProject(),
             now,
             Settings.EMPTY,
@@ -1153,7 +1173,7 @@ public void testClusterSettingsDefineSeqNoDisabledDefault() throws Exception {
         providerWithSeqNoDisabled.provideAdditionalSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            IndexMode.TIME_SERIES,
+            mode,
             emptyProject(),
             now,
             Settings.builder().put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), false).build(),
@@ -1164,7 +1184,12 @@ public void testClusterSettingsDefineSeqNoDisabledDefault() throws Exception {
         assertFalse(additionalSettings.build().hasValue(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey()));
     }
 
+    /**
+     * The synthetic {@code _id} gate is keyed off {@link IndexMode#isTsdb()}, so it must behave
+     * identically for a template index mode of {@link IndexMode#TIME_SERIES} and {@link IndexMode#TSDB}.
+     */
     public void testClusterSettingsDefineSyntheticIdEnabledDefault() throws Exception {
+        IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB);
         Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
         IndexVersion version = IndexVersionUtils.randomVersionBetween(
             IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD,
@@ -1181,7 +1206,7 @@ public void testClusterSettingsDefineSyntheticIdEnabledDefault() throws Exceptio
         providerWithSyntheticIdDisabled.provideAdditionalSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            IndexMode.TIME_SERIES,
+            mode,
             emptyProject(),
             now,
             Settings.EMPTY,
@@ -1200,7 +1225,7 @@ public void testClusterSettingsDefineSyntheticIdEnabledDefault() throws Exceptio
         providerWithSyntheticIdEnabled.provideAdditionalSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            IndexMode.TIME_SERIES,
+            mode,
             emptyProject(),
             now,
             Settings.EMPTY,
@@ -1215,7 +1240,7 @@ public void testClusterSettingsDefineSyntheticIdEnabledDefault() throws Exceptio
         providerWithSyntheticIdEnabled.provideAdditionalSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            IndexMode.TIME_SERIES,
+            mode,
             emptyProject(),
             now,
             Settings.builder().put(IndexSettings.SYNTHETIC_ID.getKey(), false).build(),

modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java

@@ -92,13 +92,13 @@ public void testSortAndRetrieve() {
         }
         // Retrieve all continuous TSDB indices in a single window
         {
+            IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB);
             Instant start1 = Instant.parse("2024-01-15T00:00:00Z");
             Instant start2 = Instant.parse("2024-01-16T00:00:00Z");
             Instant start3 = Instant.parse("2024-01-17T00:00:00Z");
             Instant end = Instant.parse("2024-01-18T00:00:00Z");
-            ProjectMetadata project = DataStreamTestHelper.getProjectWithDataStream(
-                projectId,
-                DATA_STREAM,
+            ProjectMetadata project = projectWithDataStream(
+                mode,
                 List.of(Tuple.tuple(start3, end), Tuple.tuple(start1, start2), Tuple.tuple(start2, start3))
             );
             // Updated project metadata with a consequent and unsored ts backing indices in a mixed data stream
@@ -462,15 +462,42 @@ private ClusterState stateWithNoTsdbIndices() {
     }
 
     private static IndexMetadata createIndexMetadata(String indexName, Instant startTime, Instant endTime) {
+        return createIndexMetadata(indexName, startTime, endTime, randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB));
+    }
+
+    private static IndexMetadata createIndexMetadata(String indexName, Instant startTime, Instant endTime, IndexMode indexMode) {
         Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current());
         if (startTime != null && endTime != null) {
-            settings.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName())
+            settings.put(IndexSettings.MODE.getKey(), indexMode.getName())
                 .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime.toEpochMilli())
                 .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime.toEpochMilli());
         }
         return IndexMetadata.builder(indexName).settings(settings).numberOfShards(1).numberOfReplicas(0).build();
     }
 
+    /**
+     * Builds a ProjectMetadata with a data stream in the given {@link IndexMode} whose backing indices
+     * cover the given time ranges.
+     */
+    private ProjectMetadata projectWithDataStream(IndexMode mode, List<Tuple<Instant, Instant>> timeSlices) {
+        List<IndexMetadata> backingIndices = new ArrayList<>();
+        long generation = 1L;
+        for (Tuple<Instant, Instant> slice : timeSlices) {
+            String indexName = DataStream.getDefaultBackingIndexName(DATA_STREAM, generation, slice.v1().toEpochMilli());
+            backingIndices.add(createIndexMetadata(indexName, slice.v1(), slice.v2(), mode));
+            generation++;
+        }
+        DataStream ds = DataStream.builder(DATA_STREAM, backingIndices.stream().map(IndexMetadata::getIndex).toList())
+            .setGeneration(generation)
+            .setIndexMode(mode)
+            .build();
+        ProjectMetadata.Builder builder = ProjectMetadata.builder(projectId);
+        for (IndexMetadata im : backingIndices) {
+            builder.put(im, false);
+        }
+        return builder.put(ds).build();
+    }
+
     public void testOutsideEligibleWriteWindowFails() throws Exception {
         Instant now = Instant.now();
         ClusterState clusterState = stateWithExisting(List.of(), now);

modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java

@@ -46,6 +46,7 @@
 import org.elasticsearch.datastreams.lifecycle.health.DataStreamLifecycleHealthInfoPublisher;
 import org.elasticsearch.dlm.DataStreamLifecycleErrorStore;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexVersion;
@@ -1182,6 +1183,11 @@ public void testWithTinyRetentions() {
         assertThat(failureStoreConditions.getMaxAge(), equalTo(TimeValue.timeValueHours(1))); // 12h retention -> 1h max_age
     }
 
+    /**
+     * Covers both {@link IndexMode#TIME_SERIES} and {@link IndexMode#TSDB}, the latter sharing the
+     * former's {@code isTsdb()}-gated behaviour, so {@link DataStreamLifecycleService#timeSeriesIndicesStillWithinTimeBounds}
+     * must treat both identically.
+     */
     public void testTimeSeriesIndicesStillWithinTimeBounds() {
         Instant currentTime = Instant.now().truncatedTo(ChronoUnit.MILLIS);
         // These ranges are on the edge of each other temporal boundaries.

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java

@@ -343,9 +343,14 @@ public void testMetricAndDocvalues() {
         assertThat(e.getCause().getMessage(), containsString("Field [time_series_metric] requires that [doc_values] is true"));
     }
 
+    /**
+     * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()})
+     * so it must produce the same doc-values-only defaulting for metric fields.
+     */
     public void testTimeSeriesIndexDefault() throws Exception {
         var randomMetricType = randomFrom(TimeSeriesParams.MetricType.scalar());
-        var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName())
+        var mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB);
+        var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), mode.getName())
             .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field");
         var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> {
             minimalMapping(b);