diff --git a/docs/changelog/152901.yaml b/docs/changelog/152901.yaml new file mode 100644 index 0000000000000..db632847d3c5d --- /dev/null +++ b/docs/changelog/152901.yaml @@ -0,0 +1,16 @@ +area: TSDB +highlight: + body: |- + Introduce `IndexMode.TSDB` as a new, first-class index mode for metrics. + It behaves identically to `TIME_SERIES`, so `index.mode: tsdb` + is now accepted anywhere `index.mode: time_series` is — without renaming + or changing anything about existing `time_series` indices. The querying + experience and downsampling support remains seamless, regardless of the + mix of `TSDB` and `TIME_SERIES` indices in a data stream. + + notable: true + title: Add `IndexMode.TSDB` as a preferred alias for TIME_SERIES +issues: [] +pr: 152901 +summary: Add `IndexMode.TSDB` as a preferred alias for TIME_SERIES +type: enhancement diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java index 496065e0b0f8d..bf2c52d517585 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java @@ -152,7 +152,7 @@ protected Collection> nodePlugins() { public void testInvalidIndexMode() { final var indexName = randomIdentifier(); - var randomNonTsdbIndexMode = randomValueOtherThan(IndexMode.TIME_SERIES, () -> randomFrom(IndexMode.availableModes())); + var randomNonTsdbIndexMode = randomValueOtherThanMany(m -> m.isTsdb(), () -> randomFrom(IndexMode.availableModes())); var exception = expectThrows( IllegalArgumentException.class, @@ -168,7 +168,11 @@ public void testInvalidIndexMode() { containsString( "The setting [" + IndexSettings.SYNTHETIC_ID.getKey() - + "] is only permitted when [index.mode] is set to [TIME_SERIES]. Current mode: [" + + "] is only permitted when [index.mode] is set to [" + + IndexMode.TIME_SERIES.name() + + "] or [" + + IndexMode.TSDB.name() + + "]. Current mode: [" + randomNonTsdbIndexMode.getName().toUpperCase(Locale.ROOT) + "]." ) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index a1f048fe1bbec..dce5307f14fe3 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -88,7 +88,14 @@ int maybeAdjustIndexSettingCount(int baseCount) { return count; } + /** + * {@link IndexMode#TSDB} resolves {@link IndexMode#isTsdb()} to {@code true} just like + * {@link IndexMode#TIME_SERIES}, so creating a backing index with a template index mode of + * {@link IndexMode#TSDB} must inject the same start/end time, sequence-number, synthetic id + * and dimension settings as {@link IndexMode#TIME_SERIES} does. + */ public void testGetAdditionalIndexSettings() throws Exception { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); ProjectMetadata projectMetadata = emptyProject(); String dataStreamName = "logs-app1"; @@ -130,7 +137,7 @@ public void testGetAdditionalIndexSettings() throws Exception { provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TIME_SERIES, + mode, projectMetadata, now, settings, @@ -139,11 +146,11 @@ public void testGetAdditionalIndexSettings() throws Exception { additionalSettings ); Settings result = additionalSettings.build(); - // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: + // The index.time_series.end_time setting requires index.mode to be set adding it here so that we read this setting: // (in production the index.mode setting is usually provided in an index or component template) - result = builder().put(result).put("index.mode", "time_series").build(); + result = builder().put(result).put("index.mode", mode.getName()).build(); assertThat(result.size(), equalTo(maybeAdjustIndexSettingCount(4))); - assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); + assertThat(IndexSettings.MODE.get(result), equalTo(mode)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); if (expectedIndexDimensionsTsidOptimizationEnabled) { @@ -491,7 +498,14 @@ public void testGetAdditionalIndexSettingsNonTsdbTemplate() { assertThat(result.size(), equalTo(0)); } + /** + * The migration check at index-creation time relies on {@link IndexMode#isTsdb(IndexMode)}, so + * a data stream currently in {@code standard} mode must also migrate into time series mode when + * the matching index template specifies {@link IndexMode#TSDB} directly, not just + * {@link IndexMode#TIME_SERIES}. + */ public void testGetAdditionalIndexSettingsMigrateToTsdb() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); String dataStreamName = "logs-app1"; IndexMetadata idx = createFirstBackingIndex(dataStreamName).build(); @@ -505,7 +519,7 @@ public void testGetAdditionalIndexSettingsMigrateToTsdb() { provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, - IndexMode.TIME_SERIES, + mode, projectMetadata, now, settings, @@ -514,11 +528,11 @@ public void testGetAdditionalIndexSettingsMigrateToTsdb() { additionalSettings ); Settings result = additionalSettings.build(); - // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: + // The index.time_series.end_time setting requires index.mode to be set adding it here so that we read this setting: // (in production the index.mode setting is usually provided in an index or component template) - result = builder().put(result).put("index.mode", "time_series").build(); + result = builder().put(result).put("index.mode", mode.getName()).build(); assertThat(result.size(), equalTo(maybeAdjustIndexSettingCount(3))); - assertThat(result.get(IndexSettings.MODE.getKey()), equalTo("time_series")); + assertThat(result.get(IndexSettings.MODE.getKey()), equalTo(mode.getName())); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); if (expectedDisabledSequenceNumbers) { @@ -1068,6 +1082,7 @@ private Settings generateTsdbSettings(String mapping, Instant now) throws IOExce } private Settings onUpdateMappings(String routingPath, String dimensions, String newMapping) throws IOException { + IndexMode indexMode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); String dataStreamName = "logs-app1"; Settings.Builder currentSettings = Settings.builder() .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) @@ -1076,7 +1091,7 @@ private Settings onUpdateMappings(String routingPath, String dimensions, String .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES); + .put(IndexSettings.MODE.getKey(), indexMode); IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1)) .settings(currentSettings) @@ -1102,7 +1117,12 @@ private Settings onUpdateMappings(String routingPath, String dimensions, String return additionalSettings.build(); } + /** + * The sequence-number-disabling gate is keyed off {@link IndexMode#isTsdb()}, so it must behave + * identically for a template index mode of {@link IndexMode#TIME_SERIES} and {@link IndexMode#TSDB}. + */ public void testClusterSettingsDefineSeqNoDisabledDefault() throws Exception { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); IndexVersion version = IndexVersionUtils.randomVersionBetween( IndexVersions.TIME_SERIES_DISABLE_SEQUENCE_NUMBERS_DEFAULT, @@ -1119,7 +1139,7 @@ public void testClusterSettingsDefineSeqNoDisabledDefault() throws Exception { providerWithSeqNoEnabled.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TIME_SERIES, + mode, emptyProject(), now, Settings.EMPTY, @@ -1138,7 +1158,7 @@ public void testClusterSettingsDefineSeqNoDisabledDefault() throws Exception { providerWithSeqNoDisabled.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TIME_SERIES, + mode, emptyProject(), now, Settings.EMPTY, @@ -1153,7 +1173,7 @@ public void testClusterSettingsDefineSeqNoDisabledDefault() throws Exception { providerWithSeqNoDisabled.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TIME_SERIES, + mode, emptyProject(), now, Settings.builder().put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), false).build(), @@ -1164,7 +1184,12 @@ public void testClusterSettingsDefineSeqNoDisabledDefault() throws Exception { assertFalse(additionalSettings.build().hasValue(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey())); } + /** + * The synthetic {@code _id} gate is keyed off {@link IndexMode#isTsdb()}, so it must behave + * identically for a template index mode of {@link IndexMode#TIME_SERIES} and {@link IndexMode#TSDB}. + */ public void testClusterSettingsDefineSyntheticIdEnabledDefault() throws Exception { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); IndexVersion version = IndexVersionUtils.randomVersionBetween( IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD, @@ -1181,7 +1206,7 @@ public void testClusterSettingsDefineSyntheticIdEnabledDefault() throws Exceptio providerWithSyntheticIdDisabled.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TIME_SERIES, + mode, emptyProject(), now, Settings.EMPTY, @@ -1200,7 +1225,7 @@ public void testClusterSettingsDefineSyntheticIdEnabledDefault() throws Exceptio providerWithSyntheticIdEnabled.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TIME_SERIES, + mode, emptyProject(), now, Settings.EMPTY, @@ -1215,7 +1240,7 @@ public void testClusterSettingsDefineSyntheticIdEnabledDefault() throws Exceptio providerWithSyntheticIdEnabled.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, - IndexMode.TIME_SERIES, + mode, emptyProject(), now, Settings.builder().put(IndexSettings.SYNTHETIC_ID.getKey(), false).build(), diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java index cac3ecdba85db..4ff3c629e61a4 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/PastTimeSeriesIndexCreationActionTests.java @@ -92,13 +92,13 @@ public void testSortAndRetrieve() { } // Retrieve all continuous TSDB indices in a single window { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Instant start1 = Instant.parse("2024-01-15T00:00:00Z"); Instant start2 = Instant.parse("2024-01-16T00:00:00Z"); Instant start3 = Instant.parse("2024-01-17T00:00:00Z"); Instant end = Instant.parse("2024-01-18T00:00:00Z"); - ProjectMetadata project = DataStreamTestHelper.getProjectWithDataStream( - projectId, - DATA_STREAM, + ProjectMetadata project = projectWithDataStream( + mode, List.of(Tuple.tuple(start3, end), Tuple.tuple(start1, start2), Tuple.tuple(start2, start3)) ); // Updated project metadata with a consequent and unsored ts backing indices in a mixed data stream @@ -462,15 +462,42 @@ private ClusterState stateWithNoTsdbIndices() { } private static IndexMetadata createIndexMetadata(String indexName, Instant startTime, Instant endTime) { + return createIndexMetadata(indexName, startTime, endTime, randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB)); + } + + private static IndexMetadata createIndexMetadata(String indexName, Instant startTime, Instant endTime, IndexMode indexMode) { Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()); if (startTime != null && endTime != null) { - settings.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) + settings.put(IndexSettings.MODE.getKey(), indexMode.getName()) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime.toEpochMilli()) .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime.toEpochMilli()); } return IndexMetadata.builder(indexName).settings(settings).numberOfShards(1).numberOfReplicas(0).build(); } + /** + * Builds a ProjectMetadata with a data stream in the given {@link IndexMode} whose backing indices + * cover the given time ranges. + */ + private ProjectMetadata projectWithDataStream(IndexMode mode, List> timeSlices) { + List backingIndices = new ArrayList<>(); + long generation = 1L; + for (Tuple slice : timeSlices) { + String indexName = DataStream.getDefaultBackingIndexName(DATA_STREAM, generation, slice.v1().toEpochMilli()); + backingIndices.add(createIndexMetadata(indexName, slice.v1(), slice.v2(), mode)); + generation++; + } + DataStream ds = DataStream.builder(DATA_STREAM, backingIndices.stream().map(IndexMetadata::getIndex).toList()) + .setGeneration(generation) + .setIndexMode(mode) + .build(); + ProjectMetadata.Builder builder = ProjectMetadata.builder(projectId); + for (IndexMetadata im : backingIndices) { + builder.put(im, false); + } + return builder.put(ds).build(); + } + public void testOutsideEligibleWriteWindowFails() throws Exception { Instant now = Instant.now(); ClusterState clusterState = stateWithExisting(List.of(), now); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index c1ee7c5d1f462..d37b0d37fed11 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -46,6 +46,7 @@ import org.elasticsearch.datastreams.lifecycle.health.DataStreamLifecycleHealthInfoPublisher; import org.elasticsearch.dlm.DataStreamLifecycleErrorStore; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexVersion; @@ -1182,6 +1183,11 @@ public void testWithTinyRetentions() { assertThat(failureStoreConditions.getMaxAge(), equalTo(TimeValue.timeValueHours(1))); // 12h retention -> 1h max_age } + /** + * Covers both {@link IndexMode#TIME_SERIES} and {@link IndexMode#TSDB}, the latter sharing the + * former's {@code isTsdb()}-gated behaviour, so {@link DataStreamLifecycleService#timeSeriesIndicesStillWithinTimeBounds} + * must treat both identically. + */ public void testTimeSeriesIndicesStillWithinTimeBounds() { Instant currentTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); // These ranges are on the edge of each other temporal boundaries. diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java index 6046e2a283417..8cc0371642c21 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java @@ -343,9 +343,14 @@ public void testMetricAndDocvalues() { assertThat(e.getCause().getMessage(), containsString("Field [time_series_metric] requires that [doc_values] is true")); } + /** + * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) + * so it must produce the same doc-values-only defaulting for metric fields. + */ public void testTimeSeriesIndexDefault() throws Exception { var randomMetricType = randomFrom(TimeSeriesParams.MetricType.scalar()); - var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) + var mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> { minimalMapping(b); diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 37b058199269a..cb702235ed7b8 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -129,93 +129,17 @@ public SourceFieldMapper.Mode defaultSourceMode() { TIME_SERIES("time_series") { @Override void validateWithOtherSettings(Map, Object> settings) { - if (settings.get(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING) != Integer.valueOf(1)) { - throw new IllegalArgumentException(error(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING)); - } - - var settingsWithIndexMode = Settings.builder().put(IndexSettings.MODE.getKey(), getName()).build(); - - for (Setting unsupported : TIME_SERIES_UNSUPPORTED) { - if (false == Objects.equals(unsupported.getDefault(settingsWithIndexMode), settings.get(unsupported))) { - throw new IllegalArgumentException(error(unsupported)); - } - } - if (Boolean.TRUE.equals(settings.get(IndexSettings.SLICE_ENABLED))) { - throw new IllegalArgumentException( - "The setting [" - + IndexSettings.SLICE_ENABLED.getKey() - + "] cannot be used with [" - + IndexSettings.MODE.getKey() - + "=" - + IndexMode.TIME_SERIES.getName() - + "]." - ); - } - Setting> routingPath = IndexMetadata.INDEX_ROUTING_PATH; - if (isEmpty(settings, routingPath) && isEmpty(settings, IndexMetadata.INDEX_DIMENSIONS)) { - // index.dimensions is a private setting that only gets populated for data streams. - // We don't include it in the error message to not confuse users that are manually creating time series indices - // which is the only case where this error can occur. - throw new IllegalArgumentException(tsdbMode() + " requires a non-empty [" + routingPath.getKey() + "]"); - } - } - - private static boolean isEmpty(Map, Object> settings, Setting> setting) { - return Objects.equals(setting.getDefault(Settings.EMPTY), settings.get(setting)); - } - - private static String error(Setting unsupported) { - return tsdbMode() + " is incompatible with [" + unsupported.getKey() + "]"; + validateTimeSeriesSettings(this, settings); } @Override public void validateMapping(MappingLookup lookup, Settings settings) { - if (((RoutingFieldMapper) lookup.getMapper(RoutingFieldMapper.NAME)).required()) { - throw new IllegalArgumentException(routingRequiredBad()); - } - validateTemporalityField(lookup, settings); - } - - private static void validateTemporalityField(MappingLookup lookup, Settings settings) { - String temporalityFieldName = IndexSettings.TIME_SERIES_TEMPORALITY_FIELD.get(settings); - if (temporalityFieldName == null || temporalityFieldName.isEmpty()) { - return; - } - MappedFieldType fieldType = lookup.getFieldType(temporalityFieldName); - if (fieldType == null) { - // We silently ignore this case because of composable templates: - // composable templates are verified without being merged with the default mapping - // This means the temporality field might not exist during verification, - // but we'll add it automatically when the index is created through the default mapping - return; - } - if (fieldType instanceof KeywordFieldMapper.KeywordFieldType == false) { - throw new IllegalArgumentException( - "[" - + IndexSettings.TIME_SERIES_TEMPORALITY_FIELD.getKey() - + "] field [" - + temporalityFieldName - + "] must be of type [keyword] but is [" - + fieldType.typeName() - + "]" - ); - } - if (fieldType.isDimension() == false) { - throw new IllegalArgumentException( - "[" - + IndexSettings.TIME_SERIES_TEMPORALITY_FIELD.getKey() - + "] field [" - + temporalityFieldName - + "] must be a [time_series_dimension]" - ); - } + validateTimeSeriesMapping(this, lookup, settings); } @Override public void validateAlias(@Nullable String indexRouting, @Nullable String searchRouting) { - if (indexRouting != null || searchRouting != null) { - throw new IllegalArgumentException(routingRequiredBad()); - } + validateTimeSeriesAlias(this, indexRouting, searchRouting); } @Override @@ -253,10 +177,6 @@ public TimestampBounds getTimestampBound(IndexMetadata indexMetadata) { return new TimestampBounds(indexMetadata.getTimeSeriesStart(), indexMetadata.getTimeSeriesEnd()); } - private static String routingRequiredBad() { - return "routing is forbidden on CRUD operations that target indices in " + tsdbMode(); - } - @Override public MetadataFieldMapper timeSeriesIdFieldMapper(MappingParserContext c) { return TimeSeriesIdFieldMapper.getInstance(c); @@ -306,6 +226,87 @@ public boolean isColumnar() { return true; } }, + /** + * Preferred alternative to {@link #TIME_SERIES} with the name {@code tsdb}. Behaves identically + * to {@link #TIME_SERIES} in every respect other than {@link #getName()}; existing + * {@code time_series} indices are unaffected and {@link #fromString} accepts both spellings. + */ + TSDB("tsdb") { + @Override + void validateWithOtherSettings(Map, Object> settings) { + validateTimeSeriesSettings(this, settings); + } + + @Override + public void validateMapping(MappingLookup lookup, Settings settings) { + validateTimeSeriesMapping(this, lookup, settings); + } + + @Override + public void validateAlias(@Nullable String indexRouting, @Nullable String searchRouting) { + validateTimeSeriesAlias(this, indexRouting, searchRouting); + } + + @Override + public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup mappingLookup) throws IOException { + TIME_SERIES.validateTimestampFieldMapping(isDataStream, mappingLookup); + } + + @Override + public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) { + return TIME_SERIES.getDefaultMapping(indexSettings); + } + + @Override + public TimestampBounds getTimestampBound(IndexMetadata indexMetadata) { + return TIME_SERIES.getTimestampBound(indexMetadata); + } + + @Override + public MetadataFieldMapper timeSeriesIdFieldMapper(MappingParserContext c) { + return TIME_SERIES.timeSeriesIdFieldMapper(c); + } + + @Override + public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() { + return TIME_SERIES.timeSeriesRoutingHashFieldMapper(); + } + + @Override + public Function idTransformerForReindex() { + return TIME_SERIES.idTransformerForReindex(); + } + + @Override + public RoutingFields buildRoutingFields(IndexSettings settings) { + return TIME_SERIES.buildRoutingFields(settings); + } + + @Override + public boolean shouldValidateTimestamp() { + return TIME_SERIES.shouldValidateTimestamp(); + } + + @Override + public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) { + TIME_SERIES.validateSourceFieldMapper(sourceFieldMapper); + } + + @Override + public SourceFieldMapper.Mode defaultSourceMode() { + return TIME_SERIES.defaultSourceMode(); + } + + @Override + public boolean isColumnar() { + return TIME_SERIES.isColumnar(); + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return INDEX_MODE_TSDB_ADDED; + } + }, LOGSDB("logsdb") { @Override void validateWithOtherSettings(Map, Object> settings) { @@ -816,6 +817,122 @@ private static CompressedXContent createDefaultMapping(CheckedConsumer, Object> settings) { + if (settings.get(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING) != Integer.valueOf(1)) { + throw new IllegalArgumentException(unsupportedTimeSeriesSettingError(mode, IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING)); + } + + var settingsWithIndexMode = Settings.builder().put(IndexSettings.MODE.getKey(), mode.getName()).build(); + + for (Setting unsupported : TIME_SERIES_UNSUPPORTED) { + if (false == Objects.equals(unsupported.getDefault(settingsWithIndexMode), settings.get(unsupported))) { + throw new IllegalArgumentException(unsupportedTimeSeriesSettingError(mode, unsupported)); + } + } + if (Boolean.TRUE.equals(settings.get(IndexSettings.SLICE_ENABLED))) { + throw new IllegalArgumentException( + "The setting [" + + IndexSettings.SLICE_ENABLED.getKey() + + "] cannot be used with [" + + IndexSettings.MODE.getKey() + + "=" + + mode.getName() + + "]." + ); + } + Setting> routingPath = IndexMetadata.INDEX_ROUTING_PATH; + if (isDefaultRoutingPathSetting(settings, routingPath) && isDefaultRoutingPathSetting(settings, IndexMetadata.INDEX_DIMENSIONS)) { + // index.dimensions is a private setting that only gets populated for data streams. + // We don't include it in the error message to not confuse users that are manually creating time series indices + // which is the only case where this error can occur. + throw new IllegalArgumentException( + "[" + IndexSettings.MODE.getKey() + "=" + mode.getName() + "] requires a non-empty [" + routingPath.getKey() + "]" + ); + } + } + + private static boolean isDefaultRoutingPathSetting(Map, Object> settings, Setting> setting) { + return Objects.equals(setting.getDefault(Settings.EMPTY), settings.get(setting)); + } + + private static String unsupportedTimeSeriesSettingError(IndexMode mode, Setting unsupported) { + return "[" + IndexSettings.MODE.getKey() + "=" + mode.getName() + "] is incompatible with [" + unsupported.getKey() + "]"; + } + + /** + * Shared {@code validateMapping} implementation for {@link #TIME_SERIES} and {@link #TSDB}. Takes the + * calling {@code mode} explicitly (see {@link #validateTimeSeriesSettings}) so the routing-required + * error names the mode that was actually configured. + */ + private static void validateTimeSeriesMapping(IndexMode mode, MappingLookup lookup, Settings settings) { + if (((RoutingFieldMapper) lookup.getMapper(RoutingFieldMapper.NAME)).required()) { + throw new IllegalArgumentException(routingRequiredBad(mode)); + } + validateTemporalityField(lookup, settings); + } + + private static void validateTemporalityField(MappingLookup lookup, Settings settings) { + String temporalityFieldName = IndexSettings.TIME_SERIES_TEMPORALITY_FIELD.get(settings); + if (temporalityFieldName == null || temporalityFieldName.isEmpty()) { + return; + } + MappedFieldType fieldType = lookup.getFieldType(temporalityFieldName); + if (fieldType == null) { + // We silently ignore this case because of composable templates: + // composable templates are verified without being merged with the default mapping + // This means the temporality field might not exist during verification, + // but we'll add it automatically when the index is created through the default mapping + return; + } + if (fieldType instanceof KeywordFieldMapper.KeywordFieldType == false) { + throw new IllegalArgumentException( + "[" + + IndexSettings.TIME_SERIES_TEMPORALITY_FIELD.getKey() + + "] field [" + + temporalityFieldName + + "] must be of type [keyword] but is [" + + fieldType.typeName() + + "]" + ); + } + if (fieldType.isDimension() == false) { + throw new IllegalArgumentException( + "[" + + IndexSettings.TIME_SERIES_TEMPORALITY_FIELD.getKey() + + "] field [" + + temporalityFieldName + + "] must be a [time_series_dimension]" + ); + } + } + + /** + * Shared {@code validateAlias} implementation for {@link #TIME_SERIES} and {@link #TSDB}. See + * {@link #validateTimeSeriesSettings} for why {@code mode} is passed explicitly. + */ + private static void validateTimeSeriesAlias(IndexMode mode, @Nullable String indexRouting, @Nullable String searchRouting) { + if (indexRouting != null || searchRouting != null) { + throw new IllegalArgumentException(routingRequiredBad(mode)); + } + } + + private static String routingRequiredBad(IndexMode mode) { + return "routing is forbidden on CRUD operations that target indices in [" + + IndexSettings.MODE.getKey() + + "=" + + mode.getName() + + "]"; + } + static final List> VALIDATE_WITH_SETTINGS = List.copyOf( Stream.concat( Stream.of( @@ -833,6 +950,7 @@ private static CompressedXContent createDefaultMapping(CheckedConsumer IndexMode.STANDARD; case "time_series" -> IndexMode.TIME_SERIES; + case "tsdb" -> IndexMode.TSDB; case "logsdb" -> IndexMode.LOGSDB; case "columnar" -> IndexMode.COLUMNAR; case "logsdb_columnar" -> IndexMode.LOGSDB_COLUMNAR; @@ -1032,6 +1154,7 @@ public static IndexMode readFrom(StreamInput in) throws IOException { case 4 -> COLUMNAR; case 5 -> LOGSDB_COLUMNAR; case 6 -> VECTORDB_DOCUMENT; + case 7 -> TSDB; default -> throw new IllegalStateException("unexpected index mode [" + mode + "]"); }; } @@ -1053,6 +1176,7 @@ public static void writeTo(IndexMode indexMode, StreamOutput out) throws IOExcep case COLUMNAR -> 4; case LOGSDB_COLUMNAR -> 5; case VECTORDB_DOCUMENT -> 6; + case TSDB -> 7; }; out.writeByte((byte) code); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index a890ceb46c07b..26bc6c9003072 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -778,16 +778,17 @@ public void validate(Boolean enabled) {} @Override public void validate(Boolean enabled, Map, Object> settings) { if (enabled) { - // Verify if index mode is TIME_SERIES + // Verify if index mode is a time series mode (TIME_SERIES or TSDB) var indexMode = (IndexMode) settings.get(MODE); if (indexMode.isTsdb() == false) { throw new IllegalArgumentException( String.format( Locale.ROOT, - "The setting [%s] is only permitted when [%s] is set to [%s]. Current mode: [%s].", + "The setting [%s] is only permitted when [%s] is set to [%s] or [%s]. Current mode: [%s].", SYNTHETIC_ID.getKey(), MODE.getKey(), IndexMode.TIME_SERIES.name(), + IndexMode.TSDB.name(), indexMode.name() ) ); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java index 4dfb33490a7c2..d5e65591c3592 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java @@ -37,6 +37,11 @@ public class CreateIndexCapabilities { */ private static final String VECTORDB_DOCUMENT_INDEX_MODE_CAPABILITY = "vectordb_document_index_mode"; + /** + * Support for the 'tsdb' index mode + */ + private static final String TSDB_INDEX_MODE_CAPABILITY = "tsdb_index_mode"; + private static final String NESTED_DENSE_VECTOR_SYNTHETIC_TEST = "nested_dense_vector_synthetic_test"; private static final String POORLY_FORMATTED_BAD_REQUEST = "poorly_formatted_bad_request"; @@ -63,6 +68,7 @@ public class CreateIndexCapabilities { ); caps.add(COLUMNAR_INDEX_MODES_CAPABILITY); caps.add(VECTORDB_DOCUMENT_INDEX_MODE_CAPABILITY); + caps.add(TSDB_INDEX_MODE_CAPABILITY); CAPABILITIES = Set.copyOf(caps); } } diff --git a/server/src/main/resources/transport/definitions/referable/index_mode_tsdb_added.csv b/server/src/main/resources/transport/definitions/referable/index_mode_tsdb_added.csv new file mode 100644 index 0000000000000..42a7c95629440 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/index_mode_tsdb_added.csv @@ -0,0 +1 @@ +9454000 diff --git a/server/src/main/resources/transport/upper_bounds/9.5.csv b/server/src/main/resources/transport/upper_bounds/9.5.csv index 7fb47d4f1b81c..8cf2b313623a3 100644 --- a/server/src/main/resources/transport/upper_bounds/9.5.csv +++ b/server/src/main/resources/transport/upper_bounds/9.5.csv @@ -1 +1 @@ -esql_external_warm_aggregate_profile,9453000 +index_mode_tsdb_added,9454000 diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java index 6b95a52784634..f2579002d1afc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java @@ -573,7 +573,7 @@ private static IndexMetadata createIndexMetadata( .put("index.frozen", frozen) .put("index.mode", mode.toString()); - if (mode == IndexMode.TIME_SERIES) { + if (mode.isTsdb()) { settingsBuilder.put( randomBoolean() ? IndexMetadata.INDEX_DIMENSIONS.getKey() : IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dummy_value" diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 76d342ff1c69e..2fea95f323360 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -274,6 +274,46 @@ public void testRolloverUpgradeToTsdbDataStream() { assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TIME_SERIES)); } + public void testRolloverUpgradeToTsdbDataStreamMode() { + DataStream ds = DataStreamTestHelper.randomInstance() + .copy() + .setReplicated(false) + .setIndexMode(randomBoolean() ? IndexMode.STANDARD : null) + .build(); + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + var newCoordinates = ds.nextWriteIndexAndGeneration(project, ds.getDataComponent()); + + var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), IndexMode.TSDB, null); + assertThat(rolledDs.getName(), equalTo(ds.getName())); + assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); + assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1)); + assertTrue(rolledDs.getIndices().containsAll(ds.getIndices())); + assertTrue(rolledDs.getIndices().contains(rolledDs.getWriteIndex())); + assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TSDB)); + } + + /** + * {@link IndexMode#TIME_SERIES} and {@link IndexMode#TSDB} behave identically, so a data stream + * already using one of them must also be able to roll over into the other - not just upgrade from + * {@code standard} - covering both directions of the switch. + */ + public void testRolloverBetweenTimeSeriesAndTsdbDataStreamModes() { + for (IndexMode fromMode : List.of(IndexMode.TIME_SERIES, IndexMode.TSDB)) { + IndexMode toMode = fromMode == IndexMode.TIME_SERIES ? IndexMode.TSDB : IndexMode.TIME_SERIES; + DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(fromMode).build(); + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + var newCoordinates = ds.nextWriteIndexAndGeneration(project, ds.getDataComponent()); + + var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), toMode, null); + assertThat(rolledDs.getName(), equalTo(ds.getName())); + assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); + assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1)); + assertTrue(rolledDs.getIndices().containsAll(ds.getIndices())); + assertTrue(rolledDs.getIndices().contains(rolledDs.getWriteIndex())); + assertThat(rolledDs.getIndexMode(), equalTo(toMode)); + } + } + public void testRolloverUpgradeToLogsdbDataStream() { DataStream ds = DataStreamTestHelper.randomInstance() .copy() @@ -325,6 +365,20 @@ public void testRolloverFromLogsdbToTsdb() { assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TIME_SERIES)); } + public void testRolloverFromLogsdbToTsdbMode() { + DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.LOGSDB).build(); + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + var newCoordinates = ds.nextWriteIndexAndGeneration(project, ds.getDataComponent()); + + var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), IndexMode.TSDB, null); + assertThat(rolledDs.getName(), equalTo(ds.getName())); + assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); + assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1)); + assertTrue(rolledDs.getIndices().containsAll(ds.getIndices())); + assertTrue(rolledDs.getIndices().contains(rolledDs.getWriteIndex())); + assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TSDB)); + } + public void testRolloverDowngradeFromTsdbToRegularDataStream() { DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.TIME_SERIES).build(); final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); @@ -460,6 +514,17 @@ public void testRolloverFromColumnarToTimeSeries() { assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TIME_SERIES)); } + public void testRolloverFromColumnarToTsdb() { + DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.COLUMNAR).build(); + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + var newCoordinates = ds.nextWriteIndexAndGeneration(project, ds.getDataComponent()); + + var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), IndexMode.TSDB, null); + assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); + assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1)); + assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TSDB)); + } + public void testRolloverDowngradeFromTimeSeriesDataStreamToColumnar() { DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.TIME_SERIES).build(); final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); @@ -503,6 +568,17 @@ public void testRolloverFromColumnarLogsdbToTimeSeries() { assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TIME_SERIES)); } + public void testRolloverFromColumnarLogsdbToTsdb() { + DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.LOGSDB_COLUMNAR).build(); + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + var newCoordinates = ds.nextWriteIndexAndGeneration(project, ds.getDataComponent()); + + var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), IndexMode.TSDB, null); + assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); + assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1)); + assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TSDB)); + } + public void testRolloverFromLogsdbToColumnarLogsdb() { DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.LOGSDB).build(); final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/DimensionsExtractorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/DimensionsExtractorTests.java index 3dc7f627a6dae..3911c0e80657e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/DimensionsExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/DimensionsExtractorTests.java @@ -144,6 +144,32 @@ public void testTsidIsAttachedToIndexRequest() throws IOException { assertThat("tsid should be attached to the IndexRequest after extraction", req.tsid(), notNullValue()); } + /** + * {@code index.mode: tsdb} is a preferred alternative to {@code time_series}; it must select the same + * {@link IndexRouting.ExtractFromSource.ForIndexDimensions} strategy and produce byte-identical tsids + * and shard assignments. + */ + public void testTsdbProducesSameTsidAsTimeSeries() throws IOException { + Map doc = new LinkedHashMap<>(); + doc.put("dim.host", "node-7"); + doc.put("dim.region", "us-west-2"); + doc.put("metric", "cpu.user"); + BytesReference json = toJson(doc); + + IndexRouting.ExtractFromSource.ForIndexDimensions timeSeriesStrategy = forIndexDimensions("dim.*", IndexMode.TIME_SERIES); + IndexRequest timeSeriesReq = new IndexRequest("test").id("d1").source(json, XContentType.JSON); + timeSeriesStrategy.preProcess(timeSeriesReq); + int timeSeriesShardId = timeSeriesStrategy.indexShard(timeSeriesReq); + + IndexRouting.ExtractFromSource.ForIndexDimensions tsdbStrategy = forIndexDimensions("dim.*", IndexMode.TSDB); + IndexRequest tsdbReq = new IndexRequest("test").id("d1").source(json, XContentType.JSON); + tsdbStrategy.preProcess(tsdbReq); + int tsdbShardId = tsdbStrategy.indexShard(tsdbReq); + + assertThat("tsid must be identical for time_series and tsdb", tsdbReq.tsid(), equalTo(timeSeriesReq.tsid())); + assertThat("shard id must be identical for time_series and tsdb", tsdbShardId, equalTo(timeSeriesShardId)); + } + public void testResetClearsBuilderBetweenDocuments() throws IOException { IndexRouting.ExtractFromSource.ForIndexDimensions strategyA = forIndexDimensions("dim.*"); IndexRouting.ExtractFromSource.ForIndexDimensions strategyB = forIndexDimensions("dim.*"); @@ -200,10 +226,14 @@ private void assertExtractorMatchesSourceParser(Map doc, String } private static IndexRouting.ExtractFromSource.ForIndexDimensions forIndexDimensions(String dimensionPath) { + return forIndexDimensions(dimensionPath, randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB)); + } + + private static IndexRouting.ExtractFromSource.ForIndexDimensions forIndexDimensions(String dimensionPath, IndexMode indexMode) { Settings settings = Settings.builder() .put(SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) .put(IndexMetadata.INDEX_DIMENSIONS.getKey(), dimensionPath) - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(IndexSettings.MODE.getKey(), indexMode) .build(); IndexMetadata md = IndexMetadata.builder("test").settings(settings).numberOfShards(8).numberOfReplicas(0).build(); IndexRouting routing = IndexRouting.fromIndexMetadata(md); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 29699db9bdfde..e6a5e6a7e7307 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -60,9 +60,11 @@ public class IndexRoutingTests extends ESTestCase { /** - * Result of creating time_series index routing; carries the routing and the explicit synthetic-id setting used. + * Result of creating time_series index routing; carries the routing, the explicit synthetic-id setting + * used, and the {@link IndexMode} the fixture was built with (either {@link IndexMode#TIME_SERIES} or + * {@link IndexMode#TSDB}). */ - private record TimeSeriesRoutingFixture(IndexRouting routing, boolean useSyntheticId) {} + private record TimeSeriesRoutingFixture(IndexRouting routing, boolean useSyntheticId, IndexMode indexMode) {} public void testSimpleRoutingRejectsEmptyId() { IndexRouting indexRouting = IndexRouting.fromIndexMetadata( @@ -562,16 +564,21 @@ public void testRoutingPathMismatchSource() { } public void testRoutingPathUpdate() { - IndexRouting routing = indexRoutingForPath(between(1, 5), "foo").routing(); + TimeSeriesRoutingFixture fixture = indexRoutingForPath(between(1, 5), "foo"); + IndexRouting routing = fixture.routing(); Exception e = expectThrows( IllegalArgumentException.class, () -> routing.updateShard(randomAlphaOfLength(5), randomBoolean() ? null : randomAlphaOfLength(5)) ); - assertThat(e.getMessage(), equalTo("update is not supported because the destination index [test] is in time_series mode")); + assertThat( + e.getMessage(), + equalTo("update is not supported because the destination index [test] is in " + fixture.indexMode().getName() + " mode") + ); } public void testRoutingIndexWithRouting() throws IOException { - IndexRouting indexRouting = indexRoutingForPath(5, "foo").routing(); + TimeSeriesRoutingFixture fixture = indexRoutingForPath(5, "foo"); + IndexRouting indexRouting = fixture.routing(); String value = randomAlphaOfLength(5); BytesReference source = source(Map.of("foo", value)); String docRouting = randomAlphaOfLength(5); @@ -583,16 +590,23 @@ public void testRoutingIndexWithRouting() throws IOException { ); assertThat( e.getMessage(), - equalTo("specifying routing is not supported because the destination index [test] is in time_series mode") + equalTo( + "specifying routing is not supported because the destination index [test] is in " + fixture.indexMode().getName() + " mode" + ) ); } public void testRoutingPathCollectSearchWithRouting() { - IndexRouting routing = indexRoutingForPath(between(1, 5), "foo").routing(); + TimeSeriesRoutingFixture fixture = indexRoutingForPath(between(1, 5), "foo"); + IndexRouting routing = fixture.routing(); Exception e = expectThrows(IllegalArgumentException.class, () -> routing.collectSearchShards(randomAlphaOfLength(5), null)); assertThat( e.getMessage(), - equalTo("searching with a specified routing is not supported because the destination index [test] is in time_series mode") + equalTo( + "searching with a specified routing is not supported because the destination index [test] is in " + + fixture.indexMode().getName() + + " mode" + ) ); } @@ -707,7 +721,8 @@ public void testRoutingPathBwcAfterTsidBasedRouting() throws IOException { ), 8, "dim.*,other.*,top", - randomBoolean() + randomBoolean(), + IndexMode.TIME_SERIES ); assertFalse(TsidBuilder.useSingleBytePrefixLayout(fixture.routing.creationVersion)); /* @@ -731,12 +746,18 @@ public void testRoutingPathBwcAfterTsidBasedRouting() throws IOException { assertIndexShard(fixture, Map.of("dim.a", "true"), 6); } + /** + * {@link IndexMode#TSDB} must select the same {@link IndexRouting.ExtractFromSource.ForIndexDimensions} + * strategy, track the routing hash the same way, and produce identical shard assignments as + * {@link IndexMode#TIME_SERIES}. + */ public void testRoutingPathWithSingleBytePrefixTsid() throws IOException { TimeSeriesRoutingFixture fixture = indexRoutingForTimeSeriesDimensions( IndexVersionUtils.randomVersionOnOrAfter(IndexVersions.TSID_SINGLE_PREFIX_BYTE_FEATURE_FLAG), 8, "dim.*,other.*,top", - randomBoolean() + randomBoolean(), + randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB) ); assumeTrue("require single-byte-prefix tsid", TsidBuilder.useSingleBytePrefixLayout(fixture.routing.creationVersion)); assertIndexShard(fixture, Map.of("dim", Map.of("a", "a")), 5); @@ -754,16 +775,16 @@ public void testRoutingPathWithSingleBytePrefixTsid() throws IOException { public void testRoutingPathReadWithInvalidString() { int shards = between(2, 1000); - IndexRouting indexRouting = indexRoutingForPath(shards, "foo").routing(); - Exception e = expectThrows(ResourceNotFoundException.class, () -> shardIdForReadFromSourceExtracting(indexRouting, "!@#")); - assertThat(e.getMessage(), equalTo("invalid id [!@#] for index [test] in time_series mode")); + TimeSeriesRoutingFixture fixture = indexRoutingForPath(shards, "foo"); + Exception e = expectThrows(ResourceNotFoundException.class, () -> shardIdForReadFromSourceExtracting(fixture.routing(), "!@#")); + assertThat(e.getMessage(), equalTo("invalid id [!@#] for index [test] in " + fixture.indexMode().getName() + " mode")); } public void testRoutingPathReadWithShortString() { int shards = between(2, 1000); - IndexRouting indexRouting = indexRoutingForPath(shards, "foo").routing(); - Exception e = expectThrows(ResourceNotFoundException.class, () -> shardIdForReadFromSourceExtracting(indexRouting, "")); - assertThat(e.getMessage(), equalTo("invalid id [] for index [test] in time_series mode")); + TimeSeriesRoutingFixture fixture = indexRoutingForPath(shards, "foo"); + Exception e = expectThrows(ResourceNotFoundException.class, () -> shardIdForReadFromSourceExtracting(fixture.routing(), "")); + assertThat(e.getMessage(), equalTo("invalid id [] for index [test] in " + fixture.indexMode().getName() + " mode")); } public void testRoutingPathLogsdb() { @@ -1237,9 +1258,10 @@ private TimeSeriesRoutingFixture indexRoutingForTimeSeriesDimensions( IndexVersion createdVersion, int shards, String path, - boolean useSyntheticId + boolean useSyntheticId, + IndexMode indexMode ) { - return getIndexRoutingWithSetting(createdVersion, shards, path, IndexMetadata.INDEX_DIMENSIONS.getKey(), useSyntheticId); + return getIndexRoutingWithSetting(createdVersion, shards, path, IndexMetadata.INDEX_DIMENSIONS.getKey(), useSyntheticId, indexMode); } /** @@ -1251,12 +1273,33 @@ private static TimeSeriesRoutingFixture getIndexRoutingWithSetting( String path, String setting, boolean useSyntheticId + ) { + return getIndexRoutingWithSetting( + indexVersion, + shards, + path, + setting, + useSyntheticId, + randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB) + ); + } + + /** + * Same as {@link #getIndexRoutingWithSetting(IndexVersion, int, String, String, boolean)} but lets the + * caller pick the index mode. + */ + private static TimeSeriesRoutingFixture getIndexRoutingWithSetting( + IndexVersion indexVersion, + int shards, + String path, + String setting, + boolean useSyntheticId, + IndexMode indexMode ) { if (useSyntheticId) { assert indexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94) : "Can't use synthetic id with this index version"; } - Settings.Builder settingsBuilder = settings(indexVersion).put(setting, path) - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES); + Settings.Builder settingsBuilder = settings(indexVersion).put(setting, path).put(IndexSettings.MODE.getKey(), indexMode); if (indexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94)) { settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), useSyntheticId); } @@ -1264,7 +1307,8 @@ private static TimeSeriesRoutingFixture getIndexRoutingWithSetting( IndexRouting.fromIndexMetadata( IndexMetadata.builder("test").settings(settingsBuilder).numberOfShards(shards).numberOfReplicas(1).build() ), - useSyntheticId + useSyntheticId, + indexMode ); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingPathExtractorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingPathExtractorTests.java index 95e50c92d2e88..808ab5a945d63 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingPathExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingPathExtractorTests.java @@ -122,6 +122,34 @@ public void testNoThrowOnArrayAtNonRoutingField() throws IOException { } } + /** + * {@code index.mode: tsdb} is a preferred alternative to {@code time_series}; it must flow through the same + * {@code trackTimeSeriesRoutingHash} logic in {@link IndexRouting.ExtractFromSource}, so the routing hash + * that {@link IndexRouting.ExtractFromSource#postProcess} attaches to the request must be identical + * regardless of which alias created the index. + */ + public void testTsdbProducesSameRoutingHashAsTimeSeries() throws IOException { + Map doc = new LinkedHashMap<>(); + doc.put("dim.host", "node-7"); + doc.put("dim.region", "us-west-2"); + doc.put("metric", "cpu.user"); + BytesReference json = toJson(doc); + + IndexRouting.ExtractFromSource.ForRoutingPath timeSeriesStrategy = forRoutingPath("dim.*", IndexMode.TIME_SERIES); + IndexRequest timeSeriesReq = new IndexRequest("test").id("doc-1").source(json, XContentType.JSON); + timeSeriesStrategy.preProcess(timeSeriesReq); + timeSeriesStrategy.indexShard(timeSeriesReq); + timeSeriesStrategy.postProcess(timeSeriesReq); + + IndexRouting.ExtractFromSource.ForRoutingPath tsdbStrategy = forRoutingPath("dim.*", IndexMode.TSDB); + IndexRequest tsdbReq = new IndexRequest("test").id("doc-1").source(json, XContentType.JSON); + tsdbStrategy.preProcess(tsdbReq); + tsdbStrategy.indexShard(tsdbReq); + tsdbStrategy.postProcess(tsdbReq); + + assertThat("routing hash must be identical for time_series and tsdb", tsdbReq.routing(), equalTo(timeSeriesReq.routing())); + } + public void testReuseAcrossDocumentsClearsBuilderState() throws IOException { // Same extractor, two different documents with different routing-path values should produce // the same shard ids as fresh extractors. @@ -182,10 +210,14 @@ private void assertExtractorMatchesSourceParser(Map doc, String } private static IndexRouting.ExtractFromSource.ForRoutingPath forRoutingPath(String routingPath) { + return forRoutingPath(routingPath, randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB)); + } + + private static IndexRouting.ExtractFromSource.ForRoutingPath forRoutingPath(String routingPath, IndexMode indexMode) { Settings settings = Settings.builder() .put(SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(IndexSettings.MODE.getKey(), indexMode) .build(); IndexMetadata md = IndexMetadata.builder("test").settings(settings).numberOfShards(8).numberOfReplicas(0).build(); IndexRouting routing = IndexRouting.fromIndexMetadata(md); diff --git a/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java b/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java index 4e608fb73e876..514547c37b477 100644 --- a/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java +++ b/server/src/test/java/org/elasticsearch/dlm/TimeSeriesEligibleWriteWindowLocatorTests.java @@ -51,6 +51,10 @@ public void testInfiniteWriteWindow() { assertThat(DLM_ONLY.getEligibleWriteWindowStart(dataStream, project, null, randomNonNegativeLong()), equalTo(-1L)); } + /** + * IndexMode.TSDB is a preferred alternative to IndexMode.TIME_SERIES, so a data stream configured + * with either mode must be treated identically by the eligible write window logic. + */ public void testWriteWindowDefinedByRetention() { // Configured retention { @@ -137,6 +141,9 @@ public long getEligibleWriteWindowFromPolicy(String policy, ProjectMetadata proj private static DataStream dataStream(String name, DataStreamLifecycle lifecycle) { Index index = new Index(DataStream.getDefaultBackingIndexName(name, 1), randomAlphaOfLength(10)); - return DataStream.builder(name, List.of(index)).setLifecycle(lifecycle).setIndexMode(IndexMode.TIME_SERIES).build(); + return DataStream.builder(name, List.of(index)) + .setLifecycle(lifecycle) + .setIndexMode(randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB)) + .build(); } } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index d4e4e7fd70c08..e08b5c623f99c 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -260,15 +260,21 @@ public void testSliceEnabledSettingRequiresFeatureFlag() { assertThat(exception.getMessage(), containsString("unknown setting [index.slice.enabled]")); } + /** + * {@code index.mode: tsdb} must be rejected exactly like {@code time_series}, and the rejection + * message must name the mode that was actually configured, not always the canonical + * {@code time_series} name. + */ public void testSliceEnabledSettingRejectedForTimeSeriesMode() { assumeTrue("slice indexing feature flag must be enabled", SliceIndexing.SLICE_FEATURE_FLAG.isEnabled()); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> new IndexSettings( newIndexMeta( "index", Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) + .put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim") .put(IndexSettings.SLICE_ENABLED.getKey(), true) .build() @@ -278,7 +284,7 @@ public void testSliceEnabledSettingRejectedForTimeSeriesMode() { ); assertThat(exception.getMessage(), containsString("index.slice.enabled")); assertThat(exception.getMessage(), containsString("index.mode")); - assertThat(exception.getMessage(), containsString("time_series")); + assertThat(exception.getMessage(), containsString(mode.getName())); } @TestLogging(reason = "testing warning logging", value = "org.elasticsearch.index.IndexSettings:WARN") @@ -998,7 +1004,7 @@ public void testSyntheticIdCorrectSettings() { IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94, IndexVersion.current() ); - IndexMode mode = IndexMode.TIME_SERIES; + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); String codec = version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_BEST_COMPRESSION) ? randomBoolean() ? CodecService.DEFAULT_CODEC : CodecService.BEST_COMPRESSION_CODEC : CodecService.DEFAULT_CODEC; @@ -1021,7 +1027,7 @@ public void testSyntheticIdDefaultValueTrue() { IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD, IndexVersion.current() ); - IndexMode mode = IndexMode.TIME_SERIES; + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); String codec = version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_BEST_COMPRESSION) ? randomBoolean() ? CodecService.DEFAULT_CODEC : CodecService.BEST_COMPRESSION_CODEC : CodecService.DEFAULT_CODEC; @@ -1040,7 +1046,7 @@ public void testSyntheticIdDefaultValueTrue() { public void testSyntheticIdDefaultValueFalse() { IndexVersion version = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD); - IndexMode mode = IndexMode.TIME_SERIES; + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); String codec = CodecService.DEFAULT_CODEC; Settings settings = Settings.builder() @@ -1128,7 +1134,7 @@ public void testSyntheticIdBadMode() { IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94, IndexVersion.current() ); - IndexMode badMode = randomValueOtherThan(IndexMode.TIME_SERIES, () -> randomFrom(IndexMode.availableModes())); + IndexMode badMode = randomValueOtherThanMany(m -> m.isTsdb(), () -> randomFrom(IndexMode.availableModes())); String codec = CodecService.DEFAULT_CODEC; Settings settings = Settings.builder() @@ -1144,10 +1150,11 @@ public void testSyntheticIdBadMode() { Matchers.containsString( String.format( Locale.ROOT, - "The setting [%s] is only permitted when [%s] is set to [%s]. Current mode: [%s].", + "The setting [%s] is only permitted when [%s] is set to [%s] or [%s]. Current mode: [%s].", IndexSettings.SYNTHETIC_ID.getKey(), IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name(), + IndexMode.TSDB.name(), badMode.name() ) ) @@ -1204,14 +1211,14 @@ public void testDisableSequenceNumbersRequiresDocValuesOnly() { public void testDisableSequenceNumbersRequiresDocValuesOnlyForNonStandardModes() { IndexVersion indexVersion = IndexVersionUtils.randomVersionBetween(IndexVersions.DISABLE_SEQUENCE_NUMBERS, IndexVersion.current()); - List modes = new ArrayList<>(List.of(IndexMode.TIME_SERIES, IndexMode.LOGSDB)); + List modes = new ArrayList<>(List.of(IndexMode.TIME_SERIES, IndexMode.TSDB, IndexMode.LOGSDB)); modes.addAll(List.of(IndexMode.LOGSDB_COLUMNAR, IndexMode.COLUMNAR)); IndexMode mode = randomFrom(modes); Settings.Builder builder = Settings.builder() .put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexSettings.SEQ_NO_INDEX_OPTIONS_SETTING.getKey(), SeqNoFieldMapper.SeqNoIndexOptions.POINTS_AND_DOC_VALUES) .put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true); - if (mode == IndexMode.TIME_SERIES) { + if (mode.isTsdb()) { builder.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo"); } IndexMetadata indexMetadata = newIndexMeta("some-index", builder.build(), indexVersion); @@ -1295,7 +1302,7 @@ public void testDynamicStringsAutoTextDefaultByIndexMode() { // All other modes default to true (text + keyword subfield) for (IndexMode otherMode : List.of(IndexMode.STANDARD, IndexMode.LOGSDB, IndexMode.TIME_SERIES)) { Settings.Builder builder = Settings.builder().put(IndexSettings.MODE.getKey(), otherMode.getName()); - if (otherMode == IndexMode.TIME_SERIES) { + if (otherMode.isTsdb()) { builder.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo"); } IndexSettings indexSettings = new IndexSettings(newIndexMeta(otherMode.getName() + "-index", builder.build()), Settings.EMPTY); diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index e381789f988d7..e3480d9994cda 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -348,9 +348,10 @@ public void testSortMissingValueDateNanoFieldPre714() { } public void testTimeSeriesMode() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); IndexSettings indexSettings = indexSettings( Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some_dimension") .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") @@ -363,16 +364,17 @@ public void testTimeSeriesMode() { } public void testTimeSeriesModeNoTimestamp() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); IndexSettings indexSettings = indexSettings( Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some_dimension") .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") .build() ); Exception e = expectThrows(IllegalArgumentException.class, () -> buildIndexSort(indexSettings, TimeSeriesIdFieldMapper.FIELD_TYPE)); - assertThat(e.getMessage(), equalTo("unknown index sort field:[@timestamp] required by [index.mode=time_series]")); + assertThat(e.getMessage(), equalTo("unknown index sort field:[@timestamp] required by [index.mode=" + mode.getName() + "]")); } public void testLogsdbIndexSortWithArrays() { diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index 7226b1491daf1..5f6b629356e24 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -43,45 +43,69 @@ public void testConfigureIndex() { assertSame(IndexMode.TIME_SERIES, IndexSettings.MODE.get(s)); } + /** + * {@link IndexMode#TSDB} shares its {@link IndexMode#validateWithOtherSettings} implementation with + * {@link IndexMode#TIME_SERIES}, so it must reject the same incompatible settings - naming the mode + * that was actually configured in the error message. + */ public void testPartitioned() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Settings s = Settings.builder() - .put(getSettings()) + .put(getSettingsWithMode(mode.getName(), randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) .put(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.getKey(), 2) .build(); IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.routing_partition_size]")); + assertThat(e.getMessage(), equalTo("[index.mode=" + mode.getName() + "] is incompatible with [index.routing_partition_size]")); } + /** + * Exercises the {@code validateTimeSeriesSettings} path in {@link IndexMode} that builds a synthetic + * {@code Settings} from the calling mode's own name to compute this setting's default value for + * comparison, so it must reject the same way for both {@link IndexMode#TIME_SERIES} and {@link IndexMode#TSDB}. + */ public void testSortField() { - Settings s = Settings.builder().put(getSettings()).put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "a").build(); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings s = Settings.builder() + .put(getSettingsWithMode(mode.getName(), randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "a") + .build(); IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.field]")); + assertThat(e.getMessage(), equalTo("[index.mode=" + mode.getName() + "] is incompatible with [index.sort.field]")); } public void testSortMode() { - Settings s = Settings.builder().put(getSettings()).put(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last").build(); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings s = Settings.builder() + .put(getSettingsWithMode(mode.getName(), randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) + .put(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last") + .build(); IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.missing]")); + assertThat(e.getMessage(), equalTo("[index.mode=" + mode.getName() + "] is incompatible with [index.sort.missing]")); } public void testSortOrder() { - Settings s = Settings.builder().put(getSettings()).put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "desc").build(); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings s = Settings.builder() + .put(getSettingsWithMode(mode.getName(), randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z")) + .put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "desc") + .build(); IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.order]")); + assertThat(e.getMessage(), equalTo("[index.mode=" + mode.getName() + "] is incompatible with [index.sort.order]")); } public void testWithoutRoutingPath() { - Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), mode.getName()).build(); Exception e = expectThrows( IllegalArgumentException.class, () -> new IndexSettings(IndexSettingsTests.newIndexMeta("test", s), Settings.EMPTY) ); - assertThat(e.getMessage(), containsString("[index.mode=time_series] requires a non-empty [index.routing_path]")); + assertThat(e.getMessage(), containsString("[index.mode=" + mode.getName() + "] requires a non-empty [index.routing_path]")); } public void testWithEmptyRoutingPath() { @@ -132,30 +156,44 @@ public void testSetDefaultTimeRangeValue() { } public void testRequiredRouting() { - Settings s = getSettings(); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings s = getSettingsWithMode(mode.getName(), randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z"); var mapperService = new TestMapperServiceBuilder().settings(s).applyDefaultMapping(false).build(); Exception e = expectThrows( IllegalArgumentException.class, () -> withMapping(mapperService, topMapping(b -> b.startObject("_routing").field("required", true).endObject())) ); - assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); + assertThat( + e.getMessage(), + equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=" + mode.getName() + "]") + ); } public void testValidateAlias() { - Settings s = getSettings(); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings s = getSettingsWithMode(mode.getName(), randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z"); + assertSame(mode, IndexSettings.MODE.get(s)); IndexSettings.MODE.get(s).validateAlias(null, null); // Doesn't throw exception } public void testValidateAliasWithIndexRouting() { - Settings s = getSettings(); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings s = getSettingsWithMode(mode.getName(), randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z"); Exception e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s).validateAlias("r", null)); - assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); + assertThat( + e.getMessage(), + equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=" + mode.getName() + "]") + ); } public void testValidateAliasWithSearchRouting() { - Settings s = getSettings(); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings s = getSettingsWithMode(mode.getName(), randomAlphaOfLength(5), "2021-04-28T00:00:00Z", "2021-04-29T00:00:00Z"); Exception e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s).validateAlias(null, "r")); - assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]")); + assertThat( + e.getMessage(), + equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=" + mode.getName() + "]") + ); } public void testRoutingPathMatchesObject() throws IOException { @@ -358,8 +396,16 @@ private Settings getSettings(String routingPath) { } private Settings getSettings(String routingPath, String startTime, String endTime) { + return getSettingsWithMode("time_series", routingPath, startTime, endTime); + } + + /** + * Same as {@link #getSettings(String, String, String)} but with an explicit {@code index.mode} + * value, so that tests can assert the {@code tsdb} value behaves identically to {@code time_series}. + */ + private Settings getSettingsWithMode(String mode, String routingPath, String startTime, String endTime) { return Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexSettings.MODE.getKey(), mode) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) diff --git a/server/src/test/java/org/elasticsearch/index/TsdbIndexModeTests.java b/server/src/test/java/org/elasticsearch/index/TsdbIndexModeTests.java new file mode 100644 index 0000000000000..3a5ed25454df9 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/TsdbIndexModeTests.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index; + +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.TransportVersionUtils; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +/** + * {@link IndexMode#TSDB} is a preferred alternative to {@link IndexMode#TIME_SERIES}: both parse from + * (and behave identically for) the same {@code index.mode} settings, but {@code time_series} + * remains the canonical emitted/persisted string so existing indices are unaffected. + */ +public class TsdbIndexModeTests extends ESTestCase { + + public void testFromStringAcceptsBothSpellings() { + assertThat(IndexMode.fromString("time_series"), equalTo(IndexMode.TIME_SERIES)); + assertThat(IndexMode.fromString("TIME_SERIES"), equalTo(IndexMode.TIME_SERIES)); + assertThat(IndexMode.fromString("tsdb"), equalTo(IndexMode.TSDB)); + assertThat(IndexMode.fromString("TSDB"), equalTo(IndexMode.TSDB)); + assertThat(IndexMode.fromString("Tsdb"), equalTo(IndexMode.TSDB)); + } + + public void testGetName() { + assertThat(IndexMode.TIME_SERIES.getName(), equalTo("time_series")); + assertThat(IndexMode.TIME_SERIES.toString(), equalTo("time_series")); + assertThat(IndexMode.TSDB.getName(), equalTo("tsdb")); + assertThat(IndexMode.TSDB.toString(), equalTo("tsdb")); + } + + public void testIsTsdb() { + assertTrue(IndexMode.TIME_SERIES.isTsdb()); + assertTrue(IndexMode.TSDB.isTsdb()); + for (IndexMode mode : IndexMode.values()) { + if (mode != IndexMode.TIME_SERIES && mode != IndexMode.TSDB) { + assertFalse(mode + " must not be tsdb", mode.isTsdb()); + } + } + } + + public void testIsTsdbNullSafeStatic() { + assertTrue(IndexMode.isTsdb(IndexMode.TIME_SERIES)); + assertTrue(IndexMode.isTsdb(IndexMode.TSDB)); + assertFalse(IndexMode.isTsdb(IndexMode.STANDARD)); + assertFalse(IndexMode.isTsdb(null)); + } + + public void testIsTsdbName() { + assertTrue(IndexMode.isTsdbName("time_series")); + assertTrue(IndexMode.isTsdbName("TIME_SERIES")); + assertTrue(IndexMode.isTsdbName("tsdb")); + assertTrue(IndexMode.isTsdbName("TSDB")); + assertFalse(IndexMode.isTsdbName("standard")); + assertFalse(IndexMode.isTsdbName(null)); + } + + public void testIndexModeSettingAcceptsBothSpellings() { + Settings timeSeriesSettings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("uid")) + .build(); + assertThat(IndexSettings.MODE.get(timeSeriesSettings), equalTo(IndexMode.TIME_SERIES)); + + Settings tsdbSettings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "tsdb") + .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("uid")) + .build(); + assertThat(IndexSettings.MODE.get(tsdbSettings), equalTo(IndexMode.TSDB)); + } + + public void testSerialization() throws IOException { + try (BytesStreamOutput out = new BytesStreamOutput()) { + IndexMode.writeTo(IndexMode.TSDB, out); + try (var in = out.bytes().streamInput()) { + assertThat(IndexMode.readFrom(in), equalTo(IndexMode.TSDB)); + } + } + // TIME_SERIES's own wire representation is unaffected by adding TSDB. + try (BytesStreamOutput out = new BytesStreamOutput()) { + IndexMode.writeTo(IndexMode.TIME_SERIES, out); + try (var in = out.bytes().streamInput()) { + assertThat(IndexMode.readFrom(in), equalTo(IndexMode.TIME_SERIES)); + } + } + } + + public void testSerializationFailsOnOlderTransportVersion() throws IOException { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setTransportVersion(TransportVersionUtils.getPreviousVersion(IndexMode.INDEX_MODE_TSDB_ADDED)); + IllegalStateException e = expectThrows(IllegalStateException.class, () -> IndexMode.writeTo(IndexMode.TSDB, out)); + assertThat(e.getMessage(), containsString("[tsdb] doesn't support serialization with transport version")); + } + } + + /** + * TSDB delegates every behavior to TIME_SERIES, so the two must remain indistinguishable + * other than their name/identity. + */ + public void testBehaviorParityWithTimeSeries() { + assertThat(IndexMode.TSDB.defaultSourceMode(), equalTo(IndexMode.TIME_SERIES.defaultSourceMode())); + assertThat(IndexMode.TSDB.isColumnar(), equalTo(IndexMode.TIME_SERIES.isColumnar())); + assertThat(IndexMode.TSDB.isStrictColumnar(), equalTo(IndexMode.TIME_SERIES.isStrictColumnar())); + assertThat(IndexMode.TSDB.shouldValidateTimestamp(), equalTo(IndexMode.TIME_SERIES.shouldValidateTimestamp())); + assertThat(IndexMode.TSDB.supportedSourceModes(), equalTo(IndexMode.TIME_SERIES.supportedSourceModes())); + assertThat(IndexMode.TSDB.getDefaultCodec(), equalTo(IndexMode.TIME_SERIES.getDefaultCodec())); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java index 6ce085c68bc22..d489ea056353b 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java @@ -225,7 +225,7 @@ public void testUseEs812PostingsFormatForIdField() throws IOException { for (int i = 0; i < numIterations; i++) { var indexMode = randomFrom(IndexMode.STANDARD, IndexMode.LOGSDB, IndexMode.TIME_SERIES); String mapping = randomFrom(METRIC_MAPPING, MULTI_METRIC_MAPPING, LOGS_MAPPING); - final boolean randomSyntheticId = syntheticId(indexMode.equals(IndexMode.TIME_SERIES)); + final boolean randomSyntheticId = syntheticId(indexMode.isTsdb()); PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier( randomBoolean(), randomBoolean(), @@ -409,7 +409,7 @@ private PerFieldFormatSupplier createFormatSupplier( ) throws IOException { Settings.Builder settings = Settings.builder(); settings.put(IndexSettings.MODE.getKey(), mode); - if (mode == IndexMode.TIME_SERIES) { + if (mode.isTsdb()) { settings.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field"); } if (syntheticId != null) { @@ -444,7 +444,7 @@ public void testES95OnlyUsedForTimeSeriesWhenSettingEnabled() throws IOException for (IndexMode mode : INDEX_MODES_UNDER_TEST) { final PerFieldFormatSupplier supplier = createFormatSupplierWithVersion(mode, mappingFor(mode), IndexVersion.current()); final DocValuesFormat format = supplier.getDocValuesFormatForField(fieldFor(mode)); - if (mode == IndexMode.TIME_SERIES) { + if (mode.isTsdb()) { assertThat("mode=" + mode, format, instanceOf(ES95TSDBDocValuesFormat.class)); } else { assertFalse("mode=" + mode + " expected non-ES95", format instanceof ES95TSDBDocValuesFormat); @@ -520,7 +520,7 @@ private PerFieldFormatSupplier createFormatSupplierWithVersion( final Settings.Builder settings = Settings.builder(); settings.put(IndexSettings.MODE.getKey(), mode); settings.put(IndexMetadata.SETTING_VERSION_CREATED, indexVersion); - if (mode == IndexMode.TIME_SERIES) { + if (mode.isTsdb()) { settings.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field"); } if (useTimeSeriesDocValuesFormat) { diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesFormatSelectorTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesFormatSelectorTests.java index b0368c9d252f1..93e8b45816515 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesFormatSelectorTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesFormatSelectorTests.java @@ -53,7 +53,7 @@ public void testES95OnlySelectedForTimeSeriesWhenSettingEnabled() { ); for (IndexMode mode : indexModesUnderTest()) { final DocValuesFormat format = TSDBDocValuesFormatSelector.select(indexSettings(mode, version, true), null); - if (mode == IndexMode.TIME_SERIES) { + if (mode.isTsdb()) { assertThat("mode=" + mode + " version=" + version, format.getName(), equalTo(ES95_CODEC_NAME)); } else { assertThat("mode=" + mode + " version=" + version, format.getName(), startsWith("ES819")); @@ -98,7 +98,7 @@ private static IndexSettings indexSettings(final IndexMode mode, final IndexVers if (mode != IndexMode.STANDARD) { builder.put("index.mode", mode.getName()); } - if (mode == IndexMode.TIME_SERIES) { + if (mode.isTsdb()) { builder.put("index.routing_path", "dimension"); } if (IndexSettings.ES95_CODEC_FEATURE_FLAG.isEnabled()) { diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 1980e9b5ff71e..3a71c61392622 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -8302,11 +8302,17 @@ public void testIndexBatchFastPathOnly() throws IOException { public void testIndexBatchTimeSeriesPhase2() throws IOException { // planPrimarySubBatch Phase 2 uses timeSeriesBatchLoadDocIdAndVersion for TIME_SERIES // indices, which does a single sorted segment scan with timestamp-based segment skipping. + // IndexMode.TSDB is a preferred alternative to TIME_SERIES (delegates isTsdb() etc. to + // TIME_SERIES), so it must exercise the exact same batch phase 2 behavior. + runIndexBatchTimeSeriesPhase2(randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB)); + } + + private void runIndexBatchTimeSeriesPhase2(IndexMode mode) throws IOException { IndexSettings tsSettings = IndexSettingsModule.newIndexSettings( "test", Settings.builder() .put(defaultSettings.getSettings()) - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) + .put(IndexSettings.MODE.getKey(), mode.getName()) .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") .build() ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index 214d7f2f7c9c5..dc759191960be 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -157,9 +157,14 @@ public final void testPositionMetricType() throws IOException { assertParseMinimalWarnings(); } + /** + * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) + * so it must produce the same doc-values-only defaulting for metric fields. + */ public void testTimeSeriesIndexDefault() throws Exception { var positionMetricType = TimeSeriesParams.MetricType.POSITION; - var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) + var mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> { minimalMapping(b); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexModeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexModeFieldTypeTests.java index d8ad599ceba79..f2be57a32da99 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexModeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexModeFieldTypeTests.java @@ -86,7 +86,7 @@ private SearchExecutionContext createContext(IndexMode mode) { if (mode != null) { settings.put(IndexSettings.MODE.getKey(), mode); } - if (mode == IndexMode.TIME_SERIES) { + if (IndexMode.isTsdb(mode)) { settings.putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("a,b,c")); } IndexMetadata indexMetadata = IndexMetadata.builder("index").settings(settings).numberOfShards(1).numberOfReplicas(0).build(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index e29a49ddcf2b1..09495856b6601 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -382,7 +382,7 @@ public void testIsMetadataField() throws IOException { .put(IndexMetadata.SETTING_VERSION_CREATED, version) .put(IndexSettings.MODE.getKey(), indexMode); - if (indexMode == IndexMode.TIME_SERIES) { + if (indexMode.isTsdb()) { settingsBuilder.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo"); } @@ -397,7 +397,7 @@ public void testIsMetadataField() throws IOException { continue; // Nested field does not exist in the 7x line } boolean isTimeSeriesField = builtIn.equals("_tsid") || builtIn.equals("_ts_routing_hash"); - boolean isTimeSeriesMode = mapperService.getIndexSettings().getMode().equals(IndexMode.TIME_SERIES); + boolean isTimeSeriesMode = mapperService.getIndexSettings().getMode().isTsdb(); if (isTimeSeriesField && isTimeSeriesMode == false) { assertFalse( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java index fa5344d5e4843..61134cbca6e5a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java @@ -180,7 +180,7 @@ public MetricType getMetricType() { List.of(dimMapper, metricMapper, plainMapper), emptyList(), emptyList(), - randomValueOtherThan(IndexMode.TIME_SERIES, () -> randomFrom(IndexMode.availableModes())) + randomValueOtherThanMany(m -> m.isTsdb(), () -> randomFrom(IndexMode.availableModes())) ); mappingLookup.validateDoesNotShadow("not_mapped"); mappingLookup.validateDoesNotShadow("dim"); @@ -190,7 +190,7 @@ public MetricType getMetricType() { List.of(dimMapper, metricMapper, plainMapper), emptyList(), emptyList(), - IndexMode.TIME_SERIES + randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB) ); tsMappingLookup.validateDoesNotShadow("not_mapped"); Exception e = expectThrows(MapperParsingException.class, () -> tsMappingLookup.validateDoesNotShadow("dim")); @@ -226,12 +226,17 @@ public MetricType getMetricType() { List.of(dimMapper, metricMapper), emptyList(), List.of(shadowing), - randomValueOtherThan(IndexMode.TIME_SERIES, () -> randomFrom(IndexMode.availableModes())) + randomValueOtherThanMany(m -> m.isTsdb(), () -> randomFrom(IndexMode.availableModes())) ); Exception e = expectThrows( MapperParsingException.class, - () -> createMappingLookup(List.of(dimMapper, metricMapper), emptyList(), List.of(shadowing), IndexMode.TIME_SERIES) + () -> createMappingLookup( + List.of(dimMapper, metricMapper), + emptyList(), + List.of(shadowing), + randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB) + ) ); assertThat( e.getMessage(), diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index 6fbea1e4baaf7..457774399cc4f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -512,7 +512,7 @@ public void testColumnarStoredModeRequiresColumnarIndex() { Settings.Builder builder = Settings.builder() .put(IndexSettings.MODE.getKey(), nonColumnarMode.toString()) .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.COLUMNAR_STORED.toString()); - if (nonColumnarMode == IndexMode.TIME_SERIES) { + if (nonColumnarMode.isTsdb()) { // time_series requires a routing_path; provide one so its own validation passes and our source mode validator fires builder.putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim"); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java index 4552344f7a1cc..89ed3f6d8d84d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesMetadataFieldBlockLoaderTests.java @@ -58,25 +58,33 @@ public class TimeSeriesMetadataFieldBlockLoaderTests extends MapperServiceTestCa * Plain TSDB index. {@link IndexMode#TIME_SERIES} defaults {@code _source.mode} to * {@link SourceFieldMapper.Mode#SYNTHETIC}, so synthetic source reconstructs {@code _source} as JSON. */ - private static final Settings TSDB_SYNTHETIC_SETTINGS = tsdbSettings(null, "host"); + private Settings tsdbSyntheticSettings() { + return tsdbSettings(null, "host"); + } /** * TSDB index with {@code index.mapping.source.mode: stored}. Stored source preserves the raw * indexed bytes (e.g. CBOR for Prometheus remote-write documents), so the {@code _timeseries} loader must explicitly normalize to JSON. */ - private static final Settings TSDB_STORED_SETTINGS = tsdbSettings(SourceFieldMapper.Mode.STORED, "host"); + private Settings tsdbStoredSettings() { + return tsdbSettings(SourceFieldMapper.Mode.STORED, "host"); + } /** * TSDB index that mirrors the Prometheus mapping: a {@code labels} {@code passthrough} object marked as * {@code time_series_dimension: true}. Dimension fields show up under {@code labels.*} and the index uses stored source. */ - private static final Settings TSDB_PROMETHEUS_LIKE_SETTINGS = tsdbSettings(SourceFieldMapper.Mode.STORED, "labels.*"); + private Settings tsdbPrometheusLikeSettings() { + return tsdbSettings(SourceFieldMapper.Mode.STORED, "labels.*"); + } /** * OTel-like TSDB index: an {@code attributes} passthrough object. Dimension fields show up under * {@code attributes.*} and also as root-level aliases. Synthetic source (JSON) is used. */ - private static final Settings TSDB_OTEL_LIKE_SETTINGS = tsdbSettings(null, "attributes.*"); + private Settings tsdbOtelLikeSettings() { + return tsdbSettings(null, "attributes.*"); + } private static final String MAPPING = """ { @@ -153,9 +161,13 @@ public class TimeSeriesMetadataFieldBlockLoaderTests extends MapperServiceTestCa } """; - private static Settings tsdbSettings(SourceFieldMapper.Mode sourceMode, String routingPath) { + /** + * Randomizes between {@link IndexMode#TIME_SERIES} and {@link IndexMode#TSDB} so callers exercise both, + * since {@link TimeSeriesMetadataFieldBlockLoader} must be selected and behave identically for either. + */ + private Settings tsdbSettings(SourceFieldMapper.Mode sourceMode, String routingPath) { Settings.Builder builder = Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) + .put(IndexSettings.MODE.getKey(), randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB).getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z"); @@ -167,7 +179,7 @@ private static Settings tsdbSettings(SourceFieldMapper.Mode sourceMode, String r public void testDimensionsOnly() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_SYNTHETIC_SETTINGS, + tsdbSettings(null, "host"), MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of()) ); @@ -177,7 +189,7 @@ public void testDimensionsOnly() throws IOException { public void testDimensionsAndMetrics() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_SYNTHETIC_SETTINGS, + tsdbSyntheticSettings(), MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(true, Set.of()) ); @@ -187,7 +199,7 @@ public void testDimensionsAndMetrics() throws IOException { public void testExcludedDimensions() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_SYNTHETIC_SETTINGS, + tsdbSyntheticSettings(), MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of("host", "region")) ); @@ -197,7 +209,7 @@ public void testExcludedDimensions() throws IOException { public void testExcludedDimensionsWithMetrics() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_SYNTHETIC_SETTINGS, + tsdbSyntheticSettings(), MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(true, Set.of("env")) ); @@ -214,7 +226,7 @@ public void testExcludedDimensionsWithMetrics() throws IOException { */ public void testExcludedDimensionsWithWildcardDimensionSetting() throws IOException { Settings settings = Settings.builder() - .put(TSDB_PROMETHEUS_LIKE_SETTINGS) + .put(tsdbPrometheusLikeSettings()) .putList(IndexMetadata.INDEX_DIMENSIONS.getKey(), "labels.*") .build(); BlockLoader loader = createBlockLoader( @@ -227,7 +239,7 @@ public void testExcludedDimensionsWithWildcardDimensionSetting() throws IOExcept } public void testNoConfigReturnSourceBlockLoader() throws IOException { - MapperService mapperService = createMapperService(TSDB_SYNTHETIC_SETTINGS, MAPPING); + MapperService mapperService = createMapperService(tsdbSyntheticSettings(), MAPPING); BlockLoader loader = mapperService.documentMapper() .sourceMapper() .fieldType() @@ -242,7 +254,7 @@ public void testNoConfigReturnSourceBlockLoader() throws IOException { */ public void testReadEmitsJsonForSyntheticSourceIndex() throws IOException { BytesReference json = bytes(XContentType.JSON, b -> writeTimestampAndDimensions(b, "host-1", "prod", "us-east-1")); - BytesRef value = readTimeSeriesValue(TSDB_SYNTHETIC_SETTINGS, MAPPING, sourceToParse(json, XContentType.JSON)); + BytesRef value = readTimeSeriesValue(tsdbSyntheticSettings(), MAPPING, sourceToParse(json, XContentType.JSON)); assertThat(parseJsonObject(value), equalTo(Map.of("host", "host-1", "env", "prod", "region", "us-east-1"))); } @@ -254,7 +266,7 @@ public void testReadEmitsJsonForSyntheticSourceIndex() throws IOException { */ public void testReadConvertsCborStoredSourceToJson() throws IOException { BytesReference cbor = bytes(XContentType.CBOR, b -> writeTimestampAndDimensions(b, "host-1", "prod", "us-east-1")); - BytesRef value = readTimeSeriesValue(TSDB_STORED_SETTINGS, MAPPING, sourceToParse(cbor, XContentType.CBOR)); + BytesRef value = readTimeSeriesValue(tsdbStoredSettings(), MAPPING, sourceToParse(cbor, XContentType.CBOR)); assertThat(parseJsonObject(value), equalTo(Map.of("host", "host-1", "env", "prod", "region", "us-east-1"))); } @@ -264,7 +276,7 @@ public void testReadConvertsCborStoredSourceToJson() throws IOException { */ public void testReadPassesThroughJsonStoredSource() throws IOException { BytesReference json = bytes(XContentType.JSON, b -> writeTimestampAndDimensions(b, "host-1", "prod", "us-east-1")); - BytesRef value = readTimeSeriesValue(TSDB_STORED_SETTINGS, MAPPING, sourceToParse(json, XContentType.JSON)); + BytesRef value = readTimeSeriesValue(tsdbStoredSettings(), MAPPING, sourceToParse(json, XContentType.JSON)); assertThat(parseJsonObject(value), equalTo(Map.of("host", "host-1", "env", "prod", "region", "us-east-1"))); } @@ -285,11 +297,7 @@ public void testReadJsonContainsDimensionKeyValuePairs() throws IOException { b.field("go_gc_cleanups_executed_cleanups_total", 1.0); b.endObject(); }); - BytesRef value = readTimeSeriesValue( - TSDB_PROMETHEUS_LIKE_SETTINGS, - PROMETHEUS_LIKE_MAPPING, - sourceToParse(cbor, XContentType.CBOR) - ); + BytesRef value = readTimeSeriesValue(tsdbPrometheusLikeSettings(), PROMETHEUS_LIKE_MAPPING, sourceToParse(cbor, XContentType.CBOR)); Map parsed = parseJsonObject(value); assertThat(parsed.keySet(), equalTo(Set.of("labels"))); @SuppressWarnings("unchecked") @@ -318,7 +326,7 @@ public void testWithoutFieldsExcludesLabelsFromOutput() throws IOException { b.endObject(); }); BytesRef value = readTimeSeriesValue( - TSDB_PROMETHEUS_LIKE_SETTINGS, + tsdbPrometheusLikeSettings(), PROMETHEUS_LIKE_MAPPING, sourceToParse(cbor, XContentType.CBOR), Set.of("labels.instance") @@ -340,7 +348,7 @@ public void testWithoutFieldsExcludesLabelsFromOutput() throws IOException { */ public void testOtelPassthroughAliasExcludedByShortName() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_OTEL_LIKE_SETTINGS, + tsdbOtelLikeSettings(), OTEL_LIKE_MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of("cpu")) ); @@ -355,7 +363,7 @@ public void testOtelPassthroughAliasExcludedByShortName() throws IOException { */ public void testPrometheusPassthroughAliasExcludedByShortName() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_PROMETHEUS_LIKE_SETTINGS, + tsdbPrometheusLikeSettings(), PROMETHEUS_LIKE_MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of("job")) ); @@ -368,7 +376,7 @@ public void testPrometheusPassthroughAliasExcludedByShortName() throws IOExcepti */ public void testWithoutAllDimensionsYieldsEmptySourcePaths() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_SYNTHETIC_SETTINGS, + tsdbSyntheticSettings(), MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of("host", "env", "region")) ); @@ -382,7 +390,7 @@ public void testWithoutAllDimensionsYieldsEmptySourcePaths() throws IOException */ public void testWithoutUnknownFieldIsIgnored() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_SYNTHETIC_SETTINGS, + tsdbSyntheticSettings(), MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of("does_not_exist")) ); @@ -396,7 +404,7 @@ public void testWithoutUnknownFieldIsIgnored() throws IOException { */ public void testExcludedDimensionsWithMetricsAndExclusion() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_SYNTHETIC_SETTINGS, + tsdbSyntheticSettings(), MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(true, Set.of("host")) ); @@ -410,7 +418,7 @@ public void testExcludedDimensionsWithMetricsAndExclusion() throws IOException { */ public void testOtelPassthroughConcreteNameExcludesCorrectly() throws IOException { BlockLoader loader = createBlockLoader( - TSDB_OTEL_LIKE_SETTINGS, + tsdbOtelLikeSettings(), OTEL_LIKE_MAPPING, new BlockLoaderFunctionConfig.TimeSeriesMetadata(false, Set.of("attributes.cpu")) ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java index 34b5b577bd3ff..edccad46072de 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java @@ -91,11 +91,20 @@ private static int getRoutingHash(ParsedDocument document) { return TimeSeriesRoutingHashFieldMapper.decode(Uid.decodeId(value.bytes)); } + /** + * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) + * so {@link TimeSeriesRoutingHashFieldMapper} must be enabled identically for both. + */ @SuppressWarnings("unchecked") public void testEnabledInTimeSeriesMode() throws Exception { - DocumentMapper docMapper = createMapper(mapping(b -> { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings.Builder settingsBuilder = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), mode.name()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing path is required") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-10-29T00:00:00Z"); + DocumentMapper docMapper = createMapperService(settingsBuilder.build(), mapping(b -> { b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - })); + })).documentMapper(); int hash = randomInt(); ParsedDocument doc = parseDocument(hash, docMapper, b -> b.field("a", "value")); diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index 2d1930edfe849..6cc1dc46f709a 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -575,6 +575,7 @@ public static void getClusterStateWithDataStream( String dataStreamName, List> timeSlices ) { + IndexMode indexMode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); List backingIndices = new ArrayList<>(); DataStream existing = builder.dataStream(dataStreamName); if (existing != null) { @@ -587,7 +588,7 @@ public static void getClusterStateWithDataStream( Instant start = tuple.v1(); Instant end = tuple.v2(); Settings settings = Settings.builder() - .put("index.mode", "time_series") + .put("index.mode", indexMode.getName()) .put("index.routing_path", "uid") .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(start)) .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(end)) @@ -600,7 +601,7 @@ public static void getClusterStateWithDataStream( var dataStreamBuilder = DataStream.builder( dataStreamName, backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()) - ).setGeneration(generation).setIndexMode(IndexMode.TIME_SERIES); + ).setGeneration(generation).setIndexMode(indexMode); if (existing != null) { dataStreamBuilder.setMetadata(existing.getMetadata()) .setHidden(existing.isHidden()) diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index a241eb1035dea..733bdbf6ae78f 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -159,6 +159,7 @@ protected final DocumentMapper createDocumentMapper(XContentBuilder mappings, In case STANDARD, LOOKUP -> createDocumentMapper(mappings); case VECTORDB_DOCUMENT -> createVectordbDocumentModeDocumentMapper(mappings); case TIME_SERIES -> createTimeSeriesModeDocumentMapper(mappings); + case TSDB -> createTsdbModeDocumentMapper(mappings); case LOGSDB -> createLogsModeDocumentMapper(mappings); case COLUMNAR -> createColumnarModeDocumentMapper(mappings); case LOGSDB_COLUMNAR -> createColumnarLogsdbModeDocumentMapper(mappings); @@ -177,6 +178,14 @@ protected final DocumentMapper createTimeSeriesModeDocumentMapper(XContentBuilde return createMapperService(settings, mappings).documentMapper(); } + protected final DocumentMapper createTsdbModeDocumentMapper(XContentBuilder mappings) throws IOException { + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "tsdb") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "uid") + .build(); + return createMapperService(settings, mappings).documentMapper(); + } + protected final DocumentMapper createLogsModeDocumentMapper(XContentBuilder mappings) throws IOException { Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()).build(); return createMapperService(settings, mappings).documentMapper(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index ccf0e3e96880b..927bb5ca6f3d9 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -278,7 +278,8 @@ public void testMetricType() throws IOException { public void testTimeSeriesIndexDefault() throws Exception { var randomMetricType = randomFrom(TimeSeriesParams.MetricType.scalar()); - var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> { minimalMapping(b); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java index 15632c34a7bbb..63be684caca78 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java @@ -774,12 +774,13 @@ private CheckedBiFunction nestedPa public void testProcessOnceOnPrimary() throws Exception { final Settings.Builder settingsBuilder = indexSettings(IndexVersion.current(), 1, 0).put("index.xpack.ccr.following_index", true); - boolean useSyntheticId = indexMode == IndexMode.TIME_SERIES && randomBoolean(); + boolean useSyntheticId = indexMode.isTsdb() && randomBoolean(); switch (indexMode) { case STANDARD: break; case TIME_SERIES: - settingsBuilder.put("index.mode", "time_series").put("index.routing_path", "foo"); + case TSDB: + settingsBuilder.put("index.mode", indexMode.getName()).put("index.routing_path", "foo"); settingsBuilder.put("index.seq_no.index_options", "points_and_doc_values"); settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), useSyntheticId); break; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java index 7d94e2fad5835..bc0320529268a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java @@ -269,12 +269,10 @@ public void testDownsamplingPrerequisitesStep() { assertThat(branchingStep.getNextStepKey(), is(nextStepKey)); } { - // time series indices execute the action + // time series indices execute the action, regardless of the index mode alias used BranchingStep branchingStep = getFirstBranchingStep(action, phase, nextStepKey, withForceMerge); - Settings settings = Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) - .put("index.routing_path", "uid") - .build(); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), mode).put("index.routing_path", "uid").build(); IndexMetadata indexMetadata = newIndexMeta("test", settings); ProjectState state = projectStateFromProject(ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true)); @@ -283,10 +281,11 @@ public void testDownsamplingPrerequisitesStep() { assertThat(branchingStep.getNextStepKey().name(), is(CheckNotDataStreamWriteIndexStep.NAME)); } { - // already downsampled indices for the interval skip the action + // already downsampled indices for the interval skip the action, regardless of the index mode alias used BranchingStep branchingStep = getFirstBranchingStep(action, phase, nextStepKey, withForceMerge); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Settings settings = Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(IndexSettings.MODE.getKey(), mode) .put("index.routing_path", "uid") .put(IndexMetadata.INDEX_DOWNSAMPLE_STATUS_KEY, IndexMetadata.DownsampleTaskStatus.SUCCESS) .put(IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_NAME.getKey(), "test") diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java index db30c56f1d556..fc8abc1b543b7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java @@ -10,19 +10,18 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ProjectState; -import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.Tuple; -import org.elasticsearch.index.Index; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.xcontent.ToXContentObject; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.List; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; @@ -53,33 +52,36 @@ protected WaitUntilTimeSeriesEndTimePassesStep copyInstance(WaitUntilTimeSeriesE return new WaitUntilTimeSeriesEndTimePassesStep(instance.getKey(), instance.getNextStepKey(), Instant::now); } + /** + * {@link org.elasticsearch.index.IndexMode#TSDB} is a preferred alternative to + * {@link org.elasticsearch.index.IndexMode#TIME_SERIES} and must gate {@link WaitUntilTimeSeriesEndTimePassesStep} + * identically, so the {@code index.mode} used for the time-series-specific assertions is randomized between the two. + */ public void testEvaluateCondition() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Instant currentTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); // These ranges are on the edge of each other temporal boundaries. - Instant start1 = currentTime.minus(6, ChronoUnit.HOURS); - Instant end1 = currentTime.minus(2, ChronoUnit.HOURS); - Instant start2 = currentTime.minus(2, ChronoUnit.HOURS); - Instant end2 = currentTime.plus(2, ChronoUnit.HOURS); - - String dataStreamName = "logs_my-app_prod"; - final var project = DataStreamTestHelper.getProjectWithDataStream( - randomProjectIdOrDefault(), - dataStreamName, - List.of(Tuple.tuple(start1, end1), Tuple.tuple(start2, end2)) - ); - var state = ClusterState.builder(ClusterName.DEFAULT).putProjectMetadata(project).build().projectState(project.id()); - DataStream dataStream = project.dataStreams().get(dataStreamName); + Instant startTimeLapsed = currentTime.minus(6, ChronoUnit.HOURS); + Instant endTimeLapsed = currentTime.minus(2, ChronoUnit.HOURS); + Instant startTimeFuture = currentTime.minus(2, ChronoUnit.HOURS); + Instant endTimeFuture = currentTime.plus(2, ChronoUnit.HOURS); WaitUntilTimeSeriesEndTimePassesStep step = new WaitUntilTimeSeriesEndTimePassesStep( randomStepKey(), randomStepKey(), () -> currentTime ); + { // end_time has lapsed already so condition must be met - Index previousGeneration = dataStream.getIndices().get(0); + IndexMetadata indexMeta = createTimeSeriesIndexMetadata(mode, "ts-index-lapsed", startTimeLapsed, endTimeLapsed); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMeta, true).build(); + ProjectState projectState = ClusterState.builder(ClusterName.DEFAULT) + .putProjectMetadata(project) + .build() + .projectState(project.id()); - step.evaluateCondition(state, project.index(previousGeneration), new AsyncWaitStep.Listener() { + step.evaluateCondition(projectState, indexMeta, new AsyncWaitStep.Listener() { @Override public void onResponse(boolean complete, ToXContentObject informationContext) { @@ -95,9 +97,14 @@ public void onFailure(Exception e) { { // end_time is in the future - Index writeIndex = dataStream.getIndices().get(1); + IndexMetadata indexMeta = createTimeSeriesIndexMetadata(mode, "ts-index-future", startTimeFuture, endTimeFuture); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMeta, true).build(); + ProjectState projectState = ClusterState.builder(ClusterName.DEFAULT) + .putProjectMetadata(project) + .build() + .projectState(project.id()); - step.evaluateCondition(state, project.index(writeIndex), new AsyncWaitStep.Listener() { + step.evaluateCondition(projectState, indexMeta, new AsyncWaitStep.Listener() { @Override public void onResponse(boolean complete, ToXContentObject informationContext) { @@ -107,9 +114,9 @@ public void onResponse(boolean complete, ToXContentObject informationContext) { information, containsString( "The [index.time_series.end_time] setting for index [" - + writeIndex.getName() + + indexMeta.getIndex().getName() + "] is [" - + end2.toEpochMilli() + + endTimeFuture.toEpochMilli() + "]. Waiting until the index's time series end time lapses before proceeding with action [" + step.getKey().action() + "] as the index can still accept writes." @@ -130,8 +137,12 @@ public void onFailure(Exception e) { .settings(indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()).build()) .build(); - ProjectState newState = state.updateProject(ProjectMetadata.builder(project).put(indexMeta, true).build()); - step.evaluateCondition(newState, indexMeta, new AsyncWaitStep.Listener() { + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMeta, true).build(); + ProjectState projectState = ClusterState.builder(ClusterName.DEFAULT) + .putProjectMetadata(project) + .build() + .projectState(project.id()); + step.evaluateCondition(projectState, indexMeta, new AsyncWaitStep.Listener() { @Override public void onResponse(boolean complete, ToXContentObject informationContext) { @@ -145,4 +156,14 @@ public void onFailure(Exception e) { }, MASTER_TIMEOUT); } } + + private static IndexMetadata createTimeSeriesIndexMetadata(IndexMode mode, String indexName, Instant startTime, Instant endTime) { + Settings settings = indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), mode.getName()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "uid") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(startTime)) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(endTime)) + .build(); + return IndexMetadata.builder(indexName).settings(settings).build(); + } } diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java index 263661a23572e..690fd15f058d0 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java @@ -499,7 +499,7 @@ private IndexMetadata.Builder createSourceIndexMetadata(String sourceIndex, int .settings( indexSettings(IndexVersion.current(), randomUUID(), primaryShards, replicaShards).put( IndexSettings.MODE.getKey(), - IndexMode.TIME_SERIES.getName() + randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB).getName() ) .put("index.routing_path", "dimensions") .put(IndexMetadata.SETTING_BLOCKS_WRITE, true) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java index 134f7a45234b0..216f0156b01af 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java @@ -276,6 +276,9 @@ public void createIndices() throws IOException { minVersion().supports(IndexMode.VECTORDB_DOCUMENT_INDEX_MODE) ); } + if (indexMode == IndexMode.TSDB) { + assumeTrue("Cluster has nodes that do not support index.mode=tsdb", minVersion().supports(IndexMode.INDEX_MODE_TSDB_ADDED)); + } if (indexMode == IndexMode.COLUMNAR || indexMode == IndexMode.LOGSDB_COLUMNAR) { // Gate on the cluster capability rather than the test runner build: columnar index modes are only available when the // feature flag is enabled on the nodes (e.g. they are disabled in upgrade clusters), in which case these tests skip. @@ -1408,7 +1411,7 @@ protected boolean preserveClusterUponCompletion() { private boolean syntheticSourceByDefault() { return switch (indexMode) { - case TIME_SERIES, LOGSDB, COLUMNAR, LOGSDB_COLUMNAR -> true; + case TIME_SERIES, TSDB, LOGSDB, COLUMNAR, LOGSDB_COLUMNAR -> true; case STANDARD, LOOKUP, VECTORDB_DOCUMENT -> false; }; } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java index 8b9f8033f6668..568bea3bf1ec8 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.action; +import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.TsidBuilder; import org.elasticsearch.common.Randomness; @@ -78,6 +79,91 @@ public void testEmpty() { run("TS empty_index | LIMIT 1").close(); } + public void testTsdbModeProbe() throws Exception { + Settings settings = Settings.builder().put("mode", "tsdb").putList("routing_path", List.of("host")).build(); + client().admin() + .indices() + .prepareCreate("hosts_tsdb_probe") + .setSettings(settings) + .setMapping( + "@timestamp", + "type=date", + "host", + "type=keyword,time_series_dimension=true", + "cpu", + "type=double,time_series_metric=gauge" + ) + .get(); + long timestamp = DEFAULT_DATE_TIME_FORMATTER.parseMillis("2024-04-15T00:00:00Z"); + for (int i = 0; i < 10; i++) { + client().prepareIndex("hosts_tsdb_probe") + .setSource("@timestamp", timestamp + i * 1000, "host", "h" + (i % 2), "cpu", (double) i) + .get(); + } + client().admin().indices().prepareRefresh("hosts_tsdb_probe").get(); + try (EsqlQueryResponse resp = run("TS hosts_tsdb_probe METADATA _index_mode | STATS load=avg(cpu) BY host | SORT host")) { + List> rows = EsqlTestUtils.getValuesList(resp); + assertThat(rows, hasSize(2)); + } + try (EsqlQueryResponse resp = run("TS hosts_tsdb_probe METADATA _index_mode, _index | STATS BY _index, _index_mode")) { + List> rows = EsqlTestUtils.getValuesList(resp); + assertThat(rows, equalTo(List.of(List.of("hosts_tsdb_probe", "tsdb")))); + } + // mixed wildcard: "hosts" fixture is mode=time_series, "hosts_tsdb_probe" is mode=tsdb + try (EsqlQueryResponse resp = run("TS hosts* METADATA _index_mode, _index | STATS BY _index, _index_mode | SORT _index")) { + List> rows = EsqlTestUtils.getValuesList(resp); + assertThat(rows, equalTo(List.of(List.of("hosts", "time_series"), List.of("hosts_tsdb_probe", "tsdb")))); + } + } + + /** + * Mirrors how a real data stream ends up with backing indices in both spellings after migrating + * to the new preferred {@code tsdb} spelling (e.g. OTel/Prometheus streams, see docs/changelog/152901.yaml): + * a single logical name (here, an alias) fronting one {@code time_series}-mode index and one + * {@code tsdb}-mode index. {@code TS name}, a concrete (non-wildcard) request, must resolve both. + */ + public void testTsdbModeProbeMixedAlias() throws Exception { + String mapping = """ + { + "properties": { + "@timestamp": { "type": "date" }, + "host": { "type": "keyword", "time_series_dimension": true }, + "cpu": { "type": "double", "time_series_metric": "gauge" } + } + } + """; + client().admin() + .indices() + .prepareCreate("hosts_mixed_alias_v1") + .setSettings( + Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").putList("routing_path", List.of("host")).build() + ) + .setMapping(mapping) + .addAlias(new Alias("hosts_mixed_alias")) + .get(); + client().admin() + .indices() + .prepareCreate("hosts_mixed_alias_v2") + .setSettings(Settings.builder().put(IndexSettings.MODE.getKey(), "tsdb").putList("routing_path", List.of("host")).build()) + .setMapping(mapping) + .addAlias(new Alias("hosts_mixed_alias")) + .get(); + client().prepareIndex("hosts_mixed_alias_v1").setSource("@timestamp", "2024-04-15T00:00:00Z", "host", "h0", "cpu", 1.0).get(); + client().prepareIndex("hosts_mixed_alias_v2").setSource("@timestamp", "2024-04-15T01:00:00Z", "host", "h1", "cpu", 2.0).get(); + client().admin().indices().prepareRefresh("hosts_mixed_alias").get(); + + try ( + EsqlQueryResponse resp = run("TS hosts_mixed_alias METADATA _index_mode, _index | STATS BY _index, _index_mode | SORT _index") + ) { + List> rows = EsqlTestUtils.getValuesList(resp); + assertThat(rows, equalTo(List.of(List.of("hosts_mixed_alias_v1", "time_series"), List.of("hosts_mixed_alias_v2", "tsdb")))); + } + try (EsqlQueryResponse resp = run("TS hosts_mixed_alias | STATS total=sum(cpu)")) { + List> rows = EsqlTestUtils.getValuesList(resp); + assertThat(rows, equalTo(List.of(List.of(3.0)))); + } + } + record Doc( Collection project, String host, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java index fdcac6f7479e4..b2fa4f2977eb9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java @@ -260,7 +260,7 @@ private static LogicalPlan rewriteOne( } if (relation.indexMode() != null && relation.indexMode() != IndexMode.STANDARD) { String message = switch (relation.indexMode()) { - case TIME_SERIES -> "TS command is not supported for datasets; dataset(s) requested: " + datasetNames; + case TIME_SERIES, TSDB -> "TS command is not supported for datasets; dataset(s) requested: " + datasetNames; case LOOKUP -> "LOOKUP JOIN against a dataset is not supported; dataset(s) requested: " + datasetNames; case LOGSDB -> "LOGSDB index mode on FROM is not supported; dataset(s) requested: " + datasetNames; default -> "FROM with index mode [" diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index b063db48ada3d..c4e2387dd826d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -33,7 +33,7 @@ import org.elasticsearch.index.mapper.IndexModeFieldMapper; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesExpressionGrouper; import org.elasticsearch.iplocation.api.IpDataLookupInfo; @@ -2005,14 +2005,20 @@ private void preAnalyzeFlatMainIndices( ); } - private static QueryBuilder createQueryFilter(IndexMode indexMode, QueryBuilder requestFilter) { - return switch (indexMode) { - case IndexMode.TIME_SERIES -> { - var indexModeFilter = new TermQueryBuilder(IndexModeFieldMapper.NAME, IndexMode.TIME_SERIES.getName()); - yield requestFilter != null ? new BoolQueryBuilder().filter(requestFilter).filter(indexModeFilter) : indexModeFilter; - } - default -> requestFilter; - }; + // visible for testing + static QueryBuilder createQueryFilter(IndexMode indexMode, QueryBuilder requestFilter) { + if (IndexMode.isTsdb(indexMode)) { + // Match both values: the requested indexMode is a fixed sentinel (e.g. the TS command always + // declares IndexMode.TIME_SERIES), but the concrete backing indices may be configured with + // either [index.mode=time_series] or [index.mode=tsdb]. + var indexModeFilter = new TermsQueryBuilder( + IndexModeFieldMapper.NAME, + IndexMode.TIME_SERIES.getName(), + IndexMode.TSDB.getName() + ); + return requestFilter != null ? new BoolQueryBuilder().filter(requestFilter).filter(indexModeFilter) : indexModeFilter; + } + return requestFilter; } // visible for testing diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 7a08a0e6b0f37..14e9883f9eeac 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2719,8 +2719,11 @@ private static void assertDefaultLimitForQuery(String query, int expectedLimit) } public void testImplicitTimestampSortForTsQuery() { - // TS query without STATS or SORT should have implicit sort - var plan = tsdb().query("TS test"); + // TS query without STATS or SORT should have implicit sort. AddImplicitTimestampSort is + // gated on IndexMode#isTsdb(), so this exercises both IndexMode.TIME_SERIES and + // IndexMode.TSDB, which must be treated identically. + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + var plan = analyzer().addIndex("test", "tsdb-mapping.json", mode).query("TS test"); var limit = as(plan, Limit.class); var orderBy = as(limit.child(), OrderBy.class); @@ -4988,10 +4991,13 @@ private void verifyNameAndType(String actualName, DataType actualType, String ex } public void testImplicitCastingForAggregateMetricDouble() { + // ImplicitCastAggregateMetricDoubles is gated on IndexMode#isTsdb(), so the TS branch below + // exercises both IndexMode.TIME_SERIES and IndexMode.TSDB, which must be treated identically. assumeTrue( "aggregate metric double implicit casting must be available", EsqlCapabilities.Cap.AGGREGATE_METRIC_DOUBLE_V0.isEnabled() ); + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Map mapping = Map.of( "@timestamp", new EsField("@timestamp", DATETIME, Map.of(), true, EsField.TimeSeriesFieldType.NONE), @@ -5001,13 +5007,7 @@ public void testImplicitCastingForAggregateMetricDouble() { new InvalidMappedField("metric_field", Map.of("aggregate_metric_double", Set.of("k8s-downsampled"), "double", Set.of("k8s"))) ); - var esIndex = new EsIndex( - "k8s,k8s-downsampled", - mapping, - Map.of("k8s", IndexMode.TIME_SERIES, "k8s-downsampled", IndexMode.TIME_SERIES), - Map.of(), - Map.of() - ); + var esIndex = new EsIndex("k8s,k8s-downsampled", mapping, Map.of("k8s", mode, "k8s-downsampled", mode), Map.of(), Map.of()); var testAnalyzer = analyzer().addIndex(esIndex); var stddevPlan = testAnalyzer.query(""" from k8s,k8s-downsampled | stats std_dev = std_dev(metric_field) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java index 17f125f0a9511..c18027158eb8b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java @@ -154,11 +154,13 @@ public void testMixedMultipleIndicesAndDatasetsProducesUnionAll() { } public void testIndexModeNonStandardRejected() { - // Note on coverage: only TIME_SERIES (via TS) and LOOKUP (via LOOKUP JOIN) are user-reachable + // Note on coverage: only TIME_SERIES/TSDB (via TS) and LOOKUP (via LOOKUP JOIN) are user-reachable // through ESQL syntax; LOGSDB has no user-syntax path that constructs an UnresolvedRelation // with IndexMode.LOGSDB pointing at a dataset name. The LOGSDB branch is defensive code for // any future path that might set it. There is no IT analogue for LOGSDB — this unit case // pins the rejection-message contract. + // TSDB is included alongside TIME_SERIES to prove "index.mode: tsdb" is rejected + // with the exact same message, since IndexMode.TSDB behaves identically to TIME_SERIES. DataSource parent = dataSource("s3_parent", Map.of()); Dataset dataset = new Dataset("logs", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); ProjectMetadata project = projectWith(Map.of("s3_parent", parent), Map.of("logs", dataset)); @@ -166,6 +168,8 @@ public void testIndexModeNonStandardRejected() { Map expectedFragments = Map.of( IndexMode.TIME_SERIES, "TS command is not supported for datasets", + IndexMode.TSDB, + "TS command is not supported for datasets", IndexMode.LOOKUP, "LOOKUP JOIN against a dataset is not supported", IndexMode.LOGSDB, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java index 9bfe118d9f658..7c577d93412ef 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushExpressionsToFieldLoadTests.java @@ -850,7 +850,12 @@ private static SearchStats tsSearchStats() { @Override public Map targetShards() { IndexMetadata indexMetadata = IndexMetadata.builder("k8s") - .settings(indexSettings(IndexVersion.current(), 1, 1).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())) + .settings( + indexSettings(IndexVersion.current(), 1, 1).put( + IndexSettings.MODE.getKey(), + randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB).name() + ) + ) .build(); return Map.of(new ShardId(new Index("k8s", "n/a"), 0), indexMetadata); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java index 5bf3f3c0d96e4..c15ad1f9b9d5a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/SubstituteRoundToTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; +import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.common.settings.Settings; @@ -27,11 +28,14 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.elasticsearch.xpack.esql.analysis.Analyzer; +import org.elasticsearch.xpack.esql.analysis.EnrichResolution; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; @@ -43,7 +47,11 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc; import org.elasticsearch.xpack.esql.expression.function.scalar.math.RoundTo; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.index.EsIndexGenerator; +import org.elasticsearch.xpack.esql.index.IndexResolution; import org.elasticsearch.xpack.esql.optimizer.AbstractLocalPhysicalPlanOptimizerTests; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; import org.elasticsearch.xpack.esql.optimizer.TestPlannerOptimizer; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; @@ -80,8 +88,14 @@ import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_FUNCTION_REGISTRY; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptyInferenceResolution; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.testAnalyzerContext; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.indexResolutions; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_NANOS_FORMATTER; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateNanosToLong; @@ -996,6 +1010,7 @@ public void testSubqueryWithCountStarAndDateTrunc() { * {@link PushExpressionsToFieldLoadTests#testRoundToInTsEval} and friends. */ public void testRoundToWithTimeSeriesIndices() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Map minValue = Map.of( "@timestamp", DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-10-20T12:15:03.360Z") @@ -1008,14 +1023,12 @@ public void testRoundToWithTimeSeriesIndices() { @Override public Map targetShards() { var indexMetadata = IndexMetadata.builder("test_index") - .settings( - ESTestCase.indexSettings(IndexVersion.current(), 1, 1) - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) - ) + .settings(ESTestCase.indexSettings(IndexVersion.current(), 1, 1).put(IndexSettings.MODE.getKey(), mode.getName())) .build(); return Map.of(new ShardId(new Index("id", "n/a"), 1), indexMetadata); } }; + TestPlannerOptimizer optimizer = plannerOptimizerForMode(mode); // enable filter-by-filter for rate aggregations { String q = """ @@ -1023,7 +1036,7 @@ public Map targetShards() { | STATS max(rate(network.total_bytes_in)) BY cluster, BUCKET(@timestamp, 1 hour) | LIMIT 10 """; - PhysicalPlan plan = plannerOptimizerTimeSeries.plan(q, searchStats, timeSeriesAnalyzer); + PhysicalPlan plan = optimizer.plan(q, searchStats); int queryAndTags = plainQueryAndTags(plan); assertThat(queryAndTags, equalTo(4)); } @@ -1034,12 +1047,47 @@ public Map targetShards() { | STATS max(avg_over_time(network.bytes_in)) BY cluster, BUCKET(@timestamp, 1 hour) | LIMIT 10 """; - PhysicalPlan plan = plannerOptimizerTimeSeries.plan(q, searchStats, timeSeriesAnalyzer); + PhysicalPlan plan = optimizer.plan(q, searchStats); int queryAndTags = plainQueryAndTags(plan); assertThat(queryAndTags, equalTo(1)); } } + /** + * Directly exercises {@link ReplaceRoundToWithQueryAndTags#adjustedRoundingPointsThreshold}: since + * {@link IndexMode#TSDB} is a preferred alternative to {@link IndexMode#TIME_SERIES}, both must double the + * rounding points threshold identically. + */ + public void testAdjustedRoundingPointsThreshold() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + int threshold = between(1, 1000); + int adjustedThreshold = ReplaceRoundToWithQueryAndTags.adjustedRoundingPointsThreshold(searchStats, threshold, null, mode); + assertThat(adjustedThreshold, equalTo(threshold * 2)); + } + + // Builds an analyzer/planner pair mirroring AbstractLocalPhysicalPlanOptimizerTests#init's + // plannerOptimizerTimeSeries/timeSeriesAnalyzer, but resolving the "k8s" index with the given + // IndexMode, so tests can verify IndexMode.TSDB is handled identically to IndexMode.TIME_SERIES. + private TestPlannerOptimizer plannerOptimizerForMode(IndexMode mode) { + var timeSeriesMapping = loadMapping("k8s-mappings.json"); + var timeSeriesIndex = IndexResolution.valid(EsIndexGenerator.esIndex("k8s", timeSeriesMapping, Map.of("k8s", mode))); + Analyzer analyzer = new Analyzer( + testAnalyzerContext( + EsqlTestUtils.TEST_CFG, + TEST_FUNCTION_REGISTRY, + indexResolutions(timeSeriesIndex), + new EnrichResolution(), + emptyInferenceResolution() + ), + TEST_VERIFIER + ); + return new TestPlannerOptimizer( + config, + analyzer, + new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small(), TransportVersion.current())) + ); + } + /** * Exercises the block loader fallback in {@link ReplaceRoundToWithQueryAndTags}: * when query-and-tags is disabled (threshold=0) but block loader is supported, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java index cf4e808d9e0f7..3e2ff2060954f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java @@ -13,6 +13,11 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.mapper.IndexModeFieldMapper; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; @@ -51,33 +56,64 @@ import static java.util.stream.Collectors.toMap; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_PARSER; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.sameInstance; public class EsqlSessionTests extends ESTestCase { public void testShouldRetryConcreteTimeSeriesResolution() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); assertTrue( - EsqlSession.shouldRetryConcreteTimeSeriesResolution( - IndexMode.TIME_SERIES, - IndexResolution.empty("logs"), - new IndexPattern(EMPTY, "logs") - ) + EsqlSession.shouldRetryConcreteTimeSeriesResolution(mode, IndexResolution.empty("logs"), new IndexPattern(EMPTY, "logs")) ); } public void testShouldNotRetryWildcardTimeSeriesResolution() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); assertFalse( - EsqlSession.shouldRetryConcreteTimeSeriesResolution( - IndexMode.TIME_SERIES, - IndexResolution.empty("logs*"), - new IndexPattern(EMPTY, "logs*") - ) + EsqlSession.shouldRetryConcreteTimeSeriesResolution(mode, IndexResolution.empty("logs*"), new IndexPattern(EMPTY, "logs*")) ); } + // createQueryFilter builds the _index_mode filter used to resolve the concrete backing indices of a TS/METRICS + // source. The requested indexMode is a fixed sentinel (TS always declares IndexMode.TIME_SERIES) but the + // concrete backing indices may be configured with either [index.mode=time_series] or [index.mode=tsdb], + // so the filter must match both terms regardless of which indexMode triggered it. + public void testCreateQueryFilterMatchesBothIndexModeValues() { + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + QueryBuilder filter = EsqlSession.createQueryFilter(mode, null); + TermsQueryBuilder termsFilter = as(filter, TermsQueryBuilder.class); + assertThat(termsFilter.fieldName(), equalTo(IndexModeFieldMapper.NAME)); + assertThat(termsFilter.values(), containsInAnyOrder(IndexMode.TIME_SERIES.getName(), IndexMode.TSDB.getName())); + } + + public void testCreateQueryFilterCombinesRequestFilterWithIndexModeFilter() { + QueryBuilder requestFilter = QueryBuilders.matchAllQuery(); + QueryBuilder filter = EsqlSession.createQueryFilter(IndexMode.TSDB, requestFilter); + BoolQueryBuilder boolFilter = as(filter, BoolQueryBuilder.class); + assertThat(boolFilter.filter(), hasSize(2)); + assertThat(boolFilter.filter(), hasItem(requestFilter)); + TermsQueryBuilder termsFilter = boolFilter.filter() + .stream() + .filter(TermsQueryBuilder.class::isInstance) + .map(TermsQueryBuilder.class::cast) + .findFirst() + .orElseThrow(); + assertThat(termsFilter.fieldName(), equalTo(IndexModeFieldMapper.NAME)); + assertThat(termsFilter.values(), containsInAnyOrder(IndexMode.TIME_SERIES.getName(), IndexMode.TSDB.getName())); + } + + public void testCreateQueryFilterForStandardModeReturnsRequestFilterUnchanged() { + QueryBuilder requestFilter = QueryBuilders.matchAllQuery(); + assertThat(EsqlSession.createQueryFilter(IndexMode.STANDARD, requestFilter), sameInstance(requestFilter)); + } + public void testRefineConcreteTimeSeriesResolutionReturnsHelpfulError() { IndexResolution resolution = EsqlSession.refineConcreteTimeSeriesResolution( new IndexPattern(EMPTY, "logs"), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetricTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetricTests.java index f47097519e71e..37adab33869e9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetricTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetricTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.plan.IndexPattern; +import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedExternalRelation; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; @@ -18,6 +19,8 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.relation; + /** * {@link FeatureMetric#set} throws for any plan node it cannot map and that is not on the exclusion list. Since the * exclusion list matches the {@code UnresolvedSourceRelation} marker, both {@code FROM}-style leaf shapes must be @@ -53,4 +56,28 @@ public void testExternalRelationIsExcluded() { FeatureMetric.set(external, bitset); assertTrue("excluded plans must not set any telemetry bit", bitset.isEmpty()); } + + /** + * {@link FeatureMetric#TS} and {@link FeatureMetric#FROM} branch on {@code EsRelation#indexMode()#isTsdb()}. + * {@code IndexMode.TSDB} is a preferred alternative to {@code IndexMode.TIME_SERIES} (isTsdb() is true for + * both), so both constants must flip the TS bit - never the FROM bit - exactly like the canonical TIME_SERIES + * constant. + */ + public void testTimeSeriesRelationSetsTsMetricForBothIndexModes() { + for (IndexMode mode : List.of(IndexMode.TIME_SERIES, IndexMode.TSDB)) { + EsRelation relation = relation(mode); + BitSet bitset = new BitSet(); + FeatureMetric.set(relation, bitset); + assertTrue("indexMode [" + mode + "] must set the TS telemetry bit", bitset.get(FeatureMetric.TS.ordinal())); + assertFalse("indexMode [" + mode + "] must not set the FROM telemetry bit", bitset.get(FeatureMetric.FROM.ordinal())); + } + } + + public void testStandardRelationSetsFromMetricNotTs() { + EsRelation relation = relation(IndexMode.STANDARD); + BitSet bitset = new BitSet(); + FeatureMetric.set(relation, bitset); + assertTrue("STANDARD indexMode must set the FROM telemetry bit", bitset.get(FeatureMetric.FROM.ordinal())); + assertFalse("STANDARD indexMode must not set the TS telemetry bit", bitset.get(FeatureMetric.TS.ordinal())); + } } diff --git a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TimeSeriesToTsdbIndexModeRollingUpgradeIT.java b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TimeSeriesToTsdbIndexModeRollingUpgradeIT.java new file mode 100644 index 0000000000000..df3be41a515b1 --- /dev/null +++ b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TimeSeriesToTsdbIndexModeRollingUpgradeIT.java @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import org.elasticsearch.client.Request; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.test.rest.ObjectPath; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.xpack.logsdb.TsdbIT.getTemplate; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.notNullValue; + +/** + * Verifies that a data stream's backing indices can switch from {@link IndexMode#TIME_SERIES} to + * its preferred alias {@link IndexMode#TSDB} via a template update + rollover, once the whole + * cluster has been upgraded to a version that understands {@code tsdb} (old-version clusters in + * this suite never do, since {@code tsdb} did not exist as an {@code index.mode} value before). + * The pre-switch ({@code time_series}) and post-switch ({@code tsdb}) backing indices must both + * remain queryable, together, once the switch has happened. + */ +public class TimeSeriesToTsdbIndexModeRollingUpgradeIT extends AbstractLogsdbRollingUpgradeTestCase { + + private static final String DATA_STREAM_NAME = "k10s"; + private static final int DOCS_PER_ROUND = 4 * 128; + + // Kept short (and explicit, rather than relying on the plugin's defaults) so the test can + // deterministically place documents into the pre-switch vs. post-switch backing index by + // choosing their @timestamp, rather than depending on how much real wall-clock time the + // rolling upgrade itself takes. A backing index's [start_time, end_time) window is anchored + // to its real creation time, not to any document's @timestamp; on rollover the new index's + // start_time is exactly the old index's end_time (contiguous, no overlap/gap) - see + // DataStreamIndexSettingsProvider#provideAdditionalSettings. + private static final String LOOK_AHEAD_AND_BACK_TIME = "1m"; + + private int totalDocsIndexed = 0; + + public void testSwitchFromTimeSeriesToTsdb() throws Exception { + String templateId = getClass().getSimpleName().toLowerCase(Locale.ROOT); + String timeSeriesTemplate = withLookAheadAndBack(getTemplate(null, null)); + createTemplate(DATA_STREAM_NAME, templateId, timeSeriesTemplate); + + // Anchor for the whole test: the first backing index's window is [testStart - 1m, testStart + // + 1m). Documents timestamped near testStart land there regardless of real elapsed time. + Instant testStart = Instant.now(); + + // Old cluster: only "time_series" exists as an index.mode value. + indexRound(testStart); + verifyWriteIndexMode(IndexMode.TIME_SERIES); + assertSearchAndQuery(); + + // Mixed cluster: keep indexing/querying against the still-time_series write index. + clusterRollingUpgrade(index -> { + indexRound(testStart.plusSeconds(index + 1)); + verifyWriteIndexMode(IndexMode.TIME_SERIES); + assertSearchAndQuery(); + }); + + String firstBackingIndex = getDataStreamBackingIndexNames(DATA_STREAM_NAME).getFirst(); + + // Fully upgraded: switch the template to "tsdb" and roll over onto a new backing index. + String tsdbTemplate = withLookAheadAndBack(getTemplate(null, null)).replace("\"time_series\"", "\"tsdb\""); + createTemplate(DATA_STREAM_NAME, templateId, tsdbTemplate); + rolloverDataStream(DATA_STREAM_NAME); + + List backingIndices = getDataStreamBackingIndexNames(DATA_STREAM_NAME); + assertThat(backingIndices, hasSize(2)); + assertThat(backingIndices.get(0), equalTo(firstBackingIndex)); + String secondBackingIndex = backingIndices.get(1); + + verifyIndexMode(IndexMode.TIME_SERIES, firstBackingIndex); + verifyIndexMode(IndexMode.TSDB, secondBackingIndex); + + // The second backing index's window starts exactly where the first one's ends + // (testStart + 1m); land comfortably inside it, regardless of how long the upgrade took. + indexRound(testStart.plusSeconds(90)); + assertSearchAndQuery(); + + var forceMergeRequest = new Request("POST", "/" + DATA_STREAM_NAME + "/_forcemerge"); + forceMergeRequest.addParameter("max_num_segments", "1"); + assertOK(client().performRequest(forceMergeRequest)); + ensureGreen(DATA_STREAM_NAME); + assertSearchAndQuery(); + } + + private static String withLookAheadAndBack(String template) { + return template.replace( + "\"mode\": \"time_series\"", + "\"mode\": \"time_series\", \"look_ahead_time\": \"" + + LOOK_AHEAD_AND_BACK_TIME + + "\", \"look_back_time\": \"" + + LOOK_AHEAD_AND_BACK_TIME + + "\"" + ); + } + + private void indexRound(Instant timestamp) throws Exception { + bulkIndex(DATA_STREAM_NAME, 4, 128, timestamp, TsdbIndexingRollingUpgradeIT::docSupplier); + totalDocsIndexed += DOCS_PER_ROUND; + } + + private void rolloverDataStream(String dataStreamName) throws IOException { + var request = new Request("POST", "/" + dataStreamName + "/_rollover"); + assertOK(client().performRequest(request)); + } + + private void verifyWriteIndexMode(IndexMode indexMode) throws IOException { + verifyIndexMode(indexMode, getDataStreamBackingIndexNames(DATA_STREAM_NAME).getLast()); + } + + private void verifyIndexMode(IndexMode indexMode, String index) throws IOException { + var settings = (Map) getIndexSettings(index, true).get(index); + assertThat(((Map) settings.get("settings")).get(IndexSettings.MODE.getKey()), equalTo(indexMode.getName())); + } + + private void assertSearchAndQuery() throws Exception { + search(); + query(); + } + + private void search() throws Exception { + var searchRequest = new Request("POST", "/" + DATA_STREAM_NAME + "/_search"); + searchRequest.setJsonEntity("{\"size\": 0}"); + var response = client().performRequest(searchRequest); + assertOK(response); + var responseBody = entityAsMap(response); + Integer totalCount = ObjectPath.evaluate(responseBody, "hits.total.value"); + assertThat(totalCount, equalTo(totalDocsIndexed)); + } + + /** + * Groups by {@code _index} to prove the data stream query spans every backing index - + * including both the {@code time_series} and the {@code tsdb} one once the switch has + * happened - and that their per-index counts sum to every document ever indexed. + */ + private void query() throws Exception { + var queryRequest = new Request("POST", "/_query"); + queryRequest.setJsonEntity(""" + { + "query": "FROM $ds METADATA _index | STATS count = COUNT(*) BY _index | SORT _index | LIMIT 10" + } + """.replace("$ds", DATA_STREAM_NAME)); + var response = client().performRequest(queryRequest); + assertOK(response); + var responseBody = entityAsMap(response); + + List backingIndices = getDataStreamBackingIndexNames(DATA_STREAM_NAME); + List> values = ObjectPath.evaluate(responseBody, "values"); + assertThat("values=" + values, values, hasSize(backingIndices.size())); + + long countedTotal = 0; + List queriedIndices = new ArrayList<>(); + for (List row : values) { + Number count = (Number) row.get(0); + assertThat(count, notNullValue()); + countedTotal += count.longValue(); + queriedIndices.add((String) row.get(1)); + } + assertThat(countedTotal, equalTo((long) totalDocsIndexed)); + assertThat(queriedIndices, containsInAnyOrder(backingIndices.toArray())); + } +} diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 57c2181f51a55..abac476af10f9 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -265,6 +265,8 @@ public void testOnExplicitStandardIndex() throws IOException { } public void testOnExplicitTimeSeriesIndex() throws IOException { + // Randomly exercise both IndexMode.TIME_SERIES and IndexMode.TSDB to make sure they behave identically. + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( logsdbLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() @@ -277,7 +279,7 @@ public void testOnExplicitTimeSeriesIndex() throws IOException { null, emptyProject(), Instant.now().truncatedTo(ChronoUnit.SECONDS), - Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()).build(), + Settings.builder().put(IndexSettings.MODE.getKey(), mode.getName()).build(), List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder @@ -644,6 +646,8 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException } public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { + // Randomly exercise both IndexMode.TIME_SERIES and IndexMode.TSDB to make sure they behave identically. + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); String dataStreamName = DATA_STREAM_NAME; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); String mapping = """ @@ -658,13 +662,13 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { """; LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); { - Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build(); + Settings settings = Settings.builder().put("index.mode", mode.getName()).put("index.routing_path", "my_field").build(); boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertTrue(result); } { - Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build(); + Settings settings = Settings.builder().put("index.mode", mode.getName()).put("index.routing_path", "my_field").build(); boolean result = provider.getMappingHints(indexName, null, settings, List.of()).hasSyntheticSourceUsage(); assertTrue(result); } @@ -720,6 +724,8 @@ public void testNewIndexHasSyntheticSourceUsageInvalidSettings() throws IOExcept } public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { + // Randomly exercise both IndexMode.TIME_SERIES and IndexMode.TSDB to make sure they behave identically. + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); String dataStreamName = DATA_STREAM_NAME; ProjectMetadata project = DataStreamTestHelper.getProjectWithDataStreams( List.of(Tuple.tuple(dataStreamName, 1)), @@ -770,7 +776,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, - IndexMode.TIME_SERIES, + mode, project, Instant.ofEpochMilli(1L), settings, diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java index 19b30822e4df6..04d37e985efcb 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java @@ -129,6 +129,9 @@ public void testGetAdditionalIndexSettingsProfiling() throws IOException { } public void testGetAdditionalIndexSettingsTsdb() throws IOException { + // IndexMode.TSDB is a preferred alternative to IndexMode.TIME_SERIES and must be treated identically by the + // legacy license check in LogsdbIndexModeSettingsProvider#isLegacyLicensedUsageOfSyntheticSourceAllowed. + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); @@ -136,7 +139,7 @@ public void testGetAdditionalIndexSettingsTsdb() throws IOException { provider.provideAdditionalSettings( indexName, dataStreamName, - IndexMode.TIME_SERIES, + mode, null, null, settings, @@ -172,6 +175,9 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception true ); + // IndexMode.TSDB is a preferred alternative to IndexMode.TIME_SERIES and must be treated identically by the + // legacy license check. + IndexMode mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); @@ -179,7 +185,7 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception provider.provideAdditionalSettings( indexName, dataStreamName, - IndexMode.TIME_SERIES, + mode, null, null, settings, diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index aa344dfcd13b9..c14f8e6962ec4 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -339,9 +339,14 @@ public void testMetricAndDimension() { ); } + /** + * The {@code tsdb} index mode is equivalent to {@code time_series} (see {@link IndexMode#isTsdb()}) + * so it must produce the same doc-values-only defaulting for metric fields. + */ public void testTimeSeriesIndexDefault() throws Exception { var randomMetricType = randomFrom(TimeSeriesParams.MetricType.scalar()); - var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) + var mode = randomFrom(IndexMode.TIME_SERIES, IndexMode.TSDB); + var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), mode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field"); var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> { minimalMapping(b); diff --git a/x-pack/plugin/stateless/src/test/java/org/elasticsearch/xpack/stateless/commits/IndexCommitTimestampFieldRangeTests.java b/x-pack/plugin/stateless/src/test/java/org/elasticsearch/xpack/stateless/commits/IndexCommitTimestampFieldRangeTests.java index aba85772e546c..4030e1161067c 100644 --- a/x-pack/plugin/stateless/src/test/java/org/elasticsearch/xpack/stateless/commits/IndexCommitTimestampFieldRangeTests.java +++ b/x-pack/plugin/stateless/src/test/java/org/elasticsearch/xpack/stateless/commits/IndexCommitTimestampFieldRangeTests.java @@ -116,10 +116,10 @@ public void testSoftDeletesAreAlmostAlwaysDisregardedForTimestampRange() throws IndexCommit previousCommit = null; try (Directory directory = newDirectory()) { try (IndexWriter indexWriter = new IndexWriter(directory, indexWriterConfig)) { - String docId1 = indexMode == IndexMode.TIME_SERIES ? TimeSeriesRoutingHashFieldMapper.encode(1) : "docId1"; + String docId1 = indexMode.isTsdb() ? TimeSeriesRoutingHashFieldMapper.encode(1) : "docId1"; long timestamp1 = randomLongBetween(10_000L, 1_000_000L); indexWriter.addDocument(mapper.parse(source(docId1, b -> { b.field("@timestamp", timestamp1); }, null)).rootDoc()); - String docId2 = indexMode == IndexMode.TIME_SERIES ? TimeSeriesRoutingHashFieldMapper.encode(2) : "docId2"; + String docId2 = indexMode.isTsdb() ? TimeSeriesRoutingHashFieldMapper.encode(2) : "docId2"; long timestamp2 = randomLongBetween(10_000L, 1_000_000L); indexWriter.addDocument(mapper.parse(source(docId2, b -> { b.field("@timestamp", timestamp2); }, null)).rootDoc()); boolean doc1Deleted = randomBoolean(); @@ -139,12 +139,12 @@ public void testSoftDeletesAreAlmostAlwaysDisregardedForTimestampRange() throws ); previousCommit = currentCommit; } - String docId3 = indexMode == IndexMode.TIME_SERIES ? TimeSeriesRoutingHashFieldMapper.encode(3) : "docId3"; + String docId3 = indexMode.isTsdb() ? TimeSeriesRoutingHashFieldMapper.encode(3) : "docId3"; long timestamp3 = randomLongBetween(10_000L, 1_000_000L); indexWriter.addDocument(mapper.parse(source(docId3, b -> { b.field("@timestamp", timestamp3); }, null)).rootDoc()); // create a 1-doc segment indexWriter.flush(); - String docId4 = indexMode == IndexMode.TIME_SERIES ? TimeSeriesRoutingHashFieldMapper.encode(4) : "docId4"; + String docId4 = indexMode.isTsdb() ? TimeSeriesRoutingHashFieldMapper.encode(4) : "docId4"; long timestamp4 = randomLongBetween(10_000L, 1_000_000L); indexWriter.addDocument(mapper.parse(source(docId4, b -> { b.field("@timestamp", timestamp4); }, null)).rootDoc()); deleteDoc(doc1Deleted ? docId2 : docId1, indexWriter, indexMode, mapper); @@ -193,7 +193,7 @@ public void testFieldValueRangeForBatchedCompoundCommit() throws Exception { return List.of( mapper.parse( source( - indexMode == IndexMode.TIME_SERIES + indexMode.isTsdb() ? TimeSeriesRoutingHashFieldMapper.encode(docId.incrementAndGet()) : String.valueOf(docId.incrementAndGet()), b -> { @@ -293,7 +293,7 @@ private void testFieldValueRange(boolean useCFS, IndexMode indexMode) throws Exc for (int i = 0; i < docCount; i++) { // make sure timestamp doesn't look like a year long timestamp = randomLongBetween(10_000L, 1_000_000L); - String docId = indexMode == IndexMode.TIME_SERIES ? TimeSeriesRoutingHashFieldMapper.encode(i) : randomUUID(); + String docId = indexMode.isTsdb() ? TimeSeriesRoutingHashFieldMapper.encode(i) : randomUUID(); indexWriter.addDocument(mapper.parse(source(docId, b -> { b.field("@timestamp", timestamp); }, null)).rootDoc()); flushEveryNDocCount--; commitEveryNDocCount--; @@ -396,12 +396,12 @@ private DocumentMapper getDocumentMapper(IndexMode indexMode) throws IOException b.field("doc_values", docValues).field("store", allowStore).endObject(); }), indexMode); } - } else if (indexMode == IndexMode.TIME_SERIES) { - // TIME_SERIES indexing modes use a fixed mapping for the @timestamp field + } else if (indexMode.isTsdb()) { + // TIME_SERIES/TSDB indexing modes use a fixed mapping for the @timestamp field // Synthetic id need cumbersome special care in test setup so we turn that // off in this test Settings settings = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexSettings.MODE.getKey(), indexMode.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "uid") .put(IndexSettings.SYNTHETIC_ID.getKey(), false) .build();