Merge branch '9.2' into backport/9.2/pr-138097

Author: davidkyle

docs/changelog/137652.yaml

@@ -0,0 +1,6 @@
+pr: 137652
+summary: Fix default value for some settings when filtered
+area: Infra/Settings
+type: bug
+issues:
+ - 136333

docs/changelog/138122.yaml

@@ -0,0 +1,6 @@
+pr: 138122
+summary: Add validation for updating `num_threads`
+area: Machine Learning
+type: bug
+issues:
+ - 137129

docs/changelog/138132.yaml

@@ -0,0 +1,5 @@
+pr: 138132
+summary: Fix integer overflow in block memory estimation
+area: ES|QL
+type: bug
+issues: []

muted-tests.yml

@@ -555,9 +555,6 @@ tests:
 - class: org.elasticsearch.xpack.inference.InferenceRestIT
   method: test {p0=inference/70_text_similarity_rank_retriever/Text similarity reranker with min_score zero includes all docs}
   issue: https://github.com/elastic/elasticsearch/issues/137732
-- class: org.elasticsearch.xpack.shutdown.NodeShutdownIT
-  method: testStalledShardMigrationProperlyDetected
-  issue: https://github.com/elastic/elasticsearch/issues/115697
 
 # Examples:
 #

server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsIT.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.action.admin.indices.settings.get;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.LambdaMatchers;
+
+import java.util.Map;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE)
+public class GetSettingsIT extends ESIntegTestCase {
+    public void testGetDefaultValueWhenDependentOnOtherSettings() {
+        final String indexName = "test-index-1";
+
+        Settings expectedSettings = Settings.builder()
+            .put("index.mapping.source.mode", "SYNTHETIC")
+            .put("index.mapping.total_fields.ignore_dynamic_beyond_limit", true)
+            .put("index.recovery.use_synthetic_source", true)
+            .put("index.mapping.ignore_above", 8191)
+            .build();
+
+        assertAcked(prepareCreate(indexName).setSettings(Settings.builder().put("index.mode", "logsdb")).get());
+        GetSettingsResponse unfilteredResponse = indicesAdmin().getSettings(
+            new GetSettingsRequest(TEST_REQUEST_TIMEOUT).indices(indexName).includeDefaults(true)
+        ).actionGet();
+        for (String key : expectedSettings.keySet()) {
+            assertThat(unfilteredResponse.getSetting(indexName, key), equalTo(expectedSettings.get(key)));
+            GetSettingsResponse filteredResponse = indicesAdmin().getSettings(
+                new GetSettingsRequest(TEST_REQUEST_TIMEOUT).indices(indexName).includeDefaults(true).names(key)
+            ).actionGet();
+
+            var expectedFilteredSettingsMap = Map.of(indexName, expectedSettings.filter(key::equals));
+            assertThat(
+                filteredResponse,
+                LambdaMatchers.transformedMatch(GetSettingsResponse::getIndexToDefaultSettings, equalTo(expectedFilteredSettingsMap))
+            );
+        }
+
+    }
+}

server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata;
 import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata;
@@ -39,6 +40,7 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.cluster.metadata.ReservedStateMetadata.EMPTY_VERSION;
@@ -194,22 +196,29 @@ private Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node,
         ClusterService clusterService = internalCluster().clusterService(node);
         CountDownLatch savedClusterState = new CountDownLatch(1);
         AtomicLong metadataVersion = new AtomicLong(-1);
+        Function<ClusterState, Boolean> clusterStateProcessor = clusterState -> {
+            ReservedStateMetadata reservedState = clusterState.metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
+            if (reservedState != null && reservedState.version() == fileSettingsVersion) {
+                metadataVersion.set(clusterState.metadata().version());
+                savedClusterState.countDown();
+                logger.info(
+                    "done waiting for file settings [version: {}, metadata version: {}]",
+                    clusterState.version(),
+                    clusterState.metadata().version()
+                );
+                return true;
+            }
+            return false;
+        };
         clusterService.addListener(new ClusterStateListener() {
             @Override
             public void clusterChanged(ClusterChangedEvent event) {
-                ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
-                if (reservedState != null && reservedState.version() == fileSettingsVersion) {
+                if (clusterStateProcessor.apply(event.state())) {
                     clusterService.removeListener(this);
-                    metadataVersion.set(event.state().metadata().version());
-                    savedClusterState.countDown();
-                    logger.info(
-                        "done waiting for file settings [version: {}, metadata version: {}]",
-                        event.state().version(),
-                        event.state().metadata().version()
-                    );
                 }
             }
         });
+        clusterStateProcessor.apply(clusterService.state());
 
         return new Tuple<>(savedClusterState, metadataVersion);
     }

server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java

@@ -127,7 +127,7 @@ protected void localClusterStateOperation(
 
             indexToSettings.put(concreteIndex.getName(), indexSettings);
             if (indexToDefaultSettings != null) {
-                Settings defaultSettings = settingsFilter.filter(indexScopedSettings.diff(indexSettings, Settings.EMPTY));
+                Settings defaultSettings = settingsFilter.filter(indexScopedSettings.diff(indexMetadata.getSettings(), Settings.EMPTY));
                 if (isFilteredRequest(request)) {
                     defaultSettings = defaultSettings.filter(k -> Regex.simpleMatch(request.names(), k));
                 }

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

@@ -153,6 +153,7 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
     public static final String EVENT_INGESTED_FIELD_NAME = "event.ingested";
 
     private static final TransportVersion INDEX_RESHARDING_METADATA = TransportVersion.fromName("index_resharding_metadata");
+    private static final TransportVersion INDEX_CREATED_TRANSPORT_VERSION = TransportVersion.fromName("index_created_transport_version");
 
     @Nullable
     public String getDownsamplingInterval() {
@@ -583,6 +584,7 @@ public Iterator<Setting<?>> settings() {
     public static final List<String> PARTIALLY_MOUNTED_INDEX_TIER_PREFERENCE = List.of(DataTier.DATA_FROZEN);
 
     static final String KEY_VERSION = "version";
+    static final String KEY_TRANSPORT_VERSION = "transport_version";
     static final String KEY_MAPPING_VERSION = "mapping_version";
     static final String KEY_SETTINGS_VERSION = "settings_version";
     static final String KEY_ALIASES_VERSION = "aliases_version";
@@ -623,6 +625,7 @@ public Iterator<Setting<?>> settings() {
 
     private final Index index;
     private final long version;
+    private final TransportVersion transportVersion;
 
     private final long mappingVersion;
 
@@ -708,6 +711,7 @@ public Iterator<Setting<?>> settings() {
     private IndexMetadata(
         final Index index,
         final long version,
+        final TransportVersion transportVersion,
         final long mappingVersion,
         final long settingsVersion,
         final long aliasesVersion,
@@ -758,6 +762,7 @@ private IndexMetadata(
     ) {
         this.index = index;
         this.version = version;
+        this.transportVersion = transportVersion;
         assert mappingVersion >= 0 : mappingVersion;
         this.mappingVersion = mappingVersion;
         this.mappingsUpdatedVersion = mappingsUpdatedVersion;
@@ -824,6 +829,7 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) {
         return new IndexMetadata(
             this.index,
             this.version,
+            this.transportVersion,
             this.mappingVersion,
             this.settingsVersion,
             this.aliasesVersion,
@@ -887,6 +893,7 @@ public IndexMetadata withInSyncAllocationIds(int shardId, Set<String> inSyncSet)
         return new IndexMetadata(
             this.index,
             this.version,
+            this.transportVersion,
             this.mappingVersion,
             this.settingsVersion,
             this.aliasesVersion,
@@ -958,6 +965,7 @@ public IndexMetadata withSetPrimaryTerm(int shardId, long primaryTerm) {
         return new IndexMetadata(
             this.index,
             this.version,
+            this.transportVersion,
             this.mappingVersion,
             this.settingsVersion,
             this.aliasesVersion,
@@ -1020,6 +1028,7 @@ public IndexMetadata withTimestampRanges(IndexLongFieldRange timestampRange, Ind
         return new IndexMetadata(
             this.index,
             this.version,
+            this.transportVersion,
             this.mappingVersion,
             this.settingsVersion,
             this.aliasesVersion,
@@ -1077,6 +1086,7 @@ public IndexMetadata withIncrementedVersion() {
         return new IndexMetadata(
             this.index,
             this.version + 1,
+            this.transportVersion,
             this.mappingVersion,
             this.settingsVersion,
             this.aliasesVersion,
@@ -1139,6 +1149,10 @@ public long getVersion() {
         return this.version;
     }
 
+    public TransportVersion getTransportVersion() {
+        return transportVersion;
+    }
+
     public long getMappingVersion() {
         return mappingVersion;
     }
@@ -1671,6 +1685,7 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
         private final String index;
         private final int routingNumShards;
         private final long version;
+        private final TransportVersion transportVersion;
         private final long mappingVersion;
         private final long settingsVersion;
         private final long aliasesVersion;
@@ -1704,6 +1719,7 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
         IndexMetadataDiff(IndexMetadata before, IndexMetadata after) {
             index = after.index.getName();
             version = after.version;
+            transportVersion = after.transportVersion;
             mappingVersion = after.mappingVersion;
             settingsVersion = after.settingsVersion;
             aliasesVersion = after.aliasesVersion;
@@ -1757,6 +1773,9 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
             index = in.readString();
             routingNumShards = in.readInt();
             version = in.readLong();
+            transportVersion = in.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)
+                ? TransportVersion.readVersion(in)
+                : TransportVersion.fromId(0);
             mappingVersion = in.readVLong();
             settingsVersion = in.readVLong();
             aliasesVersion = in.readVLong();
@@ -1824,6 +1843,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(index);
             out.writeInt(routingNumShards);
             out.writeLong(version);
+            if (out.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)) {
+                TransportVersion.writeVersion(transportVersion, out);
+            }
             out.writeVLong(mappingVersion);
             out.writeVLong(settingsVersion);
             out.writeVLong(aliasesVersion);
@@ -1864,6 +1886,7 @@ public void writeTo(StreamOutput out) throws IOException {
         public IndexMetadata apply(IndexMetadata part) {
             Builder builder = builder(index);
             builder.version(version);
+            builder.transportVersion(transportVersion);
             builder.mappingVersion(mappingVersion);
             builder.settingsVersion(settingsVersion);
             builder.aliasesVersion(aliasesVersion);
@@ -1907,6 +1930,11 @@ public static IndexMetadata readFrom(StreamInput in) throws IOException {
     public static IndexMetadata readFrom(StreamInput in, @Nullable Function<String, MappingMetadata> mappingLookup) throws IOException {
         Builder builder = new Builder(in.readString());
         builder.version(in.readLong());
+        builder.transportVersion(
+            in.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)
+                ? TransportVersion.readVersion(in)
+                : TransportVersion.fromId(0)
+        );
         builder.mappingVersion(in.readVLong());
         builder.settingsVersion(in.readVLong());
         builder.aliasesVersion(in.readVLong());
@@ -1974,6 +2002,9 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function<String,
     public void writeTo(StreamOutput out, boolean mappingsAsHash) throws IOException {
         out.writeString(index.getName()); // uuid will come as part of settings
         out.writeLong(version);
+        if (out.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)) {
+            TransportVersion.writeVersion(transportVersion, out);
+        }
         out.writeVLong(mappingVersion);
         out.writeVLong(settingsVersion);
         out.writeVLong(aliasesVersion);
@@ -2049,6 +2080,7 @@ public static class Builder {
         private String index;
         private State state = State.OPEN;
         private long version = 1;
+        private TransportVersion transportVersion = TransportVersion.fromId(0);
         private long mappingVersion = 1;
         private long settingsVersion = 1;
         private long aliasesVersion = 1;
@@ -2085,6 +2117,7 @@ public Builder(IndexMetadata indexMetadata) {
             this.index = indexMetadata.getIndex().getName();
             this.state = indexMetadata.state;
             this.version = indexMetadata.version;
+            this.transportVersion = indexMetadata.transportVersion;
             this.mappingVersion = indexMetadata.mappingVersion;
             this.settingsVersion = indexMetadata.settingsVersion;
             this.aliasesVersion = indexMetadata.aliasesVersion;
@@ -2293,6 +2326,15 @@ public Builder version(long version) {
             return this;
         }
 
+        public TransportVersion transportVersion() {
+            return this.transportVersion;
+        }
+
+        public Builder transportVersion(TransportVersion transportVersion) {
+            this.transportVersion = transportVersion;
+            return this;
+        }
+
         public long mappingVersion() {
             return mappingVersion;
         }
@@ -2558,6 +2600,7 @@ IndexMetadata build(boolean repair) {
             return new IndexMetadata(
                 new Index(index, uuid),
                 version,
+                transportVersion,
                 mappingVersion,
                 settingsVersion,
                 aliasesVersion,
@@ -2617,6 +2660,7 @@ public static void toXContent(IndexMetadata indexMetadata, XContentBuilder build
             builder.startObject(indexMetadata.getIndex().getName());
 
             builder.field(KEY_VERSION, indexMetadata.getVersion());
+            builder.field(KEY_TRANSPORT_VERSION, indexMetadata.getTransportVersion().toString());
             builder.field(KEY_MAPPING_VERSION, indexMetadata.getMappingVersion());
             builder.field(KEY_SETTINGS_VERSION, indexMetadata.getSettingsVersion());
             builder.field(KEY_ALIASES_VERSION, indexMetadata.getAliasesVersion());
@@ -2874,6 +2918,7 @@ public static IndexMetadata fromXContent(XContentParser parser, Map<String, Mapp
                     switch (currentFieldName) {
                         case KEY_STATE -> builder.state(State.fromString(parser.text()));
                         case KEY_VERSION -> builder.version(parser.longValue());
+                        case KEY_TRANSPORT_VERSION -> builder.transportVersion(TransportVersion.fromString(parser.text()));
                         case KEY_MAPPING_VERSION -> {
                             mappingVersion = true;
                             builder.mappingVersion(parser.longValue());

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java

@@ -627,6 +627,7 @@ private IndexMetadata buildAndValidateTemporaryIndexMetadata(
         final IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(request.index());
         tmpImdBuilder.setRoutingNumShards(routingNumShards);
         tmpImdBuilder.settings(indexSettings);
+        tmpImdBuilder.transportVersion(TransportVersion.current());
         tmpImdBuilder.system(isSystem);
 
         // Set up everything, now locally create the index to see that things are ok, and apply

server/src/main/resources/transport/definitions/referable/batched_response_might_include_reduction_failure.csv

@@ -1 +1 @@
-9213000,9185007,9112012
+9213000,9185007,9112012,8841074