Making Shard Limit Validator Index Tier Agnostic

CHANGELOG.md

@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 3.x]
 ### Added
+- Add seperate shard limit validation for local and remote indices ([#19532](https://github.com/opensearch-project/OpenSearch/pull/19532))
 - Use Lucene `pack` method for `half_float` and `usigned_long` when using `ApproximatePointRangeQuery`.
 
 ### Changed

server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java

@@ -1046,6 +1046,8 @@ public Iterator<Setting<?>> settings() {
 
     private final int indexTotalShardsPerNodeLimit;
     private final int indexTotalPrimaryShardsPerNodeLimit;
+    private final int indexTotalRemoteCapableShardsPerNodeLimit;
+    private final int indexTotalRemoteCapablePrimaryShardsPerNodeLimit;
     private final boolean isAppendOnlyIndex;
 
     private final Context context;
@@ -1080,6 +1082,8 @@ private IndexMetadata(
         final boolean isSystem,
         final int indexTotalShardsPerNodeLimit,
         final int indexTotalPrimaryShardsPerNodeLimit,
+        final int indexTotalRemoteCapableShardsPerNodeLimit,
+        final int indexTotalRemoteCapablePrimaryShardsPerNodeLimit,
         boolean isAppendOnlyIndex,
         final Context context,
         final IngestionStatus ingestionStatus
@@ -1120,6 +1124,8 @@ private IndexMetadata(
         this.isRemoteSnapshot = IndexModule.Type.REMOTE_SNAPSHOT.match(this.settings);
         this.indexTotalShardsPerNodeLimit = indexTotalShardsPerNodeLimit;
         this.indexTotalPrimaryShardsPerNodeLimit = indexTotalPrimaryShardsPerNodeLimit;
+        this.indexTotalRemoteCapableShardsPerNodeLimit = indexTotalRemoteCapableShardsPerNodeLimit;
+        this.indexTotalRemoteCapablePrimaryShardsPerNodeLimit = indexTotalRemoteCapablePrimaryShardsPerNodeLimit;
         this.isAppendOnlyIndex = isAppendOnlyIndex;
         this.context = context;
         this.ingestionStatus = ingestionStatus;
@@ -1334,10 +1340,18 @@ public int getIndexTotalShardsPerNodeLimit() {
         return this.indexTotalShardsPerNodeLimit;
     }
 
+    public int getIndexTotalRemoteCapableShardsPerNodeLimit() {
+        return this.indexTotalRemoteCapableShardsPerNodeLimit;
+    }
+
     public int getIndexTotalPrimaryShardsPerNodeLimit() {
         return this.indexTotalPrimaryShardsPerNodeLimit;
     }
 
+    public int getIndexTotalRemoteCapablePrimaryShardsPerNodeLimit() {
+        return this.indexTotalRemoteCapablePrimaryShardsPerNodeLimit;
+    }
+
     public boolean isAppendOnlyIndex() {
         return this.isAppendOnlyIndex;
     }
@@ -2175,6 +2189,10 @@ public IndexMetadata build() {
             final int indexTotalPrimaryShardsPerNodeLimit = ShardsLimitAllocationDecider.INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING.get(
                 settings
             );
+            final int indexTotalRemoteCapableShardsPerNodeLimit =
+                ShardsLimitAllocationDecider.INDEX_TOTAL_REMOTE_CAPABLE_SHARDS_PER_NODE_SETTING.get(settings);
+            final int indexTotalRemoteCapablePrimaryShardsPerNodeLimit =
+                ShardsLimitAllocationDecider.INDEX_TOTAL_REMOTE_CAPABLE_PRIMARY_SHARDS_PER_NODE_SETTING.get(settings);
             final boolean isAppendOnlyIndex = INDEX_APPEND_ONLY_ENABLED_SETTING.get(settings);
 
             final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE);
@@ -2212,6 +2230,8 @@ public IndexMetadata build() {
                 isSystem,
                 indexTotalShardsPerNodeLimit,
                 indexTotalPrimaryShardsPerNodeLimit,
+                indexTotalRemoteCapableShardsPerNodeLimit,
+                indexTotalRemoteCapablePrimaryShardsPerNodeLimit,
                 isAppendOnlyIndex,
                 context,
                 ingestionStatus

server/src/main/java/org/opensearch/cluster/metadata/Metadata.java

@@ -48,6 +48,7 @@
 import org.opensearch.cluster.block.ClusterBlockLevel;
 import org.opensearch.cluster.coordination.CoordinationMetadata;
 import org.opensearch.cluster.decommission.DecommissionAttributeMetadata;
+import org.opensearch.cluster.routing.RoutingPool;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.UUIDs;
 import org.opensearch.common.annotation.PublicApi;
@@ -270,7 +271,8 @@ static Custom fromXContent(XContentParser parser, String name) throws IOExceptio
     private final Map<String, Custom> customs;
 
     private final transient int totalNumberOfShards; // Transient ? not serializable anyway?
-    private final int totalOpenIndexShards;
+    private final int totalOpenLocalOnlyIndexShards;
+    private final int totalOpenRemoteCapableIndexShards;
 
     private final String[] allIndices;
     private final String[] visibleIndices;
@@ -315,15 +317,21 @@ static Custom fromXContent(XContentParser parser, String name) throws IOExceptio
         this.customs = Collections.unmodifiableMap(customs);
         this.templates = new TemplatesMetadata(templates);
         int totalNumberOfShards = 0;
-        int totalOpenIndexShards = 0;
+        int totalOpenLocalOnlyIndexShards = 0;
+        int totalOpenRemoteCapableIndexShards = 0;
         for (IndexMetadata cursor : indices.values()) {
             totalNumberOfShards += cursor.getTotalNumberOfShards();
             if (IndexMetadata.State.OPEN.equals(cursor.getState())) {
-                totalOpenIndexShards += cursor.getTotalNumberOfShards();
+                if (RoutingPool.getIndexPool(cursor) == RoutingPool.REMOTE_CAPABLE) {
+                    totalOpenRemoteCapableIndexShards += cursor.getTotalNumberOfShards();
+                } else {
+                    totalOpenLocalOnlyIndexShards += cursor.getTotalNumberOfShards();
+                }
             }
         }
         this.totalNumberOfShards = totalNumberOfShards;
-        this.totalOpenIndexShards = totalOpenIndexShards;
+        this.totalOpenLocalOnlyIndexShards = totalOpenLocalOnlyIndexShards;
+        this.totalOpenRemoteCapableIndexShards = totalOpenRemoteCapableIndexShards;
 
         this.allIndices = allIndices;
         this.visibleIndices = visibleIndices;
@@ -905,7 +913,16 @@ public int getTotalNumberOfShards() {
      * @return The total number of open shards from all indices.
      */
     public int getTotalOpenIndexShards() {
-        return this.totalOpenIndexShards;
+        return this.totalOpenLocalOnlyIndexShards;
+    }
+
+    /**
+     * Gets the total number of open remote capable shards from all indices. Includes
+     * replicas, but does not include shards that are part of closed indices.
+     * @return The total number of open shards from all indices.
+     */
+    public int getTotalOpenRemoteCapableIndexShards() {
+        return this.totalOpenRemoteCapableIndexShards;
     }
 
     /**

server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

@@ -155,6 +155,7 @@
 import static org.opensearch.cluster.metadata.Metadata.DEFAULT_REPLICA_COUNT_SETTING;
 import static org.opensearch.cluster.metadata.MetadataIndexTemplateService.findContextTemplateName;
 import static org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider.INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING;
+import static org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider.INDEX_TOTAL_REMOTE_CAPABLE_PRIMARY_SHARDS_PER_NODE_SETTING;
 import static org.opensearch.cluster.service.ClusterManagerTask.CREATE_INDEX;
 import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 import static org.opensearch.index.IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING;
@@ -1867,17 +1868,22 @@ public static void validateRefreshIntervalSettings(Settings requestSettings, Clu
     public static void validateIndexTotalPrimaryShardsPerNodeSetting(Settings indexSettings) {
         // Get the setting value
         int indexPrimaryShardsPerNode = INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING.get(indexSettings);
+        int indexRemoteCapablePrimaryShardsPerNode = INDEX_TOTAL_REMOTE_CAPABLE_PRIMARY_SHARDS_PER_NODE_SETTING.get(indexSettings);
 
         // If default value (-1), no validation needed
-        if (indexPrimaryShardsPerNode == -1) {
+        if (indexPrimaryShardsPerNode == -1 && indexRemoteCapablePrimaryShardsPerNode == -1) {
             return;
         }
 
         // Check if remote store is enabled
         boolean isRemoteStoreEnabled = IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(indexSettings);
         if (!isRemoteStoreEnabled) {
             throw new IllegalArgumentException(
-                "Setting [" + INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING.getKey() + "] can only be used with remote store enabled clusters"
+                "Setting ["
+                    + INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING.getKey()
+                    + "] or ["
+                    + INDEX_TOTAL_REMOTE_CAPABLE_PRIMARY_SHARDS_PER_NODE_SETTING.getKey()
+                    + "] can only be used with remote store enabled clusters"
             );
         }
     }

server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java

@@ -83,6 +83,7 @@
 import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex;
 import static org.opensearch.cluster.metadata.MetadataIndexTemplateService.findComponentTemplate;
 import static org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider.INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING;
+import static org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider.INDEX_TOTAL_REMOTE_CAPABLE_PRIMARY_SHARDS_PER_NODE_SETTING;
 import static org.opensearch.cluster.service.ClusterManagerTask.UPDATE_SETTINGS;
 import static org.opensearch.common.settings.AbstractScopedSettings.ARCHIVED_SETTINGS_PREFIX;
 import static org.opensearch.index.IndexSettings.same;
@@ -272,15 +273,11 @@ public ClusterState execute(ClusterState currentState) {
                             }
 
                             // Verify that this won't take us over the cluster shard limit.
-                            int totalNewShards = Arrays.stream(request.indices())
-                                .mapToInt(i -> getTotalNewShards(i, currentState, updatedNumberOfReplicas))
-                                .sum();
-                            Optional<String> error = shardLimitValidator.checkShardLimit(totalNewShards, currentState);
-                            if (error.isPresent()) {
-                                ValidationException ex = new ValidationException();
-                                ex.addValidationError(error.get());
-                                throw ex;
-                            }
+                            shardLimitValidator.validateShardLimitForIndices(
+                                request.indices(),
+                                currentState,
+                                index -> getTotalNewShards(index, currentState, updatedNumberOfReplicas)
+                            );
 
                             /*
                              * We do not update the in-sync allocation IDs as they will be removed upon the first index operation which makes
@@ -315,15 +312,12 @@ public ClusterState execute(ClusterState currentState) {
                             }
 
                             // Verify that this won't take us over the cluster shard limit.
-                            int totalNewShards = Arrays.stream(request.indices())
-                                .mapToInt(i -> getTotalNewShards(i, currentState, updatedNumberOfSearchReplicas))
-                                .sum();
-                            Optional<String> error = shardLimitValidator.checkShardLimit(totalNewShards, currentState);
-                            if (error.isPresent()) {
-                                ValidationException ex = new ValidationException();
-                                ex.addValidationError(error.get());
-                                throw ex;
-                            }
+                            shardLimitValidator.validateShardLimitForIndices(
+                                request.indices(),
+                                currentState,
+                                index -> getTotalNewShards(index, currentState, updatedNumberOfSearchReplicas)
+                            );
+
                             routingTableBuilder.updateNumberOfSearchReplicas(updatedNumberOfSearchReplicas, actualIndices);
                             metadataBuilder.updateNumberOfSearchReplicas(updatedNumberOfSearchReplicas, actualIndices);
                             logger.info(
@@ -571,9 +565,10 @@ private void validateSearchReplicaCountSettings(Settings requestSettings, Index[
     public static void validateIndexTotalPrimaryShardsPerNodeSetting(Settings indexSettings, ClusterService clusterService) {
         // Get the setting value
         int indexPrimaryShardsPerNode = INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING.get(indexSettings);
+        int indexRemoteCapablePrimaryShardsPerNode = INDEX_TOTAL_REMOTE_CAPABLE_PRIMARY_SHARDS_PER_NODE_SETTING.get(indexSettings);
 
         // If default value (-1), no validation needed
-        if (indexPrimaryShardsPerNode == -1) {
+        if (indexPrimaryShardsPerNode == -1 && indexRemoteCapablePrimaryShardsPerNode == -1) {
             return;
         }
 
@@ -586,7 +581,11 @@ public static void validateIndexTotalPrimaryShardsPerNodeSetting(Settings indexS
             .allMatch(DiscoveryNode::isRemoteStoreNode);
         if (!isRemoteStoreEnabled) {
             throw new IllegalArgumentException(
-                "Setting [" + INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING.getKey() + "] can only be used with remote store enabled clusters"
+                "Setting ["
+                    + INDEX_TOTAL_PRIMARY_SHARDS_PER_NODE_SETTING.getKey()
+                    + "] or ["
+                    + INDEX_TOTAL_REMOTE_CAPABLE_PRIMARY_SHARDS_PER_NODE_SETTING.getKey()
+                    + "] can only be used with remote store enabled clusters"
             );
         }
     }

server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java

@@ -74,6 +74,7 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
 
     private final Map<String, DiscoveryNode> nodes;
     private final Map<String, DiscoveryNode> dataNodes;
+    private final Map<String, DiscoveryNode> warmNodes;
     private final Map<String, DiscoveryNode> clusterManagerNodes;
     private final Map<String, DiscoveryNode> ingestNodes;
 
@@ -87,6 +88,7 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
     private DiscoveryNodes(
         final Map<String, DiscoveryNode> nodes,
         final Map<String, DiscoveryNode> dataNodes,
+        final Map<String, DiscoveryNode> warmNodes,
         final Map<String, DiscoveryNode> clusterManagerNodes,
         final Map<String, DiscoveryNode> ingestNodes,
         String clusterManagerNodeId,
@@ -98,6 +100,7 @@ private DiscoveryNodes(
     ) {
         this.nodes = Collections.unmodifiableMap(nodes);
         this.dataNodes = Collections.unmodifiableMap(dataNodes);
+        this.warmNodes = Collections.unmodifiableMap(warmNodes);
         this.clusterManagerNodes = Collections.unmodifiableMap(clusterManagerNodes);
         this.ingestNodes = Collections.unmodifiableMap(ingestNodes);
         this.clusterManagerNodeId = clusterManagerNodeId;
@@ -151,6 +154,15 @@ public Map<String, DiscoveryNode> getDataNodes() {
         return this.dataNodes;
     }
 
+    /**
+     * Get a {@link Map} of the discovered warm nodes arranged by their ids
+     *
+     * @return {@link Map} of the discovered warm nodes arranged by their ids
+     */
+    public Map<String, DiscoveryNode> getWarmNodes() {
+        return this.warmNodes;
+    }
+
     /**
      * Get a {@link Map} of the discovered cluster-manager nodes arranged by their ids
      *
@@ -802,6 +814,7 @@ private String validateAdd(DiscoveryNode node) {
 
         public DiscoveryNodes build() {
             final Map<String, DiscoveryNode> dataNodesBuilder = new HashMap<>();
+            final Map<String, DiscoveryNode> warmNodesBuilder = new HashMap<>();
             final Map<String, DiscoveryNode> clusterManagerNodesBuilder = new HashMap<>();
             final Map<String, DiscoveryNode> ingestNodesBuilder = new HashMap<>();
             Version minNodeVersion = null;
@@ -812,6 +825,9 @@ public DiscoveryNodes build() {
                 if (nodeEntry.getValue().isDataNode()) {
                     dataNodesBuilder.put(nodeEntry.getKey(), nodeEntry.getValue());
                 }
+                if (nodeEntry.getValue().isWarmNode()) {
+                    warmNodesBuilder.put(nodeEntry.getKey(), nodeEntry.getValue());
+                }
                 if (nodeEntry.getValue().isClusterManagerNode()) {
                     clusterManagerNodesBuilder.put(nodeEntry.getKey(), nodeEntry.getValue());
                 }
@@ -835,6 +851,7 @@ public DiscoveryNodes build() {
             return new DiscoveryNodes(
                 nodes,
                 dataNodesBuilder,
+                warmNodesBuilder,
                 clusterManagerNodesBuilder,
                 ingestNodesBuilder,
                 clusterManagerNodeId,