Skip to content

Commit a3007b0

Browse files
lfxylfxy
committed
Support to make dfs.namenode.fs-limits.max-directory-items reconfigurable
1 parent 3695db2 commit a3007b0

File tree

4 files changed

+61
-6
lines changed

4 files changed

+61
-6
lines changed

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ private static INodeDirectory createRoot(FSNamesystem namesystem) {
159159
private final FSNamesystem namesystem;
160160
private volatile boolean skipQuotaCheck = false; //skip while consuming edits
161161
private final int maxComponentLength;
162-
private final int maxDirItems;
162+
private int maxDirItems;
163163
private final int lsLimit; // max list limit
164164
private final int contentCountLimit; // max content summary counts per run
165165
private final long contentSleepMicroSec;
@@ -217,6 +217,8 @@ private static INodeDirectory createRoot(FSNamesystem namesystem) {
217217
// authorizeWithContext() API or not.
218218
private boolean useAuthorizationWithContextAPI = false;
219219

220+
private final int MAX_DIR_ITEMS = 64 * 100 * 1000;
221+
220222
public void setINodeAttributeProvider(
221223
@Nullable INodeAttributeProvider provider) {
222224
attributeProvider = provider;
@@ -398,7 +400,6 @@ public enum DirOp {
398400
// We need a maximum maximum because by default, PB limits message sizes
399401
// to 64MB. This means we can only store approximately 6.7 million entries
400402
// per directory, but let's use 6.4 million for some safety.
401-
final int MAX_DIR_ITEMS = 64 * 100 * 1000;
402403
Preconditions.checkArgument(
403404
maxDirItems > 0 && maxDirItems <= MAX_DIR_ITEMS, "Cannot set "
404405
+ DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY
@@ -580,6 +581,18 @@ String setProtectedDirectories(String protectedDirsString) {
580581
return Joiner.on(",").skipNulls().join(protectedDirectories);
581582
}
582583

584+
public void setMaxDirItems(int newVal) {
585+
com.google.common.base.Preconditions.checkArgument(
586+
newVal > 0 && newVal <= MAX_DIR_ITEMS, "Cannot set "
587+
+ DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY
588+
+ " to a value less than 1 or greater than " + MAX_DIR_ITEMS);
589+
maxDirItems = newVal;
590+
}
591+
592+
public int getMaxDirItems() {
593+
return maxDirItems;
594+
}
595+
583596
BlockManager getBlockManager() {
584597
return getFSNamesystem().getBlockManager();
585598
}

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,8 @@
142142
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_IMAGE_PARALLEL_LOAD_KEY;
143143
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LOCK_DETAILED_METRICS_DEFAULT;
144144
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY;
145+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_DEFAULT;
146+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY;
145147
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_DEFAULT;
146148
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY;
147149
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SLOWPEER_COLLECT_INTERVAL_DEFAULT;
@@ -384,7 +386,8 @@ public enum OperationCategory {
384386
DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY,
385387
DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY,
386388
DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY,
387-
DFS_NAMENODE_SLOWPEER_COLLECT_INTERVAL_KEY));
389+
DFS_NAMENODE_SLOWPEER_COLLECT_INTERVAL_KEY,
390+
DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY));
388391

389392
private static final String USAGE = "Usage: hdfs namenode ["
390393
+ StartupOption.BACKUP.getName() + "] | \n\t["
@@ -2387,6 +2390,8 @@ protected String reconfigurePropertyImpl(String property, String newVal)
23872390
|| property.equals(DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY)
23882391
|| property.equals(DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY)) {
23892392
return reconfigureFSNamesystemLockMetricsParameters(property, newVal);
2393+
} else if (property.equals(DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY)) {
2394+
return reconfMaxDirItems(newVal);
23902395
} else {
23912396
throw new ReconfigurationException(property, newVal, getConf().get(
23922397
property));
@@ -2805,6 +2810,23 @@ private String reconfigureFSNamesystemLockMetricsParameters(final String propert
28052810
}
28062811
}
28072812

2813+
private String reconfMaxDirItems(String newVal) throws ReconfigurationException {
2814+
int newSetting;
2815+
namesystem.writeLock(RwLockMode.BM);
2816+
try {
2817+
getNamesystem().getFSDirectory()
2818+
.setMaxDirItems(adjustNewVal(DFS_NAMENODE_MAX_DIRECTORY_ITEMS_DEFAULT, newVal));
2819+
newSetting = getNamesystem().getFSDirectory().getMaxDirItems();
2820+
LOG.info("RECONFIGURE* changed {} to {}", DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY, newSetting);
2821+
return String.valueOf(newSetting);
2822+
} catch (IllegalArgumentException e) {
2823+
throw new ReconfigurationException(DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY, newVal,
2824+
getConf().get(DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY), e);
2825+
} finally {
2826+
namesystem.writeUnlock(RwLockMode.BM, "reconfMaxDirItems");
2827+
}
2828+
}
2829+
28082830
@Override // ReconfigurableBase
28092831
protected Configuration getNewConf() {
28102832
return new HdfsConfiguration();

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY;
4040
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SLOWPEER_COLLECT_INTERVAL_KEY;
4141
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY;
42+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY;
4243

4344
import org.slf4j.Logger;
4445
import org.slf4j.LoggerFactory;
@@ -865,6 +866,23 @@ public void testReconfigureSlowPeerCollectInterval() throws Exception {
865866
assertEquals(600000, datanodeManager.getSlowPeerCollectionInterval());
866867
}
867868

869+
@Test
870+
public void testReconfigureMaxDirItems()
871+
throws ReconfigurationException {
872+
final NameNode nameNode = cluster.getNameNode();
873+
final FSDirectory fsd = nameNode.namesystem.getFSDirectory();
874+
875+
// By default, DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY is 1024*1024.
876+
assertEquals(1024*1024, fsd.getMaxDirItems());
877+
878+
// Reconfigure.
879+
nameNode.reconfigureProperty(
880+
DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY, Integer.toString(1024*1024*2));
881+
882+
// Assert DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY is 10.
883+
assertEquals(1024*1024*2, fsd.getMaxDirItems());
884+
}
885+
868886
@AfterEach
869887
public void shutDown() throws IOException {
870888
if (cluster != null) {

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@
105105
import org.slf4j.LoggerFactory;
106106

107107
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY;
108+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY;
108109
import static org.apache.hadoop.hdfs.client.HdfsAdmin.TRASH_PERMISSION;
109110
import static org.assertj.core.api.Assertions.assertThat;
110111
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -449,7 +450,7 @@ public void testNameNodeGetReconfigurableProperties() throws IOException, Interr
449450
final List<String> outs = Lists.newArrayList();
450451
final List<String> errs = Lists.newArrayList();
451452
getReconfigurableProperties("namenode", address, outs, errs);
452-
assertEquals(29, outs.size());
453+
assertEquals(30, outs.size());
453454
assertTrue(outs.get(0).contains("Reconfigurable properties:"));
454455
assertEquals(DFS_BLOCK_INVALIDATE_LIMIT_KEY, outs.get(1));
455456
assertEquals(DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY, outs.get(2));
@@ -463,8 +464,9 @@ public void testNameNodeGetReconfigurableProperties() throws IOException, Interr
463464
assertEquals(DFS_NAMENODE_BLOCKPLACEMENTPOLICY_MIN_BLOCKS_FOR_WRITE_KEY, outs.get(10));
464465
assertEquals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK, outs.get(11));
465466
assertEquals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT, outs.get(12));
466-
assertEquals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, outs.get(13));
467-
assertEquals(DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY, outs.get(14));
467+
assertEquals(DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY, outs.get(13));
468+
assertEquals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, outs.get(14));
469+
assertEquals(DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY, outs.get(15));
468470
assertEquals(errs.size(), 0);
469471
}
470472

0 commit comments

Comments
 (0)