HDFS-17331:Fix Blocks are always -1 and DataNode version are always UNKNOWN in federationhealth.html (apache#6429). Contributed by lei w.

Signed-off-by: Shuyan Zhang <[email protected]>

Author: ThinkerLei

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java

@@ -143,7 +143,7 @@ private DatanodeInfo(final String ipAddr, final String hostName,
       final int xceiverCount, final String networkLocation,
       final AdminStates adminState, final String upgradeDomain,
       final long lastBlockReportTime, final long lastBlockReportMonotonic,
-                       final int blockCount) {
+      final int blockCount, final String softwareVersion) {
     super(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort,
         ipcPort);
     this.capacity = capacity;
@@ -162,6 +162,7 @@ private DatanodeInfo(final String ipAddr, final String hostName,
     this.lastBlockReportTime = lastBlockReportTime;
     this.lastBlockReportMonotonic = lastBlockReportMonotonic;
     this.numBlocks = blockCount;
+    this.softwareVersion =  softwareVersion;
   }
 
   /** Network location name. */
@@ -699,6 +700,7 @@ public static class DatanodeInfoBuilder {
     private long lastBlockReportTime = 0L;
     private long lastBlockReportMonotonic = 0L;
     private int numBlocks = 0;
+    private String softwareVersion;
 
     // Please use setNumBlocks explicitly to set numBlocks as this method doesn't have
     // sufficient info about numBlocks
@@ -718,6 +720,9 @@ public DatanodeInfoBuilder setFrom(DatanodeInfo from) {
       this.upgradeDomain = from.getUpgradeDomain();
       this.lastBlockReportTime = from.getLastBlockReportTime();
       this.lastBlockReportMonotonic = from.getLastBlockReportMonotonic();
+      if (from.getSoftwareVersion() != null) {
+        this.softwareVersion = from.getSoftwareVersion();
+      }
       setNodeID(from);
       return this;
     }
@@ -844,18 +849,24 @@ public DatanodeInfoBuilder setLastBlockReportMonotonic(long time) {
       this.lastBlockReportMonotonic = time;
       return this;
     }
+
     public DatanodeInfoBuilder setNumBlocks(int blockCount) {
       this.numBlocks = blockCount;
       return this;
     }
 
+    public DatanodeInfoBuilder setSoftwareVersion(String dnVersion) {
+      this.softwareVersion = dnVersion;
+      return this;
+    }
+
     public DatanodeInfo build() {
       return new DatanodeInfo(ipAddr, hostName, datanodeUuid, xferPort,
           infoPort, infoSecurePort, ipcPort, capacity, dfsUsed, nonDfsUsed,
           remaining, blockPoolUsed, cacheCapacity, cacheUsed, lastUpdate,
           lastUpdateMonotonic, xceiverCount, location, adminState,
           upgradeDomain, lastBlockReportTime, lastBlockReportMonotonic,
-          numBlocks);
+          numBlocks, softwareVersion);
     }
   }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java

@@ -385,6 +385,9 @@ public static DatanodeInfoProto convert(DatanodeInfo info) {
     if (info.getUpgradeDomain() != null) {
       builder.setUpgradeDomain(info.getUpgradeDomain());
     }
+    if (info.getSoftwareVersion() != null) {
+      builder.setSoftwareVersion(info.getSoftwareVersion());
+    }
     builder
         .setId(convert((DatanodeID) info))
         .setCapacity(info.getCapacity())
@@ -786,7 +789,8 @@ static public DatanodeInfo convert(DatanodeInfoProto di) {
                 di.getLastBlockReportTime() : 0)
             .setLastBlockReportMonotonic(di.hasLastBlockReportMonotonic() ?
                 di.getLastBlockReportMonotonic() : 0)
-            .setNumBlocks(di.getNumBlocks());
+            .setNumBlocks(di.getNumBlocks())
+            .setSoftwareVersion(di.getSoftwareVersion());
 
     if (di.hasNonDfsUsed()) {
       dinfo.setNonDfsUsed(di.getNonDfsUsed());

hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto

@@ -133,6 +133,7 @@ message DatanodeInfoProto {
   optional uint64 lastBlockReportTime = 15 [default = 0];
   optional uint64 lastBlockReportMonotonic = 16 [default = 0];
   optional uint32 numBlocks = 17 [default = 0];
+  optional string softwareVersion = 18;
 }
 
 /**

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java

@@ -481,7 +481,7 @@ private String getNodesImpl(final DatanodeReportType type) {
         innerinfo.put("adminState", node.getAdminState().toString());
         innerinfo.put("nonDfsUsedSpace", node.getNonDfsUsed());
         innerinfo.put("capacity", node.getCapacity());
-        innerinfo.put("numBlocks", -1); // node.numBlocks()
+        innerinfo.put("numBlocks", node.getNumBlocks());
         innerinfo.put("version", (node.getSoftwareVersion() == null ?
                         "UNKNOWN" : node.getSoftwareVersion()));
         innerinfo.put("used", node.getDfsUsed());

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java

@@ -1101,10 +1101,15 @@ private DatanodeStorageReport[] mergeDtanodeStorageReport(
         DatanodeInfo dnInfo = dn.getDatanodeInfo();
         String nodeId = dnInfo.getXferAddr();
         DatanodeStorageReport oldDn = datanodesMap.get(nodeId);
-        if (oldDn == null ||
-            dnInfo.getLastUpdate() > oldDn.getDatanodeInfo().getLastUpdate()) {
+        if (oldDn == null) {
+          datanodesMap.put(nodeId, dn);
+        } else if (dnInfo.getLastUpdate() > oldDn.getDatanodeInfo().getLastUpdate()) {
+          dnInfo.setNumBlocks(dnInfo.getNumBlocks() +
+              oldDn.getDatanodeInfo().getNumBlocks());
           datanodesMap.put(nodeId, dn);
         } else {
+          oldDn.getDatanodeInfo().setNumBlocks(
+              oldDn.getDatanodeInfo().getNumBlocks() + dnInfo.getNumBlocks());
           LOG.debug("{} is in multiple subclusters", nodeId);
         }
       }