HDFS-17888. Fix MiniCluster startup & HttpServer service's conf issues.

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java

@@ -221,8 +221,9 @@ public static File getTestDir() {
       prop = DEFAULT_TEST_DATA_DIR;
     }
     File dir = new File(prop).getAbsoluteFile();
-    dir.mkdirs();
-    assertExists(dir);
+    if (dir.mkdirs() && !dir.exists()) {
+      throw new IllegalStateException("Directory " + dir + " not created");
+    }
     return dir;
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java

@@ -1562,7 +1562,10 @@ private void initIpcServer() throws IOException {
     DFSUtil.addInternalPBProtocol(getConf(), InterDatanodeProtocolPB.class, service,
         ipcServer);
 
-    LOG.info("Opened IPC server at {}", ipcServer.getListenerAddress());
+    InetSocketAddress listenerAddress = ipcServer.getListenerAddress();
+    LOG.info("Opened IPC server at {}", listenerAddress);
+    dnConf.getConf().set(DFS_DATANODE_IPC_ADDRESS_KEY,
+        listenerAddress.getHostName() + ":" + listenerAddress.getPort());
 
     // set service-level authorization security policy
     if (getConf().getBoolean(

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java

@@ -303,7 +303,9 @@ public void start() throws IOException {
     if (httpServer != null) {
       InetSocketAddress infoAddr = DataNode.getInfoAddr(conf);
       httpAddress = getChannelLocalAddress(httpServer, infoAddr);
-      LOG.info("Listening HTTP traffic on " + httpAddress);
+      conf.set(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY,
+          NetUtils.getHostPortString(httpAddress));
+      LOG.info("Listening for HTTP traffic on {}", httpAddress);
     }
 
     if (httpsServer != null) {
@@ -312,7 +314,9 @@ public void start() throws IOException {
               DFS_DATANODE_HTTPS_ADDRESS_KEY,
               DFS_DATANODE_HTTPS_ADDRESS_DEFAULT));
       httpsAddress = getChannelLocalAddress(httpsServer, secInfoSocAddr);
-      LOG.info("Listening HTTPS traffic on " + httpsAddress);
+      conf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY,
+          NetUtils.getHostPortString(httpsAddress));
+      LOG.info("Listening for HTTPS traffic on {}", httpsAddress);
     }
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java

@@ -54,12 +54,17 @@
 
 import org.glassfish.jersey.internal.inject.AbstractBinder;
 import org.glassfish.jersey.server.ResourceConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Encapsulates the HTTP server started by the NameNode. 
  */
 @InterfaceAudience.Private
 public class NameNodeHttpServer {
+
+  private static final Logger LOG = LoggerFactory.getLogger(NameNodeHttpServer.class);
+
   private HttpServer2 httpServer;
   private final Configuration conf;
   private final NameNode nn;
@@ -185,14 +190,20 @@ void start() throws IOException {
     int connIdx = 0;
     if (policy.isHttpEnabled()) {
       httpAddress = httpServer.getConnectorAddress(connIdx++);
-      conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY,
-          NetUtils.getHostPortString(httpAddress));
+      if (httpAddress != null) {
+        conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY,
+            NetUtils.getHostPortString(httpAddress));
+        LOG.info("Listening for HTTP traffic on {}", httpAddress);
+      }
     }
 
     if (policy.isHttpsEnabled()) {
       httpsAddress = httpServer.getConnectorAddress(connIdx);
-      conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY,
-          NetUtils.getHostPortString(httpsAddress));
+      if (httpsAddress != null) {
+        conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY,
+            NetUtils.getHostPortString(httpsAddress));
+        LOG.info("Listening for HTTPS traffic on {}", httpsAddress);
+      }
     }
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html

@@ -68,7 +68,7 @@
 
 <script type="text/x-dust-template" id="tmpl-dn">
 {#dn}
-<div class="page-header"><h1>DataNode on <small>{HostName}:{DataPort}</small></h1></div>
+<div class="page-header"><h1>DataNode on <small>{HostName}:{RpcPort}</small></h1></div>
 <table class="table table-bordered table-striped">
   <tr><th>Cluster ID:</th><td>{ClusterId}</td></tr>
   <tr><th>Started:</th><td>{DNStartedTimeInMillis|date_tostring}</td></tr>

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java

@@ -18,13 +18,15 @@
 package org.apache.hadoop.hdfs;
 
 import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
 
@@ -167,4 +169,25 @@ public void testMemlockLimit() throws Exception {
           prevLimit);
     }
   }
+
+  @Test
+  public void testDataNodeIpcAndHttpSeverConf() throws Exception {
+    Configuration conf = cluster.getConfiguration(0);
+    DataNode dn = null;
+    try {
+      dn = DataNode.createDataNode(new String[] {}, conf);
+      Configuration dnConf = dn.getConf();
+      InetSocketAddress listenerAddress = dn.ipcServer.getListenerAddress();
+      assertThat(listenerAddress.getHostName() + ":" + listenerAddress.getPort())
+          .describedAs("IPC Address is inconsistent")
+          .isEqualTo(dnConf.get(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY));
+      assertThat(listenerAddress.getHostName() + ":" + dn.getHttpPort())
+          .describedAs("HTTP Address is inconsistent")
+          .isEqualTo(dnConf.get(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY));
+    } finally {
+      if (dn != null) {
+        dn.shutdown();
+      }
+    }
+  }
 }