Skip to content

Commit 0ce07d7

Browse files
szetszwoszetszwo
committed
HADOOP-5258. Add a new DFSAdmin command to print a tree of the rack and datanode topology as seen by the namenode. (Jakob Homan via szetszwo)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@752292 13f79535-47bb-0310-9956-ffa450edef68
1 parent 85a7c85 commit 0ce07d7

File tree

11 files changed

+209
-52
lines changed

11 files changed

+209
-52
lines changed

‎CHANGES.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ Trunk (unreleased changes)
5252
HADOOP-5144. Add a new DFSAdmin command for changing the setting of restore
5353
failed storage replicas in namenode. (Boris Shkolnik via szetszwo)
5454

55+
HADOOP-5258. Add a new DFSAdmin command to print a tree of the rack and
56+
datanode topology as seen by the namenode. (Jakob Homan via szetszwo)
57+
5558
IMPROVEMENTS
5659

5760
HADOOP-4565. Added CombineFileInputFormat to use data locality information

‎src/core/org/apache/hadoop/net/NetUtils.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import java.net.UnknownHostException;
2929
import java.nio.channels.SocketChannel;
3030
import java.util.Map.Entry;
31+
import java.util.regex.Pattern;
3132
import java.util.*;
3233

3334
import javax.net.SocketFactory;
@@ -441,4 +442,39 @@ public static List<String> normalizeHostNames(Collection<String> names) {
441442
}
442443
return hostNames;
443444
}
445+
446+
/**
447+
* Attempt to obtain the host name of a name specified by ip address.
448+
* Check that the node name is an ip addr and if so, attempt to determine
449+
* its host name. If the name is not an IP addr, or the actual name cannot
450+
* be determined, return null.
451+
*
452+
* @return Host name or null
453+
*/
454+
private static final Pattern ipPattern = // Pattern for matching hostname to ip:port
455+
Pattern.compile("\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:?\\d*");
456+
public static String getHostNameOfIP(String ip) {
457+
// If name is not an ip addr, don't bother looking it up
458+
if(!ipPattern.matcher(ip).matches())
459+
return null;
460+
461+
String hostname = "";
462+
try {
463+
String n = ip.substring(0, ip.indexOf(':'));
464+
hostname = InetAddress.getByName(n).getHostName();
465+
} catch (UnknownHostException e) {
466+
return null;
467+
}
468+
469+
return hostname;
470+
}
471+
472+
/**
473+
* Return hostname without throwing exception.
474+
* @return hostname
475+
*/
476+
public static String getHostname() {
477+
try {return "" + InetAddress.getLocalHost();}
478+
catch(UnknownHostException uhe) {return "" + uhe;}
479+
}
444480
}

‎src/core/org/apache/hadoop/util/StringUtils.java‎

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -20,22 +20,21 @@
2020

2121
import java.io.PrintWriter;
2222
import java.io.StringWriter;
23-
import java.net.InetAddress;
2423
import java.net.URI;
2524
import java.net.URISyntaxException;
26-
import java.net.UnknownHostException;
2725
import java.text.DateFormat;
2826
import java.text.DecimalFormat;
2927
import java.text.NumberFormat;
30-
import java.util.Locale;
3128
import java.util.ArrayList;
3229
import java.util.Arrays;
30+
import java.util.Collection;
3331
import java.util.Date;
3432
import java.util.List;
33+
import java.util.Locale;
3534
import java.util.StringTokenizer;
36-
import java.util.Collection;
3735

38-
import org.apache.hadoop.fs.*;
36+
import org.apache.hadoop.fs.Path;
37+
import org.apache.hadoop.net.NetUtils;
3938

4039
/**
4140
* General string utils
@@ -503,15 +502,6 @@ public static String unEscapeString(String str, char escapeChar,
503502
return result.toString();
504503
}
505504

506-
/**
507-
* Return hostname without throwing exception.
508-
* @return hostname
509-
*/
510-
public static String getHostname() {
511-
try {return "" + InetAddress.getLocalHost();}
512-
catch(UnknownHostException uhe) {return "" + uhe;}
513-
}
514-
515505
/**
516506
* Return a message for logging.
517507
* @param prefix prefix keyword for the message
@@ -535,7 +525,7 @@ private static String toStartupShutdownString(String prefix, String [] msg) {
535525
*/
536526
public static void startupShutdownMessage(Class<?> clazz, String[] args,
537527
final org.apache.commons.logging.Log LOG) {
538-
final String hostname = getHostname();
528+
final String hostname = NetUtils.getHostname();
539529
final String classname = clazz.getSimpleName();
540530
LOG.info(
541531
toStartupShutdownString("STARTUP_MSG: ", new String[] {

‎src/docs/src/documentation/content/xdocs/commands_manual.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -514,6 +514,11 @@
514514
Datanodes delete their previous version working directories,
515515
followed by Namenode doing the same.
516516
This completes the upgrade process.</td>
517+
</tr>
518+
<tr>
519+
<td><code>-printTopology</code></td>
520+
<td>Print a tree of the rack/datanode topology of the
521+
cluster as seen by the NameNode.</td>
517522
</tr>
518523
<tr>
519524
<td><code>-upgradeProgress status | details | force</code></td>

‎src/docs/src/documentation/content/xdocs/hdfs_user_guide.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,11 @@
199199
dfs.host.exclude is stopped from decommissioning if it has aleady
200200
been marked for decommission. Entires not present in both the lists
201201
are decommissioned.
202+
</li>
203+
<li>
204+
<code>-printTopology</code>
205+
: Print the topology of the cluster. Display a tree of racks and
206+
datanodes attached to the tracks as viewed by the NameNode.
202207
</li>
203208
</ul>
204209
<p>

‎src/hdfs/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java‎

Lines changed: 4 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,14 @@
2020
import java.io.DataInput;
2121
import java.io.DataOutput;
2222
import java.io.IOException;
23-
import java.net.InetAddress;
24-
import java.net.UnknownHostException;
2523
import java.util.Date;
26-
import java.util.regex.Pattern;
2724

2825
import org.apache.hadoop.io.Text;
2926
import org.apache.hadoop.io.Writable;
3027
import org.apache.hadoop.io.WritableFactories;
3128
import org.apache.hadoop.io.WritableFactory;
3229
import org.apache.hadoop.io.WritableUtils;
30+
import org.apache.hadoop.net.NetUtils;
3331
import org.apache.hadoop.net.NetworkTopology;
3432
import org.apache.hadoop.net.Node;
3533
import org.apache.hadoop.net.NodeBase;
@@ -47,10 +45,8 @@ public class DatanodeInfo extends DatanodeID implements Node {
4745
protected long lastUpdate;
4846
protected int xceiverCount;
4947
protected String location = NetworkTopology.DEFAULT_RACK;
50-
static final Pattern ip = // Pattern for matching hostname to ip:port
51-
Pattern.compile("\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:?\\d*");
5248

53-
/** HostName as suplied by the datanode during registration as its
49+
/** HostName as supplied by the datanode during registration as its
5450
* name. Namenode uses datanode IP address as the name.
5551
*/
5652
protected String hostName = null;
@@ -152,7 +148,7 @@ public void setXceiverCount(int xceiverCount) {
152148
this.xceiverCount = xceiverCount;
153149
}
154150

155-
/** rack name **/
151+
/** rack name */
156152
public synchronized String getNetworkLocation() {return location;}
157153

158154
/** Sets the rack name */
@@ -177,7 +173,7 @@ public String getDatanodeReport() {
177173
long nonDFSUsed = getNonDfsUsed();
178174
float usedPercent = getDfsUsedPercent();
179175
float remainingPercent = getRemainingPercent();
180-
String hostName = getHostNameOfIP();
176+
String hostName = NetUtils.getHostNameOfIP(name);
181177

182178
buffer.append("Name: "+ name);
183179
if(hostName != null)
@@ -205,30 +201,6 @@ public String getDatanodeReport() {
205201
return buffer.toString();
206202
}
207203

208-
/**
209-
* Attempt to obtain the host name of a name specified by ip address.
210-
* Check that the node name is an ip addr and if so, attempt to determine
211-
* its host name. If the name is not an IP addr, or the actual name cannot
212-
* be determined, return null.
213-
*
214-
* @return Host name or null
215-
*/
216-
private String getHostNameOfIP() {
217-
// If name is not an ip addr, don't bother looking it up
218-
if(!ip.matcher(name).matches())
219-
return null;
220-
221-
String hostname = "";
222-
try {
223-
String n = name.substring(0, name.indexOf(':'));
224-
hostname = InetAddress.getByName(n).getHostName();
225-
} catch (UnknownHostException e) {
226-
return null;
227-
}
228-
229-
return hostname;
230-
}
231-
232204
/** A formatted string for printing the status of the DataNode. */
233205
public String dumpDatanode() {
234206
StringBuffer buffer = new StringBuffer();

‎src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java‎

Lines changed: 73 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,11 @@
1818
package org.apache.hadoop.hdfs.tools;
1919

2020
import java.io.IOException;
21+
import java.util.ArrayList;
22+
import java.util.Collections;
23+
import java.util.HashMap;
2124
import java.util.List;
25+
import java.util.TreeSet;
2226

2327
import javax.security.auth.login.LoginException;
2428

@@ -29,6 +33,7 @@
2933
import org.apache.hadoop.fs.Path;
3034
import org.apache.hadoop.fs.shell.Command;
3135
import org.apache.hadoop.fs.shell.CommandFormat;
36+
import org.apache.hadoop.hdfs.DFSClient;
3237
import org.apache.hadoop.hdfs.DistributedFileSystem;
3338
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
3439
import org.apache.hadoop.hdfs.protocol.FSConstants;
@@ -463,6 +468,7 @@ private void printHelp(String cmd) {
463468
"\t[" + SetSpaceQuotaCommand.USAGE + "]\n" +
464469
"\t[" + ClearSpaceQuotaCommand.USAGE +"]\n" +
465470
"\t[-refreshServiceAcl]\n" +
471+
"\t[-printTopology]\n" +
466472
"\t[-help [cmd]]\n";
467473

468474
String report ="-report: \tReports basic filesystem information and statistics.\n";
@@ -516,6 +522,9 @@ private void printHelp(String cmd) {
516522
String refreshServiceAcl = "-refreshServiceAcl: Reload the service-level authorization policy file\n" +
517523
"\t\tNamenode will reload the authorization policy file.\n";
518524

525+
String printTopology = "-printTopology: Print a tree of the racks and their\n" +
526+
"\t\tnodes as reported by the Namenode\n";
527+
519528
String help = "-help [cmd]: \tDisplays help for the given command or all commands if none\n" +
520529
"\t\tis specified.\n";
521530

@@ -545,6 +554,8 @@ private void printHelp(String cmd) {
545554
System.out.println(ClearSpaceQuotaCommand.DESCRIPTION);
546555
} else if ("refreshServiceAcl".equals(cmd)) {
547556
System.out.println(refreshServiceAcl);
557+
} else if ("printTopology".equals(cmd)) {
558+
System.out.println(printTopology);
548559
} else if ("help".equals(cmd)) {
549560
System.out.println(help);
550561
} else {
@@ -562,14 +573,14 @@ private void printHelp(String cmd) {
562573
System.out.println(SetSpaceQuotaCommand.DESCRIPTION);
563574
System.out.println(ClearSpaceQuotaCommand.DESCRIPTION);
564575
System.out.println(refreshServiceAcl);
576+
System.out.println(printTopology);
565577
System.out.println(help);
566578
System.out.println();
567579
ToolRunner.printGenericCommandUsage(System.out);
568580
}
569581

570582
}
571583

572-
573584
/**
574585
* Command to ask the namenode to finalize previously performed upgrade.
575586
* Usage: java DFSAdmin -finalizeUpgrade
@@ -645,6 +656,54 @@ public int metaSave(String[] argv, int idx) throws IOException {
645656
return 0;
646657
}
647658

659+
/**
660+
* Display each rack and the nodes assigned to that rack, as determined
661+
* by the NameNode, in a hierarchical manner. The nodes and racks are
662+
* sorted alphabetically.
663+
*
664+
* @throws IOException If an error while getting datanode report
665+
*/
666+
public int printTopology() throws IOException {
667+
if (fs instanceof DistributedFileSystem) {
668+
DistributedFileSystem dfs = (DistributedFileSystem)fs;
669+
DFSClient client = dfs.getClient();
670+
DatanodeInfo[] report = client.datanodeReport(DatanodeReportType.ALL);
671+
672+
// Build a map of rack -> nodes from the datanode report
673+
HashMap<String, TreeSet<String> > tree = new HashMap<String, TreeSet<String>>();
674+
for(DatanodeInfo dni : report) {
675+
String location = dni.getNetworkLocation();
676+
String name = dni.getName();
677+
678+
if(!tree.containsKey(location)) {
679+
tree.put(location, new TreeSet<String>());
680+
}
681+
682+
tree.get(location).add(name);
683+
}
684+
685+
// Sort the racks (and nodes) alphabetically, display in order
686+
ArrayList<String> racks = new ArrayList<String>(tree.keySet());
687+
Collections.sort(racks);
688+
689+
for(String r : racks) {
690+
System.out.println("Rack: " + r);
691+
TreeSet<String> nodes = tree.get(r);
692+
693+
for(String n : nodes) {
694+
System.out.print(" " + n);
695+
String hostname = NetUtils.getHostNameOfIP(n);
696+
if(hostname != null)
697+
System.out.print(" (" + hostname + ")");
698+
System.out.println();
699+
}
700+
701+
System.out.println();
702+
}
703+
}
704+
return 0;
705+
}
706+
648707
private static UnixUserGroupInformation getUGI(Configuration conf)
649708
throws IOException {
650709
UnixUserGroupInformation ugi = null;
@@ -725,6 +784,9 @@ private static void printUsage(String cmd) {
725784
} else if ("-refreshServiceAcl".equals(cmd)) {
726785
System.err.println("Usage: java DFSAdmin"
727786
+ " [-refreshServiceAcl]");
787+
} else if ("-printTopology".equals(cmd)) {
788+
System.err.println("Usage: java DFSAdmin"
789+
+ " [-printTopology]");
728790
} else {
729791
System.err.println("Usage: java DFSAdmin");
730792
System.err.println(" [-report]");
@@ -736,6 +798,7 @@ private static void printUsage(String cmd) {
736798
System.err.println(" [-upgradeProgress status | details | force]");
737799
System.err.println(" [-metasave filename]");
738800
System.err.println(" [-refreshServiceAcl]");
801+
System.err.println(" [-printTopology]");
739802
System.err.println(" ["+SetQuotaCommand.USAGE+"]");
740803
System.err.println(" ["+ClearQuotaCommand.USAGE+"]");
741804
System.err.println(" ["+SetSpaceQuotaCommand.USAGE+"]");
@@ -811,6 +874,12 @@ public int run(String[] argv) throws Exception {
811874
printUsage(cmd);
812875
return exitCode;
813876
}
877+
else if ("-printTopology".equals(cmd)) {
878+
if(argv.length != 1) {
879+
printUsage(cmd);
880+
return exitCode;
881+
}
882+
}
814883
}
815884

816885
// initialize DFSAdmin
@@ -853,6 +922,8 @@ public int run(String[] argv) throws Exception {
853922
exitCode = new SetSpaceQuotaCommand(argv, i, fs).runAll();
854923
} else if ("-refreshServiceAcl".equals(cmd)) {
855924
exitCode = refreshServiceAcl();
925+
} else if ("-printTopology".equals(cmd)) {
926+
exitCode = printTopology();
856927
} else if ("-help".equals(cmd)) {
857928
if (i < argv.length) {
858929
printHelp(argv[i]);
@@ -871,7 +942,7 @@ public int run(String[] argv) throws Exception {
871942
} catch (RemoteException e) {
872943
//
873944
// This is a error returned by hadoop server. Print
874-
// out the first line of the error mesage, ignore the stack trace.
945+
// out the first line of the error message, ignore the stack trace.
875946
exitCode = -1;
876947
try {
877948
String[] content;

‎src/test/org/apache/hadoop/cli/TestCLI.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,17 @@ public void setUp() throws Exception {
121121
conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG,
122122
true);
123123

124-
dfsCluster = new MiniDFSCluster(conf, 1, true, null);
124+
// Many of the tests expect a replication value of 1 in the output
125+
conf.setInt("dfs.replication", 1);
126+
127+
// Build racks and hosts configuration to test dfsAdmin -printTopology
128+
String [] racks = {"/rack1", "/rack1", "/rack2", "/rack2",
129+
"/rack2", "/rack3", "/rack4", "/rack4" };
130+
String [] hosts = {"host1", "host2", "host3", "host4",
131+
"host5", "host6", "host7", "host8" };
132+
133+
dfsCluster = new MiniDFSCluster(conf, 8, true, racks, hosts);
134+
125135
namenode = conf.get("fs.default.name", "file:///");
126136
clitestDataDir = new File(TEST_CACHE_DATA_DIR).
127137
toURI().toString().replace(' ', '+');

0 commit comments

Comments
 (0)