Skip to content

Commit 6c45a6c

Browse files
Jing9Jing9
committed
HDFS-5629. Support HTTPS in JournalNode and SecondaryNameNode. Contributed by Haohui Mai.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1549692 13f79535-47bb-0310-9956-ffa450edef68
1 parent 79a2fec commit 6c45a6c

File tree

13 files changed

+270
-193
lines changed

13 files changed

+270
-193
lines changed

‎hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,9 @@ Trunk (Unreleased)
239239
HDFS-5554. Flatten INodeFile hierarchy: Replace INodeFileWithSnapshot with
240240
FileWithSnapshotFeature. (jing9 via szetszwo)
241241

242+
HDFS-5629. Support HTTPS in JournalNode and SecondaryNameNode.
243+
(Haohui Mai via jing9)
244+
242245
OPTIMIZATIONS
243246
HDFS-5349. DNA_CACHE and DNA_UNCACHE should be by blockId only. (cmccabe)
244247

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
130130
public static final int DFS_NAMENODE_SAFEMODE_MIN_DATANODES_DEFAULT = 0;
131131
public static final String DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY = "dfs.namenode.secondary.http-address";
132132
public static final String DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_DEFAULT = "0.0.0.0:50090";
133+
public static final String DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY = "dfs.namenode.secondary.https-address";
134+
public static final String DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_DEFAULT = "0.0.0.0:50091";
133135
public static final String DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY = "dfs.namenode.checkpoint.check.period";
134136
public static final long DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_DEFAULT = 60;
135137
public static final String DFS_NAMENODE_CHECKPOINT_PERIOD_KEY = "dfs.namenode.checkpoint.period";
@@ -504,6 +506,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
504506
public static final String DFS_JOURNALNODE_HTTP_ADDRESS_KEY = "dfs.journalnode.http-address";
505507
public static final int DFS_JOURNALNODE_HTTP_PORT_DEFAULT = 8480;
506508
public static final String DFS_JOURNALNODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_JOURNALNODE_HTTP_PORT_DEFAULT;
509+
public static final String DFS_JOURNALNODE_HTTPS_ADDRESS_KEY = "dfs.journalnode.https-address";
510+
public static final int DFS_JOURNALNODE_HTTPS_PORT_DEFAULT = 8481;
511+
public static final String DFS_JOURNALNODE_HTTPS_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_JOURNALNODE_HTTPS_PORT_DEFAULT;
507512

508513
public static final String DFS_JOURNALNODE_KEYTAB_FILE_KEY = "dfs.journalnode.keytab.file";
509514
public static final String DFS_JOURNALNODE_USER_NAME_KEY = "dfs.journalnode.kerberos.principal";

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
package org.apache.hadoop.hdfs;
2020

21+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN;
2122
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT;
2223
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_KEY;
2324
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
@@ -89,6 +90,7 @@
8990
import org.apache.hadoop.net.NodeBase;
9091
import org.apache.hadoop.security.SecurityUtil;
9192
import org.apache.hadoop.security.UserGroupInformation;
93+
import org.apache.hadoop.security.authorize.AccessControlList;
9294
import org.apache.hadoop.util.StringUtils;
9395
import org.apache.hadoop.util.ToolRunner;
9496

@@ -1573,4 +1575,67 @@ public static long parseRelativeTime(String relTime) throws IOException {
15731575
}
15741576
return ttl*1000;
15751577
}
1578+
1579+
/**
1580+
* Load HTTPS-related configuration.
1581+
*/
1582+
public static Configuration loadSslConfiguration(Configuration conf) {
1583+
Configuration sslConf = new Configuration(false);
1584+
1585+
sslConf.addResource(conf.get(
1586+
DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY,
1587+
DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_DEFAULT));
1588+
1589+
boolean requireClientAuth = conf.getBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY,
1590+
DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT);
1591+
sslConf.setBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY, requireClientAuth);
1592+
return sslConf;
1593+
}
1594+
1595+
/**
1596+
* Return a HttpServer.Builder that the journalnode / namenode / secondary
1597+
* namenode can use to initialize their HTTP / HTTPS server.
1598+
*
1599+
*/
1600+
public static HttpServer.Builder httpServerTemplateForNNAndJN(
1601+
Configuration conf, final InetSocketAddress httpAddr,
1602+
final InetSocketAddress httpsAddr, String name, String spnegoUserNameKey,
1603+
String spnegoKeytabFileKey) throws IOException {
1604+
HttpConfig.Policy policy = getHttpPolicy(conf);
1605+
1606+
HttpServer.Builder builder = new HttpServer.Builder().setName(name)
1607+
.setConf(conf).setACL(new AccessControlList(conf.get(DFS_ADMIN, " ")))
1608+
.setSecurityEnabled(UserGroupInformation.isSecurityEnabled())
1609+
.setUsernameConfKey(spnegoUserNameKey)
1610+
.setKeytabConfKey(getSpnegoKeytabKey(conf, spnegoKeytabFileKey));
1611+
1612+
// initialize the webserver for uploading/downloading files.
1613+
LOG.info("Starting web server as: "
1614+
+ SecurityUtil.getServerPrincipal(conf.get(spnegoUserNameKey),
1615+
httpAddr.getHostName()));
1616+
1617+
if (policy.isHttpEnabled()) {
1618+
if (httpAddr.getPort() == 0) {
1619+
builder.setFindPort(true);
1620+
}
1621+
1622+
URI uri = URI.create("http://" + NetUtils.getHostPortString(httpAddr));
1623+
builder.addEndpoint(uri);
1624+
LOG.info("Starting Web-server for " + name + " at: " + uri);
1625+
}
1626+
1627+
if (policy.isHttpsEnabled() && httpsAddr != null) {
1628+
Configuration sslConf = loadSslConfiguration(conf);
1629+
loadSslConfToHttpServerBuilder(builder, sslConf);
1630+
1631+
if (httpsAddr.getPort() == 0) {
1632+
builder.setFindPort(true);
1633+
}
1634+
1635+
URI uri = URI.create("https://" + NetUtils.getHostPortString(httpsAddr));
1636+
builder.addEndpoint(uri);
1637+
LOG.info("Starting Web-server for " + name + " at: " + uri);
1638+
}
1639+
return builder;
1640+
}
15761641
}

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java‎

Lines changed: 53 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import java.io.IOException;
2121
import java.net.InetSocketAddress;
2222
import java.net.MalformedURLException;
23+
import java.net.URI;
2324
import java.net.URL;
2425
import java.security.PrivilegedExceptionAction;
2526
import java.util.concurrent.Callable;
@@ -84,8 +85,9 @@ public class IPCLoggerChannel implements AsyncLogger {
8485

8586
private final String journalId;
8687
private final NamespaceInfo nsInfo;
87-
private int httpPort = -1;
88-
88+
89+
private URL httpServerURL;
90+
8991
private final IPCLoggerChannelMetrics metrics;
9092

9193
/**
@@ -241,13 +243,12 @@ protected ExecutorService createExecutor() {
241243
public URL buildURLToFetchLogs(long segmentTxId) {
242244
Preconditions.checkArgument(segmentTxId > 0,
243245
"Invalid segment: %s", segmentTxId);
244-
Preconditions.checkState(httpPort != -1,
245-
"HTTP port not set yet");
246+
Preconditions.checkState(hasHttpServerEndPoint(), "No HTTP/HTTPS endpoint");
246247

247248
try {
248249
String path = GetJournalEditServlet.buildPath(
249250
journalId, segmentTxId, nsInfo);
250-
return new URL("http", addr.getHostName(), httpPort, path.toString());
251+
return new URL(httpServerURL, path);
251252
} catch (MalformedURLException e) {
252253
// should never get here.
253254
throw new RuntimeException(e);
@@ -313,7 +314,7 @@ public ListenableFuture<GetJournalStateResponseProto> getJournalState() {
313314
public GetJournalStateResponseProto call() throws IOException {
314315
GetJournalStateResponseProto ret =
315316
getProxy().getJournalState(journalId);
316-
httpPort = ret.getHttpPort();
317+
constructHttpServerURI(ret);
317318
return ret;
318319
}
319320
});
@@ -528,7 +529,7 @@ public RemoteEditLogManifest call() throws IOException {
528529
journalId, fromTxnId, forReading, inProgressOk);
529530
// Update the http port, since we need this to build URLs to any of the
530531
// returned logs.
531-
httpPort = ret.getHttpPort();
532+
constructHttpServerURI(ret);
532533
return PBHelper.convert(ret.getManifest());
533534
}
534535
});
@@ -540,10 +541,12 @@ public ListenableFuture<PrepareRecoveryResponseProto> prepareRecovery(
540541
return executor.submit(new Callable<PrepareRecoveryResponseProto>() {
541542
@Override
542543
public PrepareRecoveryResponseProto call() throws IOException {
543-
if (httpPort < 0) {
544-
// If the HTTP port hasn't been set yet, force an RPC call so we know
545-
// what the HTTP port should be.
546-
httpPort = getProxy().getJournalState(journalId).getHttpPort();
544+
if (!hasHttpServerEndPoint()) {
545+
// force an RPC call so we know what the HTTP port should be if it
546+
// haven't done so.
547+
GetJournalStateResponseProto ret = getProxy().getJournalState(
548+
journalId);
549+
constructHttpServerURI(ret);
547550
}
548551
return getProxy().prepareRecovery(createReqInfo(), segmentTxId);
549552
}
@@ -594,4 +597,43 @@ public synchronized long getLagTimeMillis() {
594597
Math.max(lastCommitNanos - lastAckNanos, 0),
595598
TimeUnit.NANOSECONDS);
596599
}
600+
601+
private void constructHttpServerURI(GetEditLogManifestResponseProto ret) {
602+
if (ret.hasFromURL()) {
603+
URI uri = URI.create(ret.getFromURL());
604+
httpServerURL = getHttpServerURI(uri.getScheme(), uri.getPort());
605+
} else {
606+
httpServerURL = getHttpServerURI("http", ret.getHttpPort());;
607+
}
608+
}
609+
610+
private void constructHttpServerURI(GetJournalStateResponseProto ret) {
611+
if (ret.hasFromURL()) {
612+
URI uri = URI.create(ret.getFromURL());
613+
httpServerURL = getHttpServerURI(uri.getScheme(), uri.getPort());
614+
} else {
615+
httpServerURL = getHttpServerURI("http", ret.getHttpPort());;
616+
}
617+
}
618+
619+
/**
620+
* Construct the http server based on the response.
621+
*
622+
* The fromURL field in the response specifies the endpoint of the http
623+
* server. However, the address might not be accurate since the server can
624+
* bind to multiple interfaces. Here the client plugs in the address specified
625+
* in the configuration and generates the URI.
626+
*/
627+
private URL getHttpServerURI(String scheme, int port) {
628+
try {
629+
return new URL(scheme, addr.getHostName(), port, "");
630+
} catch (MalformedURLException e) {
631+
// Unreachable
632+
throw new RuntimeException(e);
633+
}
634+
}
635+
636+
private boolean hasHttpServerEndPoint() {
637+
return httpServerURL != null;
638+
}
597639
}

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public class JournalNode implements Tool, Configurable, JournalNodeMXBean {
6464
private JournalNodeHttpServer httpServer;
6565
private Map<String, Journal> journalsById = Maps.newHashMap();
6666
private ObjectName journalNodeInfoBeanName;
67-
67+
private String httpServerURI;
6868
private File localDir;
6969

7070
static {
@@ -140,6 +140,8 @@ public void start() throws IOException {
140140
httpServer = new JournalNodeHttpServer(conf, this);
141141
httpServer.start();
142142

143+
httpServerURI = httpServer.getServerURI().toString();
144+
143145
rpcServer = new JournalNodeRpcServer(conf, this);
144146
rpcServer.start();
145147
}
@@ -155,11 +157,14 @@ public InetSocketAddress getBoundIpcAddress() {
155157
return rpcServer.getAddress();
156158
}
157159

158-
160+
@Deprecated
159161
public InetSocketAddress getBoundHttpAddress() {
160162
return httpServer.getAddress();
161163
}
162164

165+
public String getHttpServerURI() {
166+
return httpServerURI;
167+
}
163168

164169
/**
165170
* Stop the daemon with the given status code

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java‎

Lines changed: 25 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -17,42 +17,28 @@
1717
*/
1818
package org.apache.hadoop.hdfs.qjournal.server;
1919

20-
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN;
21-
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_JOURNALNODE_KEYTAB_FILE_KEY;
22-
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_JOURNALNODE_INTERNAL_SPNEGO_USER_NAME_KEY;
23-
2420
import java.io.IOException;
2521
import java.net.InetSocketAddress;
2622
import java.net.URI;
27-
import java.net.URISyntaxException;
2823

2924
import javax.servlet.ServletContext;
3025

31-
import org.apache.commons.logging.Log;
32-
import org.apache.commons.logging.LogFactory;
3326
import org.apache.hadoop.classification.InterfaceAudience;
3427
import org.apache.hadoop.conf.Configuration;
3528
import org.apache.hadoop.hdfs.DFSConfigKeys;
3629
import org.apache.hadoop.hdfs.DFSUtil;
3730
import org.apache.hadoop.hdfs.server.common.JspHelper;
3831
import org.apache.hadoop.http.HttpServer;
3932
import org.apache.hadoop.net.NetUtils;
40-
import org.apache.hadoop.security.authorize.AccessControlList;
41-
import org.apache.hadoop.security.SecurityUtil;
42-
import org.apache.hadoop.security.UserGroupInformation;
4333

4434
/**
4535
* Encapsulates the HTTP server started by the Journal Service.
4636
*/
4737
@InterfaceAudience.Private
4838
public class JournalNodeHttpServer {
49-
public static final Log LOG = LogFactory.getLog(
50-
JournalNodeHttpServer.class);
51-
5239
public static final String JN_ATTRIBUTE_KEY = "localjournal";
5340

5441
private HttpServer httpServer;
55-
private int infoPort;
5642
private JournalNode localJournalNode;
5743

5844
private final Configuration conf;
@@ -63,40 +49,24 @@ public class JournalNodeHttpServer {
6349
}
6450

6551
void start() throws IOException {
66-
final InetSocketAddress bindAddr = getAddress(conf);
67-
68-
// initialize the webserver for uploading/downloading files.
69-
LOG.info("Starting web server as: "+ SecurityUtil.getServerPrincipal(conf
70-
.get(DFS_JOURNALNODE_INTERNAL_SPNEGO_USER_NAME_KEY),
71-
bindAddr.getHostName()));
72-
73-
int tmpInfoPort = bindAddr.getPort();
74-
URI httpEndpoint;
75-
try {
76-
httpEndpoint = new URI("http://" + NetUtils.getHostPortString(bindAddr));
77-
} catch (URISyntaxException e) {
78-
throw new IOException(e);
79-
}
52+
final InetSocketAddress httpAddr = getAddress(conf);
8053

81-
httpServer = new HttpServer.Builder().setName("journal")
82-
.addEndpoint(httpEndpoint)
83-
.setFindPort(tmpInfoPort == 0).setConf(conf).setACL(
84-
new AccessControlList(conf.get(DFS_ADMIN, " ")))
85-
.setSecurityEnabled(UserGroupInformation.isSecurityEnabled())
86-
.setUsernameConfKey(
87-
DFS_JOURNALNODE_INTERNAL_SPNEGO_USER_NAME_KEY)
88-
.setKeytabConfKey(DFSUtil.getSpnegoKeytabKey(conf,
89-
DFS_JOURNALNODE_KEYTAB_FILE_KEY)).build();
54+
final String httpsAddrString = conf.get(
55+
DFSConfigKeys.DFS_JOURNALNODE_HTTPS_ADDRESS_KEY,
56+
DFSConfigKeys.DFS_JOURNALNODE_HTTPS_ADDRESS_DEFAULT);
57+
InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString);
58+
59+
HttpServer.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
60+
httpAddr, httpsAddr, "journal",
61+
DFSConfigKeys.DFS_JOURNALNODE_INTERNAL_SPNEGO_USER_NAME_KEY,
62+
DFSConfigKeys.DFS_JOURNALNODE_KEYTAB_FILE_KEY);
63+
64+
httpServer = builder.build();
9065
httpServer.setAttribute(JN_ATTRIBUTE_KEY, localJournalNode);
9166
httpServer.setAttribute(JspHelper.CURRENT_CONF, conf);
9267
httpServer.addInternalServlet("getJournal", "/getJournal",
9368
GetJournalEditServlet.class, true);
9469
httpServer.start();
95-
96-
// The web-server port can be ephemeral... ensure we have the correct info
97-
infoPort = httpServer.getConnectorAddress(0).getPort();
98-
99-
LOG.info("Journal Web-server up at: " + bindAddr + ":" + infoPort);
10070
}
10171

10272
void stop() throws IOException {
@@ -112,12 +82,25 @@ void stop() throws IOException {
11282
/**
11383
* Return the actual address bound to by the running server.
11484
*/
85+
@Deprecated
11586
public InetSocketAddress getAddress() {
11687
InetSocketAddress addr = httpServer.getConnectorAddress(0);
11788
assert addr.getPort() != 0;
11889
return addr;
11990
}
12091

92+
/**
93+
* Return the URI that locates the HTTP server.
94+
*/
95+
URI getServerURI() {
96+
// getHttpClientScheme() only returns https for HTTPS_ONLY policy. This
97+
// matches the behavior that the first connector is a HTTPS connector only
98+
// for HTTPS_ONLY policy.
99+
InetSocketAddress addr = httpServer.getConnectorAddress(0);
100+
return URI.create(DFSUtil.getHttpClientScheme(conf) + "://"
101+
+ NetUtils.getHostPortString(addr));
102+
}
103+
121104
private static InetSocketAddress getAddress(Configuration conf) {
122105
String addr = conf.get(DFSConfigKeys.DFS_JOURNALNODE_HTTP_ADDRESS_KEY,
123106
DFSConfigKeys.DFS_JOURNALNODE_HTTP_ADDRESS_DEFAULT);

0 commit comments

Comments
 (0)