YARN-961. Changed ContainerManager to enforce Token auth irrespective of security. Contributed by Omkar Vinit Joshi.

svn merge --ignore-ancestry -c 1508216 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2.1-beta@1508218 13f79535-47bb-0310-9956-ffa450edef68

Author: vinoduec

hadoop-yarn-project/CHANGES.txt

@@ -738,6 +738,12 @@ Release 2.1.0-beta - 2013-07-02
     YARN-245. Fixed NodeManager to handle duplicate responses from
     ResourceManager. (Mayank Bansal via vinodkv)
 
+    YARN-932. TestResourceLocalizationService.testLocalizationInit can fail on
+    JDK7. (Karthik Kambatla via Sandy Ryza)
+
+    YARN-961. Changed ContainerManager to enforce Token auth irrespective of
+    security. (Omkar Vinit Joshi via vinodkv)
+
   BREAKDOWN OF HADOOP-8562/YARN-191 SUBTASKS AND RELATED JIRAS
 
     YARN-158. Yarn creating package-info.java must not depend on sh.
@@ -803,9 +809,6 @@ Release 2.1.0-beta - 2013-07-02
     YARN-909. Disable TestLinuxContainerExecutorWithMocks on Windows. (Chuan Liu
     via cnauroth)
 
-    YARN-932. TestResourceLocalizationService.testLocalizationInit can fail on
-    JDK7. (Karthik Kambatla via Sandy Ryza)
-
 Release 2.0.5-alpha - 06/06/2013
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java

@@ -40,6 +40,7 @@
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SaslRpcServer;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.PolicyProvider;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
@@ -230,6 +231,13 @@ protected void serviceStart() throws Exception {
     // Enqueue user dirs in deletion context
 
     Configuration conf = getConfig();
+    Configuration serverConf = new Configuration(conf);
+
+    // always enforce it to be token-based.
+    serverConf.set(
+      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+      SaslRpcServer.AuthMethod.TOKEN.toString());
+    
     YarnRPC rpc = YarnRPC.create(conf);
 
     InetSocketAddress initialAddress = conf.getSocketAddr(
@@ -238,8 +246,8 @@ protected void serviceStart() throws Exception {
         YarnConfiguration.DEFAULT_NM_PORT);
 
     server =
-        rpc.getServer(ContainerManagementProtocol.class, this, initialAddress, conf,
-            this.context.getNMTokenSecretManager(),
+        rpc.getServer(ContainerManagementProtocol.class, this, initialAddress, 
+            serverConf, this.context.getNMTokenSecretManager(),
             conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT, 
                 YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT));
 
@@ -249,7 +257,7 @@ protected void serviceStart() throws Exception {
         false)) {
       refreshServiceAcls(conf, new NMPolicyProvider());
     }
-
+    
     LOG.info("Blocking new container-requests as container manager rpc" +
     		" server is still starting.");
     this.setBlockNewContainerRequests(true);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java

@@ -33,6 +33,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.security.SaslRpcServer;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.PolicyProvider;
 import org.apache.hadoop.security.token.TokenIdentifier;
@@ -119,12 +120,11 @@ protected void serviceStart() throws Exception {
         YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
 
     Configuration serverConf = conf;
-    if (!UserGroupInformation.isSecurityEnabled()) {
-      // If the auth is not-simple, enforce it to be token-based.
-      serverConf = new Configuration(conf);
-      serverConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
-        UserGroupInformation.AuthenticationMethod.TOKEN.toString());
-    }
+    // If the auth is not-simple, enforce it to be token-based.
+    serverConf = new Configuration(conf);
+    serverConf.set(
+        CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+        SaslRpcServer.AuthMethod.TOKEN.toString());
     this.server =
       rpc.getServer(ApplicationMasterProtocol.class, this, masterServiceAddress,
           serverConf, this.rmContext.getAMRMTokenSecretManager(),

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java

@@ -253,15 +253,14 @@ public ApplicationMasterProtocol run() {
     } catch (Exception e) {
       // Because there are no tokens, the request should be rejected as the
       // server side will assume we are trying simple auth.
-      String availableAuthMethods;
+      String expectedMessage = "";
       if (UserGroupInformation.isSecurityEnabled()) {
-        availableAuthMethods = "[TOKEN, KERBEROS]";
+        expectedMessage = "Client cannot authenticate via:[TOKEN]";
       } else {
-        availableAuthMethods = "[TOKEN]";
+        expectedMessage =
+            "SIMPLE authentication is not enabled.  Available:[TOKEN]";
       }
-      Assert.assertTrue(e.getCause().getMessage().contains(
-        "SIMPLE authentication is not enabled.  "
-            + "Available:" + availableAuthMethods));
+      Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage));
     }
 
     // TODO: Add validation of invalid authorization when there's more data in

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java

@@ -24,6 +24,8 @@
 import java.net.InetSocketAddress;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 
 import junit.framework.Assert;
@@ -63,36 +65,45 @@
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Records;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
 
+@RunWith(Parameterized.class)
 public class TestContainerManagerSecurity {
 
   static Log LOG = LogFactory.getLog(TestContainerManagerSecurity.class);
   static final RecordFactory recordFactory = RecordFactoryProvider
       .getRecordFactory(null);
   private static MiniYARNCluster yarnCluster;
 
-  static final Configuration conf = new Configuration();
+  private Configuration conf;
 
-  @Test (timeout = 1000000)
-  public void testContainerManagerWithSecurityEnabled() throws Exception {
-    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+  @Parameters
+  public static Collection<Object[]> configs() {
+    Configuration configurationWithoutSecurity = new Configuration();
+    configurationWithoutSecurity.set(
+        CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "simple");
+    
+    Configuration configurationWithSecurity = new Configuration();
+    configurationWithSecurity.set(
+        CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
         "kerberos");
-    testContainerManager();
+    return Arrays.asList(new Object[][] { { configurationWithoutSecurity },
+        { configurationWithSecurity } });
   }
 
-  @Test (timeout=1000000)
-  public void testContainerManagerWithSecurityDisabled() throws Exception {
-    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
-        "simple");
-    testContainerManager();
+  public TestContainerManagerSecurity(Configuration conf) {
+    conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 100000L);
+    UserGroupInformation.setConfiguration(conf);
+    this.conf = conf;
   }
 
-  private void testContainerManager() throws Exception {
+  @Test (timeout = 1000000)
+  public void testContainerManager() throws Exception {
     try {
       yarnCluster = new MiniYARNCluster(TestContainerManagerSecurity.class
           .getName(), 1, 1, 1);
-      conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 100000L);
-      UserGroupInformation.setConfiguration(conf);
       yarnCluster.init(conf);
       yarnCluster.start();
 
@@ -184,6 +195,18 @@ private void testNMTokens(Configuration conf) throws Exception {
     } while (tempManager.getCurrentKey().getKeyId() == nmTokenSecretManagerRM
         .getCurrentKey().getKeyId());
 
+    // Testing that NM rejects the requests when we don't send any token.
+    if (UserGroupInformation.isSecurityEnabled()) {
+      sb = new StringBuilder("Client cannot authenticate via:[TOKEN]");
+    } else {
+      sb =
+          new StringBuilder(
+              "SIMPLE authentication is not enabled.  Available:[TOKEN]");
+    }
+    String errorMsg = testStartContainer(rpc, validAppAttemptId, validNode,
+        validContainerToken, null, true);
+    Assert.assertTrue(errorMsg.contains(sb.toString()));
+    
     org.apache.hadoop.yarn.api.records.Token invalidNMToken =
         tempManager.createNMToken(validAppAttemptId, validNode, user);
     sb = new StringBuilder("Given NMToken for application : ");
@@ -402,7 +425,9 @@ protected ContainerManagementProtocol getContainerManagementProtocolProxy(
     UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
     final InetSocketAddress addr =
         NetUtils.createSocketAddr(nodeId.getHost(), nodeId.getPort());
-    ugi.addToken(ConverterUtils.convertFromYarn(nmToken, addr));
+    if (nmToken != null) {
+      ugi.addToken(ConverterUtils.convertFromYarn(nmToken, addr));      
+    }
 
     proxy = ugi
         .doAs(new PrivilegedAction<ContainerManagementProtocol>() {