YARN-1157. Fixed ResourceManager UI to behave correctly when apps like distributed-shell do not set tracking urls. Contributed by Xuan Gong.

svn merge --ignore-ancestry -c 1526371 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1526372 13f79535-47bb-0310-9956-ffa450edef68

Author: vinoduec

hadoop-yarn-project/CHANGES.txt

@@ -68,6 +68,9 @@ Release 2.1.2 - UNRELEASED
     YARN-49. Improve distributed shell application to work on a secure cluster.
     (Vinod Kumar Vavilapalli via hitesh)
 
+    YARN-1157. Fixed ResourceManager UI to behave correctly when apps like
+    distributed-shell do not set tracking urls. (Xuan Gong via vinodkv)
+
 Release 2.1.1-beta - 2013-09-23
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java

@@ -100,11 +100,22 @@ public static FinishApplicationMasterRequest newInstance(
   public abstract String getTrackingUrl();
 
   /**
-   * Set the <em>tracking URL</em>for the <code>ApplicationMaster</code>
-   * This url if contains scheme then that will be used by resource manager
-   * web application proxy otherwise it will default to http.
-   * @param url <em>tracking URL</em>for the
-   *                   <code>ApplicationMaster</code>
+   * Set the <em>final tracking URL</em>for the <code>ApplicationMaster</code>.
+   * This is the web-URL to which ResourceManager or web-application proxy will
+   * redirect client/users once the application is finished and the
+   * <code>ApplicationMaster</code> is gone.
+   * <p>
+   * If the passed url has a scheme then that will be used by the
+   * ResourceManager and web-application proxy, otherwise the scheme will
+   * default to http.
+   * </p>
+   * <p>
+   * Empty, null, "N/A" strings are all valid besides a real URL. In case an url
+   * isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
+   * <p>
+   *
+   * @param url
+   *          <em>tracking URL</em>for the <code>ApplicationMaster</code>
    */
   @Public
   @Stable

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java

@@ -112,11 +112,22 @@ public static RegisterApplicationMasterRequest newInstance(String host,
   public abstract String getTrackingUrl();
 
   /**
-   * Set the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
-   * This url if contains scheme then that will be used by resource manager
-   * web application proxy otherwise it will default to http.
-   * @param trackingUrl <em>tracking URL</em> for the 
-   *                    <code>ApplicationMaster</code>
+   * Set the <em>tracking URL</em>for the <code>ApplicationMaster</code> while
+   * it is running. This is the web-URL to which ResourceManager or
+   * web-application proxy will redirect client/users while the application and
+   * the <code>ApplicationMaster</code> are still running.
+   * <p>
+   * If the passed url has a scheme then that will be used by the
+   * ResourceManager and web-application proxy, otherwise the scheme will
+   * default to http.
+   * </p>
+   * <p>
+   * Empty, null, "N/A" strings are all valid besides a real URL. In case an url
+   * isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
+   * <p>
+   *
+   * @param trackingUrl
+   *          <em>tracking URL</em>for the <code>ApplicationMaster</code>
    */
   @Public
   @Stable

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java

@@ -994,7 +994,7 @@ public void transition(RMAppAttemptImpl appAttempt,
     }
   }
 
-  static final class AMRegisteredTransition extends BaseTransition {
+  private static final class AMRegisteredTransition extends BaseTransition {
     @Override
     public void transition(RMAppAttemptImpl appAttempt,
         RMAppAttemptEvent event) {
@@ -1003,7 +1003,8 @@ public void transition(RMAppAttemptImpl appAttempt,
           = (RMAppAttemptRegistrationEvent) event;
       appAttempt.host = registrationEvent.getHost();
       appAttempt.rpcPort = registrationEvent.getRpcport();
-      appAttempt.origTrackingUrl = registrationEvent.getTrackingurl();
+      appAttempt.origTrackingUrl =
+          sanitizeTrackingUrl(registrationEvent.getTrackingurl());
       appAttempt.proxiedTrackingUrl = 
         appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl);
 
@@ -1138,7 +1139,8 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt,
       RMAppAttemptUnregistrationEvent unregisterEvent
         = (RMAppAttemptUnregistrationEvent) event;
       appAttempt.diagnostics.append(unregisterEvent.getDiagnostics());
-      appAttempt.origTrackingUrl = unregisterEvent.getTrackingUrl();
+      appAttempt.origTrackingUrl =
+          sanitizeTrackingUrl(unregisterEvent.getTrackingUrl());
       appAttempt.proxiedTrackingUrl = 
         appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl);
       appAttempt.finalStatus = unregisterEvent.getFinalApplicationStatus();
@@ -1292,4 +1294,8 @@ private void removeCredentials(RMAppAttemptImpl appAttempt) {
     appAttempt.rmContext.getAMRMTokenSecretManager()
       .applicationMasterFinished(appAttempt.getAppAttemptId());
   }
+
+  private static String sanitizeTrackingUrl(String url) {
+    return (url == null || url.trim().isEmpty()) ? "N/A" : url;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java

@@ -30,14 +30,18 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -85,8 +89,10 @@
 import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -261,8 +267,22 @@ public void tearDown() throws Exception {
 
 
   private String getProxyUrl(RMAppAttempt appAttempt) {
-    return pjoin(RM_WEBAPP_ADDR, "proxy",
-        appAttempt.getAppAttemptId().getApplicationId(), "");
+    String url = null;
+    try {
+      URI trackingUri =
+          StringUtils.isEmpty(appAttempt.getOriginalTrackingUrl()) ? null :
+              ProxyUriUtils
+                  .getUriFromAMUrl(appAttempt.getOriginalTrackingUrl());
+      String proxy = WebAppUtils.getProxyHostAndPort(conf);
+      URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy);
+      URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
+          appAttempt.getAppAttemptId().getApplicationId());
+      url = result.toASCIIString().substring(
+          HttpConfig.getSchemePrefix().length());
+    } catch (URISyntaxException ex) {
+      Assert.fail();
+    }
+    return url;
   }
 
   /**
@@ -448,9 +468,9 @@ private void testAppAttemptRunningState(Container container,
     assertEquals(container, applicationAttempt.getMasterContainer());
     assertEquals(host, applicationAttempt.getHost());
     assertEquals(rpcPort, applicationAttempt.getRpcPort());
-    assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+    verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
     if (unmanagedAM) {
-      assertEquals("oldtrackingurl", applicationAttempt.getTrackingUrl());
+      verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl());
     } else {
       assertEquals(getProxyUrl(applicationAttempt), 
           applicationAttempt.getTrackingUrl());
@@ -468,7 +488,7 @@ private void testAppAttemptFinishingState(Container container,
     assertEquals(RMAppAttemptState.FINISHING,
         applicationAttempt.getAppAttemptState());
     assertEquals(diagnostics, applicationAttempt.getDiagnostics());
-    assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+    verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
     assertEquals(getProxyUrl(applicationAttempt),
         applicationAttempt.getTrackingUrl());
     assertEquals(container, applicationAttempt.getMasterContainer());
@@ -487,9 +507,9 @@ private void testAppAttemptFinishedState(Container container,
     assertEquals(RMAppAttemptState.FINISHED, 
         applicationAttempt.getAppAttemptState());
     assertEquals(diagnostics, applicationAttempt.getDiagnostics());
-    assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+    verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
     if (unmanagedAM) {
-      assertEquals("mytrackingurl", applicationAttempt.getTrackingUrl());
+      verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl());
 
     } else {
       assertEquals(getProxyUrl(applicationAttempt),
@@ -603,9 +623,7 @@ private void unregisterApplicationAttempt(Container container,
         trackingUrl, diagnostics);
   }
 
-  
-  @Test
-  public void testUnmanagedAMSuccess() {
+  private void testUnmanagedAMSuccess(String url) {
     unmanagedAM = true;
     when(submissionContext.getUnmanagedAM()).thenReturn(true);
     // submit AM and check it goes to LAUNCHED state
@@ -615,21 +633,20 @@ public void testUnmanagedAMSuccess() {
         applicationAttempt.getAppAttemptId());
 
     // launch AM
-    runApplicationAttempt(null, "host", 8042, "oldtrackingurl", true);
+    runApplicationAttempt(null, "host", 8042, url, true);
 
     // complete a container
     applicationAttempt.handle(new RMAppAttemptContainerAcquiredEvent(
         applicationAttempt.getAppAttemptId(), mock(Container.class)));
     applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
         applicationAttempt.getAppAttemptId(), mock(ContainerStatus.class)));
     // complete AM
-    String trackingUrl = "mytrackingurl";
     String diagnostics = "Successful";
     FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
     applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
-        applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
+        applicationAttempt.getAppAttemptId(), url, finalStatus,
         diagnostics));
-    testAppAttemptFinishedState(null, finalStatus, trackingUrl, diagnostics, 1,
+    testAppAttemptFinishedState(null, finalStatus, url, diagnostics, 1,
         true);
   }
 
@@ -824,12 +841,42 @@ public void testUnregisterToKilledFinishing() {
         "Killed by user");
   }
 
+  @Test
+  public void testTrackingUrlUnmanagedAM() {
+    testUnmanagedAMSuccess("oldTrackingUrl");
+  }
 
   @Test
-  public void testNoTrackingUrl() {
+  public void testEmptyTrackingUrlUnmanagedAM() {
+    testUnmanagedAMSuccess("");
+  }
+
+  @Test
+  public void testNullTrackingUrlUnmanagedAM() {
+    testUnmanagedAMSuccess(null);
+  }
+
+  @Test
+  public void testManagedAMWithTrackingUrl() {
+    testTrackingUrlManagedAM("theTrackingUrl");
+  }
+
+  @Test
+  public void testManagedAMWithEmptyTrackingUrl() {
+    testTrackingUrlManagedAM("");
+  }
+
+  @Test
+  public void testManagedAMWithNullTrackingUrl() {
+    testTrackingUrlManagedAM(null);
+  }
+
+  private void testTrackingUrlManagedAM(String url) {
     Container amContainer = allocateApplicationAttempt();
     launchApplicationAttempt(amContainer);
-    runApplicationAttempt(amContainer, "host", 8042, "", false);
+    runApplicationAttempt(amContainer, "host", 8042, url, false);
+    unregisterApplicationAttempt(amContainer,
+        FinalApplicationStatus.SUCCEEDED, url, "Successful");
   }
 
   @Test
@@ -927,4 +974,12 @@ private void verifyTokenCount(ApplicationAttemptId appAttemptId, int count) {
       }
     }
   }
+
+  private void verifyUrl(String url1, String url2) {
+    if (url1 == null || url1.trim().isEmpty()) {
+      assertEquals("N/A", url2);
+    } else {
+      assertEquals(url1, url2);
+    }
+  }
 }