MAPREDUCE-5300. Fix backward incompatibility for o.a.h.mapreduce.filecache.DistributedCache. Contributed by Zhijie Shen.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1489227 13f79535-47bb-0310-9956-ffa450edef68

Author: acmurthy

hadoop-mapreduce-project/CHANGES.txt

@@ -487,6 +487,9 @@ Release 2.1.0-beta - UNRELEASED
     MAPREDUCE-5296. Fix backward incompatibility for JobControl by adding the
     omitted addJob. (Zhijie Shen via acmurthy) 
 
+    MAPREDUCE-5300. Fix backward incompatibility for
+    o.a.h.mapreduce.filecache.DistributedCache. (Zhijie Shen via acmurthy)
+
   BREAKDOWN OF HADOOP-8562 SUBTASKS
 
     MAPREDUCE-4739. Some MapReduce tests fail to find winutils.

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java

@@ -336,17 +336,6 @@ public static Path getStartJobCommitFile(Configuration conf, String user,
     return startCommitFile;
   }
 
-  private static long[] parseTimeStamps(String[] strs) {
-    if (null == strs) {
-      return null;
-    }
-    long[] result = new long[strs.length];
-    for(int i=0; i < strs.length; ++i) {
-      result[i] = Long.parseLong(strs[i]);
-    }
-    return result;
-  }
-
   public static void setupDistributedCache( 
       Configuration conf, 
       Map<String, LocalResource> localResources) 
@@ -356,7 +345,7 @@ public static void setupDistributedCache(
     parseDistributedCacheArtifacts(conf, localResources,  
         LocalResourceType.ARCHIVE, 
         DistributedCache.getCacheArchives(conf), 
-        parseTimeStamps(DistributedCache.getArchiveTimestamps(conf)), 
+        DistributedCache.getArchiveTimestamps(conf),
         getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES), 
         DistributedCache.getArchiveVisibilities(conf));
 
@@ -365,7 +354,7 @@ public static void setupDistributedCache(
         localResources,  
         LocalResourceType.FILE, 
         DistributedCache.getCacheFiles(conf),
-        parseTimeStamps(DistributedCache.getFileTimestamps(conf)),
+        DistributedCache.getFileTimestamps(conf),
         getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES),
         DistributedCache.getFileVisibilities(conf));
   }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java

@@ -245,14 +245,14 @@ public void testDeprecatedFunctions() throws Exception {
     Assert.assertEquals(1,
         DistributedCache.getArchiveTimestamps(conf).length);
     Assert.assertEquals(1234567890,
-        Long.parseLong(DistributedCache.getArchiveTimestamps(conf)[0]));
+        DistributedCache.getArchiveTimestamps(conf)[0]);
     DistributedCache.setFileTimestamps(conf, "1234567890");
     Assert.assertEquals(1234567890,
         conf.getLong(DistributedCache.CACHE_FILES_TIMESTAMPS, 0));
     Assert.assertEquals(1,
         DistributedCache.getFileTimestamps(conf).length);
     Assert.assertEquals(1234567890,
-        Long.parseLong(DistributedCache.getFileTimestamps(conf)[0]));
+        DistributedCache.getFileTimestamps(conf)[0]);
 
     DistributedCache.createAllSymlink(conf, new File("Test Job Cache Dir"),
         new File("Test Work Dir"));

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java

@@ -214,31 +214,49 @@ public static Path[] getLocalCacheFiles(Configuration conf)
     return StringUtils.stringToPath(conf.getStrings(MRJobConfig.CACHE_LOCALFILES));
   }
 
+  /**
+   * Parse a list of strings into longs.
+   * @param strs the list of strings to parse
+   * @return a list of longs that were parsed. same length as strs.
+   */
+  private static long[] parseTimestamps(String[] strs) {
+    if (strs == null) {
+      return null;
+    }
+    long[] result = new long[strs.length];
+    for(int i=0; i < strs.length; ++i) {
+      result[i] = Long.parseLong(strs[i]);
+    }
+    return result;
+  }
+
   /**
    * Get the timestamps of the archives.  Used by internal
    * DistributedCache and MapReduce code.
    * @param conf The configuration which stored the timestamps
-   * @return a string array of timestamps 
+   * @return a long array of timestamps
    * @throws IOException
    * @deprecated Use {@link JobContext#getArchiveTimestamps()} instead
    */
   @Deprecated
-  public static String[] getArchiveTimestamps(Configuration conf) {
-    return conf.getStrings(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS);
+  public static long[] getArchiveTimestamps(Configuration conf) {
+    return parseTimestamps(
+        conf.getStrings(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS));
   }
 
 
   /**
    * Get the timestamps of the files.  Used by internal
    * DistributedCache and MapReduce code.
    * @param conf The configuration which stored the timestamps
-   * @return a string array of timestamps 
+   * @return a long array of timestamps
    * @throws IOException
    * @deprecated Use {@link JobContext#getFileTimestamps()} instead
    */
   @Deprecated
-  public static String[] getFileTimestamps(Configuration conf) {
-    return conf.getStrings(MRJobConfig.CACHE_FILE_TIMESTAMPS);
+  public static long[] getFileTimestamps(Configuration conf) {
+    return parseTimestamps(
+        conf.getStrings(MRJobConfig.CACHE_FILE_TIMESTAMPS));
   }
 
   /**

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java

@@ -342,15 +342,31 @@ public Path[] getLocalCacheFiles()
   public Path[] getFileClassPaths() {
     return DistributedCache.getFileClassPaths(conf);
   }
-  
+
+  /**
+   * Parse a list of longs into strings.
+   * @param timestamps the list of longs to parse
+   * @return a list of string that were parsed. same length as timestamps.
+   */
+  private static String[] toTimestampStrs(long[] timestamps) {
+    if (timestamps == null) {
+      return null;
+    }
+    String[] result = new String[timestamps.length];
+    for(int i=0; i < timestamps.length; ++i) {
+      result[i] = Long.toString(timestamps[i]);
+    }
+    return result;
+  }
+
   /**
    * Get the timestamps of the archives.  Used by internal
    * DistributedCache and MapReduce code.
    * @return a string array of timestamps 
    * @throws IOException
    */
   public String[] getArchiveTimestamps() {
-    return DistributedCache.getArchiveTimestamps(conf);
+    return toTimestampStrs(DistributedCache.getArchiveTimestamps(conf));
   }
 
   /**
@@ -360,7 +376,7 @@ public String[] getArchiveTimestamps() {
    * @throws IOException
    */
   public String[] getFileTimestamps() {
-    return DistributedCache.getFileTimestamps(conf);
+    return toTimestampStrs(DistributedCache.getFileTimestamps(conf));
   }
 
   /**