MAPREDUCE-2483. Remove duplication of jars between Hadoop subprojects

from build artifacts. (Eric Yang via omalley)


git-svn-id: https://svn.apache.org/repos/asf/hadoop/mapreduce/trunk@1125017 13f79535-47bb-0310-9956-ffa450edef68

Author: omalley

CHANGES.txt

@@ -20,7 +20,8 @@ Trunk (unreleased changes)
     MAPREDUCE-1461. Feature to instruct rumen-folder utility to skip jobs worth 
     of specific duration. (Rajesh Balamohan via amarrk)
 
-    MAPREDUCE-2172. Added test-patch.properties required by test-patch.sh (nigel)
+    MAPREDUCE-2172. Added test-patch.properties required by test-patch.sh 
+    (nigel)
 
     MAPREDUCE-2156. Raid-aware FSCK. (Patrick Kling via dhruba)
 
@@ -112,6 +113,9 @@ Trunk (unreleased changes)
     MAPREDUCE-2449. Allow for command line arguments when performing
     "Run on Hadoop" action in Eclipse plugin. (Jeff Zemerick via todd)
 
+    MAPREDUCE-2483. Remove duplication of jars between Hadoop subprojects
+    from build artifacts. (Eric Yang via omalley)
+
   OPTIMIZATIONS
 
     MAPREDUCE-2026. Make JobTracker.getJobCounters() and

build.xml

@@ -197,6 +197,7 @@
   <property name="build.ivy.dir" location="${build.dir}/ivy" />
   <property name="build.ivy.lib.dir" location="${build.ivy.dir}/lib" />
   <property name="common.ivy.lib.dir" location="${build.ivy.lib.dir}/${ant.project.name}/common"/>
+  <property name="mapred.ivy.lib.dir" location="${build.ivy.lib.dir}/${ant.project.name}/mapred"/>
   <property name="build.ivy.report.dir" location="${build.ivy.dir}/report" />
   <property name="build.ivy.maven.dir" location="${build.ivy.dir}/maven" />
   <property name="build.ivy.maven.pom" location="${build.ivy.maven.dir}/hadoop-mapred-${version}.pom" />
@@ -255,6 +256,7 @@
     <pathelement location="${build.classes}"/>
     <pathelement location="${conf.dir}"/>
     <path refid="ivy-common.classpath"/>
+    <path refid="ivy-mapred.classpath"/>
   </path>
 
   <path id="test.classpath">
@@ -268,6 +270,7 @@
     <pathelement location="${build.classes}"/>
     <pathelement location="${test.conf.dir}"/>
     <path refid="ivy-common.classpath"/>
+    <path refid="ivy-mapred.classpath"/>
     <pathelement location="${test.mapred.build.classes}" />
     <path refid="ivy-test.classpath"/>
   </path>
@@ -298,7 +301,7 @@
   <!-- ====================================================== -->
   <!-- Stuff needed by all targets                            -->
   <!-- ====================================================== -->
-  <target name="init" depends="ivy-retrieve-common">
+  <target name="init" depends="ivy-retrieve-common,ivy-retrieve-mapred">
     <mkdir dir="${build.dir}"/>
     <mkdir dir="${build.classes}"/>
     <mkdir dir="${build.tools}"/>
@@ -1160,7 +1163,7 @@
     <mkdir dir="${dist.dir}/docs/jdiff"/>
 
     <copy todir="${dist.dir}/lib" includeEmptyDirs="false" flatten="true">
-      <fileset dir="${common.ivy.lib.dir}"/>
+      <fileset dir="${mapred.ivy.lib.dir}"/>
     </copy>
 
     <copy todir="${dist.dir}/lib" includeEmptyDirs="false">
@@ -1283,7 +1286,7 @@
     <mkdir dir="${dist.dir}/bin"/>
 
     <copy todir="${dist.dir}/lib" includeEmptyDirs="false" flatten="true">
-      <fileset dir="${common.ivy.lib.dir}"/>
+      <fileset dir="${mapred.ivy.lib.dir}"/>
     </copy>
 
     <copy todir="${dist.dir}/lib" includeEmptyDirs="false">
@@ -1938,7 +1941,7 @@
   </target>
 
   <target name="eclipse" 
-          depends="init,ant-eclipse-download,ivy-retrieve-common,ivy-retrieve-test"
+          depends="init,ant-eclipse-download,ivy-retrieve-common,ivy-retrieve-mapred,ivy-retrieve-test"
           description="Create eclipse project files">
        <pathconvert property="eclipse.project">
          <path path="${basedir}"/>
@@ -2012,6 +2015,7 @@
                 output="${build.dir.eclipse-contrib-classes}/vertica/test" />
         <output path="${build.dir.eclipse-main-classes}" />
         <library pathref="ivy-common.classpath" exported="true" />
+        <library pathref="ivy-mapred.classpath" exported="true" />
         <library pathref="ivy-test.classpath" exported="false" />
         <library path="${conf.dir}" exported="false" />
       </classpath>
@@ -2102,6 +2106,11 @@
       log="${ivyresolvelog}"/>
   </target>
 
+  <target name="ivy-resolve-mapred" depends="ivy-init">
+    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="mapred"
+      log="${ivyresolvelog}"/>
+  </target>
+
   <target name="ivy-resolve-jdiff" depends="ivy-init">
     <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="jdiff" 
       log="${ivyresolvelog}"/>
@@ -2164,6 +2173,14 @@
     <ivy:cachepath pathid="ivy-common.classpath" conf="common"/>
   </target>
 
+  <target name="ivy-retrieve-mapred" depends="ivy-resolve-mapred"
+    description="Retrieve Ivy-managed artifacts for the mapred configurations">
+    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+        log="${ivyresolvelog}"/>
+    <ivy:cachepath pathid="ivy-mapred.classpath" conf="mapred"/>
+  </target>
+
   <target name="ivy-retrieve-releaseaudit" depends="ivy-resolve-releaseaudit"
     description="Retrieve Ivy-managed artifacts for the compile configurations">
     <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"

ivy.xml

@@ -36,9 +36,10 @@
     -->
     <!--Private configurations. -->
 
-    <conf name="common" visibility="private" extends="compile,runtime" description="common artifacts"/>
+    <conf name="common" visibility="private" extends="compile" description="common artifacts"/>
+    <conf name="mapred" visibility="private" extends="compile,runtime" description="Mapred dependent artifacts"/>
     <conf name="javadoc" visibility="private" description="artiracts required while performing doc generation" extends="common"/>
-    <conf name="test" extends="common" visibility="private" description="the classpath needed to run tests"/>
+    <conf name="test" extends="master" visibility="private" description="the classpath needed to run tests"/>
     <conf name="system" extends="test" visibility="private" description="the classpath needed to run system tests"/>
 
     <conf name="test-hdfswithmr" extends="test" visibility="private" description="the classpath needed to run tests"/>
@@ -73,7 +74,7 @@
    <dependency org="org.slf4j" name="slf4j-api" rev="${slf4j-api.version}" 
                conf="common->master"/>
    <dependency org="org.slf4j" name="slf4j-log4j12" 
-               rev="${slf4j-log4j12.version}" conf="common->master"/>
+               rev="${slf4j-log4j12.version}" conf="mapred->master"/>
    <dependency org="org.apache.hadoop" name="hadoop-common-test" 
                rev="${hadoop-common.version}" conf="test->default"/>
    <dependency org="org.apache.hadoop" name="hadoop-hdfs-test" 

src/contrib/mumak/build.xml

@@ -96,11 +96,6 @@ to call at top-level: ant deploy-contrib compile-core-test
     <copy todir="${dist.dir}/contrib/${name}/conf" includeEmptyDirs="true">
       <fileset dir="${root}/conf" />
     </copy>
-    <copy todir="${dist.dir}/lib" includeEmptyDirs="false">
-      <fileset dir="${common.ivy.lib.dir}">
-        <include name="*.jar"/>
-      </fileset>
-    </copy>
     <exec executable="sed">
       <arg value="-i"/>
       <arg value="-e"/>