Added support for WormholePushJob job type

Author: rakeshmalladi

CONTRIBUTORS.md

@@ -157,3 +157,6 @@ The following were contributed by Jamie Sun. Thanks, Jamie!
 * `Fix base properties scope for subflows when generating YAML files`
 * `Fix variable substitution issue when generating YAML files`
 * `Fix variable substitution issue for dependency list`
+
+The following were contributed by Rakesh Malladi. Thanks, Rakesh!
+* `Add ability for Hadoop DSL to understand the WormholePushJob job type`

VERSIONS.md

@@ -17,6 +17,9 @@ the License.
 Note that the LinkedIn build system occasionally requires that we skip a
 version bump, so you will see a few skipped version numbers in the list below.
 
+0.14.30
+* Adding WormholePushJob job type support
+
 0.14.29
 * Fix variable substitution issue for dependency list
 

gradle.properties

@@ -1,2 +1,2 @@
 org.gradle.daemon=true
-version=0.14.29
+version=0.14.30

hadoop-plugin-test/expectedJobs/jobs1/jobs1.job

@@ -1,3 +1,3 @@
 # This file generated from the Hadoop DSL. Do not edit by hand.
 type=noop
-dependencies=jobs1_job25
+dependencies=jobs1_job26

hadoop-plugin-test/expectedJobs/jobs1/jobs1_job26.job

@@ -0,0 +1,10 @@
+# This file generated from the Hadoop DSL. Do not edit by hand.
+type=WormholePushJob
+dependencies=jobs1_job25
+dataset=model
+dataset.group=linkedin-project
+input.path=/user/input
+namespace=linkedin
+output.location=LOCAL
+preserve.input=true
+version.string=1.2.4-2018.09.04

hadoop-plugin-test/expectedOutput/negative/missingFields.out

@@ -44,5 +44,6 @@ RequiredFieldsChecker ERROR: PinotBuildAndPushJob job18 must set tableName, inpu
 RequiredFieldsChecker ERROR: TableauJob job19 must set workbookName
 RequiredFieldsChecker ERROR: HdfsWaitJob job20 must set dirPath, freshness, timeout, and forceJobToFail
 RequiredFieldsChecker ERROR: VenicePushJob job23 must set avroKeyField, avroValueField, clusterName, inputPath, veniceStoreName
+RequiredFieldsChecker ERROR: WormholePushJob job26 must set inputPath, namespace, datasetGroup and dataset
 RequiredFieldsChecker WARNING: Properties properties1 does not set any confProperties, jobProperties, jvmProperties or basePropertySetName. Nothing will be built for this properties object.
 :hadoop-plugin-test:test_missingFields FAILED

hadoop-plugin-test/src/main/gradle/negative/missingFields.gradle

@@ -122,6 +122,10 @@ hadoop {
     tensorFlowJob('job25', 'spark') {
       depends 'job24'
     }
-    targets 'job25'
+
+    wormholePushJob('job26') {
+      depends 'job25'
+    }
+    targets 'job26'
   }
 }

hadoop-plugin-test/src/main/gradle/positive/jobs1.gradle

@@ -419,6 +419,18 @@ hadoop {
       ]
       depends 'job24'
     }
-    targets 'job25'
+
+    wormholePushJob('job26') {
+      fromInputPath '/user/input'            // Required
+      toNamespace 'linkedin'                 // Required
+      toDatasetGroup 'linkedin-project'      // Required
+      toDataset 'model'                      // Required
+      withVersionString '1.2.4-2018.09.04'   // Optional
+      preserveInput true                     // Optional
+      toOutputLocation 'LOCAL'               // Optional
+
+      depends 'job25'
+    }
+    targets 'job26'
   }
 }

hadoop-plugin/src/main/groovy/com/linkedin/gradle/hadoopdsl/BaseNamedScopeContainer.groovy

@@ -39,7 +39,7 @@ import com.linkedin.gradle.hadoopdsl.job.TensorFlowTonyJob;
 import com.linkedin.gradle.hadoopdsl.job.TeradataToHdfsJob;
 import com.linkedin.gradle.hadoopdsl.job.VenicePushJob;
 import com.linkedin.gradle.hadoopdsl.job.VoldemortBuildPushJob;
-
+import com.linkedin.gradle.hadoopdsl.job.WormholePushJob;
 import org.gradle.api.Project;
 
 /**
@@ -1116,6 +1116,19 @@ abstract class BaseNamedScopeContainer implements NamedScopeContainer {
     }
   }
 
+  /**
+   * DSL wormholePushJob method. Creates a wormholePushJob in scope with the given name
+   * and configuration.
+   *
+   * @param name The job name
+   * @param configure The configuration closure
+   * @return The new job
+   */
+  @HadoopDslMethod
+  WormholePushJob wormholePushJob(String name, @DelegatesTo(WormholePushJob) Closure configure) {
+    return ((WormholePushJob)configureJob(factory.makeWormholePushJob(name), configure));
+  }
+
   /**
    * DSL trigger method. Creates a Trigger in scope with the given name and configuration.
    *

hadoop-plugin/src/main/groovy/com/linkedin/gradle/hadoopdsl/BaseVisitor.groovy

@@ -37,6 +37,7 @@ import com.linkedin.gradle.hadoopdsl.job.TensorFlowJob;
 import com.linkedin.gradle.hadoopdsl.job.TeradataToHdfsJob;
 import com.linkedin.gradle.hadoopdsl.job.VenicePushJob;
 import com.linkedin.gradle.hadoopdsl.job.VoldemortBuildPushJob;
+import com.linkedin.gradle.hadoopdsl.job.WormholePushJob;
 
 /**
  * Base implementation of Visitor to make it easy to implement visitor subclasses. By default, the
@@ -231,6 +232,11 @@ abstract class BaseVisitor implements Visitor {
     visitJob((Job)job);
   }
 
+  @Override
+  void visitJob(WormholePushJob job) {
+    visitJob((Job)job);
+  }
+
   @Override
   void visitJob(TensorFlowJob job) {
     visitJob((Job)job);