Fix base properties scope for subflows when generating YAML files.

Author: jamiesjc

CONTRIBUTORS.md

@@ -154,3 +154,4 @@ The following were contributed by Chris Li.
 
 The following were contributed by Jamie Sun. Thanks, Jamie! 
 * `Add conditional workflow feature`
+* `Fix base properties scope for subflows when generating YAML files`

VERSIONS.md

@@ -17,6 +17,9 @@ the License.
 Note that the LinkedIn build system occasionally requires that we skip a
 version bump, so you will see a few skipped version numbers in the list below.
 
+0.14.27
+* Fix base properties scope for subflows when generating YAML files
+
 0.14.26
 * Bump version to match li-hadoop-plugin MP
 

gradle.properties

@@ -1,2 +1,2 @@
 org.gradle.daemon=true
-version=0.14.26
+version=0.14.27

hadoop-plugin-test/expectedJobs/embeddedFlowWithBaseProperties/embeddedFlow.flow

@@ -0,0 +1,17 @@
+nodes:
+- name: embeddedFlow
+  type: noop
+  dependsOn:
+  - embeddedFlow1
+- name: embeddedFlow1
+  type: flow
+  nodes:
+  - name: embeddedFlow1
+    type: noop
+    dependsOn:
+    - shellEcho
+  - name: shellEcho
+    type: command
+    config:
+      command: echo "This is an echoed text from embeddedFlow1."
+      user.to.proxy: foo

hadoop-plugin-test/expectedJobs/embeddedFlowWithBaseProperties/hadoop-plugin-test.project

@@ -0,0 +1 @@
+azkaban-flow-version: 2.0

hadoop-plugin-test/expectedOutput/positive/embeddedFlowWithBaseProperties.out

@@ -0,0 +1,4 @@
+:hadoop-plugin-test:test_embeddedFlowWithBaseProperties
+Running test for the file positive/embeddedFlowWithBaseProperties.gradle
+Hadoop DSL static checker PASSED
+

hadoop-plugin-test/src/main/gradle/positive/embeddedFlowWithBaseProperties.gradle

@@ -0,0 +1,37 @@
+buildscript {
+  repositories {
+    jcenter()
+  }
+  dependencies {
+    classpath files("${project.pluginTestDir}/hadoop-plugin-${project.version}.jar", "${project.pluginTestDir}/hadoop-plugin-${project.version}-SNAPSHOT.jar")
+    // Must manually specify snakeyaml for testing, not required for users
+    classpath 'org.yaml:snakeyaml:1.18'
+  }
+}
+
+apply plugin: com.linkedin.gradle.hadoop.HadoopPlugin
+
+// Test an embedded flow with base properties set in the job for Flow 2.0
+
+hadoop {
+  buildPath "jobs/embeddedFlowWithBaseProperties"
+  cleanPath false
+
+  generateYamlOutput true
+
+  workflow('embeddedFlow') {
+    propertySet('common') {
+      set properties: ['user.to.proxy' : 'foo']
+    }
+
+    workflow('embeddedFlow1') {
+      commandJob('shellEcho') {
+        baseProperties 'common'
+        uses 'echo "This is an echoed text from embeddedFlow1."'
+      }
+      targets 'shellEcho'
+    }
+
+    targets 'embeddedFlow1'
+  }
+}

hadoop-plugin/src/main/groovy/com/linkedin/gradle/azkaban/AzkabanDslYamlCompiler.groovy

@@ -234,6 +234,14 @@ class AzkabanDslYamlCompiler extends BaseCompiler {
    * @return Map representing workflow to be output in yaml file.
    */
   Map yamlizeWorkflow(Workflow workflow, boolean isSubflow) {
+    // Save the last scope information
+    NamedScope oldParentScope = this.parentScope;
+    String oldParentDirectory = this.parentDirectory;
+
+    // Set the new parent scope information
+    this.parentScope = workflow.scope;
+    this.parentDirectory = "${this.parentDirectory}/${workflow.name}";
+
     Map yamlizedWorkflow = [:];
 
     // Add workflow name if subflow
@@ -266,6 +274,10 @@ class AzkabanDslYamlCompiler extends BaseCompiler {
       yamlizedWorkflow["nodes"] = nodes;
     }
 
+    // Restore the last parent scope
+    this.parentScope = oldParentScope;
+    this.parentDirectory = oldParentDirectory;
+
     return yamlizedWorkflow;
   }