BIGTOP-3718: Fix MapReduce2 service-check in Amabri Mpack (apache#932)

The patch is to fix the issues of checking MapReduce2 service and
"JAR does not exist".

Change-Id: I59ad4d3ecbe88af7bea25bf7177502ebb3e2cf6b
Signed-off-by: Yuqi Gu <[email protected]>

Author: guyuqi

bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/YARN/package/scripts/mapred_service_check.py

@@ -119,11 +119,18 @@ def service_check(self, env):
     env.set_params(params)
 
     jar_path = format("{hadoop_mapred2_jar_location}/{hadoopMapredExamplesJarName}")
+    source_file = format("/etc/passwd")
     input_file = format("/user/{smokeuser}/mapredsmokeinput")
     output_file = format("/user/{smokeuser}/mapredsmokeoutput")
 
-    test_cmd = format("fs -test -e {output_file}")
+    hdfs_put_cmd = format("fs -put {source_file} {input_file}")
     run_wordcount_job = format("jar {jar_path} wordcount {input_file} {output_file}")
+    test_cmd = format("fs -test -e {output_file}")
+
+    ExecuteHadoop(hdfs_put_cmd,
+                  user=params.smokeuser,
+                  bin_dir=params.execute_path,
+                  conf_dir=params.hadoop_conf_dir)
 
     params.HdfsResource(format("/user/{smokeuser}"),
                       type="directory",
@@ -136,12 +143,6 @@ def service_check(self, env):
                         type = "directory",
                         dfs_type = params.dfs_type,
     )
-    params.HdfsResource(input_file,
-                        action = "create_on_execute",
-                        type = "file",
-                        source = "/etc/passwd",
-                        dfs_type = params.dfs_type,
-    )
     params.HdfsResource(None, action="execute")
 
     # initialize the ticket

bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/YARN/package/scripts/params_linux.py

@@ -127,8 +127,11 @@ def get_spark_version(service_name, component_name, yarn_version):
 # these are used to render the classpath for picking up Spark classes
 # in the event that spark is not installed, then we must default to the vesrion of YARN installed
 # since it will still load classes from its own spark version
-spark_version = get_spark_version("SPARK", "SPARK_CLIENT", version)
-spark2_version = get_spark_version("SPARK2", "SPARK2_CLIENT", version)
+
+# No Spark services in current Mpack;
+# TODO: Add Spark into stack;
+#spark_version = get_spark_version("SPARK", "SPARK_CLIENT", version)
+#spark2_version = get_spark_version("SPARK2", "SPARK2_CLIENT", version)
 
 stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks)
 stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks)
@@ -294,8 +297,8 @@ def get_spark_version(service_name, component_name, yarn_version):
 nm_log_dir_to_mount_file = "/var/lib/ambari-agent/data/yarn/yarn_log_dir_mount.hist"
 nm_local_dir_to_mount_file = "/var/lib/ambari-agent/data/yarn/yarn_local_dir_mount.hist"
 
-distrAppJarName = "hadoop-yarn-applications-distributedshell-2.*.jar"
-hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar"
+distrAppJarName = "hadoop-yarn-applications-distributedshell-3.*.jar"
+hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-3.*.jar"
 
 entity_file_history_directory = "/tmp/entity-file-history/active"
 

bigtop-packages/src/common/bigtop-ambari-mpack/bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/YARN/package/scripts/params_windows.py

@@ -55,7 +55,7 @@
 hs_webui_address = format("{hs_host}:{hs_port}")
 
 hadoop_mapred2_jar_location = os.path.join(os.environ["HADOOP_COMMON_HOME"], "share", "hadoop", "mapreduce")
-hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar"
+hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-3.*.jar"
 
 exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
 exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")