File tree Expand file tree Collapse file tree 4 files changed +12
-12
lines changed
hadoop-plugin/src/main/groovy/com/linkedin/gradle/hadoopdsl/job Expand file tree Collapse file tree 4 files changed +12
-12
lines changed Original file line number Diff line number Diff line change @@ -4,9 +4,9 @@ dependencies=jobs1_job24
44conf.key1=value1
55conf.key2=value2
66conf.spark.driver.cores=1
7- driver-memory=2048m
7+ driver-memory=2g
88execution-jar=path/to/python/script.py
99executor-cores=2
10- executor-memory=8192m
10+ executor-memory=8g
1111num-executors=6
1212queue=marathon
Original file line number Diff line number Diff line change @@ -383,12 +383,12 @@ hadoop {
383383
384384 tensorFlowJob(' job24' , ' tony' ) {
385385 executes ' path/to/python/script.py'
386- amMemory 2g
386+ amMemory ' 2g '
387387 amCores 1
388388 amGpus 1
389- psMemory 2g
389+ psMemory ' 2g '
390390 psCores 1
391- workerMemory 8g
391+ workerMemory ' 8g '
392392 workerCores 1
393393 workerGpus 2
394394 numPs 2
@@ -403,13 +403,13 @@ hadoop {
403403
404404 tensorFlowJob(' job25' , ' spark' ) {
405405 executes ' path/to/python/script.py'
406- amMemory 2g
406+ amMemory ' 2g '
407407 amCores 1
408408 // Put ps memory before worker memory and ps cores after worker cores
409409 // to verify only worker params take effect
410- psMemory 2g
410+ psMemory ' 2g '
411411 workerCores 2
412- workerMemory 8g
412+ workerMemory ' 8g '
413413 psCores 1
414414 numPs 2
415415 numWorkers 4
Original file line number Diff line number Diff line change @@ -22,11 +22,11 @@ package com.linkedin.gradle.hadoopdsl.job
2222 * tensorFlowJob('jobName', 'spark') {
2323 * executes path/to/python/script.py
2424 * appParams params
25- * amMemory 2g
25+ * amMemory '2g'
2626 * amCores 1
27- * psMemory 2g
27+ * psMemory '2g'
2828 * psCores 1
29- * workerMemory 8g
29+ * workerMemory '8g'
3030 * workerCores 1
3131 * numPs 2
3232 * numWorkers 4
Original file line number Diff line number Diff line change @@ -204,7 +204,7 @@ class TensorFlowTonyJob extends HadoopJavaProcessJob implements TensorFlowJob {
204204 parsedMem = parsedMem[0 .. -2 ];
205205 }
206206 if (parsedMem[-1 ] == " g" ) {
207- parsedMem = parsedMem[0 .. -2 ] + " 000 " ;
207+ parsedMem = ( parsedMem[0 .. -2 ]. toInteger() * 1024 ) . toString() ;
208208 }
209209 return parsedMem;
210210 }
You can’t perform that action at this time.
0 commit comments