Modified PCR duplicate removal process to use temporary files that are

deleted on exit, rather than reusable intermediate files. This allows
multiple copies of ESAT with different parameters to run at the same
time without interfering with each other.

Author: ADerr

src/java/umms/core/utils/InDropPreprocess.java

@@ -22,6 +22,7 @@
 import net.sf.samtools.SAMRecordIterator;
 
 
+
 //import org.apache.commons.math3.util.MultidimensionalCounter.Iterator;
 import org.apache.log4j.Logger;
 
@@ -63,6 +64,7 @@ public InDropPreprocess(HashMap<String,ArrayList<File>> bamFiles,
 
 		SAMRecord r;
 		SAMFileWriterFactory sf = new SAMFileWriterFactory();
+		File outFile;
 		int readsIn = 0;
 		int readsOut = 0;
 
@@ -88,52 +90,26 @@ public InDropPreprocess(HashMap<String,ArrayList<File>> bamFiles,
 				SAMFileHeader bamHeader = bamReader.getFileHeader();    // get the header information
 
 				// create the pre-processed output BAM file:
-				// The processed file is called <original bam file base name>_nextPrep.bam
+				// The processed file is called <original bam file base name>_inPrep.bam
 				String inFile = bamFile.getCanonicalPath();
 				int extPos = inFile.length()-4;
-				File outFile = new File(inFile.substring(0,extPos)+"_inPrep"+inFile.substring(extPos));
+				// File outFile = new File(inFile.substring(0,extPos)+"_inPrep"+inFile.substring(extPos));
+				
+				// Make a temporary file for the PCR de-duplicated reads:
+				//logger.info("creating temporary file: "+inFile.substring(0,extPos)+"xxxxxxx.bam");				
+				//outFile = File.createTempFile(inFile.substring(0,extPos), ".bam");   
+				outFile = File.createTempFile("sc_umiFiltered_", ".bam");   
+				logger.info("temporary file created: "+outFile);
+				// delete after exit:
+				outFile.deleteOnExit();
 
 				// add this file to the list of files to be processed by ESAT:
 				if (!bamFiles_prep.containsKey(exp)) {
 					bamFiles_prep.put(exp, new ArrayList<File>());
 				}
 				bamFiles_prep.get(exp).add(outFile);
-				SAMProgramRecord prepProg = new SAMProgramRecord("ESAT");
-				prepProg.setProgramVersion(PROGRAM_VERSION);
-				prepProg.setAttribute("task", task);
-				prepProg.setAttribute("wExt", ""+wExt);
-				boolean makeNewPrepFile = true;     // by default, make a new file.
-				
-				// check for the existence of this file:
-				if (outFile.exists()) {
-					// open the file to read the header
-					SAMFileReader scReader = new SAMFileReader(outFile);   // open as a non-eager reader
-					SAMFileHeader scHeader = scReader.getFileHeader();    // get the header information
-					List<SAMProgramRecord> scProg = scHeader.getProgramRecords();
-					// check to make sure the ESAT parameters are the same:
-					for (SAMProgramRecord sp:scProg) {
-						String pid = sp.getProgramGroupId();
-						if (pid.equals("ESAT")) {
-							// Extract all necessary parameters:
-							String oldVer = sp.getProgramVersion();
-							String oldTask = sp.getAttribute("task");
-							int oldWExt = Integer.parseInt(sp.getAttribute("wExt"));
-							if (oldVer.equals(PROGRAM_VERSION) && oldTask.equals(task) && oldWExt==wExt) {
-								makeNewPrepFile = false;	
-							}
-						}
-					}
-				}
 
-				if (!makeNewPrepFile) {
-					// close the input file
-					bamReader.close();
-					// skip creating a new output file:
-					continue;
-				}
-				
 				// copy the header from the input BAM file:
-				bamHeader.addProgramRecord(prepProg);
 				SAMFileWriter bamWriter = sf.makeBAMWriter(bamHeader, false, outFile);
 
 				//bamReader.setValidationStringency(ValidationStringency.LENIENT);	

src/java/umms/esat/NewESAT.java

@@ -94,10 +94,8 @@ public class NewESAT {
 	private static boolean stranded;    // allow for unstranded analysis (defaults to stranded)
 
 	/* single-cell parameters */
-	private static boolean nextPreprocess;    // Nextera library reads preprocessing flag
-											  // NOTE: barcode and UMI are in the read name, separated by "_".
 	private static boolean scPreprocess;    // inDrop library reads preprocessing flag
-	  										// NOTE: barcode is encoded in filename, UMIs are in the read name, separated by "_".
+	  										// NOTE: barcode and UMIs are appended to the read name, with the format "<readName>:<barcode>:<UMI>".
 	private static int bcMin;			// minimum number of reads that must be observed for a barcode to be considered valid (after PCR duplicate removal) 
 
 	/* optional AT filter */
@@ -189,6 +187,7 @@ public NewESAT(String[] args) throws IOException, ParseException, IllegalArgumen
 			 * NOTE: This was originally specific to inDrop libraries, but is now used for ALL single-cell methods
 			 */
 			inDropData = new InDropPreprocess(bamFiles, annotations, qFilter, qThresh, multimap, windowExtend, stranded, task, filtAT, filtAtN);
+			// replace the original input file list with the pre-processed (PCR de-duplicated) files:
 			bamFiles = inDropData.getPreprocessedFiles();
 			// Fill in barcode counts from preprocessed files, if necessary:
 			//int rCount = inDropData.fillBarcodeCounts();
@@ -299,7 +298,6 @@ private static boolean validateArguments(ArgumentMap argMap) throws IOException
 		stranded = argMap.isPresent("unstranded")? false : true;
 
 		/* single-cell pre-processing? */
-		nextPreprocess = argMap.isPresent("nextPrep") ? true : false;
 		scPreprocess = argMap.isPresent("scPrep") ? true : false;
 		bcMin = argMap.isPresent("bcMin") ? argMap.getInteger("bcMin") : 0;