@SuppressWarnings("unused") private Iterator<SAMRecord> getAllAssemblies_single_threaded() { ProgressLogger progressLog = new ProgressLogger(log); List<Iterator<SAMRecord>> list = new ArrayList<>(); for (BreakendDirection direction : BreakendDirection.values()) { CloseableIterator<DirectedEvidence> it = mergedIterator(source, false); Iterator<DirectedEvidence> throttledIt = throttled(it); ProgressLoggingDirectedEvidenceIterator<DirectedEvidence> loggedIt = new ProgressLoggingDirectedEvidenceIterator<>(getContext(), throttledIt, progressLog); Iterator<SAMRecord> evidenceIt = new PositionalAssembler(getContext(), this, new SequentialIdGenerator("asm"), loggedIt, direction); list.add(evidenceIt); } return Iterators.concat(list.iterator()); } /**
@BeforeTest public void setUp(){ progress = new ProgressLogger(Log.getInstance(WgsMetricsProcessorImpl.class)); }
protected void saveVcf(File file, Iterator<IdsvVariantContext> calls) throws IOException { File tmp = gridss.Defaults.OUTPUT_TO_TEMP_FILE ? FileSystemContext.getWorkingFileFor(file) : file; final ProgressLogger writeProgress = new ProgressLogger(log); try (VariantContextWriter vcfWriter = getContext().getVariantContextWriter(tmp, getOutputHeader(), true)) { while (calls.hasNext()) { IdsvVariantContext record = calls.next(); vcfWriter.add(record); writeProgress.record(record.getContig(), record.getStart()); } } if (tmp != file) { FileHelper.move(tmp, file, true); } } protected VCFHeader getInputHeader() {
private List<ReferenceCoverageLookup> createLookup(ProcessingContext context, List<SAMEvidenceSource> sources, int windowSize) { List<ReferenceCoverageLookup> result = new ArrayList<>(); for (SAMEvidenceSource ses : sources) { assert(ses.getSourceCategory() >= 0); assert(ses.getSourceCategory() < context.getCategoryCount()); // one read-ahead thread per input file SamReader reader = SamReaderFactory.makeDefault().referenceSequence(context.getReferenceFile()).open(ses.getFile()); SAMRecordIterator rawIterator = reader.iterator(); rawIterator.assertSorted(SortOrder.coordinate); CloseableIterator<SAMRecord> sit = new AsyncBufferedIterator<SAMRecord>(rawIterator, ses.getFile().getName() + "-Coverage"); toclose.add(sit); // close the async iterator first to prevent aysnc reading from a closed stream toclose.add(rawIterator); toclose.add(reader); sit = new ProgressLoggingSAMRecordIterator(sit, new ProgressLogger(log, 10000000)); SequentialReferenceCoverageLookup sourceLookup = new SequentialReferenceCoverageLookup(sit, ses.getMetrics().getIdsvMetrics(), ses.getReadPairConcordanceCalculator(), windowSize, ses.getSourceCategory(), context.isFilterDuplicates()); context.registerBuffer(ses.getFile().getName(), sourceLookup); result.add(sourceLookup); } return result; } public SequentialCoverageAnnotator(
private void writeSortedOutput(final VCFHeader outputHeader, final SortingCollection<VariantContext> sortedOutput) { final ProgressLogger writeProgress = new ProgressLogger(log, 25000, "wrote", "records"); final EnumSet<Options> options = CREATE_INDEX ? EnumSet.of(Options.INDEX_ON_THE_FLY) : EnumSet.noneOf(Options.class); final VariantContextWriter out = new VariantContextWriterBuilder(). setReferenceDictionary(outputHeader.getSequenceDictionary()). setOptions(options). setOutputFile(OUTPUT).build(); out.writeHeader(outputHeader); for (final VariantContext variantContext : sortedOutput) { out.add(variantContext); writeProgress.record(variantContext.getContig(), variantContext.getStart()); } out.close(); } }
private void writeSortedOutput(final VCFHeader outputHeader, final SortingCollection<VariantContext> sortedOutput) { final ProgressLogger writeProgress = new ProgressLogger(log, 25000, "wrote", "records"); final EnumSet<Options> options = CREATE_INDEX ? EnumSet.of(Options.INDEX_ON_THE_FLY) : EnumSet.noneOf(Options.class); final VariantContextWriter out = new VariantContextWriterBuilder(). setReferenceDictionary(outputHeader.getSequenceDictionary()). setOptions(options). setOutputFile(OUTPUT).build(); out.writeHeader(outputHeader); for (final VariantContext variantContext : sortedOutput) { out.add(variantContext); writeProgress.record(variantContext.getContig(), variantContext.getStart()); } out.close(); } }
private void filterReads(final FilteringSamIterator filteringIterator) { // get OUTPUT header from INPUT and overwrite it if necessary final SAMFileHeader fileHeader = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).getFileHeader(INPUT); final SAMFileHeader.SortOrder inputSortOrder = fileHeader.getSortOrder(); if (SORT_ORDER != null) { fileHeader.setSortOrder(SORT_ORDER); } if (FILTER == Filter.includePairedIntervals && fileHeader.getSortOrder() != SAMFileHeader.SortOrder.coordinate) { throw new UnsupportedOperationException("Input must be coordinate sorted to use includePairedIntervals"); } final boolean presorted = inputSortOrder.equals(fileHeader.getSortOrder()); log.info("Filtering [presorted=" + presorted + "] " + INPUT.getName() + " -> OUTPUT=" + OUTPUT.getName() + " [sortorder=" + fileHeader.getSortOrder().name() + "]"); // create OUTPUT file final SAMFileWriter outputWriter = new SAMFileWriterFactory().makeSAMOrBAMWriter(fileHeader, presorted, OUTPUT); final ProgressLogger progress = new ProgressLogger(log, (int) 1e6, "Written"); while (filteringIterator.hasNext()) { final SAMRecord rec = filteringIterator.next(); outputWriter.addAlignment(rec); progress.record(rec); } filteringIterator.close(); outputWriter.close(); log.info(new DecimalFormat("#,###").format(progress.getCount()) + " SAMRecords written to " + OUTPUT.getName()); }
/** Creates a simple SAM file from a single fastq file. */ protected int doUnpaired(final FastqReader freader, final SAMFileWriter writer) { int readCount = 0; final ProgressLogger progress = new ProgressLogger(LOG); for ( ; freader.hasNext() ; readCount++) { final FastqRecord frec = freader.next(); final SAMRecord srec = createSamRecord(writer.getFileHeader(), SequenceUtil.getSamReadNameFromFastqHeader(frec.getReadHeader()) , frec, false) ; srec.setReadPairedFlag(false); writer.addAlignment(srec); progress.record(srec); } return readCount; }
/** Creates a simple SAM file from a single fastq file. */ protected int doUnpaired(final FastqReader freader, final SAMFileWriter writer) { int readCount = 0; final ProgressLogger progress = new ProgressLogger(LOG); for ( ; freader.hasNext() ; readCount++) { final FastqRecord frec = freader.next(); final SAMRecord srec = createSamRecord(writer.getFileHeader(), SequenceUtil.getSamReadNameFromFastqHeader(frec.getReadHeader()) , frec, false) ; srec.setReadPairedFlag(false); writer.addAlignment(srec); progress.record(srec); } return readCount; }
protected int doWork() { IOUtil.assertFileIsReadable(INPUT); IOUtil.assertFileIsWritable(OUTPUT); final SamReader reader = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).open(INPUT); if (reader.getFileHeader().getSortOrder() != SAMFileHeader.SortOrder.coordinate) { throw new SAMException("Input must be coordinate-sorted for this program to run. Found: " + reader.getFileHeader().getSortOrder()); } final SAMFileWriter writer = new SAMFileWriterFactory().makeSAMOrBAMWriter(reader.getFileHeader(), true, OUTPUT); writer.setProgressLogger( new ProgressLogger(log, (int) 1e7, "Wrote", "records")); final ReferenceSequenceFileWalker refSeqWalker = new ReferenceSequenceFileWalker(REFERENCE_SEQUENCE); StreamSupport.stream(reader.spliterator(), false) .peek(rec -> fixRecord(rec, refSeqWalker)) .forEach(writer::addAlignment); CloserUtil.close(reader); writer.close(); return 0; }
protected int doWork() { IOUtil.assertFileIsReadable(INPUT); IOUtil.assertFileIsWritable(OUTPUT); final SamReader reader = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).open(INPUT); if (reader.getFileHeader().getSortOrder() != SAMFileHeader.SortOrder.coordinate) { throw new SAMException("Input must be coordinate-sorted for this program to run. Found: " + reader.getFileHeader().getSortOrder()); } final SAMFileWriter writer = new SAMFileWriterFactory().makeSAMOrBAMWriter(reader.getFileHeader(), true, OUTPUT); writer.setProgressLogger( new ProgressLogger(log, (int) 1e7, "Wrote", "records")); final ReferenceSequenceFileWalker refSeqWalker = new ReferenceSequenceFileWalker(REFERENCE_SEQUENCE); StreamSupport.stream(reader.spliterator(), false) .peek(rec -> fixRecord(rec, refSeqWalker)) .forEach(writer::addAlignment); CloserUtil.close(reader); writer.close(); return 0; }
protected int doWork() { IOUtil.assertFileIsReadable(INPUT); IOUtil.assertFileIsWritable(OUTPUT); final SamReader reader = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).open(INPUT); ; reader.getFileHeader().setSortOrder(SORT_ORDER.getSortOrder()); final SAMFileWriter writer = new SAMFileWriterFactory().makeSAMOrBAMWriter(reader.getFileHeader(), false, OUTPUT); writer.setProgressLogger( new ProgressLogger(log, (int) 1e7, "Wrote", "records from a sorting collection")); final ProgressLogger progress = new ProgressLogger(log, (int) 1e7, "Read"); for (final SAMRecord rec : reader) { writer.addAlignment(rec); progress.record(rec); } log.info("Finished reading inputs, merging and writing to output now."); CloserUtil.close(reader); writer.close(); return 0; } }
protected int doWork() { IOUtil.assertFileIsReadable(INPUT); IOUtil.assertFileIsWritable(OUTPUT); final SamReader reader = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).open(INPUT); ; reader.getFileHeader().setSortOrder(SORT_ORDER.getSortOrder()); final SAMFileWriter writer = new SAMFileWriterFactory().makeSAMOrBAMWriter(reader.getFileHeader(), false, OUTPUT); writer.setProgressLogger( new ProgressLogger(log, (int) 1e7, "Wrote", "records from a sorting collection")); final ProgressLogger progress = new ProgressLogger(log, (int) 1e7, "Read"); for (final SAMRecord rec : reader) { writer.addAlignment(rec); progress.record(rec); } log.info("Finished reading inputs, merging and writing to output now."); CloserUtil.close(reader); writer.close(); return 0; } }
@Override protected int doWork() { try (final SamReader reader = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).open(IOUtil.getPath(INPUT)); final SAMFileWriter writer = new SAMFileWriterFactory().makeWriter(reader.getFileHeader(), true, IOUtil.getPath(OUTPUT), REFERENCE_SEQUENCE)) { writer.setProgressLogger( new ProgressLogger(log, (int) 1e7, "Wrote", "records")); final OverlapDetector overlapDetector = getOverlapDetectorFromIntervalListFile(INTERVAL_LIST, 0, 0); for (final SAMRecord rec : reader) { if (overlapDetector == null || overlapDetector.overlapsAny(rec)) { setOATag(rec); } writer.addAlignment(rec); } } catch (IOException e) { log.error(e); return 1; } return 0; }
@Override protected int doWork() { try (final SamReader reader = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).open(IOUtil.getPath(INPUT)); final SAMFileWriter writer = new SAMFileWriterFactory().makeWriter(reader.getFileHeader(), true, IOUtil.getPath(OUTPUT), REFERENCE_SEQUENCE)) { writer.setProgressLogger( new ProgressLogger(log, (int) 1e7, "Wrote", "records")); final OverlapDetector overlapDetector = getOverlapDetectorFromIntervalListFile(INTERVAL_LIST, 0, 0); for (final SAMRecord rec : reader) { if (overlapDetector == null || overlapDetector.overlapsAny(rec)) { setOATag(rec); } writer.addAlignment(rec); } } catch (IOException e) { log.error(e); return 1; } return 0; }
protected int doWork() { IOUtil.assertFileIsReadable(INPUT); IOUtil.assertFileIsWritable(OUTPUT); final SamReader reader = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).open(INPUT); final SAMFileWriter writer = new SAMFileWriterFactory().makeWriter(reader.getFileHeader(), true, OUTPUT, REFERENCE_SEQUENCE); if (CREATE_INDEX && writer.getFileHeader().getSortOrder() != SAMFileHeader.SortOrder.coordinate) { throw new PicardException("Can't CREATE_INDEX unless sort order is coordinate"); } final ProgressLogger progress = new ProgressLogger(Log.getInstance(SamFormatConverter.class)); for (final SAMRecord rec : reader) { writer.addAlignment(rec); progress.record(rec); } CloserUtil.close(reader); writer.close(); return 0; } }
protected int doWork() { IOUtil.assertFileIsReadable(INPUT); IOUtil.assertFileIsWritable(OUTPUT); final SamReader reader = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).open(INPUT); final SAMFileWriter writer = new SAMFileWriterFactory().makeWriter(reader.getFileHeader(), true, OUTPUT, REFERENCE_SEQUENCE); if (CREATE_INDEX && writer.getFileHeader().getSortOrder() != SAMFileHeader.SortOrder.coordinate) { throw new PicardException("Can't CREATE_INDEX unless sort order is coordinate"); } final ProgressLogger progress = new ProgressLogger(Log.getInstance(SamFormatConverter.class)); for (final SAMRecord rec : reader) { writer.addAlignment(rec); progress.record(rec); } CloserUtil.close(reader); writer.close(); return 0; } }
private void standardReheader(final SAMFileHeader replacementHeader) { final SamReader recordReader = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).validationStringency(ValidationStringency.SILENT).open(INPUT); if (replacementHeader.getSortOrder() != recordReader.getFileHeader().getSortOrder()) { throw new PicardException("Sort orders of INPUT (" + recordReader.getFileHeader().getSortOrder().name() + ") and HEADER (" + replacementHeader.getSortOrder().name() + ") do not agree."); } final SAMFileWriter writer = new SAMFileWriterFactory().makeSAMOrBAMWriter(replacementHeader, true, OUTPUT); final ProgressLogger progress = new ProgressLogger(Log.getInstance(ReplaceSamHeader.class)); for (final SAMRecord rec : recordReader) { rec.setHeader(replacementHeader); writer.addAlignment(rec); progress.record(rec); } writer.close(); CloserUtil.close(recordReader); }
private void standardReheader(final SAMFileHeader replacementHeader) { final SamReader recordReader = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).validationStringency(ValidationStringency.SILENT).open(INPUT); if (replacementHeader.getSortOrder() != recordReader.getFileHeader().getSortOrder()) { throw new PicardException("Sort orders of INPUT (" + recordReader.getFileHeader().getSortOrder().name() + ") and HEADER (" + replacementHeader.getSortOrder().name() + ") do not agree."); } final SAMFileWriter writer = new SAMFileWriterFactory().makeSAMOrBAMWriter(replacementHeader, true, OUTPUT); final ProgressLogger progress = new ProgressLogger(Log.getInstance(ReplaceSamHeader.class)); for (final SAMRecord rec : recordReader) { rec.setHeader(replacementHeader); writer.addAlignment(rec); progress.record(rec); } writer.close(); CloserUtil.close(recordReader); }
/** More complicated method that takes two fastq files and builds pairing information in the SAM. */ protected int doPaired(final FastqReader freader1, final FastqReader freader2, final SAMFileWriter writer) { int readCount = 0; final ProgressLogger progress = new ProgressLogger(LOG); for ( ; freader1.hasNext() && freader2.hasNext() ; readCount++) { final FastqRecord frec1 = freader1.next(); final FastqRecord frec2 = freader2.next(); final String frec1Name = SequenceUtil.getSamReadNameFromFastqHeader(frec1.getReadHeader()); final String frec2Name = SequenceUtil.getSamReadNameFromFastqHeader(frec2.getReadHeader()); final String baseName = getBaseName(frec1Name, frec2Name, freader1, freader2); final SAMRecord srec1 = createSamRecord(writer.getFileHeader(), baseName, frec1, true) ; srec1.setFirstOfPairFlag(true); srec1.setSecondOfPairFlag(false); writer.addAlignment(srec1); progress.record(srec1); final SAMRecord srec2 = createSamRecord(writer.getFileHeader(), baseName, frec2, true) ; srec2.setFirstOfPairFlag(false); srec2.setSecondOfPairFlag(true); writer.addAlignment(srec2); progress.record(srec2); } if (freader1.hasNext() || freader2.hasNext()) { throw new PicardException("Input paired fastq files must be the same length"); } return readCount; }