@Override public PositionalBufferedStream makeSourceFromStream(final InputStream bufferedInputStream) { if (bufferedInputStream instanceof PositionalBufferedStream) return (PositionalBufferedStream) bufferedInputStream; else return new PositionalBufferedStream(bufferedInputStream); }
public static void main(String[] args) throws Exception { final File testFile = new File(args[0]); final int iterations = Integer.valueOf(args[1]); final boolean includeInputStream = Boolean.valueOf(args[2]); final boolean doReadFileInChunks = Boolean.valueOf(args[3]); System.out.printf("Testing %s%n", args[0]); for (int i = 0; i < iterations; i++) { if ( includeInputStream ) { final InputStream is = new FileInputStream(testFile); if ( doReadFileInChunks ) readFileInChunks("InputStream", is); else readFileByLine("InputStream", is); is.close(); } final PositionalBufferedStream pbs = new PositionalBufferedStream(new FileInputStream(testFile)); if ( doReadFileInChunks ) readFileInChunks("PositionalBufferedStream", pbs); else readFileByLine("PositionalBufferedStream", pbs); pbs.close(); } }
/** For compressed BCF, unless the end has been reached, this is quite * inaccurate. */ @Override public float getProgress() { if (length == 0) return 1; if (!isBGZF) return (float)(in.getPosition() - fileStart) / length; try { if (in.peek() == -1) return 1; } catch (IOException e) { return 1; } // Add 1 to the denominator to make sure that we never report 1 here. return (float)((bci.getFilePointer() >>> 16) - fileStart) / (length + 1); } @Override public LongWritable getCurrentKey () { return key; }
public final long skip(final long nBytes) throws IOException { long remainingToSkip = nBytes; // because we have this buffer, that may be shorter than nBytes // we loop while there are bytes to skip, filling the buffer // When the buffer contains enough data that we have less than // its less left to skip we increase nextChar by the remaining // amount while ( remainingToSkip > 0 && ! isDone() ) { final long bytesLeftInBuffer = nChars - nextChar; if ( remainingToSkip > bytesLeftInBuffer ) { // we need to refill the buffer and continue our skipping remainingToSkip -= bytesLeftInBuffer; fill(); } else { // there are enough bytes in the buffer to not read again // we just push forward the pointer nextChar nextChar += remainingToSkip; remainingToSkip = 0; } } final long actuallySkipped = nBytes - remainingToSkip; position += actuallySkipped; return actuallySkipped; }
if ( inputStream.read(headerBytes) != headerSizeInBytes ) error("Couldn't read all of the bytes specified in the header length = " + headerSizeInBytes); final PositionalBufferedStream bps = new PositionalBufferedStream(new ByteArrayInputStream(headerBytes)); final LineIterator lineIterator = new LineIteratorImpl(LineReaderUtil.fromBufferedStream(bps, LineReaderUtil.LineReaderOption.SYNCHRONOUS)); final VCFCodec headerParser = new VCFCodec(); this.header = (VCFHeader) headerParser.readActualHeader(lineIterator); bps.close(); } catch ( IOException e ) { throw new TribbleException("I/O error while reading BCF2 header"); return new FeatureCodecHeader(header, inputStream.getPosition());
private void testStream(final byte[] bytes) throws IOException { final InputStream is = new ByteArrayInputStream(bytes); final PositionalBufferedStream pbs = new PositionalBufferedStream(is); int bytePos = 0; while ( ! pbs.isDone() ) { Assert.assertTrue(bytePos < bytes.length); // test position Assert.assertEquals(pbs.getPosition(), bytePos); // test peek final byte atPos = bytes[bytePos]; Assert.assertEquals(toByte(pbs.peek()), atPos); // test position Assert.assertEquals(pbs.getPosition(), bytePos); // test read Assert.assertEquals(toByte(pbs.read()), atPos); bytePos++; // test position Assert.assertEquals(pbs.getPosition(), bytePos); // test repeek if ( bytePos < bytes.length ) { Assert.assertEquals(toByte(pbs.peek()), bytes[bytePos]); // test position Assert.assertEquals(pbs.getPosition(), bytePos); } } Assert.assertEquals(bytePos, bytes.length); pbs.close(); }
@Test public void testSkip() throws IOException { for ( int skipSizeBase : Arrays.asList(0, 10, 100, 1000, 10000, 1000000)) { for ( int skipSizeAdd = 0; skipSizeAdd < 10; skipSizeAdd ++ ) { final int skipSize = skipSizeBase + skipSizeAdd; final byte[] bytes = new byte[skipSize+2]; Arrays.fill(bytes, 0, skipSize, (byte)0); bytes[skipSize] = 1; bytes[skipSize+1] = 2; final InputStream is = new ByteArrayInputStream(bytes); final PositionalBufferedStream pbs = new PositionalBufferedStream(is); pbs.skip(skipSize); // first value is 1 Assert.assertTrue(! pbs.isDone()); Assert.assertEquals(pbs.getPosition(), skipSize); Assert.assertEquals(pbs.peek(), 1); Assert.assertEquals(pbs.read(), 1); Assert.assertTrue(! pbs.isDone()); Assert.assertEquals(pbs.getPosition(), skipSize + 1); Assert.assertEquals(pbs.peek(), 2); Assert.assertEquals(pbs.read(), 2); Assert.assertTrue(pbs.isDone()); } } }
new PositionalBufferedStream(cin); byte b = 0; int prevCP = cp0; while (b < BGZF_BLOCKS_NEEDED_FOR_GUESS && pbIn.peek() != -1) while (pbIn.getPosition() - up0 < UNCOMPRESSED_BYTES_NEEDED && pbIn.peek() != -1) if (pbIn.getPosition() - up0 < UNCOMPRESSED_BYTES_NEEDED) { assert arr.length < UNCOMPRESSED_BYTES_NEEDED; if (!decodedAny) if (!(decodedAny && pbIn.peek() == -1)) continue;
@Test public void testPeek() throws Exception{ int trials = 10; PositionalBufferedStream is = new PositionalBufferedStream(FileIs); int bb = is.peek(); for(int ii=0; ii < trials; ii++){ Assert.assertEquals(is.peek(), bb); Assert.assertEquals(is.getPosition(), 0); } while((bb = is.peek()) >= 0){ Assert.assertEquals(is.read(), bb); } }
/** * read the header from the file * * @throws IOException throws an IOException if we can't open the file */ private void readHeader() throws IOException { InputStream is = null; PositionalBufferedStream pbs = null; try { is = ParsingUtils.openInputStream(path); if (path.endsWith("gz")) { // TODO -- warning I don't think this can work, the buffered input stream screws up position is = new GZIPInputStream(new BufferedInputStream(is)); } pbs = new PositionalBufferedStream(is); final SOURCE source = codec.makeSourceFromStream(pbs); header = codec.readHeader(source); } catch (Exception e) { throw new TribbleException.MalformedFeatureFile("Unable to parse header with error: " + e.getMessage(), path, e); } finally { if (pbs != null) pbs.close(); else if (is != null) is.close(); } }
@Test public void testIsDone() throws Exception{ PositionalBufferedStream is = new PositionalBufferedStream(FileIs); while(!is.isDone()){ is.read(); } Assert.assertTrue(is.isDone()); Assert.assertEquals(is.getPosition(), expectedBytes); }
final PositionalBufferedStream pbs = new PositionalBufferedStream(is); final LineReader alr = new AsciiLineReader(pbs); // AsciiLineReader must be used here because it does not read ahead. while ( ! pbs.isDone() ) { Assert.assertTrue(bytePos < bytes.length); Assert.assertEquals(pbs.getPosition(), bytePos); Assert.assertEquals(pbs.getPosition(), bytePos); pbs.close();
in = new PositionalBufferedStream(bci); initContigDict(); in = new PositionalBufferedStream( new BGZFLimitingStream(bci, virtualEnd)); } else { in = new PositionalBufferedStream( file.getFileSystem(ctx.getConfiguration()).open(file)); IOUtils.skipFully(in, fileStart - in.getPosition());
public static VariantContextContainer readAllVCs(final File input, final BCF2Codec codec) throws IOException { PositionalBufferedStream headerPbs = new PositionalBufferedStream(new FileInputStream(input)); FeatureCodecHeader header = codec.readHeader(headerPbs); headerPbs.close(); final PositionalBufferedStream pbs = new PositionalBufferedStream(new FileInputStream(input)); pbs.skip(header.getHeaderEnd()); final VCFHeader vcfHeader = (VCFHeader)header.getHeaderValue(); return new VariantContextTestProvider.VariantContextContainer(vcfHeader, new VariantContextTestProvider.VCIterable(codec, vcfHeader) { @Override public boolean hasNext() { try { return !pbs.isDone(); } catch (IOException e) { throw new RuntimeException(e); } } @Override public Object nextSource() { return pbs; } }); }
try (final PositionalBufferedStream headerPbs = new PositionalBufferedStream(new FileInputStream(bcfOutputFile)); final PositionalBufferedStream bodyPbs = new PositionalBufferedStream(new FileInputStream(bcfOutputHeaderlessFile))) { while (!bodyPbs.isDone()) { VariantContext vc = codec.decode(bodyPbs); counter++;
@Override public void close() { this.inputStream.close(); } }
@Test(dataProvider = "ReadBytesTestData") public void testReadBytes(final int byteReadSize, final int bufsize) throws Exception { final byte[] bytes = new byte[255]; for ( int i = 0; i < bytes.length; i++ ) bytes[i] = (byte)i; final ByteArrayInputStream bais = new ByteArrayInputStream(bytes); final byte[] readBytes = new byte[byteReadSize]; final PositionalBufferedStream pbs = new PositionalBufferedStream(bais, bufsize); int i = 0; while ( i < 255 ) { final int expectedBytesToRead = Math.min(255 - i, readBytes.length); final int nBytesRead = pbs.read(readBytes); Assert.assertEquals(nBytesRead, expectedBytesToRead, "Didn't read as many bytes as expected from PBS"); for ( int j = 0; j < nBytesRead; j++ ) Assert.assertEquals(readBytes[j], bytes[i+j], "Bytes read not those expected"); i += nBytesRead; } } }
private PositionalBufferedStream initStream(final File inputFile, final long skip) { try { final FileInputStream is = new FileInputStream(inputFile); final PositionalBufferedStream pbs = new PositionalBufferedStream(is); if ( skip > 0 ) pbs.skip(skip); return pbs; } catch (final FileNotFoundException e) { throw new TribbleException.FeatureFileDoesntExist("Unable to open the input file, most likely the file doesn't exist.", inputFile.getAbsolutePath()); } catch (final IOException e) { throw new TribbleException.MalformedFeatureFile("Error initializing stream", inputFile.getAbsolutePath(), e); } }