private void init(OutputStream outs) throws IOException { this.underlyingStream = outs; this.out = new BufferedFileOutputStream(outs); EncoderFactory efactory = new EncoderFactory(); this.vout = efactory.binaryEncoder(out, null); dout.setSchema(schema); buffer = new NonCopyingByteArrayOutputStream( Math.min((int)(syncInterval * 1.25), Integer.MAX_VALUE/2 -1)); this.bufOut = efactory.binaryEncoder(buffer, null); if (this.codec == null) { this.codec = CodecFactory.nullCodec().createInstance(); } this.isOpen = true; }
@Override ByteBuffer decompress(ByteBuffer data) throws IOException { if (compressionBuffer == null) { compressionBuffer = new NonCopyingByteArrayOutputStream( data.remaining()); } if (inflater == null) { inflater = new Inflater(nowrap); } InputStream uncompressed = new InflaterInputStream( new ByteArrayInputStream(data.array(), data.position() + data.arrayOffset(), data.remaining()), inflater); int read; byte[] buff = new byte[2048]; try { while (true) { read = uncompressed.read(buff); if (read < 0) break; compressionBuffer.write(buff, 0, read); } } catch (EOFException e) { // sometimes InflaterInputStream.read // throws this instead of returning -1 } ByteBuffer result = compressionBuffer.getByteArrayAsByteBuffer(); inflater.reset(); compressionBuffer.reset(); return result; }
@Override ByteBuffer compress(ByteBuffer data) throws IOException { if (compressionBuffer == null) { compressionBuffer = new NonCopyingByteArrayOutputStream( data.remaining()); } if (deflater == null) { deflater = new Deflater(compressionLevel, nowrap); } // Pass output through deflate DeflaterOutputStream deflaterStream = new DeflaterOutputStream(compressionBuffer, deflater); deflaterStream.write(data.array(), data.position() + data.arrayOffset(), data.limit() + data.arrayOffset()); deflaterStream.finish(); ByteBuffer result = compressionBuffer.getByteArrayAsByteBuffer(); deflater.reset(); compressionBuffer.reset(); return result; }
private void init(OutputStream outs) throws IOException { this.underlyingStream = outs; this.out = new BufferedFileOutputStream(outs); EncoderFactory efactory = new EncoderFactory(); this.vout = efactory.binaryEncoder(out, null); dout.setSchema(schema); buffer = new NonCopyingByteArrayOutputStream( Math.min((int)(syncInterval * 1.25), Integer.MAX_VALUE/2 -1)); this.bufOut = efactory.binaryEncoder(buffer, null); if (this.codec == null) { this.codec = CodecFactory.nullCodec().createInstance(); } this.isOpen = true; }
private int bufferInUse() { return (buffer.size() + bufOut.bytesBuffered()); }
private int bufferInUse() { return (buffer.size() + bufOut.bytesBuffered()); }
private void init(OutputStream outs) throws IOException { this.out = new BufferedFileOutputStream(outs); EncoderFactory efactory = new EncoderFactory(); this.vout = efactory.binaryEncoder(out, null); dout.setSchema(schema); buffer = new NonCopyingByteArrayOutputStream( Math.min((int)(syncInterval * 1.25), Integer.MAX_VALUE/2 -1)); this.bufOut = efactory.binaryEncoder(buffer, null); if (this.codec == null) { this.codec = CodecFactory.nullCodec().createInstance(); } this.isOpen = true; }
private void writeBlock() throws IOException { if (blockCount > 0) { vout.writeLong(blockCount); ByteBuffer uncompressed = buffer.getByteArrayAsByteBuffer(); ByteBuffer block = codec.compress(uncompressed); vout.writeLong(block.remaining()); vout.writeFixed(block.array(), block.position() + block.arrayOffset(), block.remaining()); buffer.reset(); blockCount = 0; vout.writeFixed(sync); } }
/** Append a datum to the file. */ public void append(D datum) throws IOException { assertOpen(); dout.write(datum, bufOut); blockCount++; if (buffer.size() >= syncInterval) writeBlock(); }
private void writeBlock() throws IOException { if (blockCount > 0) { bufOut.flush(); ByteBuffer uncompressed = buffer.getByteArrayAsByteBuffer(); DataBlock block = new DataBlock(uncompressed, blockCount); block.compressUsing(codec); block.writeBlockTo(vout, sync); buffer.reset(); blockCount = 0; } }
private void resetBufferTo(int size) throws IOException { bufOut.flush(); byte[] data = buffer.toByteArray(); buffer.reset(); buffer.write(data, 0, size); }
private void init(OutputStream outs) throws IOException { this.out = new BufferedFileOutputStream(outs); this.vout = new BinaryEncoder(out); dout.setSchema(schema); buffer = new NonCopyingByteArrayOutputStream( Math.min((int)(syncInterval * 1.25), Integer.MAX_VALUE/2 -1)); this.bufOut = new BinaryEncoder(buffer); if (this.codec == null) { this.codec = CodecFactory.nullCodec().createInstance(); } this.isOpen = true; }
/** Expert: Append a pre-encoded datum to the file. No validation is * performed to check that the encoding conforms to the file's schema. * Appending non-conforming data may result in an unreadable file. */ public void appendEncoded(ByteBuffer datum) throws IOException { assertOpen(); int start = datum.position(); buffer.write(datum.array(), start, datum.limit()-start); blockCount++; if (buffer.size() >= syncInterval) writeBlock(); }
private void writeBlock() throws IOException { if (blockCount > 0) { bufOut.flush(); ByteBuffer uncompressed = buffer.getByteArrayAsByteBuffer(); DataBlock block = new DataBlock(uncompressed, blockCount); block.setFlushOnWrite(flushOnEveryBlock); block.compressUsing(codec); block.writeBlockTo(vout, sync); buffer.reset(); blockCount = 0; } }
private void writeBlock() throws IOException { if (blockCount > 0) { ByteBuffer uncompressed = buffer.getByteArrayAsByteBuffer(); DataBlock block = new DataBlock(uncompressed, blockCount); block.compressUsing(codec); block.writeBlockTo(vout, sync); buffer.reset(); blockCount = 0; } }
private void resetBufferTo(int size) throws IOException { bufOut.flush(); byte[] data = buffer.toByteArray(); buffer.reset(); buffer.write(data, 0, size); }
private void init(OutputStream outs) throws IOException { this.out = new BufferedFileOutputStream(outs); this.vout = new BinaryEncoder(out); dout.setSchema(schema); buffer = new NonCopyingByteArrayOutputStream( Math.min((int)(syncInterval * 1.25), Integer.MAX_VALUE/2 -1)); this.bufOut = new BinaryEncoder(buffer); if (this.codec == null) { this.codec = CodecFactory.nullCodec().createInstance(); } this.isOpen = true; }
/** Append a datum to the file. */ public void append(D datum) throws IOException { assertOpen(); dout.write(datum, bufOut); blockCount++; if (buffer.size() >= syncInterval) writeBlock(); }
private int bufferInUse() { return (buffer.size() + bufOut.bytesBuffered()); }
private void init(OutputStream outs) throws IOException { this.underlyingStream = outs; this.out = new BufferedFileOutputStream(outs); EncoderFactory efactory = new EncoderFactory(); this.vout = efactory.binaryEncoder(out, null); dout.setSchema(schema); buffer = new NonCopyingByteArrayOutputStream( Math.min((int)(syncInterval * 1.25), Integer.MAX_VALUE/2 -1)); this.bufOut = efactory.binaryEncoder(buffer, null); if (this.codec == null) { this.codec = CodecFactory.nullCodec().createInstance(); } this.isOpen = true; }