@Override public void readFields(DataInput dataInput) throws IOException { int size = dataInput.readInt(); byteArrayOutputCache.reset(); byteArrayOutputCache.skipBytes(size); dataInput.readFully(byteArrayOutputCache.getByteArray(), 0, size); }
@Override public void readFields(DataInput dataInput) throws IOException { int size = dataInput.readInt(); byteArrayOutputCache.reset(); byteArrayOutputCache.skipBytes(size); dataInput.readFully(byteArrayOutputCache.getByteArray(), 0, size); }
public void write(DataOutput dataOutput) throws IOException { dataOutput.writeInt(byteArrayOutputCache.getPos()); dataOutput.write(byteArrayOutputCache.getByteArray(), 0, byteArrayOutputCache.getPos()); }
@Override public void write(DataOutput dataOutput) throws IOException { dataOutput.writeInt(byteArrayOutputCache.getPos()); dataOutput.write(byteArrayOutputCache.getByteArray(), 0, byteArrayOutputCache.getPos()); }
public void write(DataOutput dataOutput) throws IOException { pattern.write(dataOutput); dataOutput.writeInt(partId); dataOutput.writeInt(byteArrayOutputCache.getPos()); dataOutput.write(byteArrayOutputCache.getByteArray(), 0, byteArrayOutputCache.getPos()); }
private synchronized void decompress() { if (uncompressed) { return; } //LOG.info("Big no no"); //long start = System.currentTimeMillis(); LZ4FastDecompressor decompressor = lz4factory.fastDecompressor(); ByteBuffer dest = ByteBuffer.allocate(uncompressedSize); ByteBuffer src = ByteBuffer.wrap(byteArrayOutputCache.getByteArray(), 0, byteArrayOutputCache.getPos()); decompressor.decompress(src, 0, dest, 0, uncompressedSize); byteArrayOutputCache = new ExtendedByteArrayDataOutput(dest.array(), uncompressedSize); uncompressed = true; }
public void readStash(ODAGStash stashToReadTo) { ExtendedByteArrayDataInput byteArrayInputCache = null; try { byteArrayInputCache = new ExtendedByteArrayDataInput(byteArrayOutputCache.getByteArray(), 0, byteArrayOutputCache.getPos()); LOG.info(byteArrayInputCache.available()); stashToReadTo.readFields(byteArrayInputCache); } catch (IOException e) { LOG.info("stashToReadTo: " + stashToReadTo); LOG.info("input.available: " + byteArrayInputCache.available()); LOG.info("input.pos: " + byteArrayInputCache.getPos()); LOG.info("output.pos: " + byteArrayOutputCache.getPos()); throw new RuntimeException(e); } }
@Override public void readFields(DataInput dataInput) throws IOException { if (pattern == null) { pattern = Configuration.get().createPattern(); } pattern.readFields(dataInput); partId = dataInput.readInt(); int size = dataInput.readInt(); byteArrayOutputCache.reset(); byteArrayOutputCache.skipBytes(size); dataInput.readFully(byteArrayOutputCache.getByteArray(), 0, size); }
private void compress() { if (!uncompressed) { return; } uncompressedSize = byteArrayOutputCache.getPos(); LZ4Compressor lz4Compressor = lz4factory.fastCompressor(); int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize); ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]); int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0, uncompressedSize, compressed, 0, maxCompressedLength); byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength); uncompressed = false; }
private void compress() { if (!uncompressed) { return; } uncompressedSize = byteArrayOutputCache.getPos(); LZ4Compressor lz4Compressor = lz4factory.fastCompressor(); int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize); ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]); int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0, uncompressedSize, compressed, 0, maxCompressedLength); byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength); uncompressed = false; }
public ByteArrayObjectCacheIterator(ByteArrayObjectCache objectCache) { ExtendedByteArrayDataOutput byteArrayOutputCache = objectCache.byteArrayOutputCache; byteArrayInputCache = new ExtendedByteArrayDataInput(byteArrayOutputCache.getByteArray(), 0, byteArrayOutputCache.getPos()); configuration = Configuration.get(); }
public void readEzip(ODAG ezip) { ExtendedByteArrayDataInput byteArrayInputCache = null; try { byteArrayInputCache = new ExtendedByteArrayDataInput(byteArrayOutputCache.getByteArray(), 0, byteArrayOutputCache.getPos()); //LOG.info(byteArrayInputCache.available()); ezip.setPattern(pattern); ezip.readFields(byteArrayInputCache); } catch (IOException e) { LOG.info("pattern: " + pattern); LOG.info("partId: " + partId); LOG.info("ezip: " + ezip); LOG.info("input.available: " + byteArrayInputCache.available()); LOG.info("input.pos: " + byteArrayInputCache.getPos()); LOG.info("output.pos: " + byteArrayOutputCache.getPos()); throw new RuntimeException(e); } }
private void decompressDataInput() { if (!Configuration.get().isUseCompressedCaches() || uncompressed) { return; } //LOG.info("Big no no"); LZ4FastDecompressor decompressor = lz4factory.fastDecompressor(); ByteBuffer dest = ByteBuffer.allocate(uncompressedSize); ByteBuffer src = ByteBuffer.wrap(byteArrayOutputCache.getByteArray(), 0, byteArrayOutputCache.getPos()); decompressor.decompress(src, 0, dest, 0, uncompressedSize); byteArrayOutputCache = new ExtendedByteArrayDataOutput(dest.array(), uncompressedSize); uncompressed = true; }
private void compressDataOutput() { if (!Configuration.get().isUseCompressedCaches() || !uncompressed) { return; } uncompressedSize = byteArrayOutputCache.getPos(); LZ4Compressor lz4Compressor = lz4factory.fastCompressor(); int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize); ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]); int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0, uncompressedSize, compressed, 0, maxCompressedLength); byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength); uncompressed = false; }