public boolean isEmpty() { return byteArrayOutputCache.getPos() == 0; }
@Override public boolean isEmpty() { return byteArrayOutputCache.getPos() == 0; }
public boolean isEmpty() { return byteArrayOutputCache.getPos() == 0; }
public String debugCache() { StringBuilder sb = new StringBuilder(); if (byteArrayOutputCache != null) { sb.append('\n'); sb.append("bigDataOutputCache.pos: "); sb.append(byteArrayOutputCache.getPos()); } return sb.toString(); } }
public String debugCache() { StringBuilder sb = new StringBuilder(); if (byteArrayOutputCache != null) { sb.append('\n'); sb.append("bigDataOutputCache.pos: "); sb.append(byteArrayOutputCache.getPos()); } return sb.toString(); } }
public void readStash(ODAGStash stashToReadTo) { ExtendedByteArrayDataInput byteArrayInputCache = null; try { byteArrayInputCache = new ExtendedByteArrayDataInput(byteArrayOutputCache.getByteArray(), 0, byteArrayOutputCache.getPos()); LOG.info(byteArrayInputCache.available()); stashToReadTo.readFields(byteArrayInputCache); } catch (IOException e) { LOG.info("stashToReadTo: " + stashToReadTo); LOG.info("input.available: " + byteArrayInputCache.available()); LOG.info("input.pos: " + byteArrayInputCache.getPos()); LOG.info("output.pos: " + byteArrayOutputCache.getPos()); throw new RuntimeException(e); } }
/** * Serialize given writable object, and return it's size. * * @param w Writable object * @return it's size after serialization */ public static int size(Writable w) { try { ExtendedByteArrayDataOutput out = new ExtendedByteArrayDataOutput(); w.write(out); return out.getPos(); } catch (IOException e) { throw new RuntimeException(e); } }
public boolean overThreshold() { return byteArrayOutputCache.getPos() > Configuration.get().getCacheThresholdSize(); }
public boolean overThreshold() { return byteArrayOutputCache.getPos() > Configuration.get().getCacheThresholdSize(); }
public boolean overThreshold() { return byteArrayOutputCache.getPos() > Configuration.get().getCacheThresholdSize(); }
private synchronized void decompress() { if (uncompressed) { return; } //LOG.info("Big no no"); //long start = System.currentTimeMillis(); LZ4FastDecompressor decompressor = lz4factory.fastDecompressor(); ByteBuffer dest = ByteBuffer.allocate(uncompressedSize); ByteBuffer src = ByteBuffer.wrap(byteArrayOutputCache.getByteArray(), 0, byteArrayOutputCache.getPos()); decompressor.decompress(src, 0, dest, 0, uncompressedSize); byteArrayOutputCache = new ExtendedByteArrayDataOutput(dest.array(), uncompressedSize); uncompressed = true; }
private void compress() { if (!uncompressed) { return; } uncompressedSize = byteArrayOutputCache.getPos(); LZ4Compressor lz4Compressor = lz4factory.fastCompressor(); int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize); ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]); int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0, uncompressedSize, compressed, 0, maxCompressedLength); byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength); uncompressed = false; }
private void compress() { if (!uncompressed) { return; } uncompressedSize = byteArrayOutputCache.getPos(); LZ4Compressor lz4Compressor = lz4factory.fastCompressor(); int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize); ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]); int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0, uncompressedSize, compressed, 0, maxCompressedLength); byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength); uncompressed = false; }
@Override public void run() { part.ensureDecompressed(); totalSizeODAGs.addAndGet(part.getByteArrayOutputCache().getPos()); part.readEzip(tempEzip); targetEzip.aggregate(tempEzip); } }
public ByteArrayObjectCacheIterator(ByteArrayObjectCache objectCache) { ExtendedByteArrayDataOutput byteArrayOutputCache = objectCache.byteArrayOutputCache; byteArrayInputCache = new ExtendedByteArrayDataInput(byteArrayOutputCache.getByteArray(), 0, byteArrayOutputCache.getPos()); configuration = Configuration.get(); }
private void decompressDataInput() { if (!Configuration.get().isUseCompressedCaches() || uncompressed) { return; } //LOG.info("Big no no"); LZ4FastDecompressor decompressor = lz4factory.fastDecompressor(); ByteBuffer dest = ByteBuffer.allocate(uncompressedSize); ByteBuffer src = ByteBuffer.wrap(byteArrayOutputCache.getByteArray(), 0, byteArrayOutputCache.getPos()); decompressor.decompress(src, 0, dest, 0, uncompressedSize); byteArrayOutputCache = new ExtendedByteArrayDataOutput(dest.array(), uncompressedSize); uncompressed = true; }
private void compressDataOutput() { if (!Configuration.get().isUseCompressedCaches() || !uncompressed) { return; } uncompressedSize = byteArrayOutputCache.getPos(); LZ4Compressor lz4Compressor = lz4factory.fastCompressor(); int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize); ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]); int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0, uncompressedSize, compressed, 0, maxCompressedLength); byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength); uncompressed = false; }