DataOutputStream out = new DataOutputStream(baos); this.dataBlockEncoder.startBlockEncoding(encodingCtx, out); ByteBuffer in = getUncompressedBuffer(); in.rewind(); int klength, vlength; "Bug in encoding part of algorithm %s. " + "Probably it requested more bytes than are available.", toString()), e);
it = codec.getIterator(HFileBlock.headerSize(useHBaseChecksum)); "Algorithm '%s' decoded data to different size", codec.toString())); for (int itTime = 0; itTime < benchmarkNTimes; ++itTime) { final long startTime = System.nanoTime(); codec.encodeData(); final long finishTime = System.nanoTime(); if (itTime >= benchmarkNOmit) { System.out.println(codec.toString() + ":"); printBenchmarkResult(totalSize, encodingDurations, Manipulation.ENCODING); printBenchmarkResult(totalSize, durations, Manipulation.DECODING);
/** * Estimate size after second stage of compression (e.g. LZO). * @param comprAlgo compression algorithm to be used for compression * @param compressor compressor corresponding to the given compression * algorithm * @return Size after second stage of compression. */ public int getEncodedCompressedSize(Algorithm comprAlgo, Compressor compressor) throws IOException { byte[] compressedBytes = getEncodedData(); return getCompressedSize(comprAlgo, compressor, compressedBytes, 0, compressedBytes.length); }
outputTuplePct("Total key redundancy", totalKeyRedundancyLength); int compressedSize = EncodedDataBlock.getCompressedSize( compressionAlgorithm, compressor, rawKVs, 0, rawKVs.length); outputTuple(comprAlgo + " only size", INT_FORMAT, System.out.println(codec.toString()); long encodedBytes = codec.getSize(); outputTuple("Encoded bytes", INT_FORMAT, encodedBytes); outputSavings("Key encoding", encodedBytes - totalValueLength, outputSavings("Total encoding", encodedBytes, rawBytes); int encodedCompressedSize = codec.getEncodedCompressedSize( compressionAlgorithm, compressor); outputTuple("Encoding + " + comprAlgo + " size", INT_FORMAT,
/** * Find the size of minimal buffer that could store compressed data. * @return Size in bytes of compressed data. */ public int getSize() { return getEncodedData().length; }
/** @return encoded data with header */ private byte[] getEncodedData() { if (cachedEncodedData != null) { return cachedEncodedData; } cachedEncodedData = encodeData(); return cachedEncodedData; }
.withIncludesMvcc(includesMemstoreTS) .withIncludesTags(USE_TAG).build(); codecs.add(new EncodedDataBlock(d, encoding, rawKVs, meta ));
outputTuplePct("Total key redundancy", totalKeyRedundancyLength); int compressedSize = EncodedDataBlock.getCompressedSize( compressionAlgorithm, compressor, rawKVs, 0, rawKVs.length); outputTuple(comprAlgo + " only size", INT_FORMAT, System.out.println(codec.toString()); long encodedBytes = codec.getSize(); outputTuple("Encoded bytes", INT_FORMAT, encodedBytes); outputSavings("Key encoding", encodedBytes - totalValueLength, outputSavings("Total encoding", encodedBytes, rawBytes); int encodedCompressedSize = codec.getEncodedCompressedSize( compressionAlgorithm, compressor); outputTuple("Encoding + " + comprAlgo + " size", INT_FORMAT,
byte[] encodedDataWithHeader = getEncodedData(); int bytesToSkip = headerSize + Bytes.SIZEOF_SHORT; ByteArrayInputStream bais = new ByteArrayInputStream(encodedDataWithHeader,
/** @return encoded data with header */ private byte[] getEncodedData() { if (cachedEncodedData != null) { return cachedEncodedData; } cachedEncodedData = encodeData(); return cachedEncodedData; }
.withIncludesMvcc(includesMemstoreTS) .withIncludesTags(USE_TAG).build(); codecs.add(new EncodedDataBlock(d, encoding, rawKVs, meta ));
/** * Do the compression. * @return Compressed byte buffer. */ public byte[] doCompressData() { compressedStream.reset(); DataOutputStream dataOut = new DataOutputStream(compressedStream); try { this.dataBlockEncoder.compressKeyValues( dataOut, getUncompressedBuffer(), includesMemstoreTS); } catch (IOException e) { throw new RuntimeException(String.format( "Bug in decoding part of algorithm %s. " + "Probably it requested more bytes than are available.", toString()), e); } return compressedStream.toByteArray(); }
it = codec.getIterator(HFileBlock.headerSize(useHBaseChecksum)); "Algorithm '%s' decoded data to different size", codec.toString())); for (int itTime = 0; itTime < benchmarkNTimes; ++itTime) { final long startTime = System.nanoTime(); codec.encodeData(); final long finishTime = System.nanoTime(); if (itTime >= benchmarkNOmit) { System.out.println(codec.toString() + ":"); printBenchmarkResult(totalSize, encodingDurations, Manipulation.ENCODING); printBenchmarkResult(totalSize, durations, Manipulation.DECODING);
/** * Estimate size after second stage of compression (e.g. LZO). * @param comprAlgo compression algorithm to be used for compression * @param compressor compressor corresponding to the given compression * algorithm * @return Size after second stage of compression. */ public int getEncodedCompressedSize(Algorithm comprAlgo, Compressor compressor) throws IOException { byte[] compressedBytes = getEncodedData(); return getCompressedSize(comprAlgo, compressor, compressedBytes, 0, compressedBytes.length); }
/** * Find the size of minimal buffer that could store compressed data. * @return Size in bytes of compressed data. */ public int getSize() { return getEncodedData().length; }
/** @return encoded data with header */ private byte[] getEncodedData() { if (cachedEncodedData != null) { return cachedEncodedData; } cachedEncodedData = encodeData(); return cachedEncodedData; }
DataOutputStream out = new DataOutputStream(baos); this.dataBlockEncoder.startBlockEncoding(encodingCtx, out); ByteBuffer in = getUncompressedBuffer(); in.rewind(); int klength, vlength; "Bug in encoding part of algorithm %s. " + "Probably it requested more bytes than are available.", toString()), e);
/** * Estimate size after second stage of compression (e.g. LZO). * @param comprAlgo compression algorithm to be used for compression * @param compressor compressor corresponding to the given compression * algorithm * @return Size after second stage of compression. */ public int getEncodedCompressedSize(Algorithm comprAlgo, Compressor compressor) throws IOException { byte[] compressedBytes = getEncodedData(); return getCompressedSize(comprAlgo, compressor, compressedBytes, 0, compressedBytes.length); }
/** * Find the size of minimal buffer that could store compressed data. * @return Size in bytes of compressed data. */ public int getSize() { return getEncodedData().length; }
/** @return encoded data with header */ private byte[] getEncodedData() { if (cachedEncodedData != null) { return cachedEncodedData; } cachedEncodedData = encodeData(); return cachedEncodedData; }