@Override public void transform(byte[] buf, int valueOff, int relIndex) { long adjustment = offsetAdjustments[relIndex]; if (adjustment != 0) { long offset = EncodingHelper.decodeLittleEndianFixedWidthLong(buf, valueOff, offsetNumBytes); offset += adjustment; EncodingHelper.encodeLittleEndianFixedWidthLong(offset, buf, valueOff, offsetNumBytes); } } }
private void flushCompressedBlock() throws IOException { if (compressionOutputStream != null) { // First, close compression output stream so that they write all data compressionOutputStream.close(); compressionOutputStream = null; // Encode compressed block size and write it to record stream int valueLengthNumBytes = EncodingHelper.encodeLittleEndianVarInt(compressedBlockOutputStream.size(), valueLengthBuffer); recordFileStream.write(valueLengthBuffer, 0, valueLengthNumBytes); currentRecordOffset += valueLengthNumBytes; // Write compressed block to record stream compressedBlockOutputStream.writeTo(recordFileStream); currentRecordOffset += compressedBlockOutputStream.size(); } }
EncodingHelper.encodeLittleEndianFixedWidthLong(currentRecordOffset, valueOffsetBuffer.array()); int valueLengthNumBytes = EncodingHelper.encodeLittleEndianVarInt(valueLength, valueLengthBuffer); recordFileStream.write(valueLengthBuffer, 0, valueLengthNumBytes); currentRecordOffset += valueLengthNumBytes; int valueLengthNumBytes = EncodingHelper.encodeLittleEndianVarInt(valueLength, valueLengthBuffer); compressionOutputStream.write(valueLengthBuffer, 0, valueLengthNumBytes); EncodingHelper.encodeLittleEndianFixedWidthLong(currentRecordOffset, valueOffsetBuffer.array(), 0, offsetNumBytes); EncodingHelper.encodeLittleEndianFixedWidthLong(offsetInDecompressedBlock, valueOffsetBuffer.array(), offsetNumBytes, offsetInBlockNumBytes);
if (blockCompressionCodec == null) { long recordFileOffset = EncodingHelper.decodeLittleEndianFixedWidthLong(location); long recordFileBlockOffset = EncodingHelper.decodeLittleEndianFixedWidthLong(location.array(), location.arrayOffset() + location.position(), offsetNumBytes); long offsetInBlock = EncodingHelper.decodeLittleEndianFixedWidthLong(location.array(), location.arrayOffset() + location.position() + offsetNumBytes, offsetInBlockNumBytes); int valueSize = EncodingHelper.decodeLittleEndianVarInt(decompressedBlockByteBuffer);
public static void encodeLittleEndianFixedWidthLong(long l, byte[] array) { encodeLittleEndianFixedWidthLong(l, array, 0, array.length); }
public static long decodeLittleEndianFixedWidthLong(ByteBuffer buffer) { return decodeLittleEndianFixedWidthLong(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()); }
@Test public void testDecodeLittleEndianVarInt() throws Exception { assertEquals(1, EncodingHelper.decodeLittleEndianVarInt(ByteBuffer.wrap(new byte[]{1}))); assertEquals(10, EncodingHelper.decodeLittleEndianVarInt(ByteBuffer.wrap(new byte[]{10}))); assertEquals(127, EncodingHelper.decodeLittleEndianVarInt(ByteBuffer.wrap(new byte[]{127}))); assertEquals(128, EncodingHelper.decodeLittleEndianVarInt(ByteBuffer.wrap(new byte[]{-128, 1}))); assertEquals(10, EncodingHelper.decodeLittleEndianVarInt(ByteBuffer.wrap(new byte[]{10}))); assertEquals(0x7fffffff, EncodingHelper.decodeLittleEndianVarInt(ByteBuffer.wrap(new byte[]{-1, -1, -1, -1, 0x07}))); } }
private static ByteBuffer key(long key, int keySize) { byte[] keyBytes = new byte[keySize]; EncodingHelper.encodeLittleEndianFixedWidthLong(key, keyBytes); return ByteBuffer.wrap(keyBytes); }
public Footer(FileChannel channel, int hashIndexBits) throws IOException { fileSize = channel.size(); final int hashIndexSize = 1 << hashIndexBits; footerLength = hashIndexSize * 8 + 8; byte[] footer = new byte[getFooterLength()]; int read = channel.read(ByteBuffer.wrap(footer), getFileSize() - getFooterLength()); if (read != getFooterLength()) { throw new IOException("Tried to read " + getFooterLength() + " bytes of footer, but only got " + read + " bytes!"); } hashIndex = new long[hashIndexSize]; for (int i = 0; i < getHashIndex().length; i++) { final long offset = EncodingHelper.decodeLittleEndianFixedWidthLong(footer, i * 8, 8); if (offset < -1) { throw new IOException(String.format("Read an unexpectedly negative block offset (%d) at block position %d!", offset, i)); } if (offset != -1 && i > 0 && hashIndex[i-1] != -1 && hashIndex[i-1] >= offset) { throw new IOException(String.format("Discovered an offset inversion! block %d offset: %d, block %d offset: %d", i-1, hashIndex[i-1], i, offset)); } getHashIndex()[i] = offset; } maxUncompressedBufferSize = (int) EncodingHelper.decodeLittleEndianFixedWidthLong(footer, footer.length - 8, 4); if (maxUncompressedBufferSize < 0) { throw new IOException(String.format("Read an invalid max uncompressed buffer size of %d!", maxUncompressedBufferSize)); } maxCompressedBufferSize = (int) EncodingHelper.decodeLittleEndianFixedWidthLong(footer, footer.length - 4, 4); if (maxCompressedBufferSize < 0) { throw new IOException(String.format("Read an invalid max uncompressed buffer size of %d!", maxCompressedBufferSize)); } }
int recordSize = EncodingHelper.decodeLittleEndianVarInt(result.getBuffer());
@Override public void hash(ByteBuffer keyBytes, int keySize, byte[] hashBytes) { long key = EncodingHelper.decodeLittleEndianFixedWidthLong(keyBytes); if (hashIndexBits % 8 != 0) { throw new RuntimeException("hashIndexBits must be a multiple of 8"); } long keyBlock = key / NUM_RECORDS_PER_BLOCK; long keyIndex = key % NUM_RECORDS_PER_BLOCK; byte[] keyHashBytes = new byte[keySize]; // Note: this is valid because hashIndexBits must be a multiple of 8 int hashIndexBytes = hashIndexBits / 8; // Encode bytes for key block EncodingHelper.encodeLittleEndianFixedWidthLong(keyBlock, keyHashBytes, keyHashBytes.length - hashIndexBytes, hashIndexBytes); // Encode bytes for key index in block EncodingHelper.encodeLittleEndianFixedWidthLong(keyIndex, keyHashBytes, 0, keyHashBytes.length - hashIndexBytes); for (int i = 0; i < keyHashBytes.length; ++i) { hashBytes[i] = (byte)(0xff & keyHashBytes[keyHashBytes.length - 1 - i]); } } }
@Override public void close() throws IOException { // clear the last block, if there is one if (uncompressedOffset > 0) { clearUncompressed(); } // serialize the footer byte[] footer = new byte[8 * hashIndex.length + 4 + 4]; for (int i = 0; i < hashIndex.length; i++) { EncodingHelper.encodeLittleEndianFixedWidthLong(hashIndex[i], footer, i * 8, 8); } // write the buffer size hints EncodingHelper.encodeLittleEndianFixedWidthLong(maxUncompressedBlockSize, footer, footer.length - 8, 4); EncodingHelper.encodeLittleEndianFixedWidthLong(maxCompressedBlockSize, footer, footer.length - 4, 4); stream.write(footer); numBytesWritten += footer.length; // flush everything and close stream.flush(); stream.close(); }
@Test public void testDecodeLittleEndianFixedWidthLong() throws Exception { byte[] arr = new byte[]{1, 0, 0}; assertEquals(1, EncodingHelper.decodeLittleEndianFixedWidthLong(ByteBuffer.wrap(arr))); arr = new byte[]{-1, -1, -1}; assertEquals(0xffffffL, EncodingHelper.decodeLittleEndianFixedWidthLong(ByteBuffer.wrap(arr))); arr = new byte[]{-1, 1, 0}; assertEquals(0x0001ff, EncodingHelper.decodeLittleEndianFixedWidthLong(ByteBuffer.wrap(arr))); }
@Test public void testEncodeLitteEndianVarInt() throws Exception { byte[] buffer = new byte[EncodingHelper.MAX_VARINT_SIZE]; assertEquals(1, EncodingHelper.encodeLittleEndianVarInt(1, buffer)); assertEquals(ByteBuffer.wrap(new byte[]{1}), ByteBuffer.wrap(buffer, 0, 1)); assertEquals(1, EncodingHelper.encodeLittleEndianVarInt(10, buffer)); assertEquals(ByteBuffer.wrap(new byte[]{10}), ByteBuffer.wrap(buffer, 0, 1)); assertEquals(1, EncodingHelper.encodeLittleEndianVarInt(127, buffer)); assertEquals(ByteBuffer.wrap(new byte[]{127}), ByteBuffer.wrap(buffer, 0, 1)); assertEquals(2, EncodingHelper.encodeLittleEndianVarInt(128, buffer)); assertEquals(ByteBuffer.wrap(new byte[]{(byte)0x80, 1}), ByteBuffer.wrap(buffer, 0, 2)); assertEquals(5, EncodingHelper.encodeLittleEndianVarInt(0x7fffffff, buffer)); assertEquals(ByteBuffer.wrap(new byte[]{-1, -1, -1, -1, 0x07}), ByteBuffer.wrap(buffer, 0, 5)); }
@Test public void testEncodeLittleEndianFixedWidthLong() throws Exception { byte[] arr = new byte[3]; EncodingHelper.encodeLittleEndianFixedWidthLong(1, arr); assertEquals(ByteBuffer.wrap(new byte[]{1, 0, 0}), ByteBuffer.wrap(arr)); EncodingHelper.encodeLittleEndianFixedWidthLong(-1, arr); assertEquals(ByteBuffer.wrap(new byte[]{-1, -1, -1}), ByteBuffer.wrap(arr)); EncodingHelper.encodeLittleEndianFixedWidthLong(0x001ff, arr); assertEquals(ByteBuffer.wrap(new byte[]{-1, 1, 0}), ByteBuffer.wrap(arr)); }