@Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { int n1 = WritableUtils.decodeVIntSize(b1[s1]); int n2 = WritableUtils.decodeVIntSize(b2[s2]); return compareBytes(b1, s1+n1, l1-n1, b2, s2+n2, l2-n2); } }
public static int getTotalLength(byte[] bytes, int offset) { int len = 4; if (hasDecimalOrSecondVInt(bytes[offset])) { int firstVIntLen = WritableUtils.decodeVIntSize(bytes[offset + 4]); len += firstVIntLen; if (hasSecondVInt(bytes[offset + 4])) { len += WritableUtils.decodeVIntSize(bytes[offset + 4 + firstVIntLen]); } } return len; }
public static int getTotalLength(byte[] bytes, int offset) { int len = 4; if (hasDecimalOrSecondVInt(bytes[offset])) { int firstVIntLen = WritableUtils.decodeVIntSize(bytes[offset + 4]); len += firstVIntLen; if (hasSecondVInt(bytes[offset + 4])) { len += WritableUtils.decodeVIntSize(bytes[offset + 4 + firstVIntLen]); } } return len; }
public static int getTotalLength(byte[] bytes, int offset) { int len = 4; if (hasDecimalOrSecondVInt(bytes[offset])) { int firstVIntLen = WritableUtils.decodeVIntSize(bytes[offset + 4]); len += firstVIntLen; if (hasSecondVInt(bytes[offset + 4])) { len += WritableUtils.decodeVIntSize(bytes[offset + 4 + firstVIntLen]); } } return len; }
public void skipVLong(Position readPos) { ponderNextBufferToRead(readPos); byte firstByte = readPos.buffer[readPos.offset++]; int length = (byte) WritableUtils.decodeVIntSize(firstByte); if (length > 1) { readPos.offset += (length - 1); } int diff = readPos.offset - wbSize; while (diff >= 0) { ++readPos.bufferIndex; readPos.buffer = writeBuffers.get(readPos.bufferIndex); readPos.offset = diff; diff = readPos.offset - wbSize; } }
/** * Similar to {@link WritableUtils#readVLong(DataInput)} but reads from a * {@link ByteBuffer}. */ public static long readVLong(ByteBuffer in) { byte firstByte = in.get(); int len = WritableUtils.decodeVIntSize(firstByte); if (len == 1) { return firstByte; } long i = 0; for (int idx = 0; idx < len-1; idx++) { byte b = in.get(); i = i << 8; i = i | (b & 0xFF); } return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i); }
/** * @param buffer buffer to convert * @return vint bytes as an integer. */ public static long bytesToVint(final byte [] buffer) { int offset = 0; byte firstByte = buffer[offset++]; int len = WritableUtils.decodeVIntSize(firstByte); if (len == 1) { return firstByte; } long i = 0; for (int idx = 0; idx < len-1; idx++) { byte b = buffer[offset++]; i = i << 8; i = i | (b & 0xFF); } return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i); }
/** * Reads a zero-compressed encoded long from input buffer and returns it. * @param buffer Binary array * @param offset Offset into array at which vint begins. * @return deserialized long from buffer. */ public static long readAsVLong(final byte [] buffer, final int offset) { byte firstByte = buffer[offset]; int len = WritableUtils.decodeVIntSize(firstByte); if (len == 1) { return firstByte; } long i = 0; for (int idx = 0; idx < len-1; idx++) { byte b = buffer[offset + 1 + idx]; i = i << 8; i = i | (b & 0xFF); } return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i); }
/** {@inheritDoc} */ @Override public int compare(Text val1, long val2Ptr, int val2Len) { int len2 = WritableUtils.decodeVIntSize(GridUnsafe.getByte(val2Ptr)); return HadoopUtils.compareBytes(val1.getBytes(), val1.getLength(), val2Ptr + len2, val2Len - len2); } }
/** * Read a zero-compressed encoded long from a byte array. * * @param bytes the byte array * @param offset the offset in the byte array where the VLong is stored * @return the long */ public static long readVLongFromByteArray(final byte[] bytes, int offset) { byte firstByte = bytes[offset++]; int len = WritableUtils.decodeVIntSize(firstByte); if (len == 1) { return firstByte; } long i = 0; for (int idx = 0; idx < len-1; idx++) { byte b = bytes[offset++]; i = i << 8; i = i | (b & 0xFF); } return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i); }
/** * Reads a zero-compressed encoded long from input stream and returns it. * @param stream Binary input stream * @throws java.io.IOException * @return deserialized long from stream. */ public static long readVLong(DataInput stream) throws IOException { byte firstByte = stream.readByte(); int len = decodeVIntSize(firstByte); if (len == 1) { return firstByte; } long i = 0; for (int idx = 0; idx < len-1; idx++) { byte b = stream.readByte(); i = i << 8; i = i | (b & 0xFF); } return (isNegativeVInt(firstByte) ? (i ^ -1L) : i); }
/** * Reads a zero-compressed encoded long from a byte array and returns it. * * @param bytes * the byte array * @param offset * offset of the array to read from * @param vlong * storing the deserialized long and its size in byte */ public static void readVLong(byte[] bytes, int offset, VLong vlong) { byte firstByte = bytes[offset]; vlong.length = (byte) WritableUtils.decodeVIntSize(firstByte); if (vlong.length == 1) { vlong.value = firstByte; return; } long i = 0; for (int idx = 0; idx < vlong.length - 1; idx++) { byte b = bytes[offset + 1 + idx]; i = i << 8; i = i | (b & 0xFF); } vlong.value = (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i); }
/** * Reads a zero-compressed encoded int from a byte array and returns it. * * @param bytes * the byte array * @param offset * offset of the array to read from * @param vInt * storing the deserialized int and its size in byte */ public static void readVInt(byte[] bytes, int offset, VInt vInt) { byte firstByte = bytes[offset]; vInt.length = (byte) WritableUtils.decodeVIntSize(firstByte); if (vInt.length == 1) { vInt.value = firstByte; return; } int i = 0; for (int idx = 0; idx < vInt.length - 1; idx++) { byte b = bytes[offset + 1 + idx]; i = i << 8; i = i | (b & 0xFF); } vInt.value = (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1) : i); }
public static long getSeconds(byte[] bytes, int offset) { int firstVInt = bytesToInt(bytes, offset); if (firstVInt >= 0 || !hasSecondVInt(bytes[offset + 4])) { return firstVInt & ~DECIMAL_OR_SECOND_VINT_FLAG; } return ((long) (firstVInt & ~DECIMAL_OR_SECOND_VINT_FLAG)) | (LazyBinaryUtils.readVLongFromByteArray(bytes, offset + 4 + WritableUtils.decodeVIntSize(bytes[offset + 4])) << 31); }
/** * Similar to {@link WritableUtils#readVLong(java.io.DataInput)} but reads from a * {@link ByteBuff}. */ public static long readVLong(ByteBuff in) { byte firstByte = in.get(); int len = WritableUtils.decodeVIntSize(firstByte); if (len == 1) { return firstByte; } long i = 0; for (int idx = 0; idx < len-1; idx++) { byte b = in.get(); i = i << 8; i = i | (b & 0xFF); } return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i); }
/** * Gets seconds stored as integer at bytes[offset] * @param bytes * @param offset * @return the number of seconds */ public static long getSeconds(byte[] bytes, int offset) { int lowest31BitsOfSecondsAndFlag = bytesToInt(bytes, offset); if (lowest31BitsOfSecondsAndFlag >= 0 || // the "has decimal or second VInt" flag is not set !hasSecondVInt(bytes[offset + 4])) { // The entire seconds field is stored in the first 4 bytes. return lowest31BitsOfSecondsAndFlag & LOWEST_31_BITS_OF_SEC_MASK; } // We compose the seconds field from two parts. The lowest 31 bits come from the first four // bytes. The higher-order bits come from the second VInt that follows the nanos field. return ((long) (lowest31BitsOfSecondsAndFlag & LOWEST_31_BITS_OF_SEC_MASK)) | (LazyBinaryUtils.readVLongFromByteArray(bytes, offset + 4 + WritableUtils.decodeVIntSize(bytes[offset + 4])) << 31); }
/** * Gets seconds stored as integer at bytes[offset] * @param bytes * @param offset * @return the number of seconds */ public static long getSeconds(byte[] bytes, int offset) { int lowest31BitsOfSecondsAndFlag = bytesToInt(bytes, offset); if (lowest31BitsOfSecondsAndFlag >= 0 || // the "has decimal or second VInt" flag is not set !hasSecondVInt(bytes[offset + 4])) { // The entire seconds field is stored in the first 4 bytes. return lowest31BitsOfSecondsAndFlag & LOWEST_31_BITS_OF_SEC_MASK; } // We compose the seconds field from two parts. The lowest 31 bits come from the first four // bytes. The higher-order bits come from the second VInt that follows the nanos field. return ((long) (lowest31BitsOfSecondsAndFlag & LOWEST_31_BITS_OF_SEC_MASK)) | (LazyBinaryUtils.readVLongFromByteArray(bytes, offset + 4 + WritableUtils.decodeVIntSize(bytes[offset + 4])) << 31); }
private void populateBytes() { Arrays.fill(internalBytes, (byte) 0); long seconds = timestampTZ.getEpochSecond(); int nanos = timestampTZ.getNanos(); boolean hasSecondVInt = seconds < 0 || seconds > Integer.MAX_VALUE; boolean hasDecimal = setNanosBytes(nanos, internalBytes, offset + 4, hasSecondVInt); int firstInt = (int) seconds; if (hasDecimal || hasSecondVInt) { firstInt |= DECIMAL_OR_SECOND_VINT_FLAG; } intToBytes(firstInt, internalBytes, offset); if (hasSecondVInt) { LazyBinaryUtils.writeVLongToByteArray(internalBytes, offset + 4 + WritableUtils.decodeVIntSize(internalBytes[offset + 4]), seconds >> 31); } }
public long readVLong(Position readPos) { ponderNextBufferToRead(readPos); byte firstByte = readPos.buffer[readPos.offset++]; int length = (byte) WritableUtils.decodeVIntSize(firstByte) - 1; if (length == 0) { return firstByte; } long i = 0; if (isAllInOneReadBuffer(length, readPos)) { for (int idx = 0; idx < length; idx++) { i = (i << 8) | (readPos.buffer[readPos.offset + idx] & 0xFF); } readPos.offset += length; } else { for (int idx = 0; idx < length; idx++) { i = (i << 8) | (readNextByte(readPos) & 0xFF); } } return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i); }
/** * Writes a Timestamp's serialized value to byte array b at the given offset * @param t to convert to bytes * @param b destination byte array * @param offset destination offset in the byte array */ public static void convertTimestampToBytes(Timestamp t, byte[] b, int offset) { long seconds = t.toEpochSecond(); int nanos = t.getNanos(); boolean hasSecondVInt = seconds < 0 || seconds > Integer.MAX_VALUE; boolean hasDecimal = setNanosBytes(nanos, b, offset+4, hasSecondVInt); int firstInt = (int) seconds; if (hasDecimal || hasSecondVInt) { firstInt |= DECIMAL_OR_SECOND_VINT_FLAG; } else { firstInt &= LOWEST_31_BITS_OF_SEC_MASK; } intToBytes(firstInt, b, offset); if (hasSecondVInt) { LazyBinaryUtils.writeVLongToByteArray(b, offset + 4 + WritableUtils.decodeVIntSize(b[offset + 4]), seconds >> 31); } }