@Override public void initFromPage(int valueCount, byte[] page, int offset) throws IOException { checkArgument(page.length > offset, "Attempt to read offset not in the page"); ByteArrayInputStream in = new ByteArrayInputStream(page, offset, page.length - offset); int bitWidth = BytesUtils.readIntLittleEndianOnOneByte(in); decoder = new RunLengthBitPackingHybridDecoder(bitWidth, in); }
@Override public ValuesReader getValuesReader(ColumnDescriptor descriptor, ValuesType valuesType) { int bitWidth = BytesUtils.getWidthFromMaxInt(getMaxLevel(descriptor, valuesType)); if (bitWidth == 0) { return new ZeroIntegerValuesReader(); } return new RunLengthBitPackingHybridValuesReader(bitWidth); } },
public BinaryDictionary(DictionaryPage dictionaryPage, Integer length) throws IOException { super(dictionaryPage.getEncoding()); byte[] dictionaryBytes = dictionaryPage.getSlice().getBytes(); content = new Binary[dictionaryPage.getDictionarySize()]; int offset = 0; if (length == null) { for (int i = 0; i < content.length; i++) { int len = readIntLittleEndian(dictionaryBytes, offset); offset += 4; content[i] = Binary.fromByteArray(dictionaryBytes, offset, len); offset += len; } } else { checkArgument(length > 0, "Invalid byte array length: %s", length); for (int i = 0; i < content.length; i++) { content[i] = Binary.fromByteArray(dictionaryBytes, offset, length); offset += length; } } }
/** * * @return a new byte array materializing the contents of this input * @throws IOException */ public byte[] toByteArray() throws IOException { BAOS baos = new BAOS((int)size()); this.writeAllTo(baos); if (DEBUG) LOG.debug("converted " + size() + " to byteArray of " + baos.size() + " bytes"); return baos.getBuf(); }
/** * copies the input into a new byte array * @param bytesInput * @return * @throws IOException */ public static BytesInput copy(BytesInput bytesInput) throws IOException { return from(bytesInput.toByteArray()); }
public FixedLenByteArrayPlainValuesWriter(int length, int initialSize, int pageSize) { this.length = length; this.arrayOut = new CapacityByteArrayOutputStream(initialSize, pageSize); this.out = new LittleEndianDataOutputStream(arrayOut); }
/** * Construct a CapacityByteArrayOutputStream configured such that its initial slab size is * determined by {@link #initialSlabSizeHeuristic}, with targetCapacity == maxCapacityHint */ public static CapacityByteArrayOutputStream withTargetNumSlabs( int minSlabSize, int maxCapacityHint, int targetNumSlabs) { return new CapacityByteArrayOutputStream( initialSlabSizeHeuristic(minSlabSize, maxCapacityHint, targetNumSlabs), maxCapacityHint); }
/** * * @param in * @return a Bytes input that will write the given bytes */ public static BytesInput from(byte[] in) { if (DEBUG) LOG.debug("BytesInput from array of " + in.length + " bytes"); return new ByteArrayBytesInput(in, 0 , in.length); }
/** * @param arrayOut * @return a BytesInput that will write the content of the buffer */ public static BytesInput from(ByteArrayOutputStream baos) { return new BAOSBytesInput(baos); }
/** * uses a trick mentioned in https://developers.google.com/protocol-buffers/docs/encoding to read zigZag encoded data * @param in * @return * @throws IOException */ public static int readZigZagVarInt(InputStream in) throws IOException { int raw = readUnsignedVarInt(in); int temp = (((raw << 31) >> 31) ^ raw) >> 1; return temp ^ (raw & (1 << 31)); }
/** * @param intValue the int to write * @return a BytesInput that will write var int */ public static BytesInput fromUnsignedVarInt(int intValue) { return new UnsignedVarIntBytesInput(intValue); }
/** * @param arrayOut * @return a BytesInput that will write the content of the buffer */ public static BytesInput from(CapacityByteArrayOutputStream arrayOut) { return new CapacityBAOSBytesInput(arrayOut); }
/** * * @return a new byte array materializing the contents of this input * @throws IOException */ public byte[] toByteArray() throws IOException { BAOS baos = new BAOS((int)size()); this.writeAllTo(baos); if (DEBUG) LOG.debug("converted " + size() + " to byteArray of " + baos.size() + " bytes"); return baos.getBuf(); }
public PlainValuesWriter(int initialSize, int pageSize) { arrayOut = new CapacityByteArrayOutputStream(initialSize, pageSize); out = new LittleEndianDataOutputStream(arrayOut); }
private LevelReader buildLevelRLEReader(int maxLevel, Slice slice) { if (maxLevel == 0) { return new LevelNullReader(); } return new LevelRLEReader(new RunLengthBitPackingHybridDecoder(BytesUtils.getWidthFromMaxInt(maxLevel), new ByteArrayInputStream(slice.getBytes()))); }
/** * uses a trick mentioned in https://developers.google.com/protocol-buffers/docs/encoding to read zigZag encoded data * @param in * @return * @throws IOException */ public static int readZigZagVarInt(InputStream in) throws IOException { int raw = readUnsignedVarInt(in); int temp = (((raw << 31) >> 31) ^ raw) >> 1; return temp ^ (raw & (1 << 31)); }
/** * * @param intValue the int to write */ public static BytesInput fromZigZagVarInt(int intValue) { int zigZag = (intValue << 1) ^ (intValue >> 31); return new UnsignedVarIntBytesInput(zigZag); }
/** * @param arrayOut * @return a BytesInput that will write the content of the buffer */ public static BytesInput from(CapacityByteArrayOutputStream arrayOut) { return new CapacityBAOSBytesInput(arrayOut); }
/** * * @return a new byte array materializing the contents of this input * @throws IOException */ public byte[] toByteArray() throws IOException { BAOS baos = new BAOS((int)size()); this.writeAllTo(baos); if (DEBUG) LOG.debug("converted " + size() + " to byteArray of " + baos.size() + " bytes"); return baos.getBuf(); }
public PlainColumnWriter(int initialSize) { arrayOut = new CapacityByteArrayOutputStream(initialSize); out = new LittleEndianDataOutputStream(arrayOut); }