public BinaryDictionary(DictionaryPage dictionaryPage, Integer length) throws IOException { super(dictionaryPage.getEncoding()); byte[] dictionaryBytes = dictionaryPage.getSlice().getBytes(); content = new Binary[dictionaryPage.getDictionarySize()]; int offset = 0; if (length == null) { for (int i = 0; i < content.length; i++) { int len = readIntLittleEndian(dictionaryBytes, offset); offset += 4; content[i] = Binary.fromByteArray(dictionaryBytes, offset, len); offset += len; } } else { checkArgument(length > 0, "Invalid byte array length: %s", length); for (int i = 0; i < content.length; i++) { content[i] = Binary.fromByteArray(dictionaryBytes, offset, length); offset += length; } } }
public static int readIntLittleEndianPaddedOnBitWidth(InputStream in, int bitWidth) throws IOException { int bytesWidth = paddedByteCountFromBits(bitWidth); switch (bytesWidth) { case 0: return 0; case 1: return BytesUtils.readIntLittleEndianOnOneByte(in); case 2: return BytesUtils.readIntLittleEndianOnTwoBytes(in); case 3: return BytesUtils.readIntLittleEndianOnThreeBytes(in); case 4: return BytesUtils.readIntLittleEndian(in); default: throw new IOException( String.format("Encountered bitWidth (%d) that requires more than 4 bytes", bitWidth)); } }
public static int readIntLittleEndianPaddedOnBitWidth(InputStream in, int bitWidth) throws IOException { int bytesWidth = paddedByteCountFromBits(bitWidth); switch (bytesWidth) { case 0: return 0; case 1: return BytesUtils.readIntLittleEndianOnOneByte(in); case 2: return BytesUtils.readIntLittleEndianOnTwoBytes(in); case 3: return BytesUtils.readIntLittleEndianOnThreeBytes(in); case 4: return BytesUtils.readIntLittleEndian(in); default: throw new IOException( String.format("Encountered bitWidth (%d) that requires more than 4 bytes", bitWidth)); } }
@Override public void initFromPage(int valueCountL, byte[] page, int offset) throws IOException { ByteArrayInputStream in = new ByteArrayInputStream(page, offset, page.length - offset); int length = BytesUtils.readIntLittleEndian(in); decoder = new RunLengthBitPackingHybridDecoder(bitWidth, in); // 4 is for the length which is stored as 4 bytes little endian this.nextOffset = offset + length + 4; }
@Override public void skip() { try { int length = BytesUtils.readIntLittleEndian(in, offset); offset += 4 + length; } catch (IOException e) { throw new ParquetDecodingException("could not skip bytes at offset " + offset, e); } catch (RuntimeException e) { throw new ParquetDecodingException("could not skip bytes at offset " + offset, e); } }
int footerLength = readIntLittleEndian(f); byte[] magic = new byte[MAGIC.length]; f.readFully(magic);
@Override public Binary readBytes() { try { int length = BytesUtils.readIntLittleEndian(in, offset); int start = offset + 4; offset = start + length; return Binary.fromByteArray(in, start, length); } catch (IOException e) { throw new ParquetDecodingException("could not read bytes at offset " + offset, e); } catch (RuntimeException e) { throw new ParquetDecodingException("could not read bytes at offset " + offset, e); } }
@Override public void initFromPage(int valueCount, byte[] in, int offset) throws IOException { if (DEBUG) LOG.debug("reading size at "+ offset + ": " + in[offset] + " " + in[offset + 1] + " " + in[offset + 2] + " " + in[offset + 3] + " "); int totalBytes = BytesUtils.readIntLittleEndian(in, offset); if (DEBUG) LOG.debug("will read "+ totalBytes + " bytes"); currentValueCt = 0; currentValue = 0; bitReader.prepare(in, offset + 4, totalBytes); if (DEBUG) LOG.debug("will read next from " + (offset + totalBytes + 4)); this.nextOffset = offset + totalBytes + 4; }
@Override public int initFromPage(long valueCount, byte[] in, int offset) throws IOException { if (DEBUG) LOG.debug("reading size at "+ offset + ": " + in[offset] + " " + in[offset + 1] + " " + in[offset + 2] + " " + in[offset + 3] + " "); int totalBytes = BytesUtils.readIntLittleEndian(in, offset); if (DEBUG) LOG.debug("will read "+ totalBytes + " bytes"); currentValueCt = 0; currentValue = 0; bitReader.prepare(in, offset + 4, totalBytes); if (DEBUG) LOG.debug("will read next from " + (offset + totalBytes + 4)); return offset + totalBytes + 4; } }
int len = readIntLittleEndian(dictionaryBytes, offset);