protected PageHeader readPageHeader() throws IOException { return Util.readPageHeader(this); }
private static Optional<DictionaryPage> readDictionaryPage(byte[] data, CompressionCodecName codecName) { try { ByteArrayInputStream inputStream = new ByteArrayInputStream(data); PageHeader pageHeader = Util.readPageHeader(inputStream); if (pageHeader.type != PageType.DICTIONARY_PAGE) { return Optional.empty(); } Slice compressedData = wrappedBuffer(data, data.length - inputStream.available(), pageHeader.getCompressed_page_size()); DictionaryPageHeader dicHeader = pageHeader.getDictionary_page_header(); ParquetEncoding encoding = getParquetEncoding(Encoding.valueOf(dicHeader.getEncoding().name())); int dictionarySize = dicHeader.getNum_values(); return Optional.of(new DictionaryPage(decompress(codecName, compressedData, pageHeader.getUncompressed_page_size()), dictionarySize, encoding)); } catch (IOException ignored) { return Optional.empty(); } }
protected PageHeader readPageHeader() throws IOException { return Util.readPageHeader(this); }
protected PageHeader readPageHeader() throws IOException { return Util.readPageHeader(this); }
protected PageHeader readPageHeader() throws IOException { PageHeader pageHeader; int initialPos = this.pos; try { pageHeader = Util.readPageHeader(this); } catch (IOException e) { // this is to workaround a bug where the compressedLength // of the chunk is missing the size of the header of the dictionary // to allow reading older files (using dictionary) we need this. // usually 13 to 19 bytes are missing // if the last page is smaller than this, the page header itself is truncated in the buffer. this.pos = initialPos; // resetting the buffer to the position before we got the error LOG.info("completing the column chunk to read the page header"); pageHeader = Util.readPageHeader(new SequenceInputStream(this, f)); // trying again from the buffer + remainder of the stream. } return pageHeader; }
private static DictionaryPage readDictionaryPage(byte[] data, ParquetCodecFactory codecFactory, CompressionCodecName codecName) { try { ByteArrayInputStream inputStream = new ByteArrayInputStream(data); PageHeader pageHeader = Util.readPageHeader(inputStream); if (pageHeader.type != PageType.DICTIONARY_PAGE) { return null; } // todo this wrapper is not needed BytesInput compressedData = BytesInput.from(data, data.length - inputStream.available(), pageHeader.getCompressed_page_size()); BytesDecompressor decompressor = codecFactory.getDecompressor(codecName); BytesInput decompressed = decompressor.decompress(compressedData, pageHeader.getUncompressed_page_size()); DictionaryPageHeader dicHeader = pageHeader.getDictionary_page_header(); Encoding encoding = Encoding.valueOf(dicHeader.getEncoding().name()); int dictionarySize = dicHeader.getNum_values(); return new DictionaryPage(decompressed, dictionarySize, encoding); } catch (IOException ignored) { return null; } }