protected void readPage() throws IOException { DataPage page = pageReader.readPage(); if (page == null) { return; } page.accept(new DataPage.Visitor<Void>() { @Override public Void visit(DataPageV1 dataPageV1) { readPageV1(dataPageV1); return null; } @Override public Void visit(DataPageV2 dataPageV2) { readPageV2(dataPageV2); return null; } }); }
public BaseVectorizedColumnReader( ColumnDescriptor descriptor, PageReader pageReader, boolean skipTimestampConversion, Type parquetType, TypeInfo hiveType) throws IOException { this.descriptor = descriptor; this.type = parquetType; this.pageReader = pageReader; this.maxDefLevel = descriptor.getMaxDefinitionLevel(); this.skipTimestampConversion = skipTimestampConversion; this.hiveType = hiveType; DictionaryPage dictionaryPage = pageReader.readDictionaryPage(); if (dictionaryPage != null) { try { this.dictionary = ParquetDataColumnReaderFactory .getDataColumnReaderByTypeOnDictionary(parquetType.asPrimitiveType(), hiveType, dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage), skipTimestampConversion); this.isCurrentPageDictionaryEncoded = true; } catch (IOException e) { throw new IOException("could not decode the dictionary for " + descriptor, e); } } else { this.dictionary = null; this.isCurrentPageDictionaryEncoded = false; } }
public VectorizedColumnReader(ColumnDescriptor descriptor, PageReader pageReader) throws IOException { this.descriptor = descriptor; this.pageReader = pageReader; this.maxDefLevel = descriptor.getMaxDefinitionLevel(); DictionaryPage dictionaryPage = pageReader.readDictionaryPage(); if (dictionaryPage != null) { try { this.dictionary = dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage); this.isCurrentPageDictionaryEncoded = true; } catch (IOException e) { throw new IOException("could not decode the dictionary for " + descriptor, e); } } else { this.dictionary = null; this.isCurrentPageDictionaryEncoded = false; } this.totalValueCount = pageReader.getTotalValueCount(); if (totalValueCount == 0) { throw new IOException("totalValueCount == 0"); } }
public void setPageSource(PageReader source) { this.pageSource = source; this.triplesCount = source.getTotalValueCount(); this.triplesRead = 0L; this.advanceNextPageCount = 0L; this.pageIterator.reset(); this.pageIterator.setDictionary(readDictionary(desc, pageSource)); advance(); }
public VectorizedColumnReader( ColumnDescriptor descriptor, OriginalType originalType, PageReader pageReader, TimeZone convertTz) throws IOException { this.descriptor = descriptor; this.pageReader = pageReader; this.convertTz = convertTz; this.originalType = originalType; this.maxDefLevel = descriptor.getMaxDefinitionLevel(); DictionaryPage dictionaryPage = pageReader.readDictionaryPage(); if (dictionaryPage != null) { try { this.dictionary = dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage); this.isCurrentPageDictionaryEncoded = true; } catch (IOException e) { throw new IOException("could not decode the dictionary for " + descriptor, e); } } else { this.dictionary = null; this.isCurrentPageDictionaryEncoded = false; } this.totalValueCount = pageReader.getTotalValueCount(); if (totalValueCount == 0) { throw new IOException("totalValueCount == 0"); } }
public VectorizedColumnReader( ColumnDescriptor descriptor, OriginalType originalType, PageReader pageReader, TimeZone convertTz) throws IOException { this.descriptor = descriptor; this.pageReader = pageReader; this.convertTz = convertTz; this.originalType = originalType; this.maxDefLevel = descriptor.getMaxDefinitionLevel(); DictionaryPage dictionaryPage = pageReader.readDictionaryPage(); if (dictionaryPage != null) { try { this.dictionary = dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage); this.isCurrentPageDictionaryEncoded = true; } catch (IOException e) { throw new IOException("could not decode the dictionary for " + descriptor, e); } } else { this.dictionary = null; this.isCurrentPageDictionaryEncoded = false; } this.totalValueCount = pageReader.getTotalValueCount(); if (totalValueCount == 0) { throw new IOException("totalValueCount == 0"); } }
private void readPage() throws IOException { DataPage page = pageReader.readPage(); // TODO: Why is this a visitor? page.accept(new DataPage.Visitor<Void>() { @Override public Void visit(DataPageV1 dataPageV1) { try { readPageV1(dataPageV1); return null; } catch (IOException e) { throw new RuntimeException(e); } } @Override public Void visit(DataPageV2 dataPageV2) { try { readPageV2(dataPageV2); return null; } catch (IOException e) { throw new RuntimeException(e); } } }); }
public VectorizedColumnReader(ColumnDescriptor descriptor, PageReader pageReader) throws IOException { this.descriptor = descriptor; this.pageReader = pageReader; this.maxDefLevel = descriptor.getMaxDefinitionLevel(); DictionaryPage dictionaryPage = pageReader.readDictionaryPage(); if (dictionaryPage != null) { try { this.dictionary = dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage); this.isCurrentPageDictionaryEncoded = true; } catch (IOException e) { throw new IOException("could not decode the dictionary for " + descriptor, e); } } else { this.dictionary = null; this.isCurrentPageDictionaryEncoded = false; } }
public VectorizedColumnReader(ColumnDescriptor descriptor, PageReader pageReader) throws IOException { this.descriptor = descriptor; this.pageReader = pageReader; this.maxDefLevel = descriptor.getMaxDefinitionLevel(); DictionaryPage dictionaryPage = pageReader.readDictionaryPage(); if (dictionaryPage != null) { try { this.dictionary = dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage); this.isCurrentPageDictionaryEncoded = true; } catch (IOException e) { throw new IOException("could not decode the dictionary for " + descriptor, e); } } else { this.dictionary = null; this.isCurrentPageDictionaryEncoded = false; } this.totalValueCount = pageReader.getTotalValueCount(); if (totalValueCount == 0) { throw new IOException("totalValueCount == 0"); } }
private void readPage() { DataPage page = pageReader.readPage(); // TODO: Why is this a visitor? page.accept(new DataPage.Visitor<Void>() { @Override public Void visit(DataPageV1 dataPageV1) { try { readPageV1(dataPageV1); return null; } catch (IOException e) { throw new RuntimeException(e); } } @Override public Void visit(DataPageV2 dataPageV2) { try { readPageV2(dataPageV2); return null; } catch (IOException e) { throw new RuntimeException(e); } } }); }
private static Dictionary readDictionary(ColumnDescriptor desc, PageReader pageSource) { DictionaryPage dictionaryPage = pageSource.readDictionaryPage(); if (dictionaryPage != null) { try { return dictionaryPage.getEncoding().initDictionary(desc, dictionaryPage); // if (converter.hasDictionarySupport()) { // converter.setDictionary(dictionary); // } } catch (IOException e) { throw new ParquetDecodingException("could not decode the dictionary for " + desc, e); } } return null; } }
this.writerVersion = writerVersion; this.maxDefinitionLevel = path.getMaxDefinitionLevel(); DictionaryPage dictionaryPage = pageReader.readDictionaryPage(); if (dictionaryPage != null) { try { this.dictionary = null; this.totalValueCount = pageReader.getTotalValueCount(); if (totalValueCount <= 0) { throw new ParquetDecodingException("totalValueCount '" + totalValueCount + "' <= 0");
private void readPage() { DataPage page = pageReader.readPage(); // TODO: Why is this a visitor? page.accept(new DataPage.Visitor<Void>() { @Override public Void visit(DataPageV1 dataPageV1) { try { readPageV1(dataPageV1); return null; } catch (IOException e) { throw new RuntimeException(e); } } @Override public Void visit(DataPageV2 dataPageV2) { try { readPageV2(dataPageV2); return null; } catch (IOException e) { throw new RuntimeException(e); } } }); }
this.converter = checkNotNull(converter, "converter"); this.writerVersion = writerVersion; DictionaryPage dictionaryPage = pageReader.readDictionaryPage(); if (dictionaryPage != null) { try { this.dictionary = null; this.totalValueCount = pageReader.getTotalValueCount(); if (totalValueCount <= 0) { throw new ParquetDecodingException("totalValueCount '" + totalValueCount + "' <= 0");
private void readPage() throws IOException { DataPage page = pageReader.readPage(); page.accept(new DataPage.Visitor<Void>() { @Override public Void visit(DataPageV1 dataPageV) { readPageV1(dataPageV); return null; } @Override public Void visit(DataPageV2 dataPageV2) { readPageV2(dataPageV2); return null; } }); }
private void readPage() { LOG.debug("loading page"); DataPage page = pageReader.readPage(); page.accept(new DataPage.Visitor<Void>() { @Override public Void visit(DataPageV1 dataPageV1) { readPageV1(dataPageV1); return null; } @Override public Void visit(DataPageV2 dataPageV2) { readPageV2(dataPageV2); return null; } }); }
private void readPage() { LOG.debug("loading page"); DataPage page = pageReader.readPage(); page.accept(new DataPage.Visitor<Void>() { @Override public Void visit(DataPageV1 dataPageV1) { readPageV1(dataPageV1); return null; } @Override public Void visit(DataPageV2 dataPageV2) { readPageV2(dataPageV2); return null; } }); }
private void readPage() throws IOException { DataPage page = pageReader.readPage(); // TODO: Why is this a visitor? page.accept(new DataPage.Visitor<Void>() { @Override public Void visit(DataPageV1 dataPageV1) { try { readPageV1(dataPageV1); return null; } catch (IOException e) { throw new RuntimeException(e); } } @Override public Void visit(DataPageV2 dataPageV2) { try { readPageV2(dataPageV2); return null; } catch (IOException e) { throw new RuntimeException(e); } } }); }
private void advance() { if (triplesRead >= advanceNextPageCount) { while (!pageIterator.hasNext()) { DataPage page = pageSource.readPage(); if (page != null) { pageIterator.setPage(page); this.advanceNextPageCount += pageIterator.currentPageCount(); } else { return; } } } }
private void validateContains(MessageType schema, PageReadStore pages, String[] path, int values, BytesInput bytes) throws IOException { PageReader pageReader = pages.getPageReader(schema.getColumnDescription(path)); DataPageV1 page = (DataPageV1) pageReader.readPage(); assertEquals(values, page.getValueCount()); assertArrayEquals(bytes.toByteArray(), page.getBytes().toByteArray()); }