private long readDataPageV2(PageHeader pageHeader, int uncompressedPageSize, int compressedPageSize, List<DataPage> pages) { DataPageHeaderV2 dataHeaderV2 = pageHeader.getData_page_header_v2(); int dataSize = compressedPageSize - dataHeaderV2.getRepetition_levels_byte_length() - dataHeaderV2.getDefinition_levels_byte_length(); pages.add(new DataPageV2( dataHeaderV2.getNum_rows(), dataHeaderV2.getNum_nulls(), dataHeaderV2.getNum_values(), getSlice(dataHeaderV2.getRepetition_levels_byte_length()), getSlice(dataHeaderV2.getDefinition_levels_byte_length()), getParquetEncoding(Encoding.valueOf(dataHeaderV2.getEncoding().name())), getSlice(dataSize), uncompressedPageSize, MetadataReader.readStats( dataHeaderV2.getStatistics(), descriptor.getColumnDescriptor().getType()), dataHeaderV2.isIs_compressed())); return dataHeaderV2.getNum_values(); } }
case 8: // DATA_PAGE_HEADER_V2 if (field.type == TType.STRUCT) { this.data_page_header_v2 = new DataPageHeaderV2(); this.data_page_header_v2.read(iprot); } else { TProtocolUtil.skip(iprot, field.type);
private PageHeader newDataPageV2Header( int uncompressedSize, int compressedSize, int valueCount, int nullCount, int rowCount, parquet.column.statistics.Statistics<?> statistics, parquet.column.Encoding dataEncoding, int rlByteLength, int dlByteLength) { // TODO: pageHeader.crc = ...; DataPageHeaderV2 dataPageHeaderV2 = new DataPageHeaderV2( valueCount, nullCount, rowCount, getEncoding(dataEncoding), dlByteLength, rlByteLength); if (!statistics.isEmpty()) { dataPageHeaderV2.setStatistics(toParquetStatistics(statistics)); } PageHeader pageHeader = new PageHeader(PageType.DATA_PAGE_V2, uncompressedSize, compressedSize); pageHeader.setData_page_header_v2(dataPageHeaderV2); return pageHeader; }
@Override public boolean equals(Object that) { if (that == null) return false; if (that instanceof DataPageHeaderV2) return this.equals((DataPageHeaderV2)that); return false; }
public DataPageHeaderV2 deepCopy() { return new DataPageHeaderV2(this); }
case 8: // DATA_PAGE_HEADER_V2 if (field.type == TType.STRUCT) { this.data_page_header_v2 = new DataPageHeaderV2(); this.data_page_header_v2.read(iprot); } else { TProtocolUtil.skip(iprot, field.type);
@Override public boolean equals(Object that) { if (that == null) return false; if (that instanceof DataPageHeaderV2) return this.equals((DataPageHeaderV2)that); return false; }
public DataPageHeaderV2 deepCopy() { return new DataPageHeaderV2(this); }
public Object getFieldValue(_Fields field) { switch (field) { case NUM_VALUES: return new Integer(getNum_values()); case NUM_NULLS: return new Integer(getNum_nulls()); case NUM_ROWS: return new Integer(getNum_rows()); case ENCODING: return getEncoding(); case DEFINITION_LEVELS_BYTE_LENGTH: return new Integer(getDefinition_levels_byte_length()); case REPETITION_LEVELS_BYTE_LENGTH: return new Integer(getRepetition_levels_byte_length()); case IS_COMPRESSED: return new Boolean(isIs_compressed()); case STATISTICS: return getStatistics(); } throw new IllegalStateException(); }
if (!(this_present_data_page_header_v2 && that_present_data_page_header_v2)) return false; if (!this.data_page_header_v2.equals(that.data_page_header_v2)) return false;
/** * Performs a deep copy on <i>other</i>. */ public PageHeader(PageHeader other) { __isset_bit_vector.clear(); __isset_bit_vector.or(other.__isset_bit_vector); if (other.isSetType()) { this.type = other.type; } this.uncompressed_page_size = other.uncompressed_page_size; this.compressed_page_size = other.compressed_page_size; this.crc = other.crc; if (other.isSetData_page_header()) { this.data_page_header = new DataPageHeader(other.data_page_header); } if (other.isSetIndex_page_header()) { this.index_page_header = new IndexPageHeader(other.index_page_header); } if (other.isSetDictionary_page_header()) { this.dictionary_page_header = new DictionaryPageHeader(other.dictionary_page_header); } if (other.isSetData_page_header_v2()) { this.data_page_header_v2 = new DataPageHeaderV2(other.data_page_header_v2); } }
public Object getFieldValue(_Fields field) { switch (field) { case NUM_VALUES: return new Integer(getNum_values()); case NUM_NULLS: return new Integer(getNum_nulls()); case NUM_ROWS: return new Integer(getNum_rows()); case ENCODING: return getEncoding(); case DEFINITION_LEVELS_BYTE_LENGTH: return new Integer(getDefinition_levels_byte_length()); case REPETITION_LEVELS_BYTE_LENGTH: return new Integer(getRepetition_levels_byte_length()); case IS_COMPRESSED: return new Boolean(isIs_compressed()); case STATISTICS: return getStatistics(); } throw new IllegalStateException(); }
if (!(this_present_data_page_header_v2 && that_present_data_page_header_v2)) return false; if (!this.data_page_header_v2.equals(that.data_page_header_v2)) return false;
/** * Performs a deep copy on <i>other</i>. */ public PageHeader(PageHeader other) { __isset_bit_vector.clear(); __isset_bit_vector.or(other.__isset_bit_vector); if (other.isSetType()) { this.type = other.type; } this.uncompressed_page_size = other.uncompressed_page_size; this.compressed_page_size = other.compressed_page_size; this.crc = other.crc; if (other.isSetData_page_header()) { this.data_page_header = new DataPageHeader(other.data_page_header); } if (other.isSetIndex_page_header()) { this.index_page_header = new IndexPageHeader(other.index_page_header); } if (other.isSetDictionary_page_header()) { this.dictionary_page_header = new DictionaryPageHeader(other.dictionary_page_header); } if (other.isSetData_page_header_v2()) { this.data_page_header_v2 = new DataPageHeaderV2(other.data_page_header_v2); } }
private long readDataPageV2(PageHeader pageHeader, int uncompressedPageSize, int compressedPageSize, List<DataPage> pages) throws IOException { DataPageHeaderV2 dataHeaderV2 = pageHeader.getData_page_header_v2(); int dataSize = compressedPageSize - dataHeaderV2.getRepetition_levels_byte_length() - dataHeaderV2.getDefinition_levels_byte_length(); pages.add(new DataPageV2( dataHeaderV2.getNum_rows(), dataHeaderV2.getNum_nulls(), dataHeaderV2.getNum_values(), getBytesInput(dataHeaderV2.getRepetition_levels_byte_length()), getBytesInput(dataHeaderV2.getDefinition_levels_byte_length()), Encoding.valueOf(dataHeaderV2.getEncoding().name()), getBytesInput(dataSize), uncompressedPageSize, ParquetMetadataReader.readStats( dataHeaderV2.getStatistics(), descriptor.getColumnDescriptor().getType()), dataHeaderV2.isIs_compressed())); return dataHeaderV2.getNum_values(); } }
converter.getEncoding(dataHeaderV1.getRepetition_level_encoding()), converter.getEncoding(dataHeaderV1.getDefinition_level_encoding()), converter.getEncoding(dataHeaderV1.getEncoding()) )); valuesCountReadSoFar += dataHeaderV1.getNum_values(); case DATA_PAGE_V2: DataPageHeaderV2 dataHeaderV2 = pageHeader.getData_page_header_v2(); int dataSize = compressedPageSize - dataHeaderV2.getRepetition_levels_byte_length() - dataHeaderV2.getDefinition_levels_byte_length(); pagesInChunk.add( new DataPageV2( dataHeaderV2.getNum_rows(), dataHeaderV2.getNum_nulls(), dataHeaderV2.getNum_values(), this.readAsBytesInput(dataHeaderV2.getRepetition_levels_byte_length()), this.readAsBytesInput(dataHeaderV2.getDefinition_levels_byte_length()), converter.getEncoding(dataHeaderV2.getEncoding()), this.readAsBytesInput(dataSize), uncompressedPageSize, fromParquetStatistics(dataHeaderV2.getStatistics(), descriptor.col.getType()), dataHeaderV2.isIs_compressed() )); valuesCountReadSoFar += dataHeaderV2.getNum_values(); break; default: