public java.lang.Object getFieldValue(_Fields field) { switch (field) { case NUM_VALUES: return getNum_values(); case NUM_NULLS: return getNum_nulls(); case NUM_ROWS: return getNum_rows(); case ENCODING: return getEncoding(); case DEFINITION_LEVELS_BYTE_LENGTH: return getDefinition_levels_byte_length(); case REPETITION_LEVELS_BYTE_LENGTH: return getRepetition_levels_byte_length(); case IS_COMPRESSED: return isIs_compressed(); case STATISTICS: return getStatistics(); } throw new java.lang.IllegalStateException(); }
public DataPageHeaderV2 deepCopy() { return new DataPageHeaderV2(this); }
case NUM_VALUES: if (value == null) { unsetNum_values(); } else { setNum_values((java.lang.Integer)value); unsetNum_nulls(); } else { setNum_nulls((java.lang.Integer)value); unsetNum_rows(); } else { setNum_rows((java.lang.Integer)value); unsetEncoding(); } else { setEncoding((Encoding)value); unsetDefinition_levels_byte_length(); } else { setDefinition_levels_byte_length((java.lang.Integer)value); unsetRepetition_levels_byte_length(); } else { setRepetition_levels_byte_length((java.lang.Integer)value); unsetIs_compressed(); } else { setIs_compressed((java.lang.Boolean)value);
struct.data_page_header_v2 = new DataPageHeaderV2(); struct.data_page_header_v2.read(iprot); struct.setData_page_header_v2IsSet(true);
private PageHeader newDataPageV2Header( int uncompressedSize, int compressedSize, int valueCount, int nullCount, int rowCount, org.apache.parquet.column.statistics.Statistics<?> statistics, org.apache.parquet.column.Encoding dataEncoding, int rlByteLength, int dlByteLength) { // TODO: pageHeader.crc = ...; DataPageHeaderV2 dataPageHeaderV2 = new DataPageHeaderV2( valueCount, nullCount, rowCount, getEncoding(dataEncoding), dlByteLength, rlByteLength); if (!statistics.isEmpty()) { dataPageHeaderV2.setStatistics( toParquetStatistics(statistics)); } PageHeader pageHeader = new PageHeader(PageType.DATA_PAGE_V2, uncompressedSize, compressedSize); pageHeader.setData_page_header_v2(dataPageHeaderV2); return pageHeader; }
struct.data_page_header_v2 = new DataPageHeaderV2(); struct.data_page_header_v2.read(iprot); struct.setData_page_header_v2IsSet(true);
public Object getFieldValue(_Fields field) { switch (field) { case NUM_VALUES: return getNum_values(); case NUM_NULLS: return getNum_nulls(); case NUM_ROWS: return getNum_rows(); case ENCODING: return getEncoding(); case DEFINITION_LEVELS_BYTE_LENGTH: return getDefinition_levels_byte_length(); case REPETITION_LEVELS_BYTE_LENGTH: return getRepetition_levels_byte_length(); case IS_COMPRESSED: return isIs_compressed(); case STATISTICS: return getStatistics(); } throw new IllegalStateException(); }
case NUM_VALUES: if (value == null) { unsetNum_values(); } else { setNum_values((Integer)value); unsetNum_nulls(); } else { setNum_nulls((Integer)value); unsetNum_rows(); } else { setNum_rows((Integer)value); unsetEncoding(); } else { setEncoding((Encoding)value); unsetDefinition_levels_byte_length(); } else { setDefinition_levels_byte_length((Integer)value); unsetRepetition_levels_byte_length(); } else { setRepetition_levels_byte_length((Integer)value); unsetIs_compressed(); } else { setIs_compressed((Boolean)value);
case 8: // DATA_PAGE_HEADER_V2 if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { struct.data_page_header_v2 = new DataPageHeaderV2(); struct.data_page_header_v2.read(iprot); struct.setData_page_header_v2IsSet(true); } else {
public DataPageHeaderV2 deepCopy() { return new DataPageHeaderV2(this); }
private long readDataPageV2(PageHeader pageHeader, int uncompressedPageSize, int compressedPageSize, List<DataPage> pages) { DataPageHeaderV2 dataHeaderV2 = pageHeader.getData_page_header_v2(); int dataSize = compressedPageSize - dataHeaderV2.getRepetition_levels_byte_length() - dataHeaderV2.getDefinition_levels_byte_length(); pages.add(new DataPageV2( dataHeaderV2.getNum_rows(), dataHeaderV2.getNum_nulls(), dataHeaderV2.getNum_values(), getSlice(dataHeaderV2.getRepetition_levels_byte_length()), getSlice(dataHeaderV2.getDefinition_levels_byte_length()), getParquetEncoding(Encoding.valueOf(dataHeaderV2.getEncoding().name())), getSlice(dataSize), uncompressedPageSize, MetadataReader.readStats( dataHeaderV2.getStatistics(), descriptor.getColumnDescriptor().getType()), dataHeaderV2.isIs_compressed())); return dataHeaderV2.getNum_values(); } }
case 8: // DATA_PAGE_HEADER_V2 if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { struct.data_page_header_v2 = new DataPageHeaderV2(); struct.data_page_header_v2.read(iprot); struct.setData_page_header_v2IsSet(true); } else {
private PageHeader newDataPageV2Header( int uncompressedSize, int compressedSize, int valueCount, int nullCount, int rowCount, org.apache.parquet.column.Encoding dataEncoding, int rlByteLength, int dlByteLength) { // TODO: pageHeader.crc = ...; DataPageHeaderV2 dataPageHeaderV2 = new DataPageHeaderV2( valueCount, nullCount, rowCount, getEncoding(dataEncoding), dlByteLength, rlByteLength); PageHeader pageHeader = new PageHeader(PageType.DATA_PAGE_V2, uncompressedSize, compressedSize); pageHeader.setData_page_header_v2(dataPageHeaderV2); return pageHeader; }
private long readDataPageV2(PageHeader pageHeader, int uncompressedPageSize, int compressedPageSize, List<DataPage> pages) { DataPageHeaderV2 dataHeaderV2 = pageHeader.getData_page_header_v2(); int dataSize = compressedPageSize - dataHeaderV2.getRepetition_levels_byte_length() - dataHeaderV2.getDefinition_levels_byte_length(); pages.add(new DataPageV2( dataHeaderV2.getNum_rows(), dataHeaderV2.getNum_nulls(), dataHeaderV2.getNum_values(), getSlice(dataHeaderV2.getRepetition_levels_byte_length()), getSlice(dataHeaderV2.getDefinition_levels_byte_length()), getParquetEncoding(Encoding.valueOf(dataHeaderV2.getEncoding().name())), getSlice(dataSize), uncompressedPageSize, MetadataReader.readStats( dataHeaderV2.getStatistics(), descriptor.getColumnDescriptor().getType()), dataHeaderV2.isIs_compressed())); return dataHeaderV2.getNum_values(); } }
/** * Performs a deep copy on <i>other</i>. */ public PageHeader(PageHeader other) { __isset_bitfield = other.__isset_bitfield; if (other.isSetType()) { this.type = other.type; } this.uncompressed_page_size = other.uncompressed_page_size; this.compressed_page_size = other.compressed_page_size; this.crc = other.crc; if (other.isSetData_page_header()) { this.data_page_header = new DataPageHeader(other.data_page_header); } if (other.isSetIndex_page_header()) { this.index_page_header = new IndexPageHeader(other.index_page_header); } if (other.isSetDictionary_page_header()) { this.dictionary_page_header = new DictionaryPageHeader(other.dictionary_page_header); } if (other.isSetData_page_header_v2()) { this.data_page_header_v2 = new DataPageHeaderV2(other.data_page_header_v2); } }
valueReadSoFar += pageHeader.data_page_header_v2.getNum_values(); destBuffer = uncompressPage(pageHeader, true); DataPageHeaderV2 dataHeaderV2 = pageHeader.getData_page_header_v2(); int dataSize = compressedPageSize - dataHeaderV2.getRepetition_levels_byte_length() - dataHeaderV2.getDefinition_levels_byte_length(); return new DataPageV2( dataHeaderV2.getNum_rows(), dataHeaderV2.getNum_nulls(), dataHeaderV2.getNum_values(), BytesInput.from(destBuffer, 0, dataHeaderV2.getRepetition_levels_byte_length()), BytesInput.from(destBuffer, dataHeaderV2.getRepetition_levels_byte_length(), dataHeaderV2.getDefinition_levels_byte_length()), parquetMetadataConverter.getEncoding(dataHeaderV2.getEncoding()), BytesInput.from(destBuffer, dataHeaderV2.getRepetition_levels_byte_length() + dataHeaderV2.getDefinition_levels_byte_length(), dataSize), uncompressedPageSize, fromParquetStatistics(dataHeaderV2.getStatistics(), columnDescriptor.getType()), dataHeaderV2.isIs_compressed() ); default:
/** * Performs a deep copy on <i>other</i>. */ public PageHeader(PageHeader other) { __isset_bitfield = other.__isset_bitfield; if (other.isSetType()) { this.type = other.type; } this.uncompressed_page_size = other.uncompressed_page_size; this.compressed_page_size = other.compressed_page_size; this.crc = other.crc; if (other.isSetData_page_header()) { this.data_page_header = new DataPageHeader(other.data_page_header); } if (other.isSetIndex_page_header()) { this.index_page_header = new IndexPageHeader(other.index_page_header); } if (other.isSetDictionary_page_header()) { this.dictionary_page_header = new DictionaryPageHeader(other.dictionary_page_header); } if (other.isSetData_page_header_v2()) { this.data_page_header_v2 = new DataPageHeaderV2(other.data_page_header_v2); } if (other.isSetBloom_filter_page_header()) { this.bloom_filter_page_header = new BloomFilterPageHeader(other.bloom_filter_page_header); } }
valueReadSoFar += pageHeader.data_page_header_v2.getNum_values(); buf = allocator.buffer(pageHeader.compressed_page_size); lastPage = buf; buffer.flip(); DataPageHeaderV2 dataHeaderV2 = pageHeader.getData_page_header_v2(); int dataSize = compressedPageSize - dataHeaderV2.getRepetition_levels_byte_length() - dataHeaderV2.getDefinition_levels_byte_length(); BytesInput decompressedPageData = decompressor.decompress( ByteBuffer byteBuffer = decompressedPageData.toByteBuffer(); int limit = byteBuffer.limit(); byteBuffer.limit(dataHeaderV2.getRepetition_levels_byte_length()); BytesInput repetitionLevels = BytesInput.from(byteBuffer.slice()); byteBuffer.position(dataHeaderV2.getRepetition_levels_byte_length()); byteBuffer.limit(dataHeaderV2.getRepetition_levels_byte_length() + dataHeaderV2.getDefinition_levels_byte_length()); BytesInput definitionLevels = BytesInput.from(byteBuffer.slice()); byteBuffer.position(dataHeaderV2.getRepetition_levels_byte_length() + dataHeaderV2.getDefinition_levels_byte_length()); byteBuffer.limit(limit); BytesInput data = BytesInput.from(byteBuffer.slice()); dataHeaderV2.getNum_rows(), dataHeaderV2.getNum_nulls(), dataHeaderV2.getNum_values(), repetitionLevels, definitionLevels, parquetMetadataConverter.getEncoding(dataHeaderV2.getEncoding()), data, uncompressedPageSize,
case DATA_PAGE_V2: DataPageHeaderV2 dataHeaderV2 = pageHeader.getData_page_header_v2(); int dataSize = compressedPageSize - dataHeaderV2.getRepetition_levels_byte_length() - dataHeaderV2.getDefinition_levels_byte_length(); pagesInChunk.add( new DataPageV2( dataHeaderV2.getNum_rows(), dataHeaderV2.getNum_nulls(), dataHeaderV2.getNum_values(), this.readAsBytesInput(dataHeaderV2.getRepetition_levels_byte_length()), this.readAsBytesInput(dataHeaderV2.getDefinition_levels_byte_length()), converter.getEncoding(dataHeaderV2.getEncoding()), this.readAsBytesInput(dataSize), uncompressedPageSize, converter.fromParquetStatistics( getFileMetaData().getCreatedBy(), dataHeaderV2.getStatistics(), type), dataHeaderV2.isIs_compressed() )); valuesCountReadSoFar += dataHeaderV2.getNum_values(); break; default:
case DATA_PAGE_V2: DataPageHeaderV2 dataHeaderV2 = pageHeader.getData_page_header_v2(); int dataSize = compressedPageSize - dataHeaderV2.getRepetition_levels_byte_length() - dataHeaderV2.getDefinition_levels_byte_length(); pagesInChunk.add( new DataPageV2( dataHeaderV2.getNum_rows(), dataHeaderV2.getNum_nulls(), dataHeaderV2.getNum_values(), this.readAsBytesInput(dataHeaderV2.getRepetition_levels_byte_length()), this.readAsBytesInput(dataHeaderV2.getDefinition_levels_byte_length()), converter.getEncoding(dataHeaderV2.getEncoding()), this.readAsBytesInput(dataSize), uncompressedPageSize, converter.fromParquetStatistics( getFileMetaData().getCreatedBy(), dataHeaderV2.getStatistics(), type), dataHeaderV2.isIs_compressed() )); valuesCountReadSoFar += dataHeaderV2.getNum_values(); ++dataPageCountReadSoFar; break;