validating, conf, MAX_PADDING_SIZE_DEFAULT, ParquetProperties.builder() .withPageSize(pageSize) .withDictionaryPageSize(dictionaryPageSize) .withDictionaryEncoding(enableDictionary) .withWriterVersion(writerVersion) .build());
.withPageSize(getPageSize(conf)) .withDictionaryPageSize(getDictionaryPageSize(conf)) .withDictionaryEncoding(getEnableDictionary(conf)) .withWriterVersion(getWriterVersion(conf)) .estimateRowCountForPageSizeCheck(getEstimatePageSizeCheck(conf)) .withMinRowCountForPageSizeCheck(getMinRowCountForPageSizeCheck(conf)) .withMaxRowCountForPageSizeCheck(getMaxRowCountForPageSizeCheck(conf)) .withColumnIndexTruncateLength(getColumnIndexTruncateLength(conf)) .withPageRowCountLimit(getPageRowCountLimit(conf)) .build();
validating, conf, MAX_PADDING_SIZE_DEFAULT, ParquetProperties.builder() .withPageSize(pageSize) .withDictionaryPageSize(dictionaryPageSize) .withDictionaryEncoding(enableDictionary) .withWriterVersion(writerVersion) .build());
WriterVersion writerVersion) { ParquetProperties props = ParquetProperties.builder() .withPageSize(pageSize) .withDictionaryPageSize(dictionaryPageSize) .withDictionaryEncoding(enableDictionary) .withWriterVersion(writerVersion) .build(); internalWriter = new InternalParquetRecordWriter<T>(w, writeSupport, schema, extraMetaData, blockSize, compressor, validating, props); MemoryManager memoryManager) { ParquetProperties props = ParquetProperties.builder() .withPageSize(pageSize) .withDictionaryPageSize(dictionaryPageSize) .withDictionaryEncoding(enableDictionary) .withWriterVersion(writerVersion) .build(); internalWriter = new InternalParquetRecordWriter<T>(w, writeSupport, schema, extraMetaData, blockSize, compressor, validating, props);
WriterVersion writerVersion) { ParquetProperties props = ParquetProperties.builder() .withPageSize(pageSize) .withDictionaryPageSize(dictionaryPageSize) .withDictionaryEncoding(enableDictionary) .withWriterVersion(writerVersion) .build(); internalWriter = new InternalParquetRecordWriter<T>(w, writeSupport, schema, extraMetaData, blockSize, compressor, validating, props); MemoryManager memoryManager) { ParquetProperties props = ParquetProperties.builder() .withPageSize(pageSize) .withDictionaryPageSize(dictionaryPageSize) .withDictionaryEncoding(enableDictionary) .withWriterVersion(writerVersion) .build(); internalWriter = new InternalParquetRecordWriter<T>(w, writeSupport, schema, extraMetaData, blockSize, compressor, validating, props);
ColumnDescriptor path = schema.getColumns().get(columnIndex); ColumnChunkPageWriteStore store = new ColumnChunkPageWriteStore(compressor, schema, allocator, ParquetProperties.DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH); ColumnWriteStoreV1 columnWriteStoreV1 = new ColumnWriteStoreV1(schema, store, ParquetProperties.builder().build()); for (BlocksCombiner.SmallBlock smallBlock : smallBlocks.getBlocks()) { ParquetFileReader parquetFileReader = smallBlock.getReader();
.withPageSize(getPageSize(conf)) .withDictionaryPageSize(getDictionaryPageSize(conf)) .withDictionaryEncoding(getEnableDictionary(conf)) .withWriterVersion(getWriterVersion(conf)) .estimateRowCountForPageSizeCheck(getEstimatePageSizeCheck(conf)) .withMinRowCountForPageSizeCheck(getMinRowCountForPageSizeCheck(conf)) .withMaxRowCountForPageSizeCheck(getMaxRowCountForPageSizeCheck(conf)) .build();
pageSize, new ParquetDirectByteBufferAllocator(oContext)); ParquetProperties parquetProperties = ParquetProperties.builder() .withPageSize(pageSize) .withDictionaryEncoding(enableDictionary) .withDictionaryPageSize(initialPageBufferSize) .withWriterVersion(writerVersion) .withAllocator(new ParquetDirectByteBufferAllocator(oContext)) .withValuesWriterFactory(new DefaultV1ValuesWriterFactory()) .build(); store = new ColumnWriteStoreV1(pageStore, parquetProperties); MessageColumnIO columnIO = new ColumnIOFactory(false).getColumnIO(this.schema);
/** * Build a {@link ParquetWriter} with the accumulated configuration. * * @return a configured {@code ParquetWriter} instance. * @throws IOException if there is an error while creating the writer */ public ParquetWriter<T> build() throws IOException { if (file != null) { return new ParquetWriter<>(file, mode, getWriteSupport(conf), codecName, rowGroupSize, enableValidation, conf, maxPaddingSize, encodingPropsBuilder.build()); } else { return new ParquetWriter<>(HadoopOutputFile.fromPath(path, conf), mode, getWriteSupport(conf), codecName, rowGroupSize, enableValidation, conf, maxPaddingSize, encodingPropsBuilder.build()); } } }
/** * Build a {@link ParquetWriter} with the accumulated configuration. * * @return a configured {@code ParquetWriter} instance. * @throws IOException if there is an error while creating the writer */ public ParquetWriter<T> build() throws IOException { if (file != null) { return new ParquetWriter<>(file, mode, getWriteSupport(conf), codecName, rowGroupSize, enableValidation, conf, maxPaddingSize, encodingPropsBuilder.build()); } else { return new ParquetWriter<>(HadoopOutputFile.fromPath(path, conf), mode, getWriteSupport(conf), codecName, rowGroupSize, enableValidation, conf, maxPaddingSize, encodingPropsBuilder.build()); } } }
public static Builder builder() { return new Builder(); }
public static Builder copy(ParquetProperties toCopy) { return new Builder(toCopy); }
public static Builder copy(ParquetProperties toCopy) { return new Builder(toCopy); }
public static Builder builder() { return new Builder(); }
/** * Enable or disable dictionary encoding for the constructed writer. * * @param enableDictionary whether dictionary encoding should be enabled * @return this builder for method chaining. */ public SELF withDictionaryEncoding(boolean enableDictionary) { encodingPropsBuilder.withDictionaryEncoding(enableDictionary); return self(); }
/** * Set the Parquet format page size used by the constructed writer. * * @param pageSize an integer size in bytes * @return this builder for method chaining. */ public SELF withPageSize(int pageSize) { encodingPropsBuilder.withPageSize(pageSize); return self(); }
/** * Enables dictionary encoding for the constructed writer. * * @return this builder for method chaining. */ public SELF enableDictionaryEncoding() { encodingPropsBuilder.withDictionaryEncoding(true); return self(); }
/** * Enables dictionary encoding for the constructed writer. * * @return this builder for method chaining. */ public SELF enableDictionaryEncoding() { encodingPropsBuilder.withDictionaryEncoding(true); return self(); }
/** * Set the Parquet format dictionary page size used by the constructed * writer. * * @param dictionaryPageSize an integer size in bytes * @return this builder for method chaining. */ public SELF withDictionaryPageSize(int dictionaryPageSize) { encodingPropsBuilder.withDictionaryPageSize(dictionaryPageSize); return self(); }
/** * Enable or disable dictionary encoding for the constructed writer. * * @param enableDictionary whether dictionary encoding should be enabled * @return this builder for method chaining. */ public SELF withDictionaryEncoding(boolean enableDictionary) { encodingPropsBuilder.withDictionaryEncoding(enableDictionary); return self(); }