public static CompressionCodecName getParquetCompressionCodec(Configuration configuration) { return CompressionCodecName.fromConf(configuration.get(ParquetOutputFormat.COMPRESSION, UNCOMPRESSED.name())); }
private void initializeSerProperties(JobContext job, Properties tableProperties) { String blockSize = tableProperties.getProperty(ParquetOutputFormat.BLOCK_SIZE); Configuration conf = ContextUtil.getConfiguration(job); if (blockSize != null && !blockSize.isEmpty()) { LOG.debug("get override parquet.block.size property via tblproperties"); conf.setInt(ParquetOutputFormat.BLOCK_SIZE, Integer.valueOf(blockSize)); } String enableDictionaryPage = tableProperties.getProperty(ParquetOutputFormat.ENABLE_DICTIONARY); if (enableDictionaryPage != null && !enableDictionaryPage.isEmpty()) { LOG.debug("get override parquet.enable.dictionary property via tblproperties"); conf.setBoolean(ParquetOutputFormat.ENABLE_DICTIONARY, Boolean.valueOf(enableDictionaryPage)); } String compressionName = tableProperties.getProperty(ParquetOutputFormat.COMPRESSION); if (compressionName != null && !compressionName.isEmpty()) { //get override compression properties via "tblproperties" clause if it is set LOG.debug("get override compression properties via tblproperties"); CompressionCodecName codecName = CompressionCodecName.fromConf(compressionName); conf.set(ParquetOutputFormat.COMPRESSION, codecName.name()); } }
/** * Creates a new ParquetAppender. * * @param conf Configuration properties. * @param schema The table schema. * @param meta The table metadata. * @param path The path of the Parquet file to write to. */ public ParquetAppender(Configuration conf, Schema schema, TableMeta meta, Path path) throws IOException { super(conf, schema, meta, path); this.blockSize = Integer.parseInt( meta.getOption(ParquetOutputFormat.BLOCK_SIZE, StorageConstants.PARQUET_DEFAULT_BLOCK_SIZE)); this.pageSize = Integer.parseInt( meta.getOption(ParquetOutputFormat.PAGE_SIZE, StorageConstants.PARQUET_DEFAULT_PAGE_SIZE)); this.compressionCodecName = CompressionCodecName.fromConf( meta.getOption(ParquetOutputFormat.COMPRESSION, StorageConstants.PARQUET_DEFAULT_COMPRESSION_CODEC_NAME)); this.enableDictionary = Boolean.parseBoolean( meta.getOption(ParquetOutputFormat.ENABLE_DICTIONARY, StorageConstants.PARQUET_DEFAULT_IS_DICTIONARY_ENABLED)); this.validating = Boolean.parseBoolean( meta.getOption(ParquetOutputFormat.VALIDATION, StorageConstants.PARQUET_DEFAULT_IS_VALIDATION_ENABLED)); }