public static void configureCompression(Configuration config, HiveCompressionCodec compressionCodec) { boolean compression = compressionCodec != HiveCompressionCodec.NONE; config.setBoolean(COMPRESSRESULT.varname, compression); config.setBoolean("mapred.output.compress", compression); config.setBoolean(FileOutputFormat.COMPRESS, compression); // For DWRF config.set(HIVE_ORC_DEFAULT_COMPRESS.varname, compressionCodec.getOrcCompressionKind().name()); config.set(HIVE_ORC_COMPRESSION.varname, compressionCodec.getOrcCompressionKind().name()); // For ORC config.set(OrcTableProperties.COMPRESSION.getPropName(), compressionCodec.getOrcCompressionKind().name()); // For RCFile and Text if (compressionCodec.getCodec().isPresent()) { config.set("mapred.output.compression.codec", compressionCodec.getCodec().get().getName()); config.set(FileOutputFormat.COMPRESS_CODEC, compressionCodec.getCodec().get().getName()); } else { config.unset("mapred.output.compression.codec"); config.unset(FileOutputFormat.COMPRESS_CODEC); } // For Parquet config.set(ParquetOutputFormat.COMPRESSION, compressionCodec.getParquetCompressionCodec().name()); // For SequenceFile config.set(FileOutputFormat.COMPRESS_TYPE, BLOCK.toString()); }
for (OrcFile.OrcTableProperties property : OrcFile.OrcTableProperties.values()){ String propName = property.getPropName(); if (tableProps.containsKey(propName)){ jobProperties.put(propName,tableProps.get(propName));
for (OrcFile.OrcTableProperties property : OrcFile.OrcTableProperties.values()){ String propName = property.getPropName(); if (tableProps.containsKey(propName)){ jobProperties.put(propName,tableProps.get(propName));
for (OrcFile.OrcTableProperties property : OrcFile.OrcTableProperties.values()){ String propName = property.getPropName(); if (tableProps.containsKey(propName)){ jobProperties.put(propName,tableProps.get(propName));
String propVal ; if ((propVal = getSettingFromPropsFallingBackToConf( OrcFile.OrcTableProperties.STRIPE_SIZE.getPropName(),props,conf)) != null){ options.stripeSize(Long.parseLong(propVal)); OrcFile.OrcTableProperties.COMPRESSION.getPropName(),props,conf)) != null){ options.compress(CompressionKind.valueOf(propVal)); OrcFile.OrcTableProperties.COMPRESSION_BLOCK_SIZE.getPropName(),props,conf)) != null){ options.bufferSize(Integer.parseInt(propVal)); OrcFile.OrcTableProperties.ROW_INDEX_STRIDE.getPropName(),props,conf)) != null){ options.rowIndexStride(Integer.parseInt(propVal)); OrcFile.OrcTableProperties.ENABLE_INDEXES.getPropName(),props,conf)) != null){ if ("false".equalsIgnoreCase(propVal)) { options.rowIndexStride(0); OrcFile.OrcTableProperties.BLOCK_PADDING.getPropName(),props,conf)) != null){ options.blockPadding(Boolean.parseBoolean(propVal)); OrcFile.OrcTableProperties.ENCODING_STRATEGY.getPropName(),props,conf)) != null){ options.encodingStrategy(EncodingStrategy.valueOf(propVal)); OrcFile.OrcTableProperties.BLOOM_FILTER_COLUMNS.getPropName(), props, conf)) != null) { options.bloomFilterColumns(propVal); OrcFile.OrcTableProperties.BLOOM_FILTER_FPP.getPropName(), props, conf)) != null) { options.bloomFilterFpp(Double.parseDouble(propVal));
public static void configureCompression(Configuration config, HiveCompressionCodec compressionCodec) { boolean compression = compressionCodec != HiveCompressionCodec.NONE; config.setBoolean(COMPRESSRESULT.varname, compression); config.setBoolean("mapred.output.compress", compression); config.setBoolean(FileOutputFormat.COMPRESS, compression); // For DWRF config.set(HIVE_ORC_DEFAULT_COMPRESS.varname, compressionCodec.getOrcCompressionKind().name()); config.set(HIVE_ORC_COMPRESSION.varname, compressionCodec.getOrcCompressionKind().name()); // For ORC config.set(OrcTableProperties.COMPRESSION.getPropName(), compressionCodec.getOrcCompressionKind().name()); // For RCFile if (compressionCodec.getCodec().isPresent()) { config.set("mapred.output.compression.codec", compressionCodec.getCodec().get().getName()); config.set(FileOutputFormat.COMPRESS_CODEC, compressionCodec.getCodec().get().getName()); } else { config.unset("mapred.output.compression.codec"); config.unset(FileOutputFormat.COMPRESS_CODEC); } // For Parquet config.set(ParquetOutputFormat.COMPRESSION, compressionCodec.getParquetCompressionCodec().name()); // For SequenceFile config.set(FileOutputFormat.COMPRESS_TYPE, BLOCK.toString()); }
private void putTableProperty(Map<String, String> results, OrcTableProperties property, Object value) { if (value == null) { return; } results.put(property.getPropName(), value.toString()); }
private static CompressionKind getCompression(Properties schema, JobConf configuration, OrcEncoding orcEncoding) { String compressionName = schema.getProperty(OrcTableProperties.COMPRESSION.getPropName()); if (compressionName == null) { compressionName = configuration.get("hive.exec.orc.default.compress"); } if (compressionName == null) { return CompressionKind.ZLIB; } CompressionKind compression; try { compression = CompressionKind.valueOf(compressionName.toUpperCase(ENGLISH)); } catch (IllegalArgumentException e) { throw new PrestoException(HIVE_UNSUPPORTED_FORMAT, "Unknown " + orcEncoding + " compression type " + compressionName); } return compression; } }