public static void configureCompression(Configuration config, HiveCompressionCodec compressionCodec) { boolean compression = compressionCodec != HiveCompressionCodec.NONE; config.setBoolean(COMPRESSRESULT.varname, compression); config.setBoolean("mapred.output.compress", compression); config.setBoolean(FileOutputFormat.COMPRESS, compression); // For DWRF config.set(HIVE_ORC_DEFAULT_COMPRESS.varname, compressionCodec.getOrcCompressionKind().name()); config.set(HIVE_ORC_COMPRESSION.varname, compressionCodec.getOrcCompressionKind().name()); // For ORC config.set(OrcTableProperties.COMPRESSION.getPropName(), compressionCodec.getOrcCompressionKind().name()); // For RCFile and Text if (compressionCodec.getCodec().isPresent()) { config.set("mapred.output.compression.codec", compressionCodec.getCodec().get().getName()); config.set(FileOutputFormat.COMPRESS_CODEC, compressionCodec.getCodec().get().getName()); } else { config.unset("mapred.output.compression.codec"); config.unset(FileOutputFormat.COMPRESS_CODEC); } // For Parquet config.set(ParquetOutputFormat.COMPRESSION, compressionCodec.getParquetCompressionCodec().name()); // For SequenceFile config.set(FileOutputFormat.COMPRESS_TYPE, BLOCK.toString()); }
private static String makeFileName(File tempDir, HiveClientConfig config) { return tempDir.getAbsolutePath() + "/" + config.getHiveStorageFormat().name() + "." + config.getHiveCompressionCodec().name(); }
@Test public void testAllCompression() throws Exception { for (HiveCompressionCodec codec : HiveCompressionCodec.values()) { executeBenchmark(DataSet.LINEITEM, codec, FileFormat.PRESTO_RCBINARY); } }
assertTrue(rowsCount >= 0, "rowsCount must be greater than zero"); String compressionSuffix = compressionCodec.getCodec() .map(codec -> { try {
public PrestoOrcFormatWriter(File targetFile, List<String> columnNames, List<Type> types, DateTimeZone hiveStorageTimeZone, HiveCompressionCodec compressionCodec) throws IOException { writer = new OrcWriter( new OutputStreamOrcDataSink(new FileOutputStream(targetFile)), columnNames, types, ORC, compressionCodec.getOrcCompressionKind(), new OrcWriterOptions(), ImmutableMap.of(), hiveStorageTimeZone, false, BOTH, new OrcWriterStats()); }
HiveFileFormatBenchmark_jmhType _jmh_tryInit_f_hivefileformatbenchmark0_0(InfraControl control) throws Throwable { if (control.isFailing) throw new FailureAssistException(); HiveFileFormatBenchmark_jmhType val = f_hivefileformatbenchmark0_0; if (val == null) { val = new HiveFileFormatBenchmark_jmhType(); Field f; f = com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.class.getDeclaredField("compression"); f.setAccessible(true); f.set(val, com.facebook.presto.hive.HiveCompressionCodec.valueOf(control.getParam("compression"))); f = com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.class.getDeclaredField("dataSet"); f.setAccessible(true); f.set(val, com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.DataSet.valueOf(control.getParam("dataSet"))); f = com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.class.getDeclaredField("fileFormat"); f.setAccessible(true); f.set(val, com.facebook.presto.hive.benchmark.FileFormat.valueOf(control.getParam("fileFormat"))); val.setup(); f_hivefileformatbenchmark0_0 = val; } return val; }
public PrestoRcFileFormatWriter(File targetFile, List<Type> types, RcFileEncoding encoding, HiveCompressionCodec compressionCodec) throws IOException { writer = new RcFileWriter( new OutputStreamSliceOutput(new FileOutputStream(targetFile)), types, encoding, compressionCodec.getCodec().map(Class::getName), new AircompressorCodecFactory(new HadoopCodecFactory(getClass().getClassLoader())), ImmutableMap.of(), true); }
public PrestoDwrfFormatWriter(File targetFile, List<String> columnNames, List<Type> types, DateTimeZone hiveStorageTimeZone, HiveCompressionCodec compressionCodec) throws IOException { writer = new OrcWriter( new OutputStreamOrcDataSink(new FileOutputStream(targetFile)), columnNames, types, DWRF, compressionCodec.getOrcCompressionKind(), new OrcWriterOptions(), ImmutableMap.of(), hiveStorageTimeZone, false, BOTH, new OrcWriterStats()); }
HiveFileFormatBenchmark_jmhType _jmh_tryInit_f_hivefileformatbenchmark0_0(InfraControl control) throws Throwable { if (control.isFailing) throw new FailureAssistException(); HiveFileFormatBenchmark_jmhType val = f_hivefileformatbenchmark0_0; if (val == null) { val = new HiveFileFormatBenchmark_jmhType(); Field f; f = com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.class.getDeclaredField("compression"); f.setAccessible(true); f.set(val, com.facebook.presto.hive.HiveCompressionCodec.valueOf(control.getParam("compression"))); f = com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.class.getDeclaredField("dataSet"); f.setAccessible(true); f.set(val, com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.DataSet.valueOf(control.getParam("dataSet"))); f = com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.class.getDeclaredField("fileFormat"); f.setAccessible(true); f.set(val, com.facebook.presto.hive.benchmark.FileFormat.valueOf(control.getParam("fileFormat"))); val.setup(); f_hivefileformatbenchmark0_0 = val; } return val; }
public static void configureCompression(Configuration config, HiveCompressionCodec compressionCodec) { boolean compression = compressionCodec != HiveCompressionCodec.NONE; config.setBoolean(COMPRESSRESULT.varname, compression); config.setBoolean("mapred.output.compress", compression); config.setBoolean(FileOutputFormat.COMPRESS, compression); // For DWRF config.set(HIVE_ORC_DEFAULT_COMPRESS.varname, compressionCodec.getOrcCompressionKind().name()); config.set(HIVE_ORC_COMPRESSION.varname, compressionCodec.getOrcCompressionKind().name()); // For ORC config.set(OrcTableProperties.COMPRESSION.getPropName(), compressionCodec.getOrcCompressionKind().name()); // For RCFile if (compressionCodec.getCodec().isPresent()) { config.set("mapred.output.compression.codec", compressionCodec.getCodec().get().getName()); config.set(FileOutputFormat.COMPRESS_CODEC, compressionCodec.getCodec().get().getName()); } else { config.unset("mapred.output.compression.codec"); config.unset(FileOutputFormat.COMPRESS_CODEC); } // For Parquet config.set(ParquetOutputFormat.COMPRESSION, compressionCodec.getParquetCompressionCodec().name()); // For SequenceFile config.set(FileOutputFormat.COMPRESS_TYPE, BLOCK.toString()); }
assertTrue(rowsCount >= 0, "rowsCount must be greater than zero"); String compressionSuffix = compressionCodec.getCodec() .map(codec -> { try {
public PrestoOrcFormatWriter(File targetFile, List<String> columnNames, List<Type> types, DateTimeZone hiveStorageTimeZone, HiveCompressionCodec compressionCodec) throws IOException { writer = new OrcWriter( new OutputStreamOrcDataSink(new FileOutputStream(targetFile)), columnNames, types, ORC, compressionCodec.getOrcCompressionKind(), new OrcWriterOptions(), ImmutableMap.of(), hiveStorageTimeZone, false, BOTH, new OrcWriterStats()); }
@Test public void testAllFormats() throws Exception { HiveClientConfig config = new HiveClientConfig(); File tempDir = Files.createTempDir(); try { ExtendedHiveMetastore metastore = createTestingFileHiveMetastore(new File(tempDir, "metastore")); for (HiveStorageFormat format : HiveStorageFormat.values()) { config.setHiveStorageFormat(format); config.setHiveCompressionCodec(NONE); long uncompressedLength = writeTestFile(config, metastore, makeFileName(tempDir, config)); assertGreaterThan(uncompressedLength, 0L); for (HiveCompressionCodec codec : HiveCompressionCodec.values()) { if (codec == NONE) { continue; } config.setHiveCompressionCodec(codec); long length = writeTestFile(config, metastore, makeFileName(tempDir, config)); assertTrue(uncompressedLength > length, format("%s with %s compressed to %s which is not less than %s", format, codec, length, uncompressedLength)); } } } finally { deleteRecursively(tempDir.toPath(), ALLOW_INSECURE); } }
private static String makeFileName(File tempDir, HiveClientConfig config) { return tempDir.getAbsolutePath() + "/" + config.getHiveStorageFormat().name() + "." + config.getHiveCompressionCodec().name(); }
HiveFileFormatBenchmark_jmhType _jmh_tryInit_f_hivefileformatbenchmark0_0(InfraControl control) throws Throwable { if (control.isFailing) throw new FailureAssistException(); HiveFileFormatBenchmark_jmhType val = f_hivefileformatbenchmark0_0; if (val == null) { val = new HiveFileFormatBenchmark_jmhType(); Field f; f = com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.class.getDeclaredField("compression"); f.setAccessible(true); f.set(val, com.facebook.presto.hive.HiveCompressionCodec.valueOf(control.getParam("compression"))); f = com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.class.getDeclaredField("dataSet"); f.setAccessible(true); f.set(val, com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.DataSet.valueOf(control.getParam("dataSet"))); f = com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.class.getDeclaredField("fileFormat"); f.setAccessible(true); f.set(val, com.facebook.presto.hive.benchmark.FileFormat.valueOf(control.getParam("fileFormat"))); val.setup(); f_hivefileformatbenchmark0_0 = val; } return val; }
public PrestoRcFileFormatWriter(File targetFile, List<Type> types, RcFileEncoding encoding, HiveCompressionCodec compressionCodec) throws IOException { writer = new RcFileWriter( new OutputStreamSliceOutput(new FileOutputStream(targetFile)), types, encoding, compressionCodec.getCodec().map(Class::getName), new AircompressorCodecFactory(new HadoopCodecFactory(getClass().getClassLoader())), ImmutableMap.of(), true); }
public PrestoDwrfFormatWriter(File targetFile, List<String> columnNames, List<Type> types, DateTimeZone hiveStorageTimeZone, HiveCompressionCodec compressionCodec) throws IOException { writer = new OrcWriter( new OutputStreamOrcDataSink(new FileOutputStream(targetFile)), columnNames, types, DWRF, compressionCodec.getOrcCompressionKind(), new OrcWriterOptions(), ImmutableMap.of(), hiveStorageTimeZone, false, BOTH, new OrcWriterStats()); }
@Test public void testAllCompression() throws Exception { for (HiveCompressionCodec codec : HiveCompressionCodec.values()) { executeBenchmark(DataSet.LINEITEM, codec, FileFormat.PRESTO_RCBINARY); } }
private static String makeFileName(File tempDir, HiveClientConfig config) { return tempDir.getAbsolutePath() + "/" + config.getHiveStorageFormat().name() + "." + config.getHiveCompressionCodec().name(); }
HiveFileFormatBenchmark_jmhType _jmh_tryInit_f_hivefileformatbenchmark0_0(InfraControl control) throws Throwable { if (control.isFailing) throw new FailureAssistException(); HiveFileFormatBenchmark_jmhType val = f_hivefileformatbenchmark0_0; if (val == null) { val = new HiveFileFormatBenchmark_jmhType(); Field f; f = com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.class.getDeclaredField("compression"); f.setAccessible(true); f.set(val, com.facebook.presto.hive.HiveCompressionCodec.valueOf(control.getParam("compression"))); f = com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.class.getDeclaredField("dataSet"); f.setAccessible(true); f.set(val, com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.DataSet.valueOf(control.getParam("dataSet"))); f = com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.class.getDeclaredField("fileFormat"); f.setAccessible(true); f.set(val, com.facebook.presto.hive.benchmark.FileFormat.valueOf(control.getParam("fileFormat"))); val.setup(); f_hivefileformatbenchmark0_0 = val; } return val; }