/** * Based on compression option and configured output codec - get extension for output file. This * is only required for text files - not sequencefiles * * @param jc * Job Configuration * @param isCompressed * Whether the output file is compressed or not * @return the required file extension (example: .gz) * @deprecated Use {@link #getFileExtension(JobConf, boolean, HiveOutputFormat)} */ @Deprecated public static String getFileExtension(JobConf jc, boolean isCompressed) { return getFileExtension(jc, isCompressed, new HiveIgnoreKeyTextOutputFormat()); }
@Override public RecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProperties, Progressable progress) throws IOException { Base64RecordWriter writer = new Base64RecordWriter(super .getHiveRecordWriter(jc, finalOutPath, BytesWritable.class, isCompressed, tableProperties, progress)); writer.configure(jc); return writer; }
@Override public RecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProperties, Progressable progress) throws IOException { Base64RecordWriter writer = new Base64RecordWriter(super .getHiveRecordWriter(jc, finalOutPath, BytesWritable.class, isCompressed, tableProperties, progress)); writer.configure(jc); return writer; }
/** * Based on compression option and configured output codec - get extension for output file. This * is only required for text files - not sequencefiles * * @param jc * Job Configuration * @param isCompressed * Whether the output file is compressed or not * @return the required file extension (example: .gz) * @deprecated Use {@link #getFileExtension(JobConf, boolean, HiveOutputFormat)} */ @Deprecated public static String getFileExtension(JobConf jc, boolean isCompressed) { return getFileExtension(jc, isCompressed, new HiveIgnoreKeyTextOutputFormat()); }
@Override public RecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProperties, Progressable progress) throws IOException { Base64RecordWriter writer = new Base64RecordWriter(super .getHiveRecordWriter(jc, finalOutPath, BytesWritable.class, isCompressed, tableProperties, progress)); writer.configure(jc); return writer; }
@Test public void testGetFileExtension() { JobConf jc = new JobConf(); assertEquals("No extension for uncompressed unknown format", "", getFileExtension(jc, false, null)); assertEquals("No extension for compressed unknown format", "", getFileExtension(jc, true, null)); assertEquals("No extension for uncompressed text format", "", getFileExtension(jc, false, new HiveIgnoreKeyTextOutputFormat())); assertEquals("Deflate for uncompressed text format", ".deflate", getFileExtension(jc, true, new HiveIgnoreKeyTextOutputFormat())); assertEquals("No extension for uncompressed default format", "", getFileExtension(jc, false)); assertEquals("Deflate for uncompressed default format", ".deflate", getFileExtension(jc, true)); String extension = ".myext"; jc.set("hive.output.file.extension", extension); assertEquals("Custom extension for uncompressed unknown format", extension, getFileExtension(jc, false, null)); assertEquals("Custom extension for compressed unknown format", extension, getFileExtension(jc, true, null)); assertEquals("Custom extension for uncompressed text format", extension, getFileExtension(jc, false, new HiveIgnoreKeyTextOutputFormat())); assertEquals("Custom extension for uncompressed text format", extension, getFileExtension(jc, true, new HiveIgnoreKeyTextOutputFormat())); }
@Override public RecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProperties, Progressable progress) throws IOException { Base64RecordWriter writer = new Base64RecordWriter(super .getHiveRecordWriter(jc, finalOutPath, BytesWritable.class, isCompressed, tableProperties, progress)); writer.configure(jc); return writer; }
/** * Based on compression option and configured output codec - get extension for output file. This * is only required for text files - not sequencefiles * * @param jc * Job Configuration * @param isCompressed * Whether the output file is compressed or not * @return the required file extension (example: .gz) * @deprecated Use {@link #getFileExtension(JobConf, boolean, HiveOutputFormat)} */ @Deprecated public static String getFileExtension(JobConf jc, boolean isCompressed) { return getFileExtension(jc, isCompressed, new HiveIgnoreKeyTextOutputFormat()); }
@Override public RecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProperties, Progressable progress) throws IOException { Base64RecordWriter writer = new Base64RecordWriter(super .getHiveRecordWriter(jc, finalOutPath, BytesWritable.class, isCompressed, tableProperties, progress)); writer.configure(jc); return writer; }
/** * Based on compression option and configured output codec - get extension for output file. This * is only required for text files - not sequencefiles * * @param jc * Job Configuration * @param isCompressed * Whether the output file is compressed or not * @return the required file extension (example: .gz) * @deprecated Use {@link #getFileExtension(JobConf, boolean, HiveOutputFormat)} */ @Deprecated public static String getFileExtension(JobConf jc, boolean isCompressed) { return getFileExtension(jc, isCompressed, new HiveIgnoreKeyTextOutputFormat()); }