/** * Based on compression option and configured output codec - get extension for output file. This * is only required for text files - not sequencefiles * * @param jc * Job Configuration * @param isCompressed * Whether the output file is compressed or not * @return the required file extension (example: .gz) * @deprecated Use {@link #getFileExtension(JobConf, boolean, HiveOutputFormat)} */ @Deprecated public static String getFileExtension(JobConf jc, boolean isCompressed) { return getFileExtension(jc, isCompressed, new HiveIgnoreKeyTextOutputFormat()); }
/** * Based on compression option and configured output codec - get extension for output file. This * is only required for text files - not sequencefiles * * @param jc * Job Configuration * @param isCompressed * Whether the output file is compressed or not * @return the required file extension (example: .gz) * @deprecated Use {@link #getFileExtension(JobConf, boolean, HiveOutputFormat)} */ @Deprecated public static String getFileExtension(JobConf jc, boolean isCompressed) { return getFileExtension(jc, isCompressed, new HiveIgnoreKeyTextOutputFormat()); }
public void initializeBucketPaths(int filesIdx, String taskId, boolean isNativeTable, boolean isSkewedStoredAsSubDirectories) { if (isNativeTable) { String extension = Utilities.getFileExtension(jc, isCompressed, hiveOutputFormat); String taskWithExt = extension == null ? taskId : taskId + extension; if (!isMmTable) {
@Test public void testGetFileExtension() { JobConf jc = new JobConf(); assertEquals("No extension for uncompressed unknown format", "", getFileExtension(jc, false, null)); assertEquals("No extension for compressed unknown format", "", getFileExtension(jc, true, null)); assertEquals("No extension for uncompressed text format", "", getFileExtension(jc, false, new HiveIgnoreKeyTextOutputFormat())); assertEquals("Deflate for uncompressed text format", ".deflate", getFileExtension(jc, true, new HiveIgnoreKeyTextOutputFormat())); assertEquals("No extension for uncompressed default format", "", getFileExtension(jc, false)); assertEquals("Deflate for uncompressed default format", ".deflate", getFileExtension(jc, true)); String extension = ".myext"; jc.set("hive.output.file.extension", extension); assertEquals("Custom extension for uncompressed unknown format", extension, getFileExtension(jc, false, null)); assertEquals("Custom extension for compressed unknown format", extension, getFileExtension(jc, true, null)); assertEquals("Custom extension for uncompressed text format", extension, getFileExtension(jc, false, new HiveIgnoreKeyTextOutputFormat())); assertEquals("Custom extension for uncompressed text format", extension, getFileExtension(jc, true, new HiveIgnoreKeyTextOutputFormat())); }
String extension = Utilities.getFileExtension(jc, isCompressed, hiveOutputFormat); if (!bDynParts && !this.isSkewedStoredAsSubDirectories) { fsp.finalPaths[filesIdx] = fsp.getFinalPath(taskId, parent, extension);
/** * get the final output path of a given FileOutputFormat. * * @param parent * parent dir of the expected final output path * @param jc * job configuration * @deprecated */ @Deprecated public static Path getOutputFormatFinalPath(Path parent, String taskId, JobConf jc, HiveOutputFormat<?, ?> hiveOutputFormat, boolean isCompressed, Path defaultFinalPath) throws IOException { if (hiveOutputFormat instanceof HiveIgnoreKeyTextOutputFormat) { return new Path(parent, taskId + Utilities.getFileExtension(jc, isCompressed)); } return defaultFinalPath; }
/** * get the final output path of a given FileOutputFormat. * * @param parent * parent dir of the expected final output path * @param jc * job configuration * @deprecated */ @Deprecated public static Path getOutputFormatFinalPath(Path parent, String taskId, JobConf jc, HiveOutputFormat<?, ?> hiveOutputFormat, boolean isCompressed, Path defaultFinalPath) throws IOException { if (hiveOutputFormat instanceof HiveIgnoreKeyTextOutputFormat) { return new Path(parent, taskId + Utilities.getFileExtension(jc, isCompressed)); } return defaultFinalPath; }
/** * Based on compression option and configured output codec - get extension for output file. This * is only required for text files - not sequencefiles * * @param jc * Job Configuration * @param isCompressed * Whether the output file is compressed or not * @return the required file extension (example: .gz) * @deprecated Use {@link #getFileExtension(JobConf, boolean, HiveOutputFormat)} */ @Deprecated public static String getFileExtension(JobConf jc, boolean isCompressed) { return getFileExtension(jc, isCompressed, new HiveIgnoreKeyTextOutputFormat()); }
/** * Based on compression option and configured output codec - get extension for output file. This * is only required for text files - not sequencefiles * * @param jc * Job Configuration * @param isCompressed * Whether the output file is compressed or not * @return the required file extension (example: .gz) * @deprecated Use {@link #getFileExtension(JobConf, boolean, HiveOutputFormat)} */ @Deprecated public static String getFileExtension(JobConf jc, boolean isCompressed) { return getFileExtension(jc, isCompressed, new HiveIgnoreKeyTextOutputFormat()); }
String extension = Utilities.getFileExtension(jc, isCompressed, hiveOutputFormat); if (!bDynParts) {
String extension = Utilities.getFileExtension(jc, isCompressed, hiveOutputFormat); if (!bDynParts && !this.isSkewedStoredAsSubDirectories) { fsp.finalPaths[filesIdx] = fsp.getFinalPath(taskId, parent, extension);