public static boolean exists(JobContext job, FileSystem fs, Path inputPath) throws IOException { if (!FileOutputFormat.getCompressOutput(job)) { return fs.exists(inputPath); } else { Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(job, GzipCodec.class); CompressionCodec codec = ReflectionUtils.newInstance(codecClass, job.getConfiguration()); return fs.exists(new Path(inputPath + codec.getDefaultExtension())); } }
public static InputStream openInputStream(JobContext job, Path inputPath, final FileSystem fileSystem) throws IOException { if (!FileOutputFormat.getCompressOutput(job)) { return fileSystem.open(inputPath); } else { Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(job, GzipCodec.class); CompressionCodec codec = ReflectionUtils.newInstance(codecClass, job.getConfiguration()); inputPath = new Path(inputPath + codec.getDefaultExtension()); return codec.createInputStream(fileSystem.open(inputPath)); } }
private void addCodec(CompressionCodec codec) { String suffix = codec.getDefaultExtension(); codecs.put(new StringBuilder(suffix).reverse().toString(), codec); codecsByClassName.put(codec.getClass().getCanonicalName(), codec); String codecName = codec.getClass().getSimpleName(); codecsByName.put(StringUtils.toLowerCase(codecName), codec); if (codecName.endsWith("Codec")) { codecName = codecName.substring(0, codecName.length() - "Codec".length()); codecsByName.put(StringUtils.toLowerCase(codecName), codec); } }
public FSDataOutputStream createFile() throws IOException { Path file; file = FileOutputFormat.getTaskOutputPath(jobConf, fileName + getFileExtention(fileNum++) + codec.getDefaultExtension()); FileSystem fs = file.getFileSystem(jobConf); return fs.create(file, null); }
public static OutputStream makePathAndOutputStream(JobContext job, Path outputPath, boolean deleteExisting) throws IOException { OutputStream retVal; FileSystem fs = outputPath.getFileSystem(job.getConfiguration()); Class<? extends CompressionCodec> codecClass; CompressionCodec codec = null; if (FileOutputFormat.getCompressOutput(job)) { codecClass = FileOutputFormat.getOutputCompressorClass(job, GzipCodec.class); codec = ReflectionUtils.newInstance(codecClass, job.getConfiguration()); outputPath = new Path(outputPath + codec.getDefaultExtension()); } if (fs.exists(outputPath)) { if (deleteExisting) { fs.delete(outputPath, false); } else { throw new ISE("outputPath[%s] must not exist.", outputPath); } } if (FileOutputFormat.getCompressOutput(job)) { retVal = codec.createOutputStream(fs.create(outputPath, false)); } else { retVal = fs.create(outputPath, false); } return retVal; }
/** * Get a {@link Decompressor} for the given {@link CompressionCodec} from the * pool or a new one. * * @param codec the <code>CompressionCodec</code> for which to get the * <code>Decompressor</code> * @return <code>Decompressor</code> for the given * <code>CompressionCodec</code> the pool or a new one */ public static Decompressor getDecompressor(CompressionCodec codec) { Decompressor decompressor = borrow(decompressorPool, codec.getDecompressorType()); if (decompressor == null) { decompressor = codec.createDecompressor(); LOG.info("Got brand-new decompressor ["+codec.getDefaultExtension()+"]"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got recycled decompressor"); } } if (decompressor != null && !decompressor.getClass().isAnnotationPresent(DoNotPool.class)) { updateLeaseCount(decompressorCounts, decompressor, 1); } return decompressor; }
public static String getFileExtension(JobConf conf, StorageFormat storageFormat) { // text format files must have the correct extension when compressed if (!HiveConf.getBoolVar(conf, COMPRESSRESULT) || !HiveIgnoreKeyTextOutputFormat.class.getName().equals(storageFormat.getOutputFormat())) { return ""; } String compressionCodecClass = conf.get("mapred.output.compression.codec"); if (compressionCodecClass == null) { return new DefaultCodec().getDefaultExtension(); } try { Class<? extends CompressionCodec> codecClass = conf.getClassByName(compressionCodecClass).asSubclass(CompressionCodec.class); return ReflectionUtil.newInstance(codecClass, conf).getDefaultExtension(); } catch (ClassNotFoundException e) { throw new PrestoException(HIVE_UNSUPPORTED_FORMAT, "Compression codec not found: " + compressionCodecClass, e); } catch (RuntimeException e) { throw new PrestoException(HIVE_UNSUPPORTED_FORMAT, "Failed to load compression codec: " + compressionCodecClass, e); } }
/** * Get a {@link Compressor} for the given {@link CompressionCodec} from the * pool or a new one. * * @param codec the <code>CompressionCodec</code> for which to get the * <code>Compressor</code> * @param conf the <code>Configuration</code> object which contains confs for creating or reinit the compressor * @return <code>Compressor</code> for the given * <code>CompressionCodec</code> from the pool or a new one */ public static Compressor getCompressor(CompressionCodec codec, Configuration conf) { Compressor compressor = borrow(compressorPool, codec.getCompressorType()); if (compressor == null) { compressor = codec.createCompressor(); LOG.info("Got brand-new compressor ["+codec.getDefaultExtension()+"]"); } else { compressor.reinit(conf); if(LOG.isDebugEnabled()) { LOG.debug("Got recycled compressor"); } } if (compressor != null && !compressor.getClass().isAnnotationPresent(DoNotPool.class)) { updateLeaseCount(compressorCounts, compressor, 1); } return compressor; }
DefaultCodec.class); CompressionCodec codec = ReflectionUtil.newInstance(codecClass, jc); return codec.getDefaultExtension();
DefaultCodec.class); CompressionCodec codec = ReflectionUtil.newInstance(codecClass, jc); return codec.getDefaultExtension();
.map(codec -> { try { return codec.getConstructor().newInstance().getDefaultExtension();
fullFileName += fileSuffix; } else if (codeC != null) { fullFileName += codeC.getDefaultExtension();
byte[] buffer = new byte[100]; String inFilename = removeSuffix(args[i], codec.getDefaultExtension()); in = new java.io.FileInputStream(inFilename); int len = in.read(buffer);
public PartitionFinalizer(SecorConfig config) throws Exception { mConfig = config; Class kafkaClientClass = Class.forName(mConfig.getKafkaClientClass()); this.mKafkaClient = (KafkaClient) kafkaClientClass.newInstance(); this.mKafkaClient.init(config); mZookeeperConnector = new ZookeeperConnector(mConfig); mMessageParser = (TimestampedMessageParser) ReflectionUtil.createMessageParser( mConfig.getMessageParserClass(), mConfig); mQuboleClient = new QuboleClient(mConfig); if (mConfig.getFileExtension() != null && !mConfig.getFileExtension().isEmpty()) { mFileExtension = mConfig.getFileExtension(); } else if (mConfig.getCompressionCodec() != null && !mConfig.getCompressionCodec().isEmpty()) { CompressionCodec codec = CompressionUtil.createCompressionCodec(mConfig.getCompressionCodec()); mFileExtension = codec.getDefaultExtension(); } else { mFileExtension = ""; } mLookbackPeriods = config.getFinalizerLookbackPeriods(); LOG.info("Lookback periods: " + mLookbackPeriods); }
public MessageWriter(SecorConfig config, OffsetTracker offsetTracker, FileRegistry fileRegistry) throws Exception { mConfig = config; mOffsetTracker = offsetTracker; mFileRegistry = fileRegistry; if (mConfig.getCompressionCodec() != null && !mConfig.getCompressionCodec().isEmpty()) { mCodec = CompressionUtil.createCompressionCodec(mConfig.getCompressionCodec()); mFileExtension = mCodec.getDefaultExtension(); } if (mConfig.getFileExtension() != null && !mConfig.getFileExtension().isEmpty()) { mFileExtension = mConfig.getFileExtension(); } else if (mFileExtension == null){ mFileExtension = ""; } mLocalPrefix = mConfig.getLocalPath() + '/' + IdUtil.getLocalMessageDir(); mGeneration = mConfig.getGeneration(); }
outputFilename = StringUtils.removeEnd(originalFilename, codec.getDefaultExtension()); } else { outputFilename = originalFilename;
? putFlowFile.getAttribute(CoreAttributes.FILENAME.key()) + codec.getDefaultExtension() : putFlowFile.getAttribute(CoreAttributes.FILENAME.key());
outputFilename = StringUtils.removeEnd(originalFilename, codec.getDefaultExtension()); } else { outputFilename = originalFilename;
if (mConfig.getCompressionCodec() != null && !mConfig.getCompressionCodec().isEmpty()) { codec = CompressionUtil.createCompressionCodec(mConfig.getCompressionCodec()); extension = codec.getDefaultExtension();
@VisibleForTesting TextReader createReader(Path file, String delim, Configuration cfg, String blockPoolID) throws IOException { FileSystem fs = file.getFileSystem(cfg); if (fs instanceof LocalFileSystem) { fs = ((LocalFileSystem)fs).getRaw(); } CompressionCodecFactory factory = new CompressionCodecFactory(cfg); CompressionCodec codec = factory.getCodec(file); String filename = fileNameFromBlockPoolID(blockPoolID); if (codec != null) { filename = filename + codec.getDefaultExtension(); } Path bpidFilePath = new Path(file.getParent(), filename); return new TextReader(fs, bpidFilePath, codec, delim); }