/** * Gunzip from the input stream to the output file * * @param in The compressed input stream to read from * @param outFile The file to write the uncompressed results to * * @return A FileCopyResult of the file written */ public static FileUtils.FileCopyResult gunzip(final ByteSource in, File outFile) { return gunzip(in, outFile, FileUtils.IS_EXCEPTION); }
@Override protected InputStream wrapObjectStream(URI object, InputStream stream) throws IOException { return object.getPath().endsWith(".gz") ? CompressionUtils.gzipInputStream(stream) : stream; }
/** * Unzip the byteSource to the output directory. If cacheLocally is true, the byteSource is cached to local disk before unzipping. * This may cause more predictable behavior than trying to unzip a large file directly off a network stream, for example. * * @param byteSource The ByteSource which supplies the zip data * @param outDir The output directory to put the contents of the zip * @param cacheLocally A boolean flag to indicate if the data should be cached locally * * @return A FileCopyResult containing the result of writing the zip entries to disk * * @throws IOException */ public static FileUtils.FileCopyResult unzip( final ByteSource byteSource, final File outDir, boolean cacheLocally ) throws IOException { return unzip(byteSource, outDir, FileUtils.IS_EXCEPTION, cacheLocally); }
return result; if (CompressionUtils.isZip(sourceFile.getName())) { try { final FileUtils.FileCopyResult result = CompressionUtils.unzip( Files.asByteSource(sourceFile), dir, if (CompressionUtils.isGz(sourceFile.getName())) { final File outFile = new File(dir, CompressionUtils.getGzBaseName(sourceFile.getName())); final FileUtils.FileCopyResult result = CompressionUtils.gunzip( Files.asByteSource(sourceFile), outFile,
/** * Zip the contents of directory into the file indicated by outputZipFile. Sub directories are skipped * * @param directory The directory whose contents should be added to the zip in the output stream. * @param outputZipFile The output file to write the zipped data to * @param fsync True if the output file should be fsynced to disk * * @return The number of bytes (uncompressed) read from the input directory. * * @throws IOException */ public static long zip(File directory, File outputZipFile, boolean fsync) throws IOException { if (!isZip(outputZipFile.getName())) { log.warn("No .zip suffix[%s], putting files from [%s] into it anyway.", outputZipFile, directory); } try (final FileOutputStream out = new FileOutputStream(outputZipFile)) { long bytes = zip(directory, out); // For explanation of why fsyncing here is a good practice: // https://github.com/druid-io/druid/pull/5187#pullrequestreview-85188984 if (fsync) { out.getChannel().force(true); } return bytes; } }
/** * Get the file name without the .gz extension * * @param fname The name of the gzip file * * @return fname without the ".gz" extension * * @throws IAE if fname is not a valid "*.gz" file name */ public static String getGzBaseName(String fname) { final String reducedFname = Files.getNameWithoutExtension(fname); if (isGz(fname) && !reducedFname.isEmpty()) { return reducedFname; } throw new IAE("[%s] is not a valid gz file name", fname); } }
/** * Zip the contents of directory into the file indicated by outputZipFile. Sub directories are skipped * * @param directory The directory whose contents should be added to the zip in the output stream. * @param outputZipFile The output file to write the zipped data to * * @return The number of bytes (uncompressed) read from the input directory. * * @throws IOException */ public static long zip(File directory, File outputZipFile) throws IOException { return zip(directory, outputZipFile, false); }
if (CompressionUtils.isZip(s3Coords.path)) { final FileUtils.FileCopyResult result = CompressionUtils.unzip( byteSource, outDir, return result; if (CompressionUtils.isGz(s3Coords.path)) { final String fname = Files.getNameWithoutExtension(uri.getPath()); final File outFile = new File(outDir, fname); final FileUtils.FileCopyResult result = CompressionUtils.gunzip(byteSource, outFile, S3Utils.S3RETRY); log.info("Loaded %d bytes from [%s] to [%s]", result.size(), s3Coords.toString(), outFile.getAbsolutePath()); return result;
if (CompressionUtils.isGz(uriPath)) {
private long compressSegment(File dataSegmentFile, File dest) throws IOException { log.info("Compressing files from[%s] to [%s]", dataSegmentFile, dest); return CompressionUtils.zip(dataSegmentFile, dest, true); }
@Override protected InputStream wrapObjectStream(File object, InputStream stream) throws IOException { return object.getPath().endsWith(".gz") ? CompressionUtils.gzipInputStream(stream) : stream; } }
@Override public FileUtils.FileCopyResult call() throws Exception { return unzip(byteSource.openStream(), outDir); } },
final long indexSize = CompressionUtils.zip(indexFilesDir, zipOutFile);
/** * gunzip the file to the output file. * * @param pulledFile The source of the gz data * @param outFile A target file to put the contents * * @return The result of the file copy * * @throws IOException */ public static FileUtils.FileCopyResult gunzip(final File pulledFile, File outFile) throws IOException { return gunzip(Files.asByteSource(pulledFile), outFile); }
@Override public InputStream openStream() throws IOException { return gzipInputStream(in.openStream()); } },
DEFAULT_RETRY_COUNT ); return unzip(tmpFile, outDir);
@Override protected InputStream wrapObjectStream(S3Object object, InputStream stream) throws IOException { return object.getKey().endsWith(".gz") ? CompressionUtils.gzipInputStream(stream) : stream; }
/** * gunzip from the source stream to the destination stream. * * @param in The input stream which is to be decompressed. This stream is closed. * @param out The output stream to write to. This stream is closed * * @return The number of bytes written to the output stream. * * @throws IOException */ public static long gunzip(InputStream in, OutputStream out) throws IOException { try (GZIPInputStream gzipInputStream = gzipInputStream(in)) { final long result = ByteStreams.copy(gzipInputStream, out); out.flush(); return result; } finally { out.close(); } }
/** * Unzips the input stream via a gzip filter. use gunzip(ByteSource, File, Predicate) if possible * * @param in The input stream to run through the gunzip filter. This stream is closed * @param outFile The file to output to * * @throws IOException */ public static FileUtils.FileCopyResult gunzip(InputStream in, File outFile) throws IOException { try (GZIPInputStream gzipInputStream = gzipInputStream(in)) { Files.asByteSink(outFile).writeFrom(gzipInputStream); return new FileUtils.FileCopyResult(outFile); } }