if (sourceFile.equals(dir)) { log.info("Asked to load [%s] into itself, done!", dir); return new FileUtils.FileCopyResult(sourceFile); final FileUtils.FileCopyResult result = new FileUtils.FileCopyResult(sourceFile); for (final File oldFile : files) { if (oldFile.isDirectory()) { result.addFiles( FileUtils.retryCopy( Files.asByteSource(oldFile), shouldRetryPredicate(), DEFAULT_RETRY_COUNT ).getFiles() ); result.size(), sourceFile.getAbsolutePath(), dir.getAbsolutePath() log.info( "Unzipped %d bytes from [%s] to [%s]", result.size(), sourceFile.getAbsolutePath(), dir.getAbsolutePath() log.info( "Gunzipped %d bytes from [%s] to [%s]",
final FileUtils.FileCopyResult result = new FileUtils.FileCopyResult(); while (children.hasNext()) { final LocatedFileStatus child = children.next(); NativeIO.chunkedCopy(in, outFile); result.addFile(outFile); result.size(), path.toString(), outDir.getAbsolutePath() result.size(), path.toString(), outDir.getAbsolutePath() result.size(), path.toString(), outFile.getAbsolutePath()
if (sourceFile.equals(dir)) { log.info("Asked to load [%s] into itself, done!", dir); return new FileUtils.FileCopyResult(sourceFile); final FileUtils.FileCopyResult result = new FileUtils.FileCopyResult(sourceFile); for (final File oldFile : files) { if (oldFile.isDirectory()) { result.addFiles( FileUtils.retryCopy( Files.asByteSource(oldFile), shouldRetryPredicate(), DEFAULT_RETRY_COUNT ).getFiles() ); result.size(), sourceFile.getAbsolutePath(), dir.getAbsolutePath() log.info( "Unzipped %d bytes from [%s] to [%s]", result.size(), sourceFile.getAbsolutePath(), dir.getAbsolutePath() log.info( "Gunzipped %d bytes from [%s] to [%s]",
final FileUtils.FileCopyResult result = new FileUtils.FileCopyResult(); try (final ZipFile zipFile = new ZipFile(pulledFile)) { final Enumeration<? extends ZipEntry> enumeration = zipFile.entries(); result.addFiles( FileUtils.retryCopy( new ByteSource() FileUtils.IS_EXCEPTION, DEFAULT_RETRY_COUNT ).getFiles() );
/** * Copy input byte source to outFile. If outFile exists, it is attempted to be deleted. * * @param byteSource Supplier for an input stream that is to be copied. The resulting stream is closed each iteration * @param outFile Where the file should be written to. * @param shouldRetry Predicate indicating if an error is recoverable and should be retried. * @param maxAttempts The maximum number of assumed recoverable attempts to try before completely failing. * * @throws RuntimeException wrapping the inner exception on failure. */ public static FileCopyResult retryCopy( final ByteSource byteSource, final File outFile, final Predicate<Throwable> shouldRetry, final int maxAttempts ) { try { StreamUtils.retryCopy( byteSource, Files.asByteSink(outFile), shouldRetry, maxAttempts ); return new FileCopyResult(outFile); } catch (Exception e) { throw Throwables.propagate(e); } }
/** * Unzip from the input stream to the output directory, using the entry's file name as the file name in the output directory. * The behavior of directories in the input stream's zip is undefined. * If possible, it is recommended to use unzip(ByteStream, File) instead * * @param in The input stream of the zip data. This stream is closed * @param outDir The directory to copy the unzipped data to * * @return The FileUtils.FileCopyResult containing information on all the files which were written * * @throws IOException */ public static FileUtils.FileCopyResult unzip(InputStream in, File outDir) throws IOException { try (final ZipInputStream zipIn = new ZipInputStream(in)) { final FileUtils.FileCopyResult result = new FileUtils.FileCopyResult(); ZipEntry entry; while ((entry = zipIn.getNextEntry()) != null) { final File file = new File(outDir, entry.getName()); validateZipOutputFile("", file, outDir); NativeIO.chunkedCopy(zipIn, file); result.addFile(file); zipIn.closeEntry(); } return result; } }
/** * Gzips the input file to the output * * @param inFile The file to gzip * @param outFile A target file to copy the uncompressed contents of inFile to * @param shouldRetry Predicate on a potential throwable to determine if the copy should be attempted again. * * @return The result of the file copy * * @throws IOException */ public static FileUtils.FileCopyResult gzip(final File inFile, final File outFile, Predicate<Throwable> shouldRetry) { gzip(Files.asByteSource(inFile), Files.asByteSink(outFile), shouldRetry); return new FileUtils.FileCopyResult(outFile); }
/** * Unzips the input stream via a gzip filter. use gunzip(ByteSource, File, Predicate) if possible * * @param in The input stream to run through the gunzip filter. This stream is closed * @param outFile The file to output to * * @throws IOException */ public static FileUtils.FileCopyResult gunzip(InputStream in, File outFile) throws IOException { try (GZIPInputStream gzipInputStream = gzipInputStream(in)) { NativeIO.chunkedCopy(gzipInputStream, outFile); return new FileUtils.FileCopyResult(outFile); } }
public void addFiles(Collection<File> files) { this.addSizedFiles(files); }
public FileCopyResult(Collection<File> files) { this.addSizedFiles(files); }
@Override public LoadSpecResult loadSegment(File outDir) throws SegmentLoadingException { return new LoadSpecResult(puller.getSegmentFiles(new S3DataSegmentPuller.S3Coords(bucket, key), outDir).size()); }
@Override public LoadSpecResult loadSegment(final File outDir) throws SegmentLoadingException { return new LoadSpecResult(puller.getSegmentFiles(path.toFile(), outDir).size()); } }
public void addFile(File file) { this.addFiles(ImmutableList.of(file)); } }
.call(); return new FileUtils.FileCopyResult(tmpFile); }, Predicates.alwaysTrue(), log.info( "Pull of file[%s] completed in %,d millis (%s bytes)", key, System.currentTimeMillis() - startTime, result.size() ); return result;
); log.info("Loaded %d bytes from [%s] to [%s]", result.size(), actualBlobPath, outDir.getAbsolutePath()); return result;
final FileUtils.FileCopyResult result = new FileUtils.FileCopyResult(); try (final ZipFile zipFile = new ZipFile(pulledFile)) { final Enumeration<? extends ZipEntry> enumeration = zipFile.entries(); result.addFiles( FileUtils.retryCopy( new ByteSource() FileUtils.IS_EXCEPTION, DEFAULT_RETRY_COUNT ).getFiles() );
false ); log.info("Loaded %d bytes from [%s] to [%s]", result.size(), path, outDir.getAbsolutePath()); return result;
false ); LOG.info("Loaded %d bytes from [%s] to [%s]", result.size(), path, outDir.getAbsolutePath()); return result;