private static void copyToLocal(Metadata src, Path dst) throws IOException { FileChannel dstCh = FileChannel.open( dst, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW); ReadableByteChannel srcCh = FileSystems.open(src.resourceId()); long srcSize = src.sizeBytes(); long copied = 0; do { copied += dstCh.transferFrom(srcCh, copied, srcSize - copied); } while (copied < srcSize); dstCh.close(); srcCh.close(); Preconditions.checkState(copied == srcSize); }
long srcSize = srcMeta.sizeBytes(); boolean shouldDownload = true; if (Files.exists(dst)) {
@Override public final BoundedReader<T> createReader(PipelineOptions options) throws IOException { // Validate the current source prior to creating a reader for it. this.validate(); String fileOrPattern = fileOrPatternSpec.get(); if (mode == Mode.FILEPATTERN) { long startTime = System.currentTimeMillis(); List<Metadata> fileMetadata = FileSystems.match(fileOrPattern, emptyMatchTreatment).metadata(); LOG.info("Matched {} files for pattern {}", fileMetadata.size(), fileOrPattern); List<FileBasedReader<T>> fileReaders = new ArrayList<>(); for (Metadata metadata : fileMetadata) { long endOffset = metadata.sizeBytes(); fileReaders.add( createForSubrangeOfFile(metadata, 0, endOffset).createSingleFileReader(options)); } LOG.debug( "Creating a reader for file pattern {} took {} ms", fileOrPattern, System.currentTimeMillis() - startTime); if (fileReaders.size() == 1) { return fileReaders.get(0); } return new FilePatternReader(this, fileReaders); } else { return createSingleFileReader(options); } }
input -> { assertEquals(path, input.getMetadata().resourceId().toString()); assertEquals("Hello world".length(), input.getMetadata().sizeBytes()); assertEquals(Compression.UNCOMPRESSED, input.getCompression()); assertTrue(input.getMetadata().isReadSeekEfficient()); input -> { assertEquals(pathGZ, input.getMetadata().resourceId().toString()); assertFalse(input.getMetadata().sizeBytes() == "Hello world".length()); assertEquals(Compression.GZIP, input.getCompression()); assertFalse(input.getMetadata().isReadSeekEfficient());
List<FileBasedSource<T>> splitResults = new ArrayList<>(expandedFiles.size()); for (Metadata metadata : expandedFiles) { FileBasedSource<T> split = createForSubrangeOfFile(metadata, 0, metadata.sizeBytes()); verify( split.getMode() == Mode.SINGLE_FILE_OR_SUBRANGE,
@ProcessElement public void process(ProcessContext c) { MatchResult.Metadata metadata = c.element(); if (metadata.resourceId().isDirectory()) { switch (spec.getDirectoryTreatment()) { case SKIP: return; case PROHIBIT: throw new IllegalArgumentException( "Trying to read " + metadata.resourceId() + " which is a directory"); default: throw new UnsupportedOperationException( "Unknown DirectoryTreatment: " + spec.getDirectoryTreatment()); } } Compression compression = (spec.getCompression() == Compression.AUTO) ? Compression.detect(metadata.resourceId().getFilename()) : spec.getCompression(); c.output( new ReadableFile( MatchResult.Metadata.builder() .setResourceId(metadata.resourceId()) .setSizeBytes(metadata.sizeBytes()) .setIsReadSeekEfficient( metadata.isReadSeekEfficient() && compression == Compression.UNCOMPRESSED) .build(), compression)); } }
@Override public final long getEstimatedSizeBytes(PipelineOptions options) throws IOException { // This implementation of method getEstimatedSizeBytes is provided to simplify subclasses. Here // we perform the size estimation of files and file patterns using the interface provided by // FileSystem. String fileOrPattern = fileOrPatternSpec.get(); if (mode == Mode.FILEPATTERN) { long totalSize = 0; List<Metadata> allMatches = FileSystems.match(fileOrPattern, emptyMatchTreatment).metadata(); for (Metadata metadata : allMatches) { totalSize += metadata.sizeBytes(); } LOG.info( "Filepattern {} matched {} files with total size {}", fileOrPattern, allMatches.size(), totalSize); return totalSize; } else { long start = getStartOffset(); long end = Math.min(getEndOffset(), getMaxEndOffset(options)); return end - start; } }
private boolean alreadyStaged(PackageAttributes attributes) throws IOException { try { long remoteLength = FileSystems.matchSingleFileSpec(attributes.getDestination().getLocation()).sizeBytes(); return remoteLength == attributes.getSize(); } catch (FileNotFoundException expected) { // If the file doesn't exist, it means we need to upload it. return false; } }
@Override public void encode(Metadata value, OutputStream os) throws IOException { RESOURCE_ID_CODER.encode(value.resourceId(), os); INT_CODER.encode(value.isReadSeekEfficient() ? 1 : 0, os); LONG_CODER.encode(value.sizeBytes(), os); }
@ProcessElement public void process(ProcessContext c) { Metadata metadata = c.element().getMetadata(); if (!metadata.isReadSeekEfficient()) { c.output(KV.of(c.element(), new OffsetRange(0, metadata.sizeBytes()))); return; } for (OffsetRange range : new OffsetRange(0, metadata.sizeBytes()).split(desiredBundleSizeBytes, 0)) { c.output(KV.of(c.element(), range)); } } }
@Override public final long getMaxEndOffset(PipelineOptions options) throws IOException { checkArgument( mode != Mode.FILEPATTERN, "Cannot determine the exact end offset of a file pattern"); Metadata metadata = getSingleFileMetadata(); return metadata.sizeBytes(); }