static String resolveTempLocation( String tempLocationDir, String bigQueryOperationName, String stepUuid) { return FileSystems.matchNewResource(tempLocationDir, true) .resolve(bigQueryOperationName, ResolveOptions.StandardResolveOptions.RESOLVE_DIRECTORY) .resolve(stepUuid, ResolveOptions.StandardResolveOptions.RESOLVE_DIRECTORY) .toString(); } }
@Override public ResourceId apply(ResourceId tempDirectory) { // Temp directory has a timestamp and a unique ID String tempDirName = String.format(TEMP_DIRECTORY_PREFIX + "-%s-%s", timestamp, tempId); return tempDirectory .getCurrentDirectory() .resolve(tempDirName, StandardResolveOptions.RESOLVE_DIRECTORY); } }
@Override public ResourceId unwindowedFilename( int shardNumber, int numShards, OutputFileHints outputFileHints) { DecimalFormat df = new DecimalFormat("0000"); String prefix = baseFilename.isDirectory() ? "" : firstNonNull(baseFilename.getFilename(), ""); String filename = String.format( "%s-%s-of-%s%s%s", prefix, df.format(shardNumber), df.format(numShards), outputFileHints.getSuggestedFilenameSuffix(), suffix); return baseFilename .getCurrentDirectory() .resolve(filename, StandardResolveOptions.RESOLVE_FILE); } }
public MatchResult.Metadata apply(MatchResult.Metadata metadata) { ResourceId resourceId = metadata.resourceId(); System.out.println(resourceId.getScheme() + "://" + resourceId.getCurrentDirectory() + resourceId.getFilename()); return metadata; } }));
@Override public String apply(MatchResult.Metadata input) { return input.resourceId().toString(); } }
private static void validateFailureResolvingIds(ResourceId baseDirectory) { try { ResourceId badFile = baseDirectory.resolve("file/", RESOLVE_FILE); fail(String.format("Resolving badFile %s should have failed", badFile)); } catch (IllegalArgumentException e) { // expected } ResourceId file = baseDirectory.resolve("file", RESOLVE_FILE); try { baseDirectory.resolve("file2", RESOLVE_FILE); fail(String.format("Should not be able to resolve against file resource %s", file)); } catch (IllegalArgumentException e) { // expected } }
private static void validateResolvingIds( ResourceId baseDirectory, List<ResourceId> allResourceIds) { ResourceId file1 = baseDirectory.resolve("child1", RESOLVE_FILE); ResourceId file2 = baseDirectory.resolve("child2", RESOLVE_FILE); ResourceId file2a = baseDirectory.resolve("child2", RESOLVE_FILE); allResourceIds.add(file1); allResourceIds.add(file2); assertThat("Resolved file isDirectory()", file1.isDirectory(), is(false)); assertThat("Resolved file isDirectory()", file2.isDirectory(), is(false)); assertThat("Resolved file isDirectory()", file2a.isDirectory(), is(false)); ResourceId dir1 = baseDirectory.resolve("child1", RESOLVE_DIRECTORY); ResourceId dir2 = baseDirectory.resolve("child2", RESOLVE_DIRECTORY); ResourceId dir2a = baseDirectory.resolve("child2", RESOLVE_DIRECTORY); assertThat("Resolved directory isDirectory()", dir1.isDirectory(), is(true)); assertThat("Resolved directory isDirectory()", dir2.isDirectory(), is(true)); assertThat("Resolved directory isDirectory()", dir2a.isDirectory(), is(true)); allResourceIds.add(dir1); allResourceIds.add(dir2); .addEqualityGroup(file1) .addEqualityGroup(file2, file2a) .addEqualityGroup(dir1, dir1.getCurrentDirectory()) .addEqualityGroup(dir2, dir2a, dir2.getCurrentDirectory()) .addEqualityGroup(baseDirectory, file1.getCurrentDirectory(), file2.getCurrentDirectory()) .testEquals(); .addEqualityGroup(file1.toString()) .addEqualityGroup(file2.toString(), file2a.toString()) .addEqualityGroup(dir1.toString(), dir1.getCurrentDirectory().toString())
@ProcessElement public void processElement(ProcessContext context) { ResourceId inputFile = context.element().resourceId(); Compression compression = compressionValue.get(); // Add the compression extension to the output filename. Example: demo.txt -> demo.txt.gz String outputFilename = inputFile.getFilename() + compression.getSuggestedSuffix(); // Resolve the necessary resources to perform the transfer ResourceId outputDir = FileSystems.matchNewResource(destinationLocation.get(), true); ResourceId outputFile = outputDir.resolve(outputFilename, StandardResolveOptions.RESOLVE_FILE); ResourceId tempFile = outputDir.resolve("temp-" + outputFilename, StandardResolveOptions.RESOLVE_FILE); // Perform the copy of the compressed channel to the destination. try (ReadableByteChannel readerChannel = FileSystems.open(inputFile)) { try (WritableByteChannel writerChannel = compression.writeCompressed(FileSystems.create(tempFile, MimeTypes.BINARY))) { // Execute the copy to the temporary file ByteStreams.copy(readerChannel, writerChannel); } // Rename the temporary file to the output file FileSystems.rename(ImmutableList.of(tempFile), ImmutableList.of(outputFile)); // Output the path to the uncompressed file context.output(outputFile.toString()); } catch (IOException e) { LOG.error("Error occurred during compression of {}", inputFile.toString(), e); context.output(DEADLETTER_TAG, KV.of(inputFile.toString(), e.getMessage())); } } }
new NumberedShardedFile( output .getCurrentDirectory() .resolve(filePrefix, StandardResolveOptions.RESOLVE_FILE) .toString() + "*"));
@Override public PDone expand(PCollection<T> input) { checkArgument(getRecordClass() != null, "withRecordClass() is required"); checkArgument(getRootElement() != null, "withRootElement() is required"); checkArgument(getFilenamePrefix() != null, "to() is required"); checkArgument(getCharset() != null, "withCharset() is required"); try { JAXBContext.newInstance(getRecordClass()); } catch (JAXBException e) { throw new RuntimeException("Error binding classes to a JAXB Context.", e); } ResourceId prefix = FileSystems.matchNewResource(getFilenamePrefix(), false /* isDirectory */); input.apply( FileIO.<T>write() .via( sink(getRecordClass()) .withCharset(Charset.forName(getCharset())) .withRootElement(getRootElement())) .to(prefix.getCurrentDirectory().toString()) .withPrefix(prefix.getFilename()) .withSuffix(".xml") .withIgnoreWindowing()); return PDone.in(input.getPipeline()); }
private static String extractFilename(ResourceId input) { if (input.isDirectory()) { return ""; } else { return firstNonNull(input.getFilename(), ""); } } }
@Override public void encode(ResourceId value, OutputStream os) throws IOException { STRING_CODER.encode(value.toString(), os); BOOL_CODER.encode(value.isDirectory(), os); }
try { ResourceId sourceResourceId = FileSystems.matchNewResource(jarPath, false); File destFile = Paths.get(destRoot, sourceResourceId.getFilename()).toFile(); ResourceId destResourceId = FileSystems.matchNewResource(destFile.getAbsolutePath(), false); LOG.info( "Localized jar: " + sourceResourceId.toString() + " to: " + destResourceId.toString());
@Override public ResourceId apply(ResourceId input) { return input.getCurrentDirectory(); } }
/** * Check if total number of files is correct by comparing with the number that is parsed from * shard name using a name template. If no template is specified, "SSSS-of-NNNN" will be used as * default, and "NNNN" will be the expected total number of files. * * @return {@code true} if at least one shard name matches template and total number of given * files equals the number that is parsed from shard name. */ @VisibleForTesting boolean checkTotalNumOfFiles(Collection<Metadata> files) { for (Metadata fileMedadata : files) { String fileName = fileMedadata.resourceId().getFilename(); if (fileName == null) { // this path has zero elements continue; } Matcher matcher = shardTemplate.matcher(fileName); if (!matcher.matches()) { // shard name doesn't match the pattern, check with the next shard continue; } // once match, extract total number of shards and compare to file list return files.size() == Integer.parseInt(matcher.group("numshards")); } return false; } }
/** * Returns a write channel for the given {@link ResourceId} with {@link CreateOptions}. * * <p>The resource is not expanded; it is used verbatim. * * @param resourceId the reference of the file-like resource to create * @param createOptions the configuration of the create operation */ public static WritableByteChannel create(ResourceId resourceId, CreateOptions createOptions) throws IOException { return getFileSystemInternal(resourceId.getScheme()).create(resourceId, createOptions); }
/** * Enforces that the {@link ResourceId} implementation of {@code baseDirectory} meets the {@link * ResourceId} spec. */ public static void runResourceIdBattery(ResourceId baseDirectory) { checkArgument( baseDirectory.isDirectory(), "baseDirectory %s is not a directory", baseDirectory); List<ResourceId> allResourceIds = new ArrayList<>(); allResourceIds.add(baseDirectory); // Validate that individual resources meet the fairly restrictive spec we have. validateResourceIds(allResourceIds); // Validate operations with resolving child resources. validateResolvingIds(baseDirectory, allResourceIds); // Validate safeguards against resolving bad paths. validateFailureResolvingIds(baseDirectory); }
/** * The error messages coming from the compression library are not consistent across compression * modes. Here we'll attempt to unify the messages to inform the user more clearly when we've * encountered a file which is not compressed or malformed. Note that GZIP and ZIP compression * modes will not throw an exception when a decompression is attempted on a file which is not * compressed. * * @param errorMsg The error message thrown during decompression. * @param inputFile The input file which failed decompression. * @param compression The compression mode used during decompression. * @return The sanitized error message. If the error was not from a malformed file, the same * error message passed will be returned. */ private String sanitizeDecompressionErrorMsg( String errorMsg, ResourceId inputFile, Compression compression) { if (errorMsg != null && (errorMsg.contains("not in the BZip2 format") || errorMsg.contains("incorrect header check"))) { errorMsg = String.format(MALFORMED_ERROR_MSG, inputFile.toString(), compression); } return errorMsg; } }
/** Constructs a temporary file resource given the temporary directory and a filename. */ @Experimental(Kind.FILESYSTEM) protected static ResourceId buildTemporaryFilename(ResourceId tempDirectory, String filename) throws IOException { return tempDirectory.resolve(filename, StandardResolveOptions.RESOLVE_FILE); }