@Override public String apply(MatchResult.Metadata input) { return input.resourceId().toString(); } }
private ProcessBuilder appendExecutablePath(ProcessBuilder builder) { String executable = builder.command().get(0); if (executable == null) { throw new IllegalArgumentException( "No executable provided to the Process Builder... we will do... nothing... "); } builder .command() .set(0, FileUtils.getFileResourceId(configuration.getWorkerPath(), executable).toString()); return builder; } }
private void setSourceLocation(SubProcessConfiguration configuration) { this.destinationLocation = FileUtils.getFileResourceId(configuration.getWorkerPath(), fileName).toString(); } }
private void setDestinationLocation(SubProcessConfiguration configuration) { this.sourceGCSLocation = FileUtils.getFileResourceId(configuration.getSourcePath(), fileName).toString(); }
@Override public void validateOutputResourceSupported(ResourceId resourceId) { checkArgument( "gs".equals(resourceId.getScheme()), "Expected a valid 'gs://' path but was given: '%s'", resourceId); verifyPath(resourceId.toString()); }
String readFromFile(String passwordFile) throws IOException { MatchResult.Metadata m = FileSystems.matchSingleFileSpec(passwordFile); LOGGER.info("Reading password from file: {}", m.resourceId().toString()); InputStream inputStream = Channels.newInputStream(FileSystems.open(m.resourceId())); return CharStreams.toString(new InputStreamReader(inputStream, Charsets.UTF_8)); }
static String resolveTempLocation( String tempLocationDir, String bigQueryOperationName, String stepUuid) { return FileSystems.matchNewResource(tempLocationDir, true) .resolve(bigQueryOperationName, ResolveOptions.StandardResolveOptions.RESOLVE_DIRECTORY) .resolve(stepUuid, ResolveOptions.StandardResolveOptions.RESOLVE_DIRECTORY) .toString(); } }
@Override public void encode(ResourceId value, OutputStream os) throws IOException { STRING_CODER.encode(value.toString(), os); BOOL_CODER.encode(value.isDirectory(), os); }
/** Assert that a file contains the lines provided, in the same order as expected. */ private void assertFileContains(List<String> expected, ResourceId file) throws Exception { try (BufferedReader reader = Files.newBufferedReader(Paths.get(file.toString()), UTF_8)) { List<String> actual = new ArrayList<>(); for (; ; ) { String line = reader.readLine(); if (line == null) { break; } actual.add(line); } assertEquals("contents for " + file, expected, actual); } }
public static FileNaming relativeFileNaming( final ValueProvider<String> baseDirectory, final FileNaming innerNaming) { return (window, pane, numShards, shardIndex, compression) -> FileSystems.matchNewResource(baseDirectory.get(), true /* isDirectory */) .resolve( innerNaming.getFilename(window, pane, numShards, shardIndex, compression), RESOLVE_FILE) .toString(); }
/** Generate n temporary files using the temporary file pattern of Writer. */ private List<File> generateTemporaryFilesForFinalize(int numFiles) throws Exception { List<File> temporaryFiles = new ArrayList<>(); for (int i = 0; i < numFiles; i++) { ResourceId temporaryFile = WriteOperation.buildTemporaryFilename(getBaseTempDirectory(), "" + i); File tmpFile = new File(tmpFolder.getRoot(), temporaryFile.toString()); tmpFile.getParentFile().mkdirs(); assertTrue(tmpFile.createNewFile()); temporaryFiles.add(tmpFile); } return temporaryFiles; }
@Override public void encode(Params value, OutputStream outStream) throws IOException { if (value == null) { throw new CoderException("cannot encode a null value"); } stringCoder.encode(value.baseFilename.get().toString(), outStream); stringCoder.encode(value.shardTemplate, outStream); stringCoder.encode(value.suffix, outStream); }
private String getBaseOutputFilename() { return getBaseOutputDirectory().resolve("file", StandardResolveOptions.RESOLVE_FILE).toString(); }
@Override public void populateDisplayData(DisplayData.Builder builder) { builder.add( DisplayData.item("fileNamePrefix", outputFilePrefix.toString()) .withLabel("File Name Prefix")); } }
@Test public void testRemoveTemporaryFiles() throws Exception { int numFiles = 10; List<String> fileNames = Lists.newArrayList(); String tempFilePrefix = options.getTempLocation() + "/"; for (int i = 0; i < numFiles; ++i) { TableRowWriter writer = new TableRowWriter(tempFilePrefix); writer.close(); fileNames.add(writer.getResult().resourceId.toString()); } fileNames.add(tempFilePrefix + String.format("files%05d", numFiles)); File tempDir = new File(options.getTempLocation()); testNumFiles(tempDir, 10); WriteTables.removeTemporaryFiles(fileNames); }
@Before public void setup() throws IOException { filePattern = LocalResources.fromFile(tmpFolder.getRoot(), true) .resolve("*", StandardResolveOptions.RESOLVE_FILE) .toString(); }
@Before public void setup() throws IOException { filePattern = LocalResources.fromFile(tmpFolder.getRoot(), true) .resolve("*", StandardResolveOptions.RESOLVE_FILE) .toString(); }
public static PackageAttributes forBytesToStage( byte[] bytes, String targetName, String stagingPath) { Hasher hasher = Hashing.md5().newHasher(); String hash = Base64Variants.MODIFIED_FOR_URL.encode(hasher.putBytes(bytes).hash().asBytes()); long size = bytes.length; String uniqueName = getUniqueContentName(new File(targetName), hash); String resourcePath = FileSystems.matchNewResource(stagingPath, true) .resolve(uniqueName, StandardResolveOptions.RESOLVE_FILE) .toString(); DataflowPackage target = new DataflowPackage(); target.setName(uniqueName); target.setLocation(resourcePath); return new AutoValue_PackageUtil_PackageAttributes(null, bytes, target, size, hash); }
@Test public void testPackageUploadIsSkippedWithNonExistentResource() throws Exception { String nonExistentFile = FileSystems.matchNewResource(tmpFolder.getRoot().getPath(), true) .resolve("non-existent-file", StandardResolveOptions.RESOLVE_FILE) .toString(); assertEquals( Collections.EMPTY_LIST, defaultPackageUtil.stageClasspathElements( ImmutableList.of(nonExistentFile), STAGING_PATH, createOptions)); }
@Override public void encode(FileResult<DestinationT> value, OutputStream outStream) throws IOException { if (value == null) { throw new CoderException("cannot encode a null value"); } FILENAME_CODER.encode(value.getTempFilename().toString(), outStream); windowCoder.encode(value.getWindow(), outStream); PANE_INFO_CODER.encode(value.getPaneInfo(), outStream); SHARD_CODER.encode(value.getShard(), outStream); destinationCoder.encode(value.getDestination(), outStream); }