@Override public ResourceId apply(ResourceId input) { return input.getCurrentDirectory(); } }
@Override public ResourceId apply(ResourceId tempDirectory) { // Temp directory has a timestamp and a unique ID String tempDirName = String.format(TEMP_DIRECTORY_PREFIX + "-%s-%s", timestamp, tempId); return tempDirectory .getCurrentDirectory() .resolve(tempDirName, StandardResolveOptions.RESOLVE_DIRECTORY); } }
public static ResourceId getFileResourceId(String directory, String fileName) { ResourceId resourceID = FileSystems.matchNewResource(directory, true); return resourceID.getCurrentDirectory().resolve(fileName, StandardResolveOptions.RESOLVE_FILE); }
public MatchResult.Metadata apply(MatchResult.Metadata metadata) { ResourceId resourceId = metadata.resourceId(); System.out.println(resourceId.getScheme() + "://" + resourceId.getCurrentDirectory() + resourceId.getFilename()); return metadata; } }));
.getCurrentDirectory() .resolve(sb.toString(), StandardResolveOptions.RESOLVE_FILE);
@Override public ResourceId windowedFilename( final int shardNumber, final int numShards, final BoundedWindow window, final PaneInfo paneInfo, final FileBasedSink.OutputFileHints outputFileHints) { System.out.println("Windowd file name: " + window); final IntervalWindow intervalWindow = (IntervalWindow) window; final String filename = String.format( "%s-%s-of-%s%s", filenamePrefixForWindow(intervalWindow), shardNumber, numShards, outputFileHints.getSuggestedFilenameSuffix()); return baseFilename .getCurrentDirectory() .resolve(filename, ResolveOptions.StandardResolveOptions.RESOLVE_FILE); } @Override
@Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { IntervalWindow intervalWindow = (IntervalWindow) window; String filename = String.format( "%s-%s-of-%s%s", filenamePrefixForWindow(intervalWindow), shardNumber, numShards, outputFileHints.getSuggestedFilenameSuffix()); return prefix.getCurrentDirectory().resolve(filename, StandardResolveOptions.RESOLVE_FILE); }
@Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { IntervalWindow intervalWindow = (IntervalWindow) window; String filename = String.format( "%s-%s-of-%s%s", filenamePrefixForWindow(intervalWindow), shardNumber, numShards, outputFileHints.getSuggestedFilenameSuffix()); return baseFilename .getCurrentDirectory() .resolve(filename, StandardResolveOptions.RESOLVE_FILE); }
@Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { DecimalFormat df = new DecimalFormat("0000"); IntervalWindow intervalWindow = (IntervalWindow) window; String filename = String.format( "%s-%s-of-%s%s%s", filenamePrefixForWindow(intervalWindow), df.format(shardNumber), df.format(numShards), outputFileHints.getSuggestedFilenameSuffix(), suffix); return baseFilename .getCurrentDirectory() .resolve(filename, StandardResolveOptions.RESOLVE_FILE); }
.addEqualityGroup(file1) .addEqualityGroup(file2, file2a) .addEqualityGroup(dir1, dir1.getCurrentDirectory()) .addEqualityGroup(dir2, dir2a, dir2.getCurrentDirectory()) .addEqualityGroup(baseDirectory, file1.getCurrentDirectory(), file2.getCurrentDirectory()) .testEquals(); .addEqualityGroup(file1.toString()) .addEqualityGroup(file2.toString(), file2a.toString()) .addEqualityGroup(dir1.toString(), dir1.getCurrentDirectory().toString()) .addEqualityGroup(dir2.toString(), dir2a.toString(), dir2.getCurrentDirectory().toString()) .addEqualityGroup( baseDirectory.toString(), file1.getCurrentDirectory().toString(), file2.getCurrentDirectory().toString()) .testEquals();
@Override public ResourceId unwindowedFilename( int shardNumber, int numShards, OutputFileHints outputFileHints) { DecimalFormat df = new DecimalFormat("0000"); String prefix = baseFilename.isDirectory() ? "" : firstNonNull(baseFilename.getFilename(), ""); String filename = String.format( "%s-%s-of-%s%s%s", prefix, df.format(shardNumber), df.format(numShards), outputFileHints.getSuggestedFilenameSuffix(), suffix); return baseFilename .getCurrentDirectory() .resolve(filename, StandardResolveOptions.RESOLVE_FILE); } }
@Override public PDone expand(PCollection<String> input) { ResourceId resource = FileBasedSink.convertToFileResourceIfPossible(filenamePrefix); TextIO.Write write = TextIO.write() .to(new PerWindowFiles(resource)) .withTempDirectory(resource.getCurrentDirectory()) .withWindowedWrites(); if (numShards != null) { write = write.withNumShards(numShards); } return input.apply(write); }
@Override public PDone expand(final PCollection<String> input) { final ResourceId resource = FileBasedSink.convertToFileResourceIfPossible(filenamePrefix); TextIO.Write write = TextIO.write() .to(new PerWindowFiles(resource)) .withTempDirectory(resource.getCurrentDirectory()) .withWindowedWrites(); if (numShards != null) { write = write.withNumShards(numShards); } return input.apply(write); } /**
@Override public PDone expand(PCollection<String> input) { // Verify that the input has a compatible window type. checkArgument( input.getWindowingStrategy().getWindowFn().windowCoder() == IntervalWindow.getCoder()); ResourceId resource = FileBasedSink.convertToFileResourceIfPossible(filenamePrefix); return input.apply( TextIO.write() .to(new PerWindowFiles(resource)) .withTempDirectory(resource.getCurrentDirectory()) .withWindowedWrites() .withNumShards(3)); } }
@Override public PDone expand(PCollection<T> input) { checkArgument(getRecordClass() != null, "withRecordClass() is required"); checkArgument(getRootElement() != null, "withRootElement() is required"); checkArgument(getFilenamePrefix() != null, "to() is required"); checkArgument(getCharset() != null, "withCharset() is required"); try { JAXBContext.newInstance(getRecordClass()); } catch (JAXBException e) { throw new RuntimeException("Error binding classes to a JAXB Context.", e); } ResourceId prefix = FileSystems.matchNewResource(getFilenamePrefix(), false /* isDirectory */); input.apply( FileIO.<T>write() .via( sink(getRecordClass()) .withCharset(Charset.forName(getCharset())) .withRootElement(getRootElement())) .to(prefix.getCurrentDirectory().toString()) .withPrefix(prefix.getFilename()) .withSuffix(".xml") .withIgnoreWindowing()); return PDone.in(input.getPipeline()); }
@Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { String filenamePrefix = outputFilePrefix.isDirectory() ? "" : firstNonNull(outputFilePrefix.getFilename(), ""); IntervalWindow interval = (IntervalWindow) window; String windowStr = String.format("%s-%s", interval.start().toString(), interval.end().toString()); String filename = String.format( "%s-%s-%s-of-%s-pane-%s%s%s.avro", filenamePrefix, windowStr, shardNumber, numShards, paneInfo.getIndex(), paneInfo.isLast() ? "-last" : "", outputFileHints.getSuggestedFilenameSuffix()); return outputFilePrefix.getCurrentDirectory().resolve(filename, RESOLVE_FILE); }
new NumberedShardedFile( output .getCurrentDirectory() .resolve(filePrefix, StandardResolveOptions.RESOLVE_FILE) .toString()