public static void writeToFile(String filename, ByteBuffer contents) throws IOException { ResourceId resourceId = FileSystems.matchNewResource(filename, false); try (WritableByteChannel out = FileSystems.create(resourceId, MimeTypes.TEXT)) { out.write(contents); } }
public static void runAvroToCsv(SampleOptions options) throws IOException, IllegalArgumentException { FileSystems.setDefaultPipelineOptions(options); // Get Avro Schema String schemaJson = getSchema(options.getAvroSchema()); Schema schema = new Schema.Parser().parse(schemaJson); // Check schema field types before starting the Dataflow job checkFieldTypes(schema); // Create the Pipeline object with the options we defined above. Pipeline pipeline = Pipeline.create(options); // Convert Avro To CSV pipeline.apply("Read Avro files", AvroIO.readGenericRecords(schemaJson).from(options.getInputFile())) .apply("Convert Avro to CSV formatted data", ParDo.of(new ConvertAvroToCsv(schemaJson, options.getCsvDelimiter()))) .apply("Write CSV formatted data", TextIO.write().to(options.getOutput()) .withSuffix(".csv")); // Run the pipeline. pipeline.run().waitUntilFinish(); }
public static String getSchema(String schemaPath) throws IOException { ReadableByteChannel chan = FileSystems.open(FileSystems.matchNewResource( schemaPath, false)); try (InputStream stream = Channels.newInputStream(chan)) { BufferedReader streamReader = new BufferedReader(new InputStreamReader(stream, "UTF-8")); StringBuilder dataBuilder = new StringBuilder(); String line; while ((line = streamReader.readLine()) != null) { dataBuilder.append(line); } return dataBuilder.toString(); } }
/** Like {@link #match(String)}, but with a configurable {@link EmptyMatchTreatment}. */ public static MatchResult match(String spec, EmptyMatchTreatment emptyMatchTreatment) throws IOException { MatchResult res = match(spec); return maybeAdjustEmptyMatchResult(spec, res, emptyMatchTreatment); }
@Override public ResourceId apply(String input) { return FileSystems.matchNewResource(input, true); } }
ResourceId outputDir = FileSystems.matchNewResource(destinationLocation.get(), true); ResourceId outputFile = outputDir.resolve(outputFilename, StandardResolveOptions.RESOLVE_FILE); compression.readDecompressed(FileSystems.open(inputFile))) { try (WritableByteChannel writerChannel = FileSystems.create(tempFile, MimeTypes.TEXT)) { ByteStreams.copy(readerChannel, writerChannel); FileSystems.rename( ImmutableList.of(tempFile), ImmutableList.of(outputFile),
static void removeTemporaryFiles(Iterable<String> files) throws IOException { ImmutableList.Builder<ResourceId> fileResources = ImmutableList.builder(); for (String file : files) { fileResources.add(FileSystems.matchNewResource(file, false /* isDirectory */)); } FileSystems.delete(fileResources.build()); } }
/** * Check if a remote {@link URI} exists. */ public boolean remoteExists(URI uri) throws IOException { try { FileSystems.matchSingleFileSpec(uri.toString()); return true; } catch (FileNotFoundException e) { return false; } }
/** utility method to copy binary (jar file) data from source to dest. */ private static void copy(ResourceId source, ResourceId dest) throws IOException { try (ReadableByteChannel rbc = FileSystems.open(source)) { try (WritableByteChannel wbc = FileSystems.create(dest, MimeTypes.BINARY)) { ByteStreams.copy(rbc, wbc); } } } }
private static void copyToLocal(Metadata src, Path dst) throws IOException { FileChannel dstCh = FileChannel.open( dst, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW); ReadableByteChannel srcCh = FileSystems.open(src.resourceId()); long srcSize = src.sizeBytes(); long copied = 0; do { copied += dstCh.transferFrom(srcCh, copied, srcSize - copied); } while (copied < srcSize); dstCh.close(); srcCh.close(); Preconditions.checkState(copied == srcSize); }
String readFromFile(String passwordFile) throws IOException { MatchResult.Metadata m = FileSystems.matchSingleFileSpec(passwordFile); LOGGER.info("Reading password from file: {}", m.resourceId().toString()); InputStream inputStream = Channels.newInputStream(FileSystems.open(m.resourceId())); return CharStreams.toString(new InputStreamReader(inputStream, Charsets.UTF_8)); }
/** * Like {@link #match(List)}, but for a single resource specification. * * <p>The function {@link #match(List)} is preferred when matching multiple patterns, as it allows * for bulk API calls to remote filesystems. */ public static MatchResult match(String spec) throws IOException { List<MatchResult> matches = match(Collections.singletonList(spec)); verify( matches.size() == 1, "FileSystem implementation for %s did not return exactly one MatchResult: %s", spec, matches); return matches.get(0); }
/** * Returns a write channel for the given {@link ResourceId}. * * <p>The resource is not expanded; it is used verbatim. * * @param resourceId the reference of the file-like resource to create * @param mimeType the mine type of the file-like resource to create */ public static WritableByteChannel create(ResourceId resourceId, String mimeType) throws IOException { return create(resourceId, StandardCreateOptions.builder().setMimeType(mimeType).build()); }
public final void cleanup() throws Exception { if (outputFile != null) { LOG.info("Deleting temporary file {}", outputFile); // outputFile may be null if open() was not called or failed. FileSystems.delete( Collections.singletonList(outputFile), StandardMoveOptions.IGNORE_MISSING_FILES); } }
MatchResult singleMatch = Iterables.getOnlyElement( FileSystems.match(Collections.singletonList(tempDir.toString() + "*"))); for (Metadata matchResult : singleMatch.metadata()) { if (allMatches.add(matchResult.resourceId())) { FileSystems.delete(allMatches, StandardMoveOptions.IGNORE_MISSING_FILES); FileSystems.delete( Collections.singletonList(tempDir), StandardMoveOptions.IGNORE_MISSING_FILES); } catch (Exception e) {
@Override public ResourceId apply(String input) { return FileSystems.matchNewResource(input, true); } }
@ProcessElement public void processElement(ProcessContext context) { ResourceId inputFile = context.element().resourceId(); Compression compression = compressionValue.get(); // Add the compression extension to the output filename. Example: demo.txt -> demo.txt.gz String outputFilename = inputFile.getFilename() + compression.getSuggestedSuffix(); // Resolve the necessary resources to perform the transfer ResourceId outputDir = FileSystems.matchNewResource(destinationLocation.get(), true); ResourceId outputFile = outputDir.resolve(outputFilename, StandardResolveOptions.RESOLVE_FILE); ResourceId tempFile = outputDir.resolve("temp-" + outputFilename, StandardResolveOptions.RESOLVE_FILE); // Perform the copy of the compressed channel to the destination. try (ReadableByteChannel readerChannel = FileSystems.open(inputFile)) { try (WritableByteChannel writerChannel = compression.writeCompressed(FileSystems.create(tempFile, MimeTypes.BINARY))) { // Execute the copy to the temporary file ByteStreams.copy(readerChannel, writerChannel); } // Rename the temporary file to the output file FileSystems.rename(ImmutableList.of(tempFile), ImmutableList.of(outputFile)); // Output the path to the uncompressed file context.output(outputFile.toString()); } catch (IOException e) { LOG.error("Error occurred during compression of {}", inputFile.toString(), e); context.output(DEADLETTER_TAG, KV.of(inputFile.toString(), e.getMessage())); } } }
private static Metadata getMetadata(URI src) throws IOException { return FileSystems.matchSingleFileSpec(src.toString()); }
public static String copyFile(ResourceId sourceFile, ResourceId destinationFile) throws IOException { try (WritableByteChannel writeChannel = FileSystems.create(destinationFile, "text/plain")) { try (ReadableByteChannel readChannel = FileSystems.open(sourceFile)) { final ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024); while (readChannel.read(buffer) != -1) { buffer.flip(); writeChannel.write(buffer); buffer.compact(); } buffer.flip(); while (buffer.hasRemaining()) { writeChannel.write(buffer); } } } return destinationFile.toString(); }