/** utility method to copy binary (jar file) data from source to dest. */ private static void copy(ResourceId source, ResourceId dest) throws IOException { try (ReadableByteChannel rbc = FileSystems.open(source)) { try (WritableByteChannel wbc = FileSystems.create(dest, MimeTypes.BINARY)) { ByteStreams.copy(rbc, wbc); } } } }
public static void writeToFile(String filename, ByteBuffer contents) throws IOException { ResourceId resourceId = FileSystems.matchNewResource(filename, false); try (WritableByteChannel out = FileSystems.create(resourceId, MimeTypes.TEXT)) { out.write(contents); } }
private static void copyToRemote(Path src, URI dst) throws IOException { ResourceId dstId = FileSystems.matchNewResource(dst.toString(), false); WritableByteChannel dstCh = FileSystems.create(dstId, MimeTypes.BINARY); FileChannel srcCh = FileChannel.open(src, StandardOpenOption.READ); long srcSize = srcCh.size(); long copied = 0; do { copied += srcCh.transferTo(copied, srcSize - copied, dstCh); } while (copied < srcSize); dstCh.close(); srcCh.close(); Preconditions.checkState(copied == srcSize); }
private void saveSchema(String content, String schemaPath) { LOG.info("Schema: " + content); try { WritableByteChannel chan = FileSystems.create(FileSystems.matchNewResource(schemaPath, false), "text/plain"); try (OutputStream stream = Channels.newOutputStream(chan)) { stream.write(content.getBytes()); } } catch (IOException e) { throw new RuntimeException("Failed to write schema", e); } } }
public static String copyFile(ResourceId sourceFile, ResourceId destinationFile) throws IOException { try (WritableByteChannel writeChannel = FileSystems.create(destinationFile, "text/plain")) { try (ReadableByteChannel readChannel = FileSystems.open(sourceFile)) { final ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024); while (readChannel.read(buffer) != -1) { buffer.flip(); writeChannel.write(buffer); buffer.compact(); } buffer.flip(); while (buffer.hasRemaining()) { writeChannel.write(buffer); } } } return destinationFile.toString(); }
TableRowWriter(String basename) throws Exception { String uId = UUID.randomUUID().toString(); resourceId = FileSystems.matchNewResource(basename + uId, false); LOG.info("Opening TableRowWriter to {}.", resourceId); channel = FileSystems.create(resourceId, MimeTypes.TEXT); out = new CountingOutputStream(Channels.newOutputStream(channel)); }
private static void writeTextToFileSideEffect(String text, String filename) throws IOException { ResourceId rid = FileSystems.matchNewResource(filename, false); WritableByteChannel chan = FileSystems.create(rid, "text/plain"); chan.write(ByteBuffer.wrap(text.getBytes(StandardCharsets.UTF_8))); chan.close(); } }
/** * Returns a write channel for the given {@link ResourceId}. * * <p>The resource is not expanded; it is used verbatim. * * @param resourceId the reference of the file-like resource to create * @param mimeType the mine type of the file-like resource to create */ public static WritableByteChannel create(ResourceId resourceId, String mimeType) throws IOException { return create(resourceId, StandardCreateOptions.builder().setMimeType(mimeType).build()); }
private void writeRowsHelper( List<TableRow> rows, Schema avroSchema, String destinationPattern, int shard) { String filename = destinationPattern.replace("*", String.format("%012d", shard)); try (WritableByteChannel channel = FileSystems.create( FileSystems.matchNewResource(filename, false /* isDirectory */), MimeTypes.BINARY); DataFileWriter<GenericRecord> tableRowWriter = new DataFileWriter<>(new GenericDatumWriter<GenericRecord>(avroSchema)) .create(avroSchema, Channels.newOutputStream(channel))) { for (Map<String, Object> record : rows) { GenericRecordBuilder genericRecordBuilder = new GenericRecordBuilder(avroSchema); for (Map.Entry<String, Object> field : record.entrySet()) { genericRecordBuilder.set(field.getKey(), field.getValue()); } tableRowWriter.append(genericRecordBuilder.build()); } } catch (IOException e) { throw new IllegalStateException( String.format("Could not create destination for extract job %s", filename), e); } } }
try (WritableByteChannel writerChannel = FileSystems.create(tempFile, MimeTypes.TEXT)) { ByteStreams.copy(readerChannel, writerChannel);
WritableByteChannel tempChannel = FileSystems.create(outputFile, channelMimeType); try { channel = factory.create(tempChannel);
@ProcessElement public void processElement(ProcessContext context) { ResourceId inputFile = context.element().resourceId(); Compression compression = compressionValue.get(); // Add the compression extension to the output filename. Example: demo.txt -> demo.txt.gz String outputFilename = inputFile.getFilename() + compression.getSuggestedSuffix(); // Resolve the necessary resources to perform the transfer ResourceId outputDir = FileSystems.matchNewResource(destinationLocation.get(), true); ResourceId outputFile = outputDir.resolve(outputFilename, StandardResolveOptions.RESOLVE_FILE); ResourceId tempFile = outputDir.resolve("temp-" + outputFilename, StandardResolveOptions.RESOLVE_FILE); // Perform the copy of the compressed channel to the destination. try (ReadableByteChannel readerChannel = FileSystems.open(inputFile)) { try (WritableByteChannel writerChannel = compression.writeCompressed(FileSystems.create(tempFile, MimeTypes.BINARY))) { // Execute the copy to the temporary file ByteStreams.copy(readerChannel, writerChannel); } // Rename the temporary file to the output file FileSystems.rename(ImmutableList.of(tempFile), ImmutableList.of(outputFile)); // Output the path to the uncompressed file context.output(outputFile.toString()); } catch (IOException e) { LOG.error("Error occurred during compression of {}", inputFile.toString(), e); context.output(DEADLETTER_TAG, KV.of(inputFile.toString(), e.getMessage())); } } }
private StagingResult tryStagePackage(PackageAttributes attributes, CreateOptions createOptions) throws IOException, InterruptedException { String sourceDescription = attributes.getSourceDescription(); String target = attributes.getDestination().getLocation(); LOG.info("Uploading {} to {}", sourceDescription, target); try (WritableByteChannel writer = FileSystems.create(FileSystems.matchNewResource(target, false), createOptions)) { if (attributes.getBytes() != null) { ByteSource.wrap(attributes.getBytes()).copyTo(Channels.newOutputStream(writer)); } else { File sourceFile = attributes.getSource(); checkState( sourceFile != null, "Internal inconsistency: we tried to stage something to %s, but neither a source file " + "nor the byte content was specified", target); if (sourceFile.isDirectory()) { ZipFiles.zipDirectory(sourceFile, Channels.newOutputStream(writer)); } else { Files.asByteSource(sourceFile).copyTo(Channels.newOutputStream(writer)); } } } return StagingResult.uploaded(attributes); }
new BufferedWriter( new OutputStreamWriter( Channels.newOutputStream(FileSystems.create(fileResource, MimeTypes.TEXT)), UTF_8)))) { printWriter.print(workSpecJson);