/** * Opens an FSDataOutputStream at the indicated Path. * * @param f * the file name to open * @param overwrite * if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * * @throws IOException Thrown, if the stream could not be opened because of an I/O, or because * a file already exists at that path and the write mode indicates to not * overwrite the file. * * @deprecated Use {@link #create(Path, WriteMode)} instead. */ @Deprecated public FSDataOutputStream create(Path f, boolean overwrite) throws IOException { return create(f, overwrite ? WriteMode.OVERWRITE : WriteMode.NO_OVERWRITE); }
@Override @SuppressWarnings("deprecation") public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize) throws IOException { FSDataOutputStream innerStream = unsafeFileSystem.create(f, overwrite, bufferSize, replication, blockSize); return ClosingFSDataOutputStream.wrapSafe(innerStream, registry, String.valueOf(f)); }
@Override @Deprecated @SuppressWarnings("deprecation") public FSDataOutputStream create( Path f, boolean overwrite, int bufferSize, short replication, long blockSize) throws IOException { return createOutputStream(() -> originalFs.create(f, overwrite, bufferSize, replication, blockSize)); }
@Override public FSDataOutputStream create(Path f, WriteMode overwrite) throws IOException { FSDataOutputStream innerStream = unsafeFileSystem.create(f, overwrite); return ClosingFSDataOutputStream.wrapSafe(innerStream, registry, String.valueOf(f)); }
long blockSize) throws IOException { return create(f, overwrite ? WriteMode.OVERWRITE : WriteMode.NO_OVERWRITE);
private static void internalCopyFile(Path sourcePath, Path targetPath, boolean executable, FileSystem sFS, FileSystem tFS) throws IOException { try (FSDataOutputStream lfsOutput = tFS.create(targetPath, FileSystem.WriteMode.NO_OVERWRITE); FSDataInputStream fsInput = sFS.open(sourcePath)) { IOUtils.copyBytes(fsInput, lfsOutput); //noinspection ResultOfMethodCallIgnored new File(targetPath.toString()).setExecutable(executable); } }
private static void unzipPythonLibrary(Path targetDir) throws IOException { FileSystem targetFs = targetDir.getFileSystem(); ClassLoader classLoader = PythonPlanBinder.class.getClassLoader(); try (ZipInputStream zis = new ZipInputStream(classLoader.getResourceAsStream("python-source.zip"))) { ZipEntry entry = zis.getNextEntry(); while (entry != null) { String fileName = entry.getName(); Path newFile = new Path(targetDir, fileName); if (entry.isDirectory()) { targetFs.mkdirs(newFile); } else { try { LOG.debug("Unzipping to {}.", newFile); FSDataOutputStream fsDataOutputStream = targetFs.create(newFile, FileSystem.WriteMode.NO_OVERWRITE); IOUtils.copyBytes(zis, fsDataOutputStream, false); } catch (Exception e) { zis.closeEntry(); throw new IOException("Failed to unzip flink python library.", e); } } zis.closeEntry(); entry = zis.getNextEntry(); } zis.closeEntry(); } }
public static Path compressDirectory(Path directory, Path target) throws IOException { FileSystem sourceFs = directory.getFileSystem(); FileSystem targetFs = target.getFileSystem(); try (ZipOutputStream out = new ZipOutputStream(targetFs.create(target, FileSystem.WriteMode.NO_OVERWRITE))) { addToZip(directory, sourceFs, directory.getParent(), out); } return target; }
private void createFile(Path file) throws IOException { try (FSDataOutputStream out = fs.create(file, WriteMode.NO_OVERWRITE)) { out.write(new byte[] {1, 2, 3, 4, 5, 6, 7, 8}); } }
/** * Handles entropy injection across regular and entropy-aware file systems. * * <p>If the given file system is entropy-aware (a implements {@link EntropyInjectingFileSystem}), * then this method replaces the entropy marker in the path with random characters. * The entropy marker is defined by {@link EntropyInjectingFileSystem#getEntropyInjectionKey()}. * * <p>If the given file system does not implement {@code EntropyInjectingFileSystem}, * then this method delegates to {@link FileSystem#create(Path, WriteMode)} and * returns the same path in the resulting {@code OutputStreamAndPath}. */ public static OutputStreamAndPath createEntropyAware( FileSystem fs, Path path, WriteMode writeMode) throws IOException { // check and possibly inject entropy into the path final EntropyInjectingFileSystem efs = getEntropyFs(fs); final Path processedPath = efs == null ? path : resolveEntropy(path, efs, true); // create the stream on the original file system to let the safety net // take its effect final FSDataOutputStream out = fs.create(processedPath, writeMode); return new OutputStreamAndPath(out, processedPath); }
@Override public void flatMap(FileCopyTask task, Collector<Object> out) throws Exception { LOGGER.info("Processing task: " + task); Path outPath = new Path(targetPath, task.getRelativePath()); FileSystem targetFs = targetPath.getFileSystem(); // creating parent folders in case of a local FS if (!targetFs.isDistributedFS()) { //dealing with cases like file:///tmp or just /tmp File outFile = outPath.toUri().isAbsolute() ? new File(outPath.toUri()) : new File(outPath.toString()); File parentFile = outFile.getParentFile(); if (!parentFile.mkdirs() && !parentFile.exists()) { throw new RuntimeException("Cannot create local file system directories: " + parentFile); } } FSDataOutputStream outputStream = null; FSDataInputStream inputStream = null; try { outputStream = targetFs.create(outPath, FileSystem.WriteMode.OVERWRITE); inputStream = task.getPath().getFileSystem().open(task.getPath()); int bytes = IOUtils.copy(inputStream, outputStream); bytesCounter.add(bytes); } finally { IOUtils.closeQuietly(inputStream); IOUtils.closeQuietly(outputStream); } fileCounter.add(1L); } });
public static Path expandDirectory(Path file, Path targetDirectory) throws IOException { FileSystem sourceFs = file.getFileSystem(); FileSystem targetFs = targetDirectory.getFileSystem(); Path rootDir = null; try (ZipInputStream zis = new ZipInputStream(sourceFs.open(file))) { ZipEntry entry; while ((entry = zis.getNextEntry()) != null) { Path relativePath = new Path(entry.getName()); if (rootDir == null) { // the first entry contains the name of the original directory that was zipped rootDir = relativePath; } Path newFile = new Path(targetDirectory, relativePath); if (entry.isDirectory()) { targetFs.mkdirs(newFile); } else { try (FSDataOutputStream fileStream = targetFs.create(newFile, FileSystem.WriteMode.NO_OVERWRITE)) { // do not close the streams here as it prevents access to further zip entries IOUtils.copyBytes(zis, fileStream, false); } } zis.closeEntry(); } } return new Path(targetDirectory, rootDir); }
this.stream = fs.create(this.actualFilePath, writeMode);
@Override public void go() throws Exception { fs.initOutPathLocalFS(path.getParent(), WriteMode.OVERWRITE, true); try (FSDataOutputStream out = fs.create(path, WriteMode.OVERWRITE)) { out.write(11); } } }
@Test public void testSimpleFileWriteAndRead() throws Exception { final String testLine = "Hello Upload!"; final Path path = new Path(basePath, "test.txt"); try { try (FSDataOutputStream out = fs.create(path, FileSystem.WriteMode.OVERWRITE); OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) { writer.write(testLine); } // just in case, wait for the path to exist checkPathExistence(path, true, deadline); try (FSDataInputStream in = fs.open(path); InputStreamReader ir = new InputStreamReader(in, StandardCharsets.UTF_8); BufferedReader reader = new BufferedReader(ir)) { String line = reader.readLine(); assertEquals(testLine, line); } } finally { fs.delete(path, false); } checkPathExistence(path, false, deadline); }
@Test public void testDelegateOutStreamMethods() throws IOException { // mock the output stream final FSDataOutputStream mockOut = mock(FSDataOutputStream.class); final long outPos = 46651L; when(mockOut.getPos()).thenReturn(outPos); final FileSystem fs = mock(FileSystem.class); when(fs.create(any(Path.class), any(WriteMode.class))).thenReturn(mockOut); final LimitedConnectionsFileSystem lfs = new LimitedConnectionsFileSystem(fs, 100); final FSDataOutputStream out = lfs.create(mock(Path.class), WriteMode.OVERWRITE); // validate the output stream out.write(77); verify(mockOut).write(77); { byte[] bytes = new byte[1786]; out.write(bytes, 100, 111); verify(mockOut).write(bytes, 100, 111); } assertEquals(outPos, out.getPos()); out.flush(); verify(mockOut).flush(); out.sync(); verify(mockOut).sync(); out.close(); verify(mockOut).close(); }
when(fs.open(any(Path.class))).thenReturn(mock(FSDataInputStream.class)); when(fs.open(any(Path.class), anyInt())).thenReturn(mock(FSDataInputStream.class)); when(fs.create(any(Path.class), anyBoolean())).thenReturn(mock(FSDataOutputStream.class)); when(fs.create(any(Path.class), any(WriteMode.class))).thenReturn(mock(FSDataOutputStream.class)); when(fs.create(any(Path.class), anyBoolean(), anyInt(), anyShort(), anyLong())).thenReturn(mock(FSDataOutputStream.class)); verify(fs).create(path, overwrite, bufferSize, replication, blockSize); WriteMode mode = rnd.nextBoolean() ? WriteMode.OVERWRITE : WriteMode.NO_OVERWRITE; lfs.create(path, mode); verify(fs).create(path, mode);
closeableRegistry.registerCloseable(inputStream); outputStream = restoreFileSystem.create(restoreFilePath, FileSystem.WriteMode.OVERWRITE); closeableRegistry.registerCloseable(outputStream);
for (int i = 0; i < numFiles; i++) { Path file = new Path(directory, "/file-" + i); try (FSDataOutputStream out = fs.create(file, FileSystem.WriteMode.OVERWRITE); OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) { writer.write("hello-" + i + "\n");
@Override public FSDataOutputStream create(Path f, WriteMode overwrite) throws IOException { FSDataOutputStream innerStream = unsafeFileSystem.create(f, overwrite); return ClosingFSDataOutputStream.wrapSafe(innerStream, registry, String.valueOf(f)); }