@Override public FSDataInputStream open(Path f) throws IOException { FSDataInputStream innerStream = unsafeFileSystem.open(f); return ClosingFSDataInputStream.wrapSafe(innerStream, registry, String.valueOf(f)); }
@Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { FSDataInputStream innerStream = unsafeFileSystem.open(f, bufferSize); return ClosingFSDataInputStream.wrapSafe(innerStream, registry, String.valueOf(f)); }
@Override public FSDataInputStream open(Path f) throws IOException { return createInputStream(() -> originalFs.open(f)); }
@Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { return createInputStream(() -> originalFs.open(f, bufferSize)); }
private static void internalCopyFile(Path sourcePath, Path targetPath, boolean executable, FileSystem sFS, FileSystem tFS) throws IOException { try (FSDataOutputStream lfsOutput = tFS.create(targetPath, FileSystem.WriteMode.NO_OVERWRITE); FSDataInputStream fsInput = sFS.open(sourcePath)) { IOUtils.copyBytes(fsInput, lfsOutput); //noinspection ResultOfMethodCallIgnored new File(targetPath.toString()).setExecutable(executable); } }
@Override public void flatMap(Tuple3<String, Long, Long> value, Collector<String> out) throws Exception { FSDataInputStream stream = FileSystem.get(new URI(value.f0)).open(new Path(value.f0)); stream.seek(value.f1); BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); String line; try { while ((line = reader.readLine()) != null && (value.f2 == -1L || stream.getPos() <= value.f2)) { out.collect(line); } } finally { reader.close(); } } }
private static void addToZip(Path fileOrDirectory, FileSystem fs, Path rootDir, ZipOutputStream out) throws IOException { String relativePath = fileOrDirectory.getPath().replace(rootDir.getPath() + '/', ""); if (fs.getFileStatus(fileOrDirectory).isDir()) { out.putNextEntry(new ZipEntry(relativePath + '/')); for (FileStatus containedFile : fs.listStatus(fileOrDirectory)) { addToZip(containedFile.getPath(), fs, rootDir, out); } } else { ZipEntry entry = new ZipEntry(relativePath); out.putNextEntry(entry); try (FSDataInputStream in = fs.open(fileOrDirectory)) { IOUtils.copyBytes(in, out, false); } out.closeEntry(); } }
@Override public void flatMap(FileCopyTask task, Collector<Object> out) throws Exception { LOGGER.info("Processing task: " + task); Path outPath = new Path(targetPath, task.getRelativePath()); FileSystem targetFs = targetPath.getFileSystem(); // creating parent folders in case of a local FS if (!targetFs.isDistributedFS()) { //dealing with cases like file:///tmp or just /tmp File outFile = outPath.toUri().isAbsolute() ? new File(outPath.toUri()) : new File(outPath.toString()); File parentFile = outFile.getParentFile(); if (!parentFile.mkdirs() && !parentFile.exists()) { throw new RuntimeException("Cannot create local file system directories: " + parentFile); } } FSDataOutputStream outputStream = null; FSDataInputStream inputStream = null; try { outputStream = targetFs.create(outPath, FileSystem.WriteMode.OVERWRITE); inputStream = task.getPath().getFileSystem().open(task.getPath()); int bytes = IOUtils.copy(inputStream, outputStream); bytesCounter.add(bytes); } finally { IOUtils.closeQuietly(inputStream); IOUtils.closeQuietly(outputStream); } fileCounter.add(1L); } });
public static Path expandDirectory(Path file, Path targetDirectory) throws IOException { FileSystem sourceFs = file.getFileSystem(); FileSystem targetFs = targetDirectory.getFileSystem(); Path rootDir = null; try (ZipInputStream zis = new ZipInputStream(sourceFs.open(file))) { ZipEntry entry; while ((entry = zis.getNextEntry()) != null) { Path relativePath = new Path(entry.getName()); if (rootDir == null) { // the first entry contains the name of the original directory that was zipped rootDir = relativePath; } Path newFile = new Path(targetDirectory, relativePath); if (entry.isDirectory()) { targetFs.mkdirs(newFile); } else { try (FSDataOutputStream fileStream = targetFs.create(newFile, FileSystem.WriteMode.NO_OVERWRITE)) { // do not close the streams here as it prevents access to further zip entries IOUtils.copyBytes(zis, fileStream, false); } } zis.closeEntry(); } } return new Path(targetDirectory, rootDir); }
@Override public void run() { try { final FileSystem fs = FileSystem.get(this.split.getPath().toUri()); this.fdis = fs.open(this.split.getPath()); // check for canceling and close the stream in that case, because no one will obtain it if (this.aborted) { final FSDataInputStream f = this.fdis; this.fdis = null; f.close(); } } catch (Throwable t) { this.error = t; } }
@Test public void testSimpleFileWriteAndRead() throws Exception { final String testLine = "Hello Upload!"; final Path path = new Path(basePath, "test.txt"); try { try (FSDataOutputStream out = fs.create(path, FileSystem.WriteMode.OVERWRITE); OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) { writer.write(testLine); } // just in case, wait for the path to exist checkPathExistence(path, true, deadline); try (FSDataInputStream in = fs.open(path); InputStreamReader ir = new InputStreamReader(in, StandardCharsets.UTF_8); BufferedReader reader = new BufferedReader(ir)) { String line = reader.readLine(); assertEquals(testLine, line); } } finally { fs.delete(path, false); } checkPathExistence(path, false, deadline); }
private Map<Path, String> getFileContentByPath(Path directory) throws Exception { Map<Path, String> contents = new HashMap<>(); final FileStatus[] filesInBucket = getFileSystem().listStatus(directory); for (FileStatus file : filesInBucket) { final long fileLength = file.getLen(); byte[] serContents = new byte[(int) fileLength]; FSDataInputStream stream = getFileSystem().open(file.getPath()); stream.read(serContents); contents.put(file.getPath(), new String(serContents, StandardCharsets.UTF_8)); } return contents; }
when(fs.open(any(Path.class))).thenReturn(mockIn);
public void testDelegateFsMethods() throws IOException { final FileSystem fs = mock(FileSystem.class); when(fs.open(any(Path.class))).thenReturn(mock(FSDataInputStream.class)); when(fs.open(any(Path.class), anyInt())).thenReturn(mock(FSDataInputStream.class)); when(fs.create(any(Path.class), anyBoolean())).thenReturn(mock(FSDataOutputStream.class)); when(fs.create(any(Path.class), any(WriteMode.class))).thenReturn(mock(FSDataOutputStream.class)); int bufferSize = rnd.nextInt(); lfs.open(path, bufferSize); verify(fs).open(path, bufferSize); verify(fs).open(path);
final FSDataInputStream stream = fs.open(path); try { ctx.write(new ChunkedStream(stream));
try (FSDataInputStream fdis = fs.open(file.getPath(), blockInfo.getInfoSize())) { fdis.seek(file.getLen() - blockInfo.getInfoSize());
inputStream = backupFileSystem.open(filePath); closeableRegistry.registerCloseable(inputStream);
@Override public FSDataInputStream open(Path f) throws IOException { FSDataInputStream innerStream = unsafeFileSystem.open(f); return ClosingFSDataInputStream.wrapSafe(innerStream, registry, String.valueOf(f)); }
@Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { FSDataInputStream innerStream = unsafeFileSystem.open(f, bufferSize); return ClosingFSDataInputStream.wrapSafe(innerStream, registry, String.valueOf(f)); }
private static void internalCopyFile(Path sourcePath, Path targetPath, boolean executable, FileSystem sFS, FileSystem tFS) throws IOException { try (FSDataOutputStream lfsOutput = tFS.create(targetPath, FileSystem.WriteMode.NO_OVERWRITE); FSDataInputStream fsInput = sFS.open(sourcePath)) { IOUtils.copyBytes(fsInput, lfsOutput); //noinspection ResultOfMethodCallIgnored new File(targetPath.toString()).setExecutable(executable); } }