Refine search
public void testDeleteRecursively_nonDirectoryFile() throws IOException { try (FileSystem fs = newTestFileSystem(SECURE_DIRECTORY_STREAM)) { Path file = fs.getPath("dir/a"); assertTrue(Files.isRegularFile(file, NOFOLLOW_LINKS)); MoreFiles.deleteRecursively(file); assertFalse(Files.exists(file, NOFOLLOW_LINKS)); Path symlink = fs.getPath("/symlinktodir"); assertTrue(Files.isSymbolicLink(symlink)); Path realSymlinkTarget = symlink.toRealPath(); assertTrue(Files.isDirectory(realSymlinkTarget, NOFOLLOW_LINKS)); MoreFiles.deleteRecursively(symlink); assertFalse(Files.exists(symlink, NOFOLLOW_LINKS)); assertTrue(Files.isDirectory(realSymlinkTarget, NOFOLLOW_LINKS)); } }
private static Iterable<Path> fileTreeChildren(Path dir) { if (Files.isDirectory(dir, NOFOLLOW_LINKS)) { try { return listFiles(dir); } catch (IOException e) { // the exception thrown when iterating a DirectoryStream if an I/O exception occurs throw new DirectoryIteratorException(e); } } return ImmutableList.of(); }
@Override public ByteSink createSink() throws IOException { Path file = createFile(); if (initialBytes != null) { java.nio.file.Files.write(file, initialBytes); return MoreFiles.asByteSink(file, StandardOpenOption.APPEND); } return MoreFiles.asByteSink(file); }
/** * Simple, insecure method for deleting the contents of a directory for file systems that don't * support {@code SecureDirectoryStream}. Returns a collection of exceptions that occurred or null * if no exceptions were thrown. */ private static @Nullable Collection<IOException> deleteDirectoryContentsInsecure( DirectoryStream<Path> dir) { Collection<IOException> exceptions = null; try { for (Path entry : dir) { exceptions = concat(exceptions, deleteRecursivelyInsecure(entry)); } return exceptions; } catch (DirectoryIteratorException e) { return addException(exceptions, e.getCause()); } }
/** * Secure method for deleting the contents of a directory using {@code SecureDirectoryStream}. * Returns a collection of exceptions that occurred or null if no exceptions were thrown. */ private static @Nullable Collection<IOException> deleteDirectoryContentsSecure( SecureDirectoryStream<Path> dir) { Collection<IOException> exceptions = null; try { for (Path path : dir) { exceptions = concat(exceptions, deleteRecursivelySecure(dir, path.getFileName())); } return exceptions; } catch (DirectoryIteratorException e) { return addException(exceptions, e.getCause()); } }
/** * Insecure recursive delete for file systems that don't support {@code SecureDirectoryStream}. * Returns a collection of exceptions that occurred or null if no exceptions were thrown. */ private static @Nullable Collection<IOException> deleteRecursivelyInsecure(Path path) { Collection<IOException> exceptions = null; try { if (Files.isDirectory(path, NOFOLLOW_LINKS)) { try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) { exceptions = deleteDirectoryContentsInsecure(stream); } } // If exceptions is not null, something went wrong trying to delete the contents of the // directory, so we shouldn't try to delete the directory as it will probably fail. if (exceptions == null) { Files.delete(path); } return exceptions; } catch (IOException e) { return addException(exceptions, e); } }
@Test public void testCreateWithNonexistentStagingDirectory() throws Exception { java.nio.file.Path stagingParent = createTempDirectory("test"); java.nio.file.Path staging = Paths.get(stagingParent.toString(), "staging"); // stagingParent = /tmp/testXXX // staging = /tmp/testXXX/staging try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); Configuration conf = new Configuration(); conf.set(S3_STAGING_DIRECTORY, staging.toString()); fs.initialize(new URI("s3n://test-bucket/"), conf); fs.setS3Client(s3); FSDataOutputStream stream = fs.create(new Path("s3n://test-bucket/test")); stream.close(); assertTrue(Files.exists(staging)); } finally { deleteRecursively(stagingParent, ALLOW_INSECURE); } }
try (FileSystem fs = newTestFileSystem(SECURE_DIRECTORY_STREAM)) { Path dirToDelete = fs.getPath("dir/b/i"); Path changingFile = dirToDelete.resolve("j/l"); Path symlinkTarget = fs.getPath("/dontdelete"); for (int i = 0; i < 5000; i++) { try { Files.createDirectories(changingFile); Files.createFile(dirToDelete.resolve("j/k")); } catch (FileAlreadyExistsException expected) { assertEquals(3, MoreFiles.listFiles(symlinkTarget).size());
List<Path> spillPaths = ImmutableList.of(spillPath1.toPath(), spillPath2.toPath()); spillPath1.mkdirs(); spillPath2.mkdirs(); java.nio.file.Files.createTempFile(spillPath1.toPath(), SPILL_FILE_PREFIX, SPILL_FILE_SUFFIX); java.nio.file.Files.createTempFile(spillPath1.toPath(), SPILL_FILE_PREFIX, SPILL_FILE_SUFFIX); java.nio.file.Files.createTempFile(spillPath1.toPath(), SPILL_FILE_PREFIX, "blah"); java.nio.file.Files.createTempFile(spillPath2.toPath(), SPILL_FILE_PREFIX, SPILL_FILE_SUFFIX); java.nio.file.Files.createTempFile(spillPath2.toPath(), "blah", SPILL_FILE_SUFFIX); java.nio.file.Files.createTempFile(spillPath2.toPath(), "blah", "blah"); assertEquals(listFiles(spillPath1.toPath()).size(), 3); assertEquals(listFiles(spillPath2.toPath()).size(), 3); spillerFactory.cleanupOldSpillFiles(); assertEquals(listFiles(spillPath1.toPath()).size(), 1); assertEquals(listFiles(spillPath2.toPath()).size(), 2);
@Test public void testDistributesSpillOverPaths() throws Exception { List<Type> types = ImmutableList.of(BIGINT); BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager(new TypeRegistry()); List<Path> spillPaths = ImmutableList.of(spillPath1.toPath(), spillPath2.toPath()); FileSingleStreamSpillerFactory spillerFactory = new FileSingleStreamSpillerFactory( executor, // executor won't be closed, because we don't call destroy() on the spiller factory blockEncodingSerde, new SpillerStats(), spillPaths, 1.0); assertEquals(listFiles(spillPath1.toPath()).size(), 0); assertEquals(listFiles(spillPath2.toPath()).size(), 0); Page page = buildPage(); List<SingleStreamSpiller> spillers = new ArrayList<>(); for (int i = 0; i < 10; ++i) { SingleStreamSpiller singleStreamSpiller = spillerFactory.create(types, bytes -> {}, newSimpleAggregatedMemoryContext().newLocalMemoryContext("test")); getUnchecked(singleStreamSpiller.spill(page)); spillers.add(singleStreamSpiller); } assertEquals(listFiles(spillPath1.toPath()).size(), 5); assertEquals(listFiles(spillPath2.toPath()).size(), 5); spillers.forEach(SingleStreamSpiller::close); assertEquals(listFiles(spillPath1.toPath()).size(), 0); assertEquals(listFiles(spillPath2.toPath()).size(), 0); }
spiller.spill(page).get(); spiller.spill(Iterators.forArray(page, page, page)).get(); assertEquals(listFiles(spillPath.toPath()).size(), 1); ImmutableList<Page> spilledPages = ImmutableList.copyOf(spilledPagesIterator); assertEquals(4, spilledPages.size()); for (int i = 0; i < 4; ++i) { PageAssertions.assertPageEquals(TYPES, page, spilledPages.get(i)); assertEquals(listFiles(spillPath.toPath()).size(), 0); assertEquals(memoryContext.getBytes(), 0);
public void testDeleteRecursively_symlinkToDir_sdsNotSupported_allowInsecure() throws IOException { try (FileSystem fs = newTestFileSystem()) { Path symlink = fs.getPath("/symlinktodir"); Path dir = fs.getPath("dir"); assertEquals(6, MoreFiles.listFiles(dir).size()); MoreFiles.deleteRecursively(symlink, ALLOW_INSECURE); assertFalse(Files.exists(symlink)); assertTrue(Files.exists(dir)); assertEquals(6, MoreFiles.listFiles(dir).size()); } }
public void testPredicates() throws IOException { Path file = createTempFile(); Path dir = tempDir.resolve("dir"); Files.createDirectory(dir); assertTrue(MoreFiles.isDirectory().apply(dir)); assertFalse(MoreFiles.isRegularFile().apply(dir)); assertFalse(MoreFiles.isDirectory().apply(file)); assertTrue(MoreFiles.isRegularFile().apply(file)); Path symlinkToDir = tempDir.resolve("symlinkToDir"); Path symlinkToFile = tempDir.resolve("symlinkToFile"); Files.createSymbolicLink(symlinkToDir, dir); Files.createSymbolicLink(symlinkToFile, file); assertTrue(MoreFiles.isDirectory().apply(symlinkToDir)); assertFalse(MoreFiles.isRegularFile().apply(symlinkToDir)); assertFalse(MoreFiles.isDirectory().apply(symlinkToFile)); assertTrue(MoreFiles.isRegularFile().apply(symlinkToFile)); assertFalse(MoreFiles.isDirectory(NOFOLLOW_LINKS).apply(symlinkToDir)); assertFalse(MoreFiles.isRegularFile(NOFOLLOW_LINKS).apply(symlinkToFile)); }
public void testDeleteDirectoryContents_symlinkToDir() throws IOException { try (FileSystem fs = newTestFileSystem(SECURE_DIRECTORY_STREAM)) { Path symlink = fs.getPath("/symlinktodir"); Path dir = fs.getPath("dir"); assertEquals(6, MoreFiles.listFiles(symlink).size()); MoreFiles.deleteDirectoryContents(symlink); assertTrue(Files.exists(symlink, NOFOLLOW_LINKS)); assertTrue(Files.exists(symlink)); assertTrue(Files.exists(dir)); assertEquals(0, MoreFiles.listFiles(symlink).size()); } }
public void testDirectoryDeletion_sdsNotSupported_fails() throws IOException { for (DirectoryDeleteMethod method : EnumSet.allOf(DirectoryDeleteMethod.class)) { try (FileSystem fs = newTestFileSystem()) { Path dir = fs.getPath("dir"); assertEquals(6, MoreFiles.listFiles(dir).size()); try { method.delete(dir); fail("expected InsecureRecursiveDeleteException"); } catch (InsecureRecursiveDeleteException expected) { } assertTrue(Files.exists(dir)); assertEquals(6, MoreFiles.listFiles(dir).size()); } } }
public void testByteSource_size_ofSymlinkToRegularFile() throws IOException { try (FileSystem fs = Jimfs.newFileSystem(Configuration.unix())) { Path file = fs.getPath("file"); Files.write(file, new byte[10]); Path link = fs.getPath("link"); Files.createSymbolicLink(link, file); ByteSource source = MoreFiles.asByteSource(link); assertEquals(10L, (long) source.sizeIfKnown().get()); assertEquals(10L, source.size()); } }
public void testEqual() throws IOException { try (FileSystem fs = Jimfs.newFileSystem(Configuration.unix())) { Path fooPath = fs.getPath("foo"); Path barPath = fs.getPath("bar"); MoreFiles.asCharSink(fooPath, UTF_8).write("foo"); MoreFiles.asCharSink(barPath, UTF_8).write("barbar"); assertThat(MoreFiles.equal(fooPath, barPath)).isFalse(); assertThat(MoreFiles.equal(fooPath, fooPath)).isTrue(); assertThat(MoreFiles.asByteSource(fooPath).contentEquals(MoreFiles.asByteSource(fooPath))) .isTrue(); Path fooCopy = Files.copy(fooPath, fs.getPath("fooCopy")); assertThat(Files.isSameFile(fooPath, fooCopy)).isFalse(); assertThat(MoreFiles.equal(fooPath, fooCopy)).isTrue(); MoreFiles.asCharSink(fooCopy, UTF_8).write("boo"); assertThat(MoreFiles.asByteSource(fooPath).size()) .isEqualTo(MoreFiles.asByteSource(fooCopy).size()); assertThat(MoreFiles.equal(fooPath, fooCopy)).isFalse(); // should also assert that a Path that erroneously reports a size 0 can still be compared, // not sure how to do that with the Path API } }
@Override public ByteSource createSource(byte[] bytes) throws IOException { checkNotNull(bytes); Path file = createFile(); java.nio.file.Files.write(file, bytes); return MoreFiles.asByteSource(file); }
@AfterMethod(alwaysRun = true) public void tearDown() throws Exception { deleteRecursively(spillPath.toPath(), ALLOW_INSECURE); }