@Override public FileSystem initializeFileSystem() { return FileSystem.getLocalFileSystem(); } }
@Test public void testKind() { final FileSystem fs = FileSystem.getLocalFileSystem(); assertEquals(FileSystemKind.FILE_SYSTEM, fs.getKind()); }
@Test public void testRenameNonExistingFile() throws IOException { final FileSystem fs = FileSystem.getLocalFileSystem(); final File srcFile = new File(temporaryFolder.newFolder(), "someFile.txt"); final File destFile = new File(temporaryFolder.newFolder(), "target"); final Path srcFilePath = new Path(srcFile.toURI()); final Path destFilePath = new Path(destFile.toURI()); // this cannot succeed because the source file does not exist assertFalse(fs.rename(srcFilePath, destFilePath)); }
try { FileSystem local = FileSystem.getLocalFileSystem(); local.delete(new Path(localTmpPath), true); } catch (IOException ioe) {
@Test public void testRenameFileWithNoAccess() throws IOException { final FileSystem fs = FileSystem.getLocalFileSystem(); final File srcFile = temporaryFolder.newFile("someFile.txt"); final File destFile = new File(temporaryFolder.newFolder(), "target"); // we need to make the file non-modifiable so that the rename fails Assume.assumeTrue(srcFile.getParentFile().setWritable(false, false)); Assume.assumeTrue(srcFile.setWritable(false, false)); try { final Path srcFilePath = new Path(srcFile.toURI()); final Path destFilePath = new Path(destFile.toURI()); // this cannot succeed because the source folder has no permission to remove the file assertFalse(fs.rename(srcFilePath, destFilePath)); } finally { // make sure we give permission back to ensure proper cleanup //noinspection ResultOfMethodCallIgnored srcFile.getParentFile().setWritable(true, false); //noinspection ResultOfMethodCallIgnored srcFile.setWritable(true, false); } }
@Test public void testConcurrentMkdirs() throws Exception { final FileSystem fs = FileSystem.getLocalFileSystem(); final File root = temporaryFolder.getRoot(); final int directoryDepth = 10;
@Test public void testRenameToNonEmptyTargetDir() throws IOException { final FileSystem fs = FileSystem.getLocalFileSystem(); // a source folder with a file final File srcFolder = temporaryFolder.newFolder(); final File srcFile = new File(srcFolder, "someFile.txt"); assertTrue(srcFile.createNewFile()); // a non-empty destination folder final File dstFolder = temporaryFolder.newFolder(); final File dstFile = new File(dstFolder, "target"); assertTrue(dstFile.createNewFile()); // this cannot succeed because the destination folder is not empty assertFalse(fs.rename(new Path(srcFolder.toURI()), new Path(dstFolder.toURI()))); // retry after deleting the occupying target file assertTrue(dstFile.delete()); assertTrue(fs.rename(new Path(srcFolder.toURI()), new Path(dstFolder.toURI()))); assertTrue(new File(dstFolder, srcFile.getName()).exists()); }
@Test public void testGenericRecord() throws IOException { final Path outputPath = new Path(File.createTempFile("avro-output-file", "generic.avro").getAbsolutePath()); final AvroOutputFormat<GenericRecord> outputFormat = new AvroOutputFormat<>(outputPath, GenericRecord.class); Schema schema = new Schema.Parser().parse("{\"type\":\"record\", \"name\":\"user\", \"fields\": [{\"name\":\"user_name\", \"type\":\"string\"}, {\"name\":\"favorite_number\", \"type\":\"int\"}, {\"name\":\"favorite_color\", \"type\":\"string\"}]}"); outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE); outputFormat.setSchema(schema); output(outputFormat, schema); GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema); DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(new File(outputPath.getPath()), reader); while (dataFileReader.hasNext()) { GenericRecord record = dataFileReader.next(); assertEquals(record.get("user_name").toString(), "testUser"); assertEquals(record.get("favorite_number"), 1); assertEquals(record.get("favorite_color").toString(), "blue"); } //cleanup FileSystem fs = FileSystem.getLocalFileSystem(); fs.delete(outputPath, false); }
@Test public void testDeletePathIfEmpty() throws IOException { final FileSystem localFs = FileSystem.getLocalFileSystem(); final File dir = tmp.newFolder(); assertTrue(dir.exists()); final Path dirPath = new Path(dir.toURI()); // deleting an empty directory should work assertTrue(FileUtils.deletePathIfEmpty(localFs, dirPath)); // deleting a non existing directory should work assertTrue(FileUtils.deletePathIfEmpty(localFs, dirPath)); // create a non-empty dir final File nonEmptyDir = tmp.newFolder(); final Path nonEmptyDirPath = new Path(nonEmptyDir.toURI()); new FileOutputStream(new File(nonEmptyDir, "filename")).close(); assertFalse(FileUtils.deletePathIfEmpty(localFs, nonEmptyDirPath)); }
@Test public void testMakeQualified() throws IOException { // make relative path qualified String path = "test/test"; Path p = new Path(path).makeQualified(FileSystem.getLocalFileSystem()); URI u = p.toUri(); assertEquals("file", u.getScheme()); assertEquals(null, u.getAuthority()); String q = new Path(FileSystem.getLocalFileSystem().getWorkingDirectory().getPath(), path).getPath(); assertEquals(q, u.getPath()); // make absolute path qualified path = "/test/test"; p = new Path(path).makeQualified(FileSystem.getLocalFileSystem()); u = p.toUri(); assertEquals("file", u.getScheme()); assertEquals(null, u.getAuthority()); assertEquals(path, u.getPath()); } }
try { FileSystem local = FileSystem.getLocalFileSystem(); local.delete(new Path(tmpPlanFilesDir), true); } catch (IOException ioe) {
tempFile.deleteOnExit(); expectedFiles.add(new Path(tempFile.getAbsolutePath()).makeQualified(FileSystem.getLocalFileSystem()).toString());
final Path destFilePath = new Path(destFile.toURI()); FileSystem fs = FileSystem.getLocalFileSystem();
@Test public void testCompression() throws Exception { // given final Path outputPath = new Path(File.createTempFile("avro-output-file", "avro").getAbsolutePath()); final AvroOutputFormat<User> outputFormat = new AvroOutputFormat<>(outputPath, User.class); outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE); final Path compressedOutputPath = new Path(File.createTempFile("avro-output-file", "compressed.avro").getAbsolutePath()); final AvroOutputFormat<User> compressedOutputFormat = new AvroOutputFormat<>(compressedOutputPath, User.class); compressedOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE); compressedOutputFormat.setCodec(AvroOutputFormat.Codec.SNAPPY); // when output(outputFormat); output(compressedOutputFormat); // then assertTrue(fileSize(outputPath) > fileSize(compressedOutputPath)); // cleanup FileSystem fs = FileSystem.getLocalFileSystem(); fs.delete(outputPath, false); fs.delete(compressedOutputPath, false); }
/** * Test that {@link FileUtils#deletePathIfEmpty(FileSystem, Path)} deletes the path if it is * empty. A path can only be empty if it is a directory which does not contain any * files/directories. */ @Test public void testDeletePathIfEmpty() throws IOException { File file = temporaryFolder.newFile(); File directory = temporaryFolder.newFolder(); File directoryFile = new File(directory, UUID.randomUUID().toString()); assertTrue(directoryFile.createNewFile()); Path filePath = new Path(file.toURI()); Path directoryPath = new Path(directory.toURI()); Path directoryFilePath = new Path(directoryFile.toURI()); FileSystem fs = FileSystem.getLocalFileSystem(); // verify that the files have been created assertTrue(fs.exists(filePath)); assertTrue(fs.exists(directoryFilePath)); // delete the single file assertFalse(FileUtils.deletePathIfEmpty(fs, filePath)); assertTrue(fs.exists(filePath)); // try to delete the non-empty directory assertFalse(FileUtils.deletePathIfEmpty(fs, directoryPath)); assertTrue(fs.exists(directoryPath)); // delete the file contained in the directory assertTrue(fs.delete(directoryFilePath, false)); // now the deletion should work assertTrue(FileUtils.deletePathIfEmpty(fs, directoryPath)); assertFalse(fs.exists(directoryPath)); }
@Test public void testGetStatisticsMultipleFilesWithCachedVersion() { try { FileSystem fs = FileSystem.getLocalFileSystem();
@Test public void testGetStatisticsMultipleOneFileWithCachedVersion() throws IOException { FileSystem fs = FileSystem.getLocalFileSystem();
private void deletePartitionDirOnFailure() { // currently we only support local file FileSystem fileSystem = FileSystem.getLocalFileSystem(); boolean deleteSuccess = false; try { deleteSuccess = fileSystem.delete(new Path(partitionRootPath), true); } catch (Throwable e) { LOG.error("Exception occurred on deletePartitionDirOnFailure.", e); } if (!deleteSuccess) { LOG.error("Failed to delete dirty data, directory path " + partitionRootPath); } }
public Path getLastCheckpointPath() throws IOException { FileStatus[] listStatus = FileSystem.getLocalFileSystem() .listStatus(new Path(getCheckpointDir(), jobID.toString())); return Arrays.stream(listStatus) .filter(s -> s.getPath().getName().startsWith("chk")) .sorted((s1, s2) -> -Integer.compare( Integer.parseInt(s1.getPath().getName().split("-")[1]), Integer.parseInt(s2.getPath().getName().split("-")[1]))) .findFirst() .map(s -> s.getPath()) .orElseThrow(() -> new IllegalStateException("Cannot find any checkpoints")); }
@Override public void finish() throws IOException, InterruptedException { sortMerger.finishAdding(); List<SortedDataFile<Tuple2<Integer, T>>> remainFiles = sortMerger.getRemainingSortedDataFiles(); int nextFileId = 0; FileSystem localFileSystem = FileSystem.getLocalFileSystem(); for (SortedDataFile<Tuple2<Integer, T>> file : remainFiles) { // rename file localFileSystem.rename( new Path(file.getChannelID().getPath()), new Path(ExternalBlockShuffleUtils.generateDataPath(partitionDataRootPath, nextFileId++))); } LOG.info("Finish external result partition writing."); }