/** * Adds a suffix to the final name in the path. * * @param suffix The suffix to be added * @return the new path including the suffix */ public Path suffix(String suffix) { return new Path(getParent(), getName() + suffix); }
public static Path compressDirectory(Path directory, Path target) throws IOException { FileSystem sourceFs = directory.getFileSystem(); FileSystem targetFs = target.getFileSystem(); try (ZipOutputStream out = new ZipOutputStream(targetFs.create(target, FileSystem.WriteMode.NO_OVERWRITE))) { addToZip(directory, sourceFs, directory.getParent(), out); } return target; }
@Override public FSDataOutputStream create(final Path filePath, final WriteMode overwrite) throws IOException { checkNotNull(filePath, "filePath"); if (exists(filePath) && overwrite == WriteMode.NO_OVERWRITE) { throw new FileAlreadyExistsException("File already exists: " + filePath); } final Path parent = filePath.getParent(); if (parent != null && !mkdirs(parent)) { throw new IOException("Mkdirs failed to create " + parent); } final File file = pathToFile(filePath); return new LocalDataOutputStream(file); }
@Test public void testGetParent() { Path p = new Path("/my/fancy/path"); assertEquals("/my/fancy", p.getParent().toUri().getPath()); p = new Path("/my/other/fancy/path/"); assertEquals("/my/other/fancy", p.getParent().toUri().getPath()); p = new Path("hdfs:///my/path"); assertEquals("/my", p.getParent().toUri().getPath()); p = new Path("hdfs:///myPath/"); assertEquals("/", p.getParent().toUri().getPath()); p = new Path("/"); assertNull(p.getParent()); p = new Path("C:/my/windows/path"); assertEquals("/C:/my/windows", p.getParent().toUri().getPath()); }
@Override public void go() throws Exception { fs.initOutPathLocalFS(path.getParent(), WriteMode.OVERWRITE, true); try (FSDataOutputStream out = fs.create(path, WriteMode.OVERWRITE)) { out.write(11); } } }
@Test public void testSerializationOnlyInProgress() throws IOException { final File testFolder = tempFolder.newFolder(); final FileSystem fs = FileSystem.get(testFolder.toURI()); final Path testBucket = new Path(testFolder.getPath(), "test"); final RecoverableWriter writer = fs.createRecoverableWriter(); final RecoverableFsDataOutputStream stream = writer.open(testBucket); stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8"))); final RecoverableWriter.ResumeRecoverable current = stream.persist(); final BucketState<String> bucketState = new BucketState<>( "test", testBucket, Long.MAX_VALUE, current, new HashMap<>()); final SimpleVersionedSerializer<BucketState<String>> serializer = new BucketStateSerializer<>( writer.getResumeRecoverableSerializer(), writer.getCommitRecoverableSerializer(), SimpleVersionedStringSerializer.INSTANCE ); final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState); // to simulate that everything is over for file. stream.close(); final BucketState<String> recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes); Assert.assertEquals(testBucket, recoveredState.getBucketPath()); FileStatus[] statuses = fs.listStatus(testBucket.getParent()); Assert.assertEquals(1L, statuses.length); Assert.assertTrue( statuses[0].getPath().getPath().startsWith( (new Path(testBucket.getParent(), ".test.inprogress")).toString()) ); }
(new Path(testBucket.getParent(), ".test-2.inprogress").toString())));
/** * Adds a suffix to the final name in the path. * * @param suffix The suffix to be added * @return the new path including the suffix */ public Path suffix(String suffix) { return new Path(getParent(), getName() + suffix); }
/** * Adds a suffix to the final name in the path. * * @param suffix The suffix to be added * @return the new path including the suffix */ public Path suffix(String suffix) { return new Path(getParent(), getName() + suffix); }
@VisibleForTesting static Path getTempMetaPath(Path path) { return new Path(path.getParent(), "_tmp_" + path.getName()); } }
public static Path compressDirectory(Path directory, Path target) throws IOException { FileSystem sourceFs = directory.getFileSystem(); FileSystem targetFs = target.getFileSystem(); try (ZipOutputStream out = new ZipOutputStream(targetFs.create(target, FileSystem.WriteMode.NO_OVERWRITE))) { addToZip(directory, sourceFs, directory.getParent(), out); } return target; }
private void delete(String blobPath) { try { LOG.debug("Deleting {}.", blobPath); Path path = new Path(blobPath); fileSystem.delete(path, true); // send a call to delete the directory containing the file. This will // fail (and be ignored) when some files still exist. try { fileSystem.delete(path.getParent(), false); fileSystem.delete(new Path(basePath), false); } catch (IOException ignored) {} } catch (Exception e) { LOG.warn("Failed to delete blob at " + blobPath); } }
private boolean delete(String blobPath) { try { LOG.debug("Deleting {}.", blobPath); Path path = new Path(blobPath); boolean result = fileSystem.delete(path, true); // send a call to delete the directory containing the file. This will // fail (and be ignored) when some files still exist. try { fileSystem.delete(path.getParent(), false); fileSystem.delete(new Path(basePath), false); } catch (IOException ignored) {} return result; } catch (Exception e) { LOG.warn("Failed to delete blob at " + blobPath); return false; } }
private boolean delete(String blobPath) { try { LOG.debug("Deleting {}.", blobPath); Path path = new Path(blobPath); boolean result = fileSystem.delete(path, true); // send a call to delete the directory containing the file. This will // fail (and be ignored) when some files still exist. try { fileSystem.delete(path.getParent(), false); fileSystem.delete(new Path(basePath), false); } catch (IOException ignored) {} return result; } catch (Exception e) { LOG.warn("Failed to delete blob at " + blobPath); return false; } }
private boolean delete(String blobPath) { try { LOG.debug("Deleting {}.", blobPath); Path path = new Path(blobPath); boolean result = fileSystem.delete(path, true); // send a call to delete the directory containing the file. This will // fail (and be ignored) when some files still exist. try { fileSystem.delete(path.getParent(), false); fileSystem.delete(new Path(basePath), false); } catch (IOException ignored) {} return result; } catch (Exception e) { LOG.warn("Failed to delete blob at " + blobPath); return false; } }
/** * Stores the savepoint metadata file. * * @param <T> Savepoint type * @param directory Target directory to store savepoint in * @param savepoint Savepoint to be stored * @return Path of stored savepoint * @throws IOException Failures during store are forwarded */ public static <T extends Savepoint> String storeSavepoint(String directory, T savepoint) throws IOException { // write and create the file handle FileStateHandle metadataFileHandle = storeSavepointToHandle(directory, SAVEPOINT_METADATA_FILE, savepoint); // we return the savepoint directory path here! // The directory path also works to resume from and is more elegant than the direct // metadata file pointer return metadataFileHandle.getFilePath().getParent().toString(); }
/** * Discard the state by deleting the file that stores the state. If the parent directory * of the state is empty after deleting the state file, it is also deleted. * * @throws Exception Thrown, if the file deletion (not the directory deletion) fails. */ @Override public void discardState() throws Exception { getFileSystem().delete(filePath, false); try { FileUtils.deletePathIfEmpty(getFileSystem(), filePath.getParent()); } catch (Exception ignored) {} }
@Override public FSDataOutputStream create(final Path filePath, final WriteMode overwrite) throws IOException { checkNotNull(filePath, "filePath"); if (exists(filePath) && overwrite == WriteMode.NO_OVERWRITE) { throw new FileAlreadyExistsException("File already exists: " + filePath); } final Path parent = filePath.getParent(); if (parent != null && !mkdirs(parent)) { throw new IOException("Mkdirs failed to create " + parent); } final File file = pathToFile(filePath); return new LocalDataOutputStream(file); }
@Override public FSDataOutputStream create(final Path filePath, final WriteMode overwrite) throws IOException { checkNotNull(filePath, "filePath"); if (exists(filePath) && overwrite == WriteMode.NO_OVERWRITE) { throw new FileAlreadyExistsException("File already exists: " + filePath); } final Path parent = filePath.getParent(); if (parent != null && !mkdirs(parent)) { throw new IOException("Mkdirs failed to create " + parent); } final File file = pathToFile(filePath); return new LocalDataOutputStream(file); }
/** * Discard the state by deleting the file that stores the state. If the parent directory * of the state is empty after deleting the state file, it is also deleted. * * @throws Exception Thrown, if the file deletion (not the directory deletion) fails. */ @Override public void discardState() throws Exception { FileSystem fs = getFileSystem(); fs.delete(filePath, false); if (fs.getKind() == FileSystemKind.FILE_SYSTEM) { try { FileUtils.deletePathIfEmpty(fs, filePath.getParent()); } catch (Exception ignored) {} } }