protected FileSystem getSourceFileSystem (State state) throws IOException { Configuration conf = HadoopUtils.getConfFromState(state); String uri = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, ConfigurationKeys.LOCAL_FS_URI); return HadoopUtils.getOptionallyThrottledFileSystem(FileSystem.get(URI.create(uri), conf), state); } }
public AvroFsHelper(State state) { this(state, HadoopUtils.newConfiguration()); }
/** * A wrapper around {@link FileContext#rename(Path, Path, Options.Rename...)}. */ public static void renamePath(FileContext fc, Path oldName, Path newName) throws IOException { renamePath(fc, oldName, newName, false); }
/** * Calls deletePath() on each directory in the given list of directories to delete. * If moveToTrash is set, it will be moved to trash according to the file system trash policy. */ public static void deleteDirectories(FileSystem fs, List<String> directoriesToDelete, boolean recursive, boolean moveToTrash) throws IOException { for (String directory : directoriesToDelete) { if (moveToTrash) { moveToTrash(fs, new Path(directory)); } else { deletePath(fs, new Path(directory), recursive); } } }
public static void moveSelectFiles(String extension, String source, String destination) throws IOException { FileSystem fs = getFileSystem(); fs.mkdirs(new Path(destination)); FileStatus[] fileStatuses = fs.listStatus(new Path(source)); for (FileStatus fileStatus : fileStatuses) { Path path = fileStatus.getPath(); if (!fileStatus.isDirectory() && path.toString().toLowerCase().endsWith(extension.toLowerCase())) { HadoopUtils.deleteIfExists(fs, new Path(destination), true); HadoopUtils.copyPath(fs, path, fs, new Path(destination), getConfiguration()); } } } }
public HadoopFsHelper(State state) { this(state, HadoopUtils.getConfFromState(state)); }
@Test public void testStateToConfiguration() throws IOException { Map<String, String> vals = Maps.newHashMap(); vals.put("test_key1", "test_val1"); vals.put("test_key2", "test_val2"); Configuration expected = HadoopUtils.newConfiguration(); State state = new State(); for (Map.Entry<String, String> entry : vals.entrySet()) { state.setProp(entry.getKey(), entry.getValue()); expected.set(entry.getKey(), entry.getValue()); } Assert.assertEquals(HadoopUtils.getConfFromState(state), expected); Assert.assertEquals(HadoopUtils.getConfFromState(state, Optional.<String>absent()), expected); Assert.assertEquals(HadoopUtils.getConfFromState(state, Optional.of("dummy")), expected); }
copyFile(srcFs, src, dstFs, dst, overwriteDst, conf); } else { copyFile(srcFs, src, dstFs, tmp, overwriteDst, conf); try { boolean renamed = false; if (overwriteDst && dstFs.exists(dst)) { try { deletePath(dstFs, dst, true); } finally { renamePath(dstFs, tmp, dst); renamed = true; renamePath(dstFs, tmp, dst); deletePath(dstFs, tmp, true);
/** * Writes the <code>config</code> to {@link #storeMetadataFilePath}. Creates a backup file at * <code>storeMetadataFilePath + ".bkp"</code> to recover old metadata in case of unexpected deployment failures * * @param config to be serialized * @throws IOException if there was any problem writing the <code>config</code> to the store metadata file. */ void writeMetadata(Config config) throws IOException { Path storeMetadataFileBkpPath = new Path(this.storeMetadataFilePath.getParent(), this.storeMetadataFilePath.getName() + ".bkp"); // Delete old backup file if exists HadoopUtils.deleteIfExists(this.fs, storeMetadataFileBkpPath, true); // Move current storeMetadataFile to backup if (this.fs.exists(this.storeMetadataFilePath)) { HadoopUtils.renamePath(this.fs, this.storeMetadataFilePath, storeMetadataFileBkpPath); } // Write new storeMetadataFile try (FSDataOutputStream outputStream = FileSystem.create(this.fs, this.storeMetadataFilePath, FsDeploymentConfig.DEFAULT_STORE_PERMISSIONS);) { outputStream.write(config.root().render(ConfigRenderOptions.concise()).getBytes(Charsets.UTF_8)); } catch (Exception e) { // Restore from backup HadoopUtils.deleteIfExists(this.fs, this.storeMetadataFilePath, true); HadoopUtils.renamePath(this.fs, storeMetadataFileBkpPath, this.storeMetadataFilePath); throw new IOException( String.format("Failed to write store metadata at %s. Restored existing store metadata file from backup", this.storeMetadataFilePath), e); } }
/** * A wrapper around {@link FileSystem#delete(Path, boolean)} that only deletes a given {@link Path} if it is present * on the given {@link FileSystem}. */ public static void deleteIfExists(FileSystem fs, Path path, boolean recursive) throws IOException { if (fs.exists(path)) { deletePath(fs, path, recursive); } }
HadoopUtils.deletePath(fs, jobStagingPath, true); HadoopUtils.deletePath(fs, jobStagingPath.getParent(), true); HadoopUtils.deletePath(fs, jobOutputPath, true); HadoopUtils.deletePath(fs, jobOutputPath.getParent(), true); Path jobErrPath = new Path(state.getProp(ConfigurationKeys.ROW_LEVEL_ERR_FILE)); log.info("Cleaning up err directory : " + jobErrPath); HadoopUtils.deleteIfExists(fs, jobErrPath, true);
/** * Replace a null or empty dataset URN with {@link #DEFAULT_DATASET_URN}, and replaces illegal HDFS * characters with '_'. */ private static String sanitizeDatasetUrn(String datasetUrn) { return Strings.isNullOrEmpty(datasetUrn) ? DEFAULT_DATASET_URN : HadoopUtils.sanitizePath(datasetUrn, "_"); }
/** * Delete the temporary {@link #schemaDir} */ public void cleanupTempSchemas() throws IOException { HadoopUtils.deleteIfExists(this.fs, this.schemaDir, true); }
HadoopUtils.deletePath(fs, filePath, true); HadoopUtils.movePath(fs, tempFilePath, fs, filePath, true, fs.getConf());
/** * Moves a src {@link Path} from a srcFs {@link FileSystem} to a dst {@link Path} on a dstFs {@link FileSystem}. If * the srcFs and the dstFs have the same scheme, and neither of them or S3 schemes, then the {@link Path} is simply * renamed. Otherwise, the data is from the src {@link Path} to the dst {@link Path}. So this method can handle copying * data between different {@link FileSystem} implementations. * * @param srcFs the source {@link FileSystem} where the src {@link Path} exists * @param src the source {@link Path} which will me moved * @param dstFs the destination {@link FileSystem} where the dst {@link Path} should be created * @param dst the {@link Path} to move data to * @param overwrite true if the destination should be overwritten; otherwise, false */ public static void movePath(FileSystem srcFs, Path src, FileSystem dstFs, Path dst, boolean overwrite, Configuration conf) throws IOException { if (srcFs.getUri().getScheme().equals(dstFs.getUri().getScheme()) && !FS_SCHEMES_NON_ATOMIC.contains(srcFs.getUri().getScheme()) && !FS_SCHEMES_NON_ATOMIC.contains(dstFs.getUri().getScheme())) { renamePath(srcFs, src, dst); } else { copyPath(srcFs, src, dstFs, dst, true, overwrite, conf); } }
@Override public Void call() throws Exception { Lock lock = ParallelRunner.this.locks.get(src.toString()); lock.lock(); try { if (ParallelRunner.this.fs.exists(src)) { HadoopUtils.renamePath(ParallelRunner.this.fs, src, dst); if (group.isPresent()) { HadoopUtils.setGroup(ParallelRunner.this.fs, dst, group.get()); } } return null; } catch (FileAlreadyExistsException e) { LOGGER.warn(String.format("Failed to rename %s to %s: dst already exists", src, dst), e); return null; } finally { lock.unlock(); } } }), "Rename " + src + " to " + dst));
/** * Moves a src {@link Path} from a srcFs {@link FileSystem} to a dst {@link Path} on a dstFs {@link FileSystem}. If * the srcFs and the dstFs have the same scheme, and neither of them or S3 schemes, then the {@link Path} is simply * renamed. Otherwise, the data is from the src {@link Path} to the dst {@link Path}. So this method can handle copying * data between different {@link FileSystem} implementations. * * @param srcFs the source {@link FileSystem} where the src {@link Path} exists * @param src the source {@link Path} which will me moved * @param dstFs the destination {@link FileSystem} where the dst {@link Path} should be created * @param dst the {@link Path} to move data to */ public static void movePath(FileSystem srcFs, Path src, FileSystem dstFs, Path dst, Configuration conf) throws IOException { movePath(srcFs, src, dstFs, dst, false, conf); }
/** * @deprecated use {@link HadoopUtils#getWriterFileSystem(State, int, int)}. */ @Deprecated private static FileSystem getTargetFileSystem(State state) throws IOException { return HadoopUtils.getOptionallyThrottledFileSystem(WriterUtils.getWriterFS(state, 1, 0), state); }
@Override public void initialize(URI name, Configuration conf) throws IOException { super.initialize(name, conf); State state = HadoopUtils.getStateFromConf(conf); this.fsHelper = new SftpFsHelper(state); try { this.fsHelper.connect(); } catch (FileBasedHelperException e) { throw new IOException(e); } }
@Override public Void call() throws Exception { Lock lock = ParallelRunner.this.locks.get(src.toString()); lock.lock(); try { if (ParallelRunner.this.fs.exists(src)) { HadoopUtils.movePath(ParallelRunner.this.fs, src, dstFs, dst, overwrite, dstFs.getConf()); if (group.isPresent()) { HadoopUtils.setGroup(dstFs, dst, group.get()); } } return null; } catch (FileAlreadyExistsException e) { LOGGER.warn(String.format("Failed to move %s to %s: dst already exists", src, dst), e); return null; } finally { lock.unlock(); } } }), "Move " + src + " to " + dst));