/** * Copy a file to the local filesystem, then delete it from the * remote filesystem (if successfully copied). * @param src path src file in the remote filesystem * @param dst path local destination * @throws IOException IO failure */ public void moveToLocalFile(Path src, Path dst) throws IOException { copyToLocalFile(true, src, dst); }
/** * The src file is under FS, and the dst is on the local disk. * Copy it from FS control to the local dst name. * delSrc indicates if the src will be removed or not. */ @Override public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { fs.copyToLocalFile(delSrc, src, dst); }
/** * Copy it a file from the remote filesystem to the local one. * @param src path src file in the remote filesystem * @param dst path local destination * @throws IOException IO failure */ public void copyToLocalFile(Path src, Path dst) throws IOException { copyToLocalFile(false, src, dst); }
/** * Copy it a file from a remote filesystem to the local one. * delSrc indicates if the src will be removed or not. * @param delSrc whether to delete the src * @param src path src file in the remote filesystem * @param dst path local destination * @throws IOException IO failure */ public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { copyToLocalFile(delSrc, src, dst, false); }
private void copyLocal(String var, Configuration conf) throws IOException { String[] filenames = TempletonUtils.decodeArray(conf.get(var)); if (filenames != null) { for (String filename : filenames) { Path src = new Path(filename); Path dst = new Path(src.getName()); FileSystem fs = src.getFileSystem(conf); LOG.info("templeton: copy " + src + " => " + dst); fs.copyToLocalFile(src, dst); } } }
private ResolverUrlClassLoader createClassLoader(ClassLoader parent, org.apache.hadoop.fs.Path path) throws IOException { Path tempPath = Files.createTempFile("hbase-coprocessor-", ".jar"); org.apache.hadoop.fs.Path destination = new org.apache.hadoop.fs.Path(tempPath.toString()); LOG.debug("Copying coprocessor jar '{}' to '{}'.", path, tempPath); FileSystem fileSystem = FileSystem.get(getConf()); fileSystem.copyToLocalFile(path, destination); URL url = tempPath.toUri().toURL(); return createClassLoader(new URL[] { url }, parent); }
/** * Copy file in DFS from p to newPath * @param fs file system * @param p old path * @param newPath new path * @throws IOException exception */ protected void copyFile(FileSystem fs, Path p, Path newPath) throws IOException { File f = File.createTempFile("data", "meta"); Path localPath = new Path(f.getAbsolutePath()); fs.copyToLocalFile(p, localPath); fs.copyFromLocalFile(localPath, newPath); boolean exists = fs.exists(newPath); if (!exists) { throw new IOException("Failed to copy meta file to: "+ newPath); } }
private String downloadResource(URI srcUri, String subDir, boolean convertToUnix) throws IOException, URISyntaxException { LOG.debug("Converting to local {}", srcUri); File destinationDir = (subDir == null) ? resourceDir : new File(resourceDir, subDir); ensureDirectory(destinationDir); File destinationFile = new File(destinationDir, new Path(srcUri.toString()).getName()); String dest = destinationFile.getCanonicalPath(); if (destinationFile.exists()) { return dest; } FileSystem fs = FileSystem.get(srcUri, conf); fs.copyToLocalFile(new Path(srcUri.toString()), new Path(dest)); // add "execute" permission to downloaded resource file (needed when loading dll file) FileUtil.chmod(dest, "ugo+rx", true); return dest; }
private static File copyToLocal(FileSystem fs, Path keyPath) throws IOException { java.nio.file.Path tmpKeyPath = Files.createTempFile(GoogleCommon.class.getSimpleName(), "tmp", PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------"))); File copied = tmpKeyPath.toFile(); copied.deleteOnExit(); fs.copyToLocalFile(keyPath, new Path(copied.getAbsolutePath())); return copied; }
@Override public Void call() throws Exception { synchronized (fs) { String tezLibs = conf.get(TezConfiguration.TEZ_LIB_URIS); if (tezLibs == null) { LOG.warn("Missing tez.lib.uris in tez-site.xml"); } if (LOG.isDebugEnabled()) { LOG.debug("Copying tez libs from " + tezLibs); } lfs.mkdirs(tezDir); fs.copyToLocalFile(new Path(tezLibs), new Path(libDir, "tez.tar.gz")); CompressionUtils.unTar(new Path(libDir, "tez.tar.gz").toString(), tezDir.toString(), true); lfs.delete(new Path(libDir, "tez.tar.gz"), false); } return null; } };
@Override public String retrieve(Config config, String targetDir) throws IOException { URI uri = URI.create(this.fsUri); FileSystem fs = FileSystem.get(uri, HadoopUtils.getConfFromState(ConfigUtils.configToState(config))); final Path sourceFile = new Path(path); final String zipFile = appendSlash(targetDir) + StringUtils.substringAfterLast(this.path, File.separator); LOGGER.debug("Downloading to zip: " + zipFile + " from uri: " + sourceFile); fs.copyToLocalFile(sourceFile, new Path(zipFile)); return zipFile; } }
private String downloadResource(URI srcUri, String subDir, boolean convertToUnix) throws IOException, URISyntaxException { LOG.info("converting to local " + srcUri); File destinationDir = (subDir == null) ? resourceDir : new File(resourceDir, subDir); ensureDirectory(destinationDir); File destinationFile = new File(destinationDir, new Path(srcUri.toString()).getName()); String dest = destinationFile.getCanonicalPath(); if (destinationFile.exists()) { return dest; } FileSystem fs = FileSystem.get(srcUri, conf); fs.copyToLocalFile(new Path(srcUri.toString()), new Path(dest)); // add "execute" permission to downloaded resource file (needed when loading dll file) FileUtil.chmod(dest, "ugo+rx", true); return dest; }
@Override public void copyToLocalFile(URI srcUri, File dstFile) throws Exception { LOGGER.debug("starting to fetch segment from hdfs"); final String dstFilePath = dstFile.getAbsolutePath(); try { final Path remoteFile = new Path(srcUri); final Path localFile = new Path(dstFile.toURI()); RetryPolicy fixedDelayRetryPolicy = RetryPolicies.fixedDelayRetryPolicy(_retryCount, _retryWaitMs); fixedDelayRetryPolicy.attempt(() -> { try { if (_hadoopFS == null) { throw new RuntimeException("_hadoopFS client is not initialized when trying to copy files"); } long startMs = System.currentTimeMillis(); _hadoopFS.copyToLocalFile(remoteFile, localFile); LOGGER.debug("copied {} from hdfs to {} in local for size {}, take {} ms", srcUri, dstFilePath, dstFile.length(), System.currentTimeMillis() - startMs); return true; } catch (IOException e) { LOGGER.warn("failed to fetch segment {} from hdfs, might retry", srcUri, e); return false; } }); } catch (Exception e) { LOGGER.error("failed to fetch {} from hdfs to local {}", srcUri, dstFilePath, e); throw e; } }
@Override public void fetchSegmentToLocal(final String uri, final File tempFile) throws Exception { LOGGER.debug("starting to fetch segment from hdfs"); final String dstFilePath = tempFile.getAbsolutePath(); try { final Path remoteFile = new Path(uri); final Path localFile = new Path(tempFile.toURI()); RetryPolicy fixedDelayRetryPolicy = RetryPolicies.fixedDelayRetryPolicy(_retryCount, _retryWaitMs); fixedDelayRetryPolicy.attempt(() -> { try { if (_hadoopFS == null) { throw new RuntimeException("_hadoopFS client is not initialized when trying to copy files"); } long startMs = System.currentTimeMillis(); _hadoopFS.copyToLocalFile(remoteFile, localFile); LOGGER .debug("copied {} from hdfs to {} in local for size {}, take {} ms", uri, dstFilePath, tempFile.length(), System.currentTimeMillis() - startMs); return true; } catch (IOException e) { LOGGER.warn("failed to fetch segment {} from hdfs, might retry", uri, e); return false; } }); } catch (Exception e) { LOGGER.error("failed to fetch {} from hdfs to local {}", uri, dstFilePath, e); throw e; } }
public void copyToLocalFile(boolean delSrc, Path src, Path dst, boolean useRawLocalFileSystem) throws java.io.IOException { this.underlyingFs.copyToLocalFile(delSrc, replaceScheme(src, this.replacementScheme, this.underlyingScheme), replaceScheme(dst, this.replacementScheme, this.underlyingScheme), useRawLocalFileSystem); }
public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws java.io.IOException { this.underlyingFs.copyToLocalFile(delSrc, replaceScheme(src, this.replacementScheme, this.underlyingScheme), replaceScheme(dst, this.replacementScheme, this.underlyingScheme)); }
public void copyToLocalFile(Path src, Path dst) throws java.io.IOException { this.underlyingFs.copyToLocalFile(replaceScheme(src, this.replacementScheme, this.underlyingScheme), replaceScheme(dst, this.replacementScheme, this.underlyingScheme)); }
LOG.info("Copying " + remoteFile + " to " + localFile); FileSystem remoteFS = remoteFile.getFileSystem(conf); remoteFS.copyToLocalFile(remoteFile, localFile); downloadedFiles.put(path, timeStamp);
private void moveFileFromDfsToLocal(Path sourcePath, Path targetPath, FileSystem fs, FileSystem dstFs) throws HiveException, IOException { // RawLocalFileSystem seems not able to get the right permissions for a local file, it // always returns hdfs default permission (00666). So we can not overwrite a directory // by deleting and recreating the directory and restoring its permissions. We should // delete all its files and subdirectories instead. if (dstFs.exists(targetPath)) { if (dstFs.isDirectory(targetPath)) { FileStatus[] destFiles = dstFs.listStatus(targetPath); for (FileStatus destFile : destFiles) { if (!dstFs.delete(destFile.getPath(), true)) { throw new IOException("Unable to clean the destination directory: " + targetPath); } } } else { throw new HiveException("Target " + targetPath + " is not a local directory."); } } else { if (!FileUtils.mkdir(dstFs, targetPath, conf)) { throw new HiveException("Failed to create local target directory " + targetPath); } } if (fs.exists(sourcePath)) { FileStatus[] srcs = fs.listStatus(sourcePath, FileUtils.HIDDEN_FILES_PATH_FILTER); for (FileStatus status : srcs) { fs.copyToLocalFile(status.getPath(), targetPath); } } }
/** * Publish job execution results into local file system, so you can view them. * * @param fs Distributed file sytem used in job. * @throws IOException If failed. */ private void publishResults(FileSystem fs) throws IOException { X.println(">>> Cleaning up DFS input directory: " + BOOKS_DFS_DIR); fs.delete(BOOKS_DFS_DIR, true); X.println(">>> Cleaning up LOCAL result directory: " + RESULT_LOCAL_DIR); fs.delete(RESULT_LOCAL_DIR, true); X.println(">>> Moving job results into LOCAL result directory: " + RESULT_LOCAL_DIR); fs.copyToLocalFile(true, RESULT_DFS_DIR, RESULT_LOCAL_DIR); }