@Override public Path getHomeDirectory() { return fs.getHomeDirectory(); }
@Override public Path getHomeDirectory() { return fsImpl.getHomeDirectory(); }
/** * Returns the Path where the YARN application files should be uploaded to. * * @param appId YARN application id */ private Path getYarnFilesDir(final ApplicationId appId) throws IOException { final FileSystem fileSystem = FileSystem.get(yarnConfiguration); final Path homeDir = fileSystem.getHomeDirectory(); return new Path(homeDir, ".flink/" + appId + '/'); }
private Path getServiceWorkDirPath(FileSystem fs, String serviceName, String serviceId) { return new Path(fs.getHomeDirectory(), serviceName + Path.SEPARATOR + serviceId); }
@Override protected URI getDefaultRootDir(Config factoryConfig, FileSystem defaultFileSystem, Optional<URI> configDefinedDefaultURI) { return configDefinedDefaultURI.or(defaultFileSystem.getHomeDirectory().toUri()); }
public YarnContainerSecurityManager(Config config, FileSystem fs, EventBus eventBus) { this.fs = fs; this.tokenFilePath = new Path(this.fs.getHomeDirectory(), config.getString(GobblinYarnConfigurationKeys.APPLICATION_NAME_KEY) + Path.SEPARATOR + GobblinYarnConfigurationKeys.TOKEN_FILE_NAME); this.eventBus = eventBus; }
public Path getHomeDirectory() { return new Path(this.fs.getHomeDirectory().toUri()); }
/** * Get the root directory of Trash for current user when the path specified * is deleted. * * @param path the trash root of the path to be determined. * @return the default implementation returns {@code /user/$USER/.Trash} */ public Path getTrashRoot(Path path) { return this.makeQualified(new Path(getHomeDirectory().toUri().getPath(), TRASH_PREFIX)); }
public Path getUploadPath(String baseName) { String dfsDirStr = config.getString(DrillOnYarnConfig.DFS_APP_DIR); Path appDir; if (dfsDirStr.startsWith("/")) { appDir = new Path(dfsDirStr); } else { Path home = fs.getHomeDirectory(); appDir = new Path(home, dfsDirStr); } return new Path(appDir, baseName); }
/** * Get the application working directory {@link Path}. * * @param fs a {@link FileSystem} instance on which {@link FileSystem#getHomeDirectory()} is called * to get the home directory of the {@link FileSystem} of the application working directory * @param applicationName the application name * @param applicationId the application ID in string form * @return the cluster application working directory {@link Path} */ public static Path getAppWorkDirPath(FileSystem fs, String applicationName, String applicationId) { return new Path(fs.getHomeDirectory(), getAppWorkDirPath(applicationName, applicationId)); }
private static ParallelRunner getParallelRunner(FileSystem fs, Closer closer, int parallelRunnerThreads, Map<String, ParallelRunner> parallelRunners) { String uriAndHomeDir = new Path(new Path(fs.getUri()), fs.getHomeDirectory()).toString(); if (!parallelRunners.containsKey(uriAndHomeDir)) { parallelRunners.put(uriAndHomeDir, closer.register(new ParallelRunner(parallelRunnerThreads, fs))); } return parallelRunners.get(uriAndHomeDir); } }
private YarnAppSecurityManager buildYarnAppSecurityManager() throws IOException { Path tokenFilePath = new Path(this.fs.getHomeDirectory(), this.applicationName + Path.SEPARATOR + GobblinYarnConfigurationKeys.TOKEN_FILE_NAME); return new YarnAppSecurityManager(this.config, this.helixManager, this.fs, tokenFilePath); }
public static Path getAppWorkDirPathFromConfig(Config config, FileSystem fs, String applicationName, String applicationId) { if (config.hasPath(CLUSTER_WORK_DIR)) { return new Path(config.getString(CLUSTER_WORK_DIR)); } return new Path(fs.getHomeDirectory(), getAppWorkDirPath(applicationName, applicationId)); }
public Path getHomeDirectory() { return replaceScheme(this.underlyingFs.getHomeDirectory(), this.underlyingScheme, this.replacementScheme); }
public static Path getBulkOutputDir(String tableName, Configuration conf, boolean deleteOnExit) throws IOException { FileSystem fs = FileSystem.get(conf); String tmp = conf.get(HConstants.TEMPORARY_FS_DIRECTORY_KEY, fs.getHomeDirectory() + "/hbase-staging"); Path path = new Path(tmp + Path.SEPARATOR + "bulk_output-" + tableName + "-" + EnvironmentEdgeManager.currentTime()); if (deleteOnExit) { fs.deleteOnExit(path); } return path; }
protected Path uploadFileIfNecessary(FileSystem localFS, Path localPath, FileSystem destFS) throws IOException { // Fast path for local FS -- DistributedCache + local JobRunner seems copy/link files automatically if (destFS.equals(localFS)) { log.debug("Skipping file upload for {} (destination filesystem {} equals local filesystem)", localPath, destFS); return localPath; } Path destPath = new Path(destFS.getHomeDirectory() + "/" + HDFS_TMP_LIB_DIR + "/" + localPath.getName()); Stats fileStats = null; try { fileStats = compareModtimes(localFS, localPath, destFS, destPath); } catch (IOException e) { log.warn("Unable to read or stat file: localPath={}, destPath={}, destFS={}", localPath, destPath, destFS); } if (!fileStats.isRemoteCopyCurrent()) { log.debug("Copying {} to {}", localPath, destPath); destFS.copyFromLocalFile(localPath, destPath); if (null != fileStats.local) { final long mtime = fileStats.local.getModificationTime(); log.debug("Setting modtime on {} to {}", destPath, mtime); destFS.setTimes(destPath, mtime, -1); // -1 means leave atime alone } } return destPath; }
@Override public Path run() { try (FileSystem fs = FileSystem.get(conf)) { return fs.getHomeDirectory(); } catch (IOException ioe) { LOG.error("Failed to get foo's home directory", ioe); } return null; } });
@Override public Path getCurrentTrashPath(Configuration conf, FileSystem fs) { TrashPolicy tp = TrashPolicy.getInstance(conf, fs, fs.getHomeDirectory()); return tp.getCurrentTrashDir(); }
private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId, Map<String, LocalResource> localResources, String resources) throws IOException { String suffix = jstormClientContext.appName + JOYConstants.BACKLASH + appId + JOYConstants.BACKLASH + fileDstPath; Path dst = new Path(fs.getHomeDirectory(), suffix); if (fileSrcPath == null) { FSDataOutputStream ostream = null; try { ostream = FileSystem .create(fs, dst, new FsPermission(JOYConstants.FS_PERMISSION)); ostream.writeUTF(resources); } finally { IOUtils.closeQuietly(ostream); } } else { fs.copyFromLocalFile(new Path(fileSrcPath), dst); } FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance( ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put(fileDstPath, scRsrc); }
@Test public void work_dir_should_get_default_calculated_value_when_not_specified() throws Exception { Map<String, String> configMap = new HashMap<>(); Config config = ConfigFactory.parseMap(configMap); when(fs.getHomeDirectory()).thenReturn(new Path("/home/")); Path workDirPath = GobblinClusterUtils .getAppWorkDirPathFromConfig(config, fs, "appName", "appid"); assertEquals(new Path("/home/appName/appid"), workDirPath); } }