/** * Return the default container log directory permissions. * * @return the default container log directory permissions */ @VisibleForTesting public String getLogDirPermissions() { if (this.logDirPermissions==null) { this.logDirPermissions = getConf().get( YarnConfiguration.NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS, YarnConfiguration.NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS_DEFAULT); } return this.logDirPermissions; }
protected CommandExecutor buildCommandExecutor(String wrapperScriptPath, String containerIdStr, String user, Path pidFile, Resource resource, File wordDir, Map<String, String> environment) throws IOException { String[] command = getRunCommand(wrapperScriptPath, containerIdStr, user, pidFile, this.getConf(), resource); LOG.info("launchContainer: " + Arrays.toString(command)); return new ShellCommandExecutor( command, wordDir, environment, 0L, false); }
protected CommandExecutor buildCommandExecutor(String wrapperScriptPath, String containerIdStr, String user, Path pidFile, Resource resource, File wordDir, Map<String, String> environment) throws IOException { String[] command = getRunCommand(wrapperScriptPath, containerIdStr, user, pidFile, this.getConf(), resource); LOG.info("launchContainer: " + Arrays.toString(command)); return new ShellCommandExecutor( command, wordDir, environment, 0L, false); }
protected CommandExecutor buildCommandExecutor(String wrapperScriptPath, String containerIdStr, String user, Path pidFile, Resource resource, File wordDir, Map<String, String> environment) throws IOException { String[] command = getRunCommand(wrapperScriptPath, containerIdStr, user, pidFile, this.getConf(), resource); LOG.info("launchContainer: " + Arrays.toString(command)); return new ShellCommandExecutor( command, wordDir, environment, 0L, false); }
/** * Create a new {@link ShellCommandExecutor} using the parameters. * * @param wrapperScriptPath the path to the script to execute * @param containerIdStr the container ID * @param user the application owner's username * @param pidFile the path to the container's PID file * @param resource this parameter controls memory and CPU limits. * @param workDir If not-null, specifies the directory which should be set * as the current working directory for the command. If null, * the current working directory is not modified. * @param environment the container environment * @return the new {@link ShellCommandExecutor} * @see ShellCommandExecutor */ protected CommandExecutor buildCommandExecutor(String wrapperScriptPath, String containerIdStr, String user, Path pidFile, Resource resource, File workDir, Map<String, String> environment) { String[] command = getRunCommand(wrapperScriptPath, containerIdStr, user, pidFile, this.getConf(), resource); LOG.info("launchContainer: " + Arrays.toString(command)); return new ShellCommandExecutor( command, workDir, environment, 0L, false); }
@Private @VisibleForTesting protected ContainerLocalizer createContainerLocalizer(String user, String appId, String locId, List<String> localDirs, FileContext localizerFc, String userFolder) throws IOException { ContainerLocalizer localizer = new ContainerLocalizer(localizerFc, user, appId, locId, getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf()), userFolder); return localizer; }
/** * Create a new {@link ContainerLocalizer} instance. * * @param user the user who owns the job for which the localization is being * run * @param appId the ID of the application for which the localization is being * run * @param locId the ID of the container for which the localization is being * run * @param localDirs a list of directories to use as destinations for the * localization * @param localizerFc the {@link FileContext} to use when localizing files * @return the new {@link ContainerLocalizer} instance * @throws IOException if {@code user} or {@code locId} is {@code null} or if * the container localizer has an initialization failure */ @Private @VisibleForTesting protected ContainerLocalizer createContainerLocalizer(String user, String appId, String locId, List<String> localDirs, FileContext localizerFc) throws IOException { ContainerLocalizer localizer = new ContainerLocalizer(localizerFc, user, appId, locId, getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf())); return localizer; }
localDirs, super.getConf());
@Override public void startLocalizer(Path nmPrivateContainerTokensPath, InetSocketAddress nmAddr, String user, String appId, String locId, LocalDirsHandlerService dirsHandler) throws IOException, InterruptedException { List<String> localDirs = dirsHandler.getLocalDirs(); List<String> logDirs = dirsHandler.getLogDirs(); createUserLocalDirs(localDirs, user); createUserCacheDirs(localDirs, user); createAppDirs(localDirs, user, appId); createAppLogDirs(appId, logDirs, user); // randomly choose the local directory Path appStorageDir = getWorkingDir(localDirs, user, appId); String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId); Path tokenDst = new Path(appStorageDir, tokenFn); copyFile(nmPrivateContainerTokensPath, tokenDst, user); LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst); FileContext localizerFc = FileContext.getFileContext( lfs.getDefaultFileSystem(), getConf()); localizerFc.setUMask(lfs.getUMask()); localizerFc.setWorkingDirectory(appStorageDir); LOG.info("Localizer CWD set to " + appStorageDir + " = " + localizerFc.getWorkingDirectory()); ContainerLocalizer localizer = new ContainerLocalizer(localizerFc, user, appId, locId, getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf())); // TODO: DO it over RPC for maintaining similarity? localizer.runLocalization(nmAddr); }
@Override public void startLocalizer(Path nmPrivateContainerTokensPath, InetSocketAddress nmAddr, String user, String appId, String locId, LocalDirsHandlerService dirsHandler) throws IOException, InterruptedException { List<String> localDirs = dirsHandler.getLocalDirs(); List<String> logDirs = dirsHandler.getLogDirs(); createUserLocalDirs(localDirs, user); createUserCacheDirs(localDirs, user); createAppDirs(localDirs, user, appId); createAppLogDirs(appId, logDirs, user); // randomly choose the local directory Path appStorageDir = getWorkingDir(localDirs, user, appId); String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId); Path tokenDst = new Path(appStorageDir, tokenFn); copyFile(nmPrivateContainerTokensPath, tokenDst, user); LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst); FileContext localizerFc = FileContext.getFileContext( lfs.getDefaultFileSystem(), getConf()); localizerFc.setUMask(lfs.getUMask()); localizerFc.setWorkingDirectory(appStorageDir); LOG.info("Localizer CWD set to " + appStorageDir + " = " + localizerFc.getWorkingDirectory()); ContainerLocalizer localizer = new ContainerLocalizer(localizerFc, user, appId, locId, getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf())); // TODO: DO it over RPC for maintaining similarity? localizer.runLocalization(nmAddr); }
FileContext.getFileContext(lfs.getDefaultFileSystem(), getConf()); localizerFc.setUMask(lfs.getUMask()); localizerFc.setWorkingDirectory(appStorageDir);
lfs.getDefaultFileSystem(), getConf()); localizerFc.setUMask(lfs.getUMask()); localizerFc.setWorkingDirectory(appStorageDir);