@Override public Path getRemoteAppLogDir(ApplicationId appId, String user) throws IOException { return LogAggregationUtils.getRemoteAppLogDir(conf, appId, user, this.remoteRootLogDir, this.remoteRootLogDirSuffix); }
/** * Get the remote application directory for log aggregation. * @param appId the Application ID * @param appOwner the Application Owner * @return the remote application directory * @throws IOException if can not find the remote application directory */ public Path getRemoteAppLogDir(ApplicationId appId, String appOwner) throws IOException { return LogAggregationUtils.getRemoteAppLogDir(conf, appId, appOwner, this.remoteRootLogDir, this.remoteRootLogDirSuffix); }
Path getRemoteAppLogDir(ApplicationId appId, String user) { return LogAggregationUtils.getRemoteAppLogDir(this.remoteRootLogDir, appId, user, this.remoteRootLogDirSuffix); }
Path getRemoteAppLogDir(ApplicationId appId, String user) { return LogAggregationUtils.getRemoteAppLogDir(this.remoteRootLogDir, appId, user, this.remoteRootLogDirSuffix); }
Path getRemoteAppLogDir(ApplicationId appId, String user) { return LogAggregationUtils.getRemoteAppLogDir(this.remoteRootLogDir, appId, user, this.remoteRootLogDirSuffix); }
/** * Constructs the full filename for an application's log file per node. * @param remoteRootLogDir * @param appId * @param user * @param nodeId * @param suffix * @return the remote log file. */ public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir, ApplicationId appId, String user, NodeId nodeId, String suffix) { return new Path(getRemoteAppLogDir(remoteRootLogDir, appId, user, suffix), getNodeString(nodeId)); }
/** * Constructs the full filename for an application's log file per node. * @param remoteRootLogDir the aggregated remote root log dir * @param appId the application Id * @param user the application owner * @param nodeId the node id * @param suffix the log dir suffix * @return the remote log file. */ public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir, ApplicationId appId, String user, NodeId nodeId, String suffix) { return new Path(getRemoteAppLogDir(remoteRootLogDir, appId, user, suffix), getNodeString(nodeId)); }
/** * Constructs the full filename for an application's log file per node. * @param remoteRootLogDir * @param appId * @param user * @param nodeId * @param suffix * @return the remote log file. */ public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir, ApplicationId appId, String user, NodeId nodeId, String suffix) { return new Path(getRemoteAppLogDir(remoteRootLogDir, appId, user, suffix), getNodeString(nodeId)); }
/** * Return the remote application log directory. * @param conf the configuration * @param appId the application * @param appOwner the application owner * @return the remote application log directory path * @throws IOException if we can not find remote application log directory */ public static org.apache.hadoop.fs.Path getRemoteAppLogDir( Configuration conf, ApplicationId appId, String appOwner) throws IOException { String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); org.apache.hadoop.fs.Path remoteRootLogDir = new org.apache.hadoop.fs.Path(conf.get( YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); return getRemoteAppLogDir(conf, appId, appOwner, remoteRootLogDir, suffix); }
qualifiedRemoteRootLogDir.toUri(), conf); org.apache.hadoop.fs.Path toMatch = LogAggregationUtils .getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix); FileStatus[] matching = fc.util().globStatus(toMatch); if (matching == null || matching.length != 1) { remoteAppDir = LogAggregationUtils.getRemoteAppLogDir( remoteRootLogDir, appId, appOwner, suffix);
/** * Get all available log files under remote app log directory. * @param conf the configuration * @param appId the applicationId * @param appOwner the application owner * @param remoteRootLogDir the remote root log directory * @param suffix the log directory suffix * @return the list of available log files * @throws IOException if there is no log file available */ public static List<FileStatus> getRemoteNodeFileList( Configuration conf, ApplicationId appId, String appOwner, org.apache.hadoop.fs.Path remoteRootLogDir, String suffix) throws IOException { Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner, remoteRootLogDir, suffix); List<FileStatus> nodeFiles = new ArrayList<>(); Path qualifiedLogDir = FileContext.getFileContext(conf).makeQualified(remoteAppLogDir); nodeFiles.addAll(Arrays.asList(FileContext.getFileContext( qualifiedLogDir.toUri(), conf).util().listStatus(remoteAppLogDir))); return nodeFiles; }
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); Path fullPath = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir, appId, bestGuess, suffix); FileContext fc = getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix); pathAccess = toMatch.toString(); FileStatus[] matching = fc.util().globStatus(toMatch);
/** * Get all available log files under remote app log directory. * @param conf the configuration * @param appId the applicationId * @param appOwner the application owner * @param remoteRootLogDir the remote root log directory * @param suffix the log directory suffix * @return the iterator of available log files * @throws IOException if there is no log file available */ public static RemoteIterator<FileStatus> getRemoteNodeFileDir( Configuration conf, ApplicationId appId, String appOwner, org.apache.hadoop.fs.Path remoteRootLogDir, String suffix) throws IOException { Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner, remoteRootLogDir, suffix); RemoteIterator<FileStatus> nodeFiles = null; Path qualifiedLogDir = FileContext.getFileContext(conf).makeQualified(remoteAppLogDir); nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(), conf).listStatus(remoteAppLogDir); return nodeFiles; }
Path appDir = LogAggregationUtils.getRemoteAppLogDir( LogAggregationService.this.remoteRootLogDir, appId, user, LogAggregationService.this.remoteRootLogDirSuffix);
Path appDir = LogAggregationUtils.getRemoteAppLogDir( LogAggregationService.this.remoteRootLogDir, appId, user, LogAggregationService.this.remoteRootLogDirSuffix);
fileFormat.getRemoteRootLogDirSuffix()); } catch (FileNotFoundException fnf) { logDirNotExist(LogAggregationUtils.getRemoteAppLogDir( conf, appId, appOwner).toString()); } catch (AccessControlException | AccessDeniedException ace) { logDirNoAccessPermission(LogAggregationUtils.getRemoteAppLogDir( conf, appId, appOwner).toString(), appOwner, ace.getMessage());
/** * Get all available log files under remote app log directory. * @param conf the configuration * @param appId the applicationId * @param appOwner the application owner * @return the iterator of available log files * @throws IOException if there is no log file available */ public static RemoteIterator<FileStatus> getRemoteNodeFileDir( Configuration conf, ApplicationId appId, String appOwner) throws IOException { Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner); RemoteIterator<FileStatus> nodeFiles = null; Path qualifiedLogDir = FileContext.getFileContext(conf).makeQualified(remoteAppLogDir); nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(), conf).listStatus(remoteAppLogDir); return nodeFiles; } }
Path appDir = LogAggregationUtils.getRemoteAppLogDir( remoteRootLogDir, appId, user, remoteRootLogDirSuffix);
@Private public int dumpAllContainersLogs(ContainerLogsRequest options) throws IOException { LogAggregationFileController fc = null; try { fc = this.getFileController( options.getAppId(), options.getAppOwner()); } catch (IOException ex) { System.err.println(ex); } boolean foundAnyLogs = false; if (fc != null) { foundAnyLogs = fc.readAggregatedLogs(options, null); } if (!foundAnyLogs) { emptyLogDir(LogAggregationUtils.getRemoteAppLogDir( conf, options.getAppId(), options.getAppOwner()) .toString()); return -1; } return 0; }
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); Path remoteAppLogDir = LogAggregationUtils.getRemoteAppLogDir( remoteRootLogDir, ConverterUtils.toApplicationId(appId), jobOwner, suffix);