@Override public boolean apply(FileStatus next) { return next.getPath().getName() .contains(LogAggregationUtils.getNodeString(nodeId)) && !next.getPath().getName().endsWith( LogAggregationUtils.TMP_FILE_SUFFIX); } });
/** * Constructs the full filename for an application's log file per node. * @param remoteRootLogDir * @param appId * @param user * @param nodeId * @param suffix * @return the remote log file. */ public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir, ApplicationId appId, String user, NodeId nodeId, String suffix) { return new Path(getRemoteAppLogDir(remoteRootLogDir, appId, user, suffix), getNodeString(nodeId)); }
/** * Gets the remote app log dir. * @param remoteRootLogDir the aggregated log remote root log dir * @param appId the application id * @param user the application owner * @param suffix the log directory suffix * @return the remote application specific log dir. */ public static Path getRemoteAppLogDir(Path remoteRootLogDir, ApplicationId appId, String user, String suffix) { return new Path(getRemoteLogSuffixedDir(remoteRootLogDir, user, suffix), appId.toString()); }
/** * Return the remote application log directory. * @param conf the configuration * @param appId the application * @param appOwner the application owner * @return the remote application log directory path * @throws IOException if we can not find remote application log directory */ public static org.apache.hadoop.fs.Path getRemoteAppLogDir( Configuration conf, ApplicationId appId, String appOwner) throws IOException { String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); org.apache.hadoop.fs.Path remoteRootLogDir = new org.apache.hadoop.fs.Path(conf.get( YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); return getRemoteAppLogDir(conf, appId, appOwner, remoteRootLogDir, suffix); }
Path appDir = LogAggregationUtils.getRemoteAppLogDir( LogAggregationService.this.remoteRootLogDir, appId, user, LogAggregationService.this.remoteRootLogDirSuffix); Path suffixDir = LogAggregationUtils.getRemoteLogSuffixedDir( LogAggregationService.this.remoteRootLogDir, user, LogAggregationService.this.remoteRootLogDirSuffix); Path userDir = LogAggregationUtils.getRemoteLogUserDir( LogAggregationService.this.remoteRootLogDir, user); userDir = userDir.makeQualified(remoteFS.getUri(),
YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); Path remoteAppLogDir = LogAggregationUtils.getRemoteAppLogDir( remoteRootLogDir, ConverterUtils.toApplicationId(appId), jobOwner, suffix); FileStatus thisNodeFile = nodeFiles.next(); String fileName = thisNodeFile.getPath().getName(); if (fileName.contains(LogAggregationUtils.getNodeString(nodeId)) && !fileName.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { AggregatedLogFormat.LogReader reader = null;
boolean getAllContainers = (containerIdStr == null); String nodeIdStr = (nodeId == null) ? null : LogAggregationUtils.getNodeString(nodeId); RemoteIterator<FileStatus> nodeFiles = LogAggregationUtils .getRemoteNodeFileDir(conf, appId, appOwner); if (nodeFiles == null) { throw new IOException("There is no available log file for "
@Override public Path getRemoteAppLogDir(ApplicationId appId, String user) throws IOException { return LogAggregationUtils.getRemoteAppLogDir(conf, appId, user, this.remoteRootLogDir, this.remoteRootLogDirSuffix); }
/** * Gets the remote suffixed log dir for the user. * @param remoteRootLogDir the aggregated log remote root log dir * @param user the application owner * @param suffix the log dir suffix * @return the remote suffixed log dir. */ public static Path getRemoteLogSuffixedDir(Path remoteRootLogDir, String user, String suffix) { if (suffix == null || suffix.isEmpty()) { return getRemoteLogUserDir(remoteRootLogDir, user); } // TODO Maybe support suffix to be more than a single file. return new Path(getRemoteLogUserDir(remoteRootLogDir, user), suffix); }
nodeFiles = LogAggregationUtils.getRemoteNodeFileDir(conf, appId, appOwner, fileFormat.getRemoteRootLogDir(), fileFormat.getRemoteRootLogDirSuffix()); } catch (FileNotFoundException fnf) { logDirNotExist(LogAggregationUtils.getRemoteAppLogDir( conf, appId, appOwner).toString()); } catch (AccessControlException | AccessDeniedException ace) { logDirNoAccessPermission(LogAggregationUtils.getRemoteAppLogDir( conf, appId, appOwner).toString(), appOwner, ace.getMessage());
Path getRemoteNodeLogFileForApp(ApplicationId appId, String user) { return LogAggregationUtils.getRemoteNodeLogFileForApp( this.remoteRootLogDir, appId, user, this.nodeId, this.remoteRootLogDirSuffix); }
public LogDeletionTask(Configuration conf, long retentionSecs, ApplicationClientProtocol rmClient) { this.conf = conf; this.retentionMillis = retentionSecs * 1000; this.suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); this.remoteRootLogDir = new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); this.rmClient = rmClient; }
Path appDir = LogAggregationUtils.getRemoteAppLogDir( LogAggregationService.this.remoteRootLogDir, appId, user, LogAggregationService.this.remoteRootLogDirSuffix); Path suffixDir = LogAggregationUtils.getRemoteLogSuffixedDir( LogAggregationService.this.remoteRootLogDir, user, LogAggregationService.this.remoteRootLogDirSuffix); Path userDir = LogAggregationUtils.getRemoteLogUserDir( LogAggregationService.this.remoteRootLogDir, user); userDir = userDir.makeQualified(remoteFS.getUri(),
YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); Path remoteAppLogDir = LogAggregationUtils.getRemoteAppLogDir( remoteRootLogDir, ConverterUtils.toApplicationId(appId), jobOwner, suffix); FileStatus thisNodeFile = nodeFiles.next(); String fileName = thisNodeFile.getPath().getName(); if (fileName.contains(LogAggregationUtils.getNodeString(nodeId)) && !fileName.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { AggregatedLogFormat.LogReader reader = null;
long size = logRequest.getBytes(); RemoteIterator<FileStatus> nodeFiles = LogAggregationUtils .getRemoteNodeFileDir(conf, appId, logRequest.getAppOwner()); byte[] buf = new byte[65535]; while (nodeFiles != null && nodeFiles.hasNext()) { .getNodeString(nodeId))) && !nodeName.endsWith( LogAggregationUtils.TMP_FILE_SUFFIX)) { AggregatedLogFormat.LogReader reader = null;
/** * Get the remote application directory for log aggregation. * @param appId the Application ID * @param appOwner the Application Owner * @return the remote application directory * @throws IOException if can not find the remote application directory */ public Path getRemoteAppLogDir(ApplicationId appId, String appOwner) throws IOException { return LogAggregationUtils.getRemoteAppLogDir(conf, appId, appOwner, this.remoteRootLogDir, this.remoteRootLogDirSuffix); }
YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); Path fullPath = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir, appId, bestGuess, suffix); FileContext fc = getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix); pathAccess = toMatch.toString(); FileStatus[] matching = fc.util().globStatus(toMatch);
/** * Gets the remote suffixed log dir for the user. * @param remoteRootLogDir * @param user * @param suffix * @return the remote suffixed log dir. */ public static Path getRemoteLogSuffixedDir(Path remoteRootLogDir, String user, String suffix) { if (suffix == null || suffix.isEmpty()) { return getRemoteLogUserDir(remoteRootLogDir, user); } // TODO Maybe support suffix to be more than a single file. return new Path(getRemoteLogUserDir(remoteRootLogDir, user), suffix); }
Path getRemoteNodeLogFileForApp(ApplicationId appId, String user) { return LogAggregationUtils.getRemoteNodeLogFileForApp( this.remoteRootLogDir, appId, user, this.nodeId, this.remoteRootLogDirSuffix); }
public LogDeletionTask(Configuration conf, long retentionSecs, ApplicationClientProtocol rmClient) { this.conf = conf; this.retentionMillis = retentionSecs * 1000; this.suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); this.remoteRootLogDir = new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); this.rmClient = rmClient; }