@Override public boolean apply(FileStatus next) { return next.getPath().getName() .contains(LogAggregationUtils.getNodeString(nodeId)) && !next.getPath().getName().endsWith( LogAggregationUtils.TMP_FILE_SUFFIX); } });
@Override public boolean apply(FileStatus next) { return next.getPath().getName() .contains(LogAggregationUtils.getNodeString(nodeId)) && !next.getPath().getName().endsWith( LogAggregationUtils.TMP_FILE_SUFFIX); } });
@Override public boolean apply(FileStatus next) { return next.getPath().getName() .contains(LogAggregationUtils.getNodeString(nodeId)) && !next.getPath().getName().endsWith( LogAggregationUtils.TMP_FILE_SUFFIX); } });
@Override public boolean apply(FileStatus next) { return next.getPath().getName() .contains(LogAggregationUtils.getNodeString(nodeId)) && !next.getPath().getName().endsWith( LogAggregationUtils.TMP_FILE_SUFFIX); } });
/** * Constructs the full filename for an application's log file per node. * @param remoteRootLogDir * @param appId * @param user * @param nodeId * @param suffix * @return the remote log file. */ public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir, ApplicationId appId, String user, NodeId nodeId, String suffix) { return new Path(getRemoteAppLogDir(remoteRootLogDir, appId, user, suffix), getNodeString(nodeId)); }
/** * Constructs the full filename for an application's log file per node. * @param remoteRootLogDir * @param appId * @param user * @param nodeId * @param suffix * @return the remote log file. */ public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir, ApplicationId appId, String user, NodeId nodeId, String suffix) { return new Path(getRemoteAppLogDir(remoteRootLogDir, appId, user, suffix), getNodeString(nodeId)); }
/** * Constructs the full filename for an application's log file per node. * @param remoteRootLogDir the aggregated remote root log dir * @param appId the application Id * @param user the application owner * @param nodeId the node id * @param suffix the log dir suffix * @return the remote log file. */ public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir, ApplicationId appId, String user, NodeId nodeId, String suffix) { return new Path(getRemoteAppLogDir(remoteRootLogDir, appId, user, suffix), getNodeString(nodeId)); }
/** * Create the container log file under given (local directory/nodeId) and * return the PrintStream object. * @param localDir the Local Dir * @param nodeId the NodeId * @param containerId the ContainerId * @return the printStream object * @throws IOException if an I/O error occurs */ public static PrintStream createPrintStream(String localDir, String nodeId, String containerId) throws IOException { PrintStream out = System.out; if(localDir != null && !localDir.isEmpty()) { Path nodePath = new Path(localDir, LogAggregationUtils .getNodeString(nodeId)); Files.createDirectories(Paths.get(nodePath.toString())); Path containerLogPath = new Path(nodePath, containerId); out = new PrintStream(containerLogPath.toString(), "UTF-8"); } return out; } }
@Private public List<FileStatus> getNodeLogFileToRead( List<FileStatus> nodeFiles, String nodeId, ApplicationId appId) throws IOException { List<FileStatus> listOfFiles = new ArrayList<>(); for (FileStatus thisNodeFile : nodeFiles) { String nodeName = thisNodeFile.getPath().getName(); if ((nodeId == null || nodeId.isEmpty() || nodeName.contains(LogAggregationUtils .getNodeString(nodeId))) && !nodeName.endsWith( LogAggregationUtils.TMP_FILE_SUFFIX) && !nodeName.endsWith(CHECK_SUM_FILE_SUFFIX)) { listOfFiles.add(thisNodeFile); } } return listOfFiles; }
private Path getCurrentRemoteLogFile(final FileContext fc, final Path parent, final String nodeId) throws IOException { RemoteIterator<FileStatus> files = fc.listStatus(parent); long maxTime = 0L; Path returnPath = null; while(files.hasNext()) { FileStatus candidate = files.next(); String fileName = candidate.getPath().getName(); if (fileName.contains(LogAggregationUtils.getNodeString(nodeId)) && !fileName.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX) && !fileName.endsWith(CHECK_SUM_FILE_SUFFIX)) { if (candidate.getModificationTime() > maxTime) { maxTime = candidate.getModificationTime(); returnPath = candidate.getPath(); } } } return returnPath; }
private byte[] loadUUIDFromLogFile(final FileContext fc, final Path parent, final ApplicationId appId, final String nodeId) throws Exception { byte[] id = null; RemoteIterator<FileStatus> files = fc.listStatus(parent); FSDataInputStream fsDataInputStream = null; byte[] uuid = createUUID(appId); while(files.hasNext()) { try { Path checkPath = files.next().getPath(); if (checkPath.getName().contains(LogAggregationUtils .getNodeString(nodeId)) && !checkPath.getName() .endsWith(CHECK_SUM_FILE_SUFFIX)) { fsDataInputStream = fc.open(checkPath); byte[] b = new byte[uuid.length]; int actual = fsDataInputStream.read(b); if (actual != uuid.length || Arrays.equals(b, uuid)) { deleteFileWithRetries(fc, checkPath); } else if (id == null){ id = uuid; } } } finally { IOUtils.cleanupWithLogger(LOG, fsDataInputStream); } } return id == null ? uuid : id; }
boolean getAllContainers = (containerIdStr == null); String nodeIdStr = (nodeId == null) ? null : LogAggregationUtils.getNodeString(nodeId); RemoteIterator<FileStatus> nodeFiles = LogAggregationUtils .getRemoteNodeFileDir(conf, appId, appOwner);
.getNodeString(nodeId))) && !nodeName.endsWith( LogAggregationUtils.TMP_FILE_SUFFIX)) { AggregatedLogFormat.LogReader reader = null;
containerIdStr.isEmpty()); String nodeIdStr = (nodeId == null || nodeId.isEmpty()) ? null : LogAggregationUtils.getNodeString(nodeId); RemoteIterator<FileStatus> nodeFiles = LogAggregationUtils .getRemoteNodeFileDir(conf, appId, appOwner, this.remoteRootLogDir,
FileStatus thisNodeFile = nodeFiles.next(); String fileName = thisNodeFile.getPath().getName(); if (fileName.contains(LogAggregationUtils.getNodeString(nodeId)) && !fileName.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { AggregatedLogFormat.LogReader reader = null;
String nodeId = logRequest.getNodeId(); String nodeIdStr = (nodeId == null || nodeId.isEmpty()) ? null : LogAggregationUtils.getNodeString(nodeId); List<String> logTypes = new ArrayList<>(); if (logRequest.getLogTypes() != null && !logRequest
FileStatus thisNodeFile = nodeFiles.next(); String fileName = thisNodeFile.getPath().getName(); if (fileName.contains(LogAggregationUtils.getNodeString(nodeId)) && !fileName.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { AggregatedLogFormat.LogReader reader = null;
.contains(LogAggregationUtils.getNodeString(nodeId)) || thisNodeFile.getPath().getName() .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
throw new Exception("Log uploaded failed for Application: " + record.getAppId() + " in NodeManager: " + LogAggregationUtils.getNodeString(record.getNodeId()) + " at " + Times.format(record.getLogUploadTimeStamp()) + "\n");
private static void uploadContainerLogIntoRemoteDir(UserGroupInformation ugi, Configuration configuration, List<String> rootLogDirs, NodeId nodeId, ContainerId containerId, Path appDir, FileSystem fs) throws Exception { Path path = new Path(appDir, LogAggregationUtils.getNodeString(nodeId)); LogAggregationFileControllerFactory factory = new LogAggregationFileControllerFactory(configuration); LogAggregationFileController fileController = factory .getFileControllerForWrite(); try { Map<ApplicationAccessType, String> appAcls = new HashMap<>(); appAcls.put(ApplicationAccessType.VIEW_APP, ugi.getUserName()); ApplicationId appId = containerId.getApplicationAttemptId() .getApplicationId(); LogAggregationFileControllerContext context = new LogAggregationFileControllerContext( path, path, true, 1000, appId, appAcls, nodeId, ugi); fileController.initializeWriter(context); fileController.write(new AggregatedLogFormat.LogKey(containerId), new AggregatedLogFormat.LogValue(rootLogDirs, containerId, ugi.getShortUserName())); } finally { fileController.closeWriter(); } } }