@Private @VisibleForTesting public int dumpAContainersLogs(String appId, String containerId, String nodeId, String jobOwner) throws IOException { ContainerLogsRequest options = new ContainerLogsRequest(); options.setAppId(ApplicationId.fromString(appId)); options.setContainerId(containerId); options.setNodeId(nodeId); options.setAppOwner(jobOwner); Set<String> logs = new HashSet<String>(); options.setLogTypes(logs); options.setBytes(Long.MAX_VALUE); return dumpAContainerLogsForLogType(options, false); }
public ContainerLogsRequest(ContainerLogsRequest request) { this.setAppId(request.getAppId()); this.setAppFinished(request.isAppFinished()); this.setAppOwner(request.getAppOwner()); this.setNodeId(request.getNodeId()); this.setNodeHttpAddress(request.getNodeHttpAddress()); this.setContainerId(request.getContainerId()); this.setOutputLocalDir(request.getOutputLocalDir()); this.setLogTypes(request.getLogTypes()); this.setBytes(request.getBytes()); this.setContainerState(request.getContainerState()); }
public ContainerLogsRequest(ApplicationId applicationId, boolean isAppFinished, String owner, String address, String httpAddress, String container, String localDir, Set<String> logs, long bytes, ContainerState containerState) { this.setAppId(applicationId); this.setAppFinished(isAppFinished); this.setAppOwner(owner); this.setNodeId(address); this.setNodeHttpAddress(httpAddress); this.setContainerId(container); this.setOutputLocalDir(localDir); this.setLogTypes(logs); this.setBytes(bytes); this.setContainerState(containerState); }
new ArrayList<ContainerLogsRequest>(); boolean getAMContainerLists = false; String appId = request.getAppId().toString(); StringBuilder errorMessage = new StringBuilder(); getAMContainerLists = true; for (JSONObject amContainer : amContainersList) { ContainerLogsRequest amRequest = new ContainerLogsRequest(request); amRequest.setContainerId(amContainer.getString("containerId")); String httpAddress = amContainer.getString("nodeHttpAddress"); if (httpAddress != null && !httpAddress.isEmpty()) { amRequest.setNodeHttpAddress(httpAddress); amRequest.setNodeId(amContainer.getString("nodeId")); requests.add(amRequest); if (request.isAppFinished()) { if (!conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { getAMContainerLists = true; for (JSONObject amContainer : amContainersList) { ContainerLogsRequest amRequest = new ContainerLogsRequest( request); amRequest.setContainerId( amContainer.getString("amContainerId")); requests.add(amRequest); System.err.println("Can not get AMContainers logs for "
throws IOException, ClientHandlerException, UniformInterfaceException, JSONException { String appIdStr = request.getAppId().toString(); String containerIdStr = request.getContainerId(); String nodeAddress = request.getNodeId(); String appOwner = request.getAppOwner(); boolean isAppFinished = request.isAppFinished(); if (newOptions == null) { System.err.println("Can not find any log file matching the pattern: " + request.getLogTypes() + " for the container: " + request.getContainerId() + " within the application: " + request.getAppId()); return -1; nodeHttpAddress = nodeHttpAddress.replaceFirst( WebAppUtils.getHttpSchemePrefix(getConf()), ""); request.setNodeHttpAddress(nodeHttpAddress); request.setNodeId(nodeId); request.setContainerState(report.getContainerState()); } catch (IOException | YarnException ex) { nodeHttpAddress = getNodeHttpAddressFromRMWebString(request); if (nodeHttpAddress != null && !nodeHttpAddress.isEmpty()) { request.setNodeHttpAddress(nodeHttpAddress); } else { if (newOptions == null) { System.err.println("Can not find any log file matching the pattern: "
boolean findLogs = false; boolean createPrintStream = (os == null); ApplicationId appId = logRequest.getAppId(); String nodeId = logRequest.getNodeId(); List<String> logTypes = new ArrayList<>(); if (logRequest.getLogTypes() != null && !logRequest .getLogTypes().isEmpty()) { logTypes.addAll(logRequest.getLogTypes()); String containerIdStr = logRequest.getContainerId(); boolean getAllContainers = (containerIdStr == null || containerIdStr.isEmpty()); long size = logRequest.getBytes(); RemoteIterator<FileStatus> nodeFiles = LogAggregationUtils .getRemoteNodeFileDir(conf, appId, logRequest.getAppOwner()); byte[] buf = new byte[65535]; while (nodeFiles != null && nodeFiles.hasNext()) { if (createPrintStream) { os = LogToolUtils.createPrintStream( logRequest.getOutputLocalDir(), thisNodeFile.getPath().getName(), key.toString());
private List<ContainerLogsRequest> getContainersLogRequestForRunningApplication( ContainerLogsRequest options) throws YarnException, IOException { List<ContainerLogsRequest> newOptionsList = new ArrayList<ContainerLogsRequest>(); List<ContainerReport> reports = getContainerReportsFromRunningApplication(options); for (ContainerReport container : reports) { ContainerLogsRequest newOptions = new ContainerLogsRequest(options); newOptions.setContainerId(container.getContainerId().toString()); newOptions.setNodeId(container.getAssignedNode().toString()); String httpAddress = container.getNodeHttpAddress(); if (httpAddress != null && !httpAddress.isEmpty()) { newOptions.setNodeHttpAddress(httpAddress .replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), "")); } newOptions.setContainerState(container.getContainerState()); newOptionsList.add(newOptions); } return newOptionsList; }
PrintStream out, PrintStream err) throws IOException { String nodeId = options.getNodeId(); String containerIdStr = options.getContainerId(); List<ContainerLogMeta> containersLogMeta; try { containersLogMeta = getFileController(options.getAppId(), options.getAppOwner()).readAggregatedLogsMeta( options); } catch (Exception ex) {
@Private @VisibleForTesting public ContainerLogsRequest getMatchedOptionForRunningApp( ContainerLogsRequest container, boolean useRegex, boolean ignoreSizeLimit) throws IOException { String containerIdStr = container.getContainerId().toString(); String nodeHttpAddress = container.getNodeHttpAddress(); if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { System.err.println("Can not get the logs for the container: " + containerIdStr); System.err.println("The node http address is required to get container " + "logs for the Running application."); return null; } Set<String> matchedFiles = getMatchedContainerLogFiles(container, useRegex, ignoreSizeLimit); if (matchedFiles.isEmpty()) { System.err.println("Can not find any log file matching the pattern: " + container.getLogTypes() + " for the container: " + containerIdStr + " within the application: " + container.getAppId()); return null; } container.setLogTypes(matchedFiles); return container; }
if (options.isAppFinished()) { ContainerLogsRequest newOptions = getMatchedLogOptions( options, logCliHelper, useRegex, ignoreSizeLimit); if (newOptions == null) { System.err.println("Can not find any log file matching the pattern: " + options.getLogTypes() + " for the application: " + options.getAppId()); } else { resultCode = + options.getAppId() + " with the appOwner: " + options.getAppOwner());
@Private public int dumpAContainerLogsForLogTypeWithoutNodeId( ContainerLogsRequest options) throws IOException { LogAggregationFileController fc = null; try { fc = this.getFileController( options.getAppId(), options.getAppOwner()); } catch (IOException ex) { System.err.println(ex); } boolean foundAnyLogs = false; if (fc != null) { foundAnyLogs = fc.readAggregatedLogs(options, null); } if (!foundAnyLogs) { containerLogNotFound(options.getContainerId()); return -1; } return 0; }
private ContainerLogsRequest getMatchedLogOptions( ContainerLogsRequest request, LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit) throws IOException { ContainerLogsRequest newOptions = new ContainerLogsRequest(request); Set<ContainerLogFileInfo> files = logCliHelper.listContainerLogs( request); Set<String> matchedFiles = getMatchedLogFiles(request, files, useRegex, ignoreSizeLimit); if (matchedFiles.isEmpty()) { return null; } else { newOptions.setLogTypes(matchedFiles); return newOptions; } }
private Map<String, ContainerLogsRequest> getMatchedLogTypesForFinishedApp( List<ContainerLogsRequest> containerLogRequests, LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit) { Map<String, ContainerLogsRequest> containerMatchedLog = new HashMap<>(); for (ContainerLogsRequest container : containerLogRequests) { try { ContainerLogsRequest request = getMatchedLogOptions(container, logCliHelper, useRegex, ignoreSizeLimit); if (request == null) { System.err.println("Can not find any log file matching the pattern: " + container.getLogTypes() + " for the container: " + container.getContainerId() + " within the application: " + container.getAppId()); continue; } containerMatchedLog.put(container.getContainerId(), request); } catch (IOException ex) { System.err.println(ex); continue; } } return containerMatchedLog; } }
@Private public int dumpAllContainersLogs(ContainerLogsRequest options) throws IOException { LogAggregationFileController fc = null; try { fc = this.getFileController( options.getAppId(), options.getAppOwner()); } catch (IOException ex) { System.err.println(ex); } boolean foundAnyLogs = false; if (fc != null) { foundAnyLogs = fc.readAggregatedLogs(options, null); } if (!foundAnyLogs) { emptyLogDir(LogAggregationUtils.getRemoteAppLogDir( conf, options.getAppId(), options.getAppOwner()) .toString()); return -1; } return 0; }
@Private public void printContainersList(ContainerLogsRequest options, PrintStream out, PrintStream err) throws IOException { ApplicationId appId = options.getAppId(); String nodeId = options.getNodeId(); boolean foundAnyLogs = false; List<ContainerLogMeta> containersLogMeta = new ArrayList<>(); try { containersLogMeta = getFileController(options.getAppId(), options.getAppOwner()).readAggregatedLogsMeta( options); } catch (Exception ex) { err.println(ex.getMessage()); } for(ContainerLogMeta logMeta : containersLogMeta) { out.println(String.format(CONTAINER_ON_NODE_PATTERN, logMeta.getContainerId(), logMeta.getNodeId())); foundAnyLogs = true; } if (!foundAnyLogs) { if (nodeId != null) { err.println("Can not find information for any containers on " + nodeId); } else { err.println("Can not find any container information for " + "the application: " + appId); } } }
@VisibleForTesting public ClientResponse getResponeFromNMWebService(Configuration conf, Client webServiceClient, ContainerLogsRequest request, String logFile) { WebResource webResource = webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf) + request.getNodeHttpAddress()); return webResource.path("ws").path("v1").path("node") .path("containers").path(request.getContainerId()).path("logs") .path(logFile) .queryParam("size", Long.toString(request.getBytes())) .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); }
throws YarnException, IOException, ClientHandlerException, UniformInterfaceException, JSONException { String containerIdStr = options.getContainerId(); String nodeIdStr = options.getNodeId(); List<ContainerReport> reports = getContainerReportsFromRunningApplication(options); if (options.getContainerId() != null && !options.getContainerId().isEmpty()) { nodeHttpAddress = getNodeHttpAddressFromRMWebString(options); outputContainerLogMeta(options.getContainerId(), options.getNodeId(), nodeHttpAddress); return 0; + options.getAppId()); System.err.println(sb.toString());
String appOwner = app == null ? null : app.getUser(); try { ContainerLogsRequest logRequest = new ContainerLogsRequest(); logRequest.setAppId(appId); logRequest.setAppOwner(appOwner);
private int showApplicationLogInfo(ContainerLogsRequest request, LogCLIHelpers logCliHelper) throws IOException, YarnException { String appState = "Application State: " + (request.isAppFinished() ? "Completed." : "Running."); if (!request.isAppFinished()) { List<ContainerReport> reports = getContainerReportsFromRunningApplication(request); List<ContainerReport> filterReports = filterContainersInfo( request, reports); if (filterReports.isEmpty()) { System.err.println("Can not find any containers for the application:" + request.getAppId() + "."); return -1; } outStream.println(appState); for (ContainerReport report : filterReports) { outStream.println(String.format(LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, report.getContainerId(), report.getAssignedNode())); } return 0; } else { outStream.println(appState); logCliHelper.printContainersList(request, System.out, System.err); return 0; } }
boolean ignoreSizeLimit) throws IOException { Set<String> matchedFiles = new HashSet<String>(); Set<String> filePattern = options.getLogTypes(); long size = options.getBytes(); boolean getAll = options.getLogTypes().contains("ALL"); Iterator<ContainerLogFileInfo> iterator = candidate.iterator(); while(iterator.hasNext()) {