@Private @VisibleForTesting public Map<String, ContainerLogsRequest> getMatchedLogTypesForRunningApp( List<ContainerLogsRequest> containerLogRequests, boolean useRegex, boolean ignoreSizeLimit) { Map<String, ContainerLogsRequest> containerMatchedLog = new HashMap<>(); for (ContainerLogsRequest container : containerLogRequests) { try { ContainerLogsRequest request = getMatchedOptionForRunningApp( container, useRegex, ignoreSizeLimit); if (request == null) { continue; } containerMatchedLog.put(container.getContainerId(), request); } catch(IOException ex) { System.err.println(ex); continue; } } return containerMatchedLog; }
private Map<String, ContainerLogsRequest> getMatchedLogTypesForFinishedApp( List<ContainerLogsRequest> containerLogRequests, LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit) { Map<String, ContainerLogsRequest> containerMatchedLog = new HashMap<>(); for (ContainerLogsRequest container : containerLogRequests) { try { ContainerLogsRequest request = getMatchedLogOptions(container, logCliHelper, useRegex, ignoreSizeLimit); if (request == null) { System.err.println("Can not find any log file matching the pattern: " + container.getLogTypes() + " for the container: " + container.getContainerId() + " within the application: " + container.getAppId()); continue; } containerMatchedLog.put(container.getContainerId(), request); } catch (IOException ex) { System.err.println(ex); continue; } } return containerMatchedLog; } }
@VisibleForTesting public Set<String> getMatchedContainerLogFiles(ContainerLogsRequest request, boolean useRegex, boolean ignoreSizeLimit) throws IOException { // fetch all the log files for the container // filter the log files based on the given -log_files pattern List<Pair<ContainerLogFileInfo, String>> allLogFileInfos= getContainerLogFiles(getConf(), request.getContainerId(), request.getNodeHttpAddress()); List<ContainerLogFileInfo> fileNames = new ArrayList< ContainerLogFileInfo>(); for (Pair<ContainerLogFileInfo, String> fileInfo : allLogFileInfos) { fileNames.add(fileInfo.getKey()); } return getMatchedLogFiles(request, fileNames, useRegex, ignoreSizeLimit); }
private List<ContainerReport> filterContainersInfo( ContainerLogsRequest options, List<ContainerReport> containers) { List<ContainerReport> filterReports = new ArrayList<ContainerReport>( containers); String nodeId = options.getNodeId(); boolean filterBasedOnNodeId = (nodeId != null && !nodeId.isEmpty()); String containerId = options.getContainerId(); boolean filterBasedOnContainerId = (containerId != null && !containerId.isEmpty()); if (filterBasedOnNodeId || filterBasedOnContainerId) { // filter the reports based on the containerId and.or nodeId for(ContainerReport report : containers) { if (filterBasedOnContainerId) { if (!report.getContainerId().toString() .equalsIgnoreCase(containerId)) { filterReports.remove(report); } } if (filterBasedOnNodeId) { if (!report.getAssignedNode().toString().equalsIgnoreCase(nodeId)) { filterReports.remove(report); } } } } return filterReports; }
@Private @VisibleForTesting public ContainerLogsRequest getMatchedOptionForRunningApp( ContainerLogsRequest container, boolean useRegex, boolean ignoreSizeLimit) throws IOException { String containerIdStr = container.getContainerId().toString(); String nodeHttpAddress = container.getNodeHttpAddress(); if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { System.err.println("Can not get the logs for the container: " + containerIdStr); System.err.println("The node http address is required to get container " + "logs for the Running application."); return null; } Set<String> matchedFiles = getMatchedContainerLogFiles(container, useRegex, ignoreSizeLimit); if (matchedFiles.isEmpty()) { System.err.println("Can not find any log file matching the pattern: " + container.getLogTypes() + " for the container: " + containerIdStr + " within the application: " + container.getAppId()); return null; } container.setLogTypes(matchedFiles); return container; }
throws IOException { String nodeId = options.getNodeId(); String containerIdStr = options.getContainerId(); List<ContainerLogMeta> containersLogMeta; try {
throws YarnException, IOException, ClientHandlerException, UniformInterfaceException, JSONException { String containerIdStr = options.getContainerId(); String nodeIdStr = options.getNodeId(); List<ContainerReport> reports = if (options.getContainerId() != null && !options.getContainerId().isEmpty()) { nodeHttpAddress = getNodeHttpAddressFromRMWebString(options); outputContainerLogMeta(options.getContainerId(), options.getNodeId(), nodeHttpAddress); return 0;
@Private @VisibleForTesting public int dumpAContainerLogsForLogType(ContainerLogsRequest options, boolean outputFailure) throws IOException { LogAggregationFileController fc = null; try { fc = this.getFileController( options.getAppId(), options.getAppOwner()); } catch (IOException ex) { System.err.println(ex); } boolean foundAnyLogs = false; if (fc != null) { foundAnyLogs = fc.readAggregatedLogs(options, null); } if (!foundAnyLogs) { if (outputFailure) { containerLogNotFound(options.getContainerId()); } return -1; } return 0; }
private void outputAMContainerLogs(ContainerLogsRequest request, Configuration conf, LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit) throws Exception { String nodeHttpAddress = request.getNodeHttpAddress(); String containerId = request.getContainerId(); String nodeId = request.getNodeId(); if (request.isAppFinished()) { if (containerId != null && !containerId.isEmpty()) { if (nodeId != null && !nodeId.isEmpty()) { logCliHelper.dumpAContainerLogsForLogType(request); } else { logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId( request); } } } else { if (nodeHttpAddress != null && containerId != null && !nodeHttpAddress.isEmpty() && !containerId.isEmpty()) { ContainerState containerState = getContainerReport(containerId) .getContainerState(); request.setContainerState(containerState); printContainerLogsFromRunningApplication(conf, request, logCliHelper, useRegex, ignoreSizeLimit); } } }
@Private public int dumpAContainerLogsForLogTypeWithoutNodeId( ContainerLogsRequest options) throws IOException { LogAggregationFileController fc = null; try { fc = this.getFileController( options.getAppId(), options.getAppOwner()); } catch (IOException ex) { System.err.println(ex); } boolean foundAnyLogs = false; if (fc != null) { foundAnyLogs = fc.readAggregatedLogs(options, null); } if (!foundAnyLogs) { containerLogNotFound(options.getContainerId()); return -1; } return 0; }
ContainerLogsRequest request, LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit) throws IOException { String containerIdStr = request.getContainerId().toString(); String localDir = request.getOutputLocalDir(); String nodeId = request.getNodeId();
@VisibleForTesting public ClientResponse getResponeFromNMWebService(Configuration conf, Client webServiceClient, ContainerLogsRequest request, String logFile) { WebResource webResource = webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf) + request.getNodeHttpAddress()); return webResource.path("ws").path("v1").path("node") .path("containers").path(request.getContainerId()).path("logs") .path(logFile) .queryParam("size", Long.toString(request.getBytes())) .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); }
ContainerLogsRequest logRequest) throws IOException { List<ContainerLogMeta> containersLogMeta = new ArrayList<>(); String containerIdStr = logRequest.getContainerId(); String nodeId = logRequest.getNodeId(); ApplicationId appId = logRequest.getAppId();
logTypes.addAll(logRequest.getLogTypes()); String containerIdStr = logRequest.getContainerId(); boolean getAllContainers = (containerIdStr == null || containerIdStr.isEmpty());
List<IndexedLogsMeta> listOfLogsMeta = new ArrayList<>(); List<ContainerLogMeta> containersLogMeta = new ArrayList<>(); String containerIdStr = logRequest.getContainerId(); String nodeId = logRequest.getNodeId(); ApplicationId appId = logRequest.getAppId();
logTypes.addAll(logRequest.getLogTypes()); String containerIdStr = logRequest.getContainerId(); boolean getAllContainers = (containerIdStr == null || containerIdStr.isEmpty());
JSONException { String appIdStr = request.getAppId().toString(); String containerIdStr = request.getContainerId(); String nodeAddress = request.getNodeId(); String appOwner = request.getAppOwner(); System.err.println("Can not find any log file matching the pattern: " + request.getLogTypes() + " for the container: " + request.getContainerId() + " within the application: " + request.getAppId()); return -1; System.err.println("Can not find any log file matching the pattern: " + request.getLogTypes() + " for the container: " + request.getContainerId() + " within the application: " + request.getAppId()); } else {
public ContainerLogsRequest(ContainerLogsRequest request) { this.setAppId(request.getAppId()); this.setAppFinished(request.isAppFinished()); this.setAppOwner(request.getAppOwner()); this.setNodeId(request.getNodeId()); this.setNodeHttpAddress(request.getNodeHttpAddress()); this.setContainerId(request.getContainerId()); this.setOutputLocalDir(request.getOutputLocalDir()); this.setLogTypes(request.getLogTypes()); this.setBytes(request.getBytes()); this.setContainerState(request.getContainerState()); }