/** {@inheritDoc} */ public synchronized void onNodesUpdated(List<NodeReport> updated) { for (NodeReport node : updated) { // If node unusable. if (node.getNodeState().isUnusable()) { for (IgniteContainer cont : containers.values()) { if (cont.nodeId().equals(node.getNodeId())) { containers.remove(cont.id()); log.log(Level.WARNING, "Node is unusable. Node: {0}, state: {1}.", new Object[]{node.getNodeId().getHost(), node.getNodeState()}); } } log.log(Level.WARNING, "Node is unusable. Node: {0}, state: {1}.", new Object[]{node.getNodeId().getHost(), node.getNodeState()}); } } }
totalMemory += res.getMemory(); totalCores += res.getVirtualCores(); ps.format(format, "NodeID", rep.getNodeId()); ps.format(format, "Memory", res.getMemory() + " MB"); ps.format(format, "vCores", res.getVirtualCores());
private void buildNodeMap() throws YarnFacadeException { List<NodeReport> nodes = yarn.getNodeReports(); for (NodeReport node : nodes) { String hostName = node.getNodeId().getHost(); nodeMap.put(hostName, node.getHttpAddress()); yarnNodes.put(hostName, node); } if (LOG.isInfoEnabled()) { LOG.info("YARN Node report"); for (NodeReport node : nodes) { LOG.info("Node: " + node.getHttpAddress() + ", Rack: " + node.getRackName() + " has " + node.getCapability().getMemory() + " MB, " + node.getCapability().getVirtualCores() + " vcores, labels: " + node.getNodeLabels()); } } }
for (NodeReport node : clusterNodeReports) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName()
public AMNodeEventStateChanged(NodeReport nodeReport, int sourceId) { super(nodeReport.getNodeId(), sourceId, (nodeReport.getNodeState().isUnusable() ? AMNodeEventType.N_TURNED_UNHEALTHY : AMNodeEventType.N_TURNED_HEALTHY)); this.nodeReport = nodeReport; }
public static TaskTrackerInfo fromYarn(NodeReport node) { TaskTrackerInfo taskTracker = new TaskTrackerInfo(TT_NAME_PREFIX + node.getNodeId().toString()); return taskTracker; }
public static TaskTrackerInfo fromYarn(NodeReport node) { TaskTrackerInfo taskTracker = new TaskTrackerInfo(TT_NAME_PREFIX + node.getNodeId().toString()); return taskTracker; }
public static TaskTrackerInfo fromYarn(NodeReport node) { TaskTrackerInfo taskTracker = new TaskTrackerInfo(TT_NAME_PREFIX + node.getNodeId().toString()); return taskTracker; }
public static TaskTrackerInfo fromYarn(NodeReport node) { TaskTrackerInfo taskTracker = new TaskTrackerInfo(TT_NAME_PREFIX + node.getNodeId().toString()); return taskTracker; }
private void addSupervisorsRequest() { NodeReport[] availableNodes; try { Set<String> hosts = containers.stream().map(c -> c.getNodeId().getHost()).collect(Collectors.toSet()); hosts.addAll(requestingNodes); availableNodes = yarnClient.getNodeReports(NodeState.RUNNING).stream() .filter(n -> !hosts.contains(n.getNodeId().getHost())).toArray(NodeReport[]::new); } catch (Exception e) { LOG.warn("can not get node list", e); return; } int requiredNumSupervisors = numSupervisors.getAndSet(0); int num = Math.min(requiredNumSupervisors, availableNodes.length); LOG.info("Require {} supervisors, and {} nodes available", requiredNumSupervisors, availableNodes.length); for (int i = 0; i < num; i++) { String node = availableNodes[i].getNodeId().getHost(); requestingNodes.add(node); ContainerRequest req = new ContainerRequest(this.maxResourceCapability, new String[]{node}, // String[] nodes, null, // String[] racks, DEFAULT_PRIORITY, false); super.addContainerRequest(req); } }
/** * Tracks update to available resources. Resource availability is used to make decisions about where to request new * containers. * * @param nodeReports */ public void updateNodeReports(List<NodeReport> nodeReports) { for (NodeReport nr : nodeReports) { LOG.debug("Node report: rackName={}, nodeid={}, numContainers={}, capability={}, used={}, state={}", nr.getRackName(), nr.getNodeId(), nr.getNumContainers(), nr.getCapability(), nr.getUsed(), nr.getNodeState()); nodeReportMap.put(nr.getNodeId().getHost(), nr); nodeToRack.put(nr.getNodeId().getHost(), nr.getRackName()); } }
@Override public void onNodesUpdated(List<NodeReport> nodeReports) { LOG.debug("Received nodes update for '{}' nodes", nodeReports.size()); for (NodeReport nodeReport : nodeReports) { if (nodeReport.getNodeState() == NodeState.RUNNING) { String nodeKey = getNodeName(nodeReport.getNodeId()); nodes.put(nodeKey, nodeReport.getCapability()); LOG.debug("Added node '{}' with '{}' cpus and '{}' memory", nodeKey, nodeReport.getCapability().getVirtualCores(), nodeReport.getCapability().getMemory()); } else { LOG.debug("Removed node '{}'", nodeReport.getNodeId()); nodes.remove(getNodeName(nodeReport.getNodeId())); } } }
/** * RM Callback: RM reports status change of some nodes. * @param nodeReports list of nodes with changed status. */ @Override public void onNodesUpdated(final List<NodeReport> nodeReports) { for (final NodeReport nodeReport : nodeReports) { this.nodeIdToRackName.put(nodeReport.getNodeId().toString(), nodeReport.getRackName()); this.onNodeReport(nodeReport); } }
/** * Test helper: build or update a cluster from a list of node reports * @param reports the list of reports * @return true if this has been considered to have changed the cluster */ @VisibleForTesting public boolean buildOrUpdate(List<NodeReport> reports) { boolean updated = false; for (NodeReport report : reports) { updated |= getOrCreate(report.getNodeId().getHost()).updateNode(report); } return updated; }
/** * Test helper: build or update a cluster from a list of node reports * @param reports the list of reports * @return true if this has been considered to have changed the cluster */ @VisibleForTesting public boolean buildOrUpdate(List<NodeReport> reports) { boolean updated = false; for (NodeReport report : reports) { updated |= getOrCreate(report.getNodeId().getHost()).updateNode(report); } return updated; }
@Override public void transition(JobImpl job, JobEvent event) { JobUpdatedNodesEvent updateEvent = (JobUpdatedNodesEvent) event; for(NodeReport nr: updateEvent.getUpdatedNodes()) { NodeState nodeState = nr.getNodeState(); if(nodeState.isUnusable()) { // act on the updates job.actOnUnusableNode(nr.getNodeId(), nodeState); } } } }
@Override public void transition(JobImpl job, JobEvent event) { JobUpdatedNodesEvent updateEvent = (JobUpdatedNodesEvent) event; for(NodeReport nr: updateEvent.getUpdatedNodes()) { NodeState nodeState = nr.getNodeState(); if(nodeState.isUnusable()) { // act on the updates job.actOnUnusableNode(nr.getNodeId(), nodeState); } } } }
@Override public void transition(JobImpl job, JobEvent event) { JobUpdatedNodesEvent updateEvent = (JobUpdatedNodesEvent) event; for(NodeReport nr: updateEvent.getUpdatedNodes()) { NodeState nodeState = nr.getNodeState(); if(nodeState.isUnusable()) { // act on the updates job.actOnUnusableNode(nr.getNodeId(), nodeState); } } } }
private void onNodeReport(final NodeReport nodeReport) { LOG.log(Level.FINE, "Send node descriptor: {0}", nodeReport); this.reefEventHandlers.onNodeDescriptor(NodeDescriptorEventImpl.newBuilder() .setIdentifier(nodeReport.getNodeId().toString()) .setHostName(nodeReport.getNodeId().getHost()) .setPort(nodeReport.getNodeId().getPort()) .setMemorySize(nodeReport.getCapability().getMemory()) .setRackName(nodeReport.getRackName()) .build()); }
private void onNodeReport(final NodeReport nodeReport) { LOG.log(Level.FINE, "Send node descriptor: {0}", nodeReport); this.reefEventHandlers.onNodeDescriptor(NodeDescriptorProto.newBuilder() .setIdentifier(nodeReport.getNodeId().toString()) .setHostName(nodeReport.getNodeId().getHost()) .setPort(nodeReport.getNodeId().getPort()) .setMemorySize(nodeReport.getCapability().getMemory()) .setRackName(nodeReport.getRackName()) .build()); }