/** {@inheritDoc} */ @Override public String toString() { return "IgniteTask [host=" + nodeId.getHost() + ", cpuCores=" + cpuCores + ", mem=" + mem + ']'; } }
/** * @return Address running nodes. */ private String getAddress(String addr) { if (containers.isEmpty()) { if (addr != null && !addr.isEmpty()) return addr + DEFAULT_PORT; return ""; } StringBuilder sb = new StringBuilder(); for (IgniteContainer cont : containers.values()) sb.append(cont.nodeId.getHost()).append(DEFAULT_PORT).append(DELIM); return sb.substring(0, sb.length() - 1); }
@Override public String apply(Container container) { return container.getNodeId().getHost(); } }));
public String getHostName() { if (container == null) { return null; } return container.getNodeId().getHost(); }
public String getHost( ) { if ( container == null ) { return ""; } return container.getNodeId().getHost(); }
public static String labelContainer(Container container) { StringBuilder buf = new StringBuilder() .append("[id: ") .append(container.getId()) .append(", host: ") .append(container.getNodeId().getHost()) .append(", priority: ") .append(container.getPriority()) .append("]"); return buf.toString(); }
public void reserve(Container container) { reserve(container.getNodeId().getHost()); }
@Override public void onContainersAllocated(List<Container> containers) { for (Container acquiredContainer : containers) { LOG.info("Acquired container " + acquiredContainer.getId() + " on host " + acquiredContainer.getNodeId().getHost() + " , with the resource " + acquiredContainer.getResource().toString()); String host = acquiredContainer.getNodeId().getHost(); if (!blackHosts.contains(host)) { if (workerContainersAllocating.get()) { acquiredWorkerContainers.add(acquiredContainer); acquiredWorkerContainersCount.incrementAndGet(); } else { acquiredPsContainers.add(acquiredContainer); acquiredPsContainersCount.incrementAndGet(); } } else { LOG.info("Add container " + acquiredContainer.getId() + " to cancel list"); cancelContainers.add(acquiredContainer); } } LOG.info("Current acquired worker container " + acquiredWorkerContainersCount.get() + " / " + neededWorkerContainersCount + " ps container " + acquiredPsContainersCount.get() + " / " + neededPsContainersCount); }
public void release(Container container) { release(container.getNodeId().getHost()); }
@Override public String info() throws TException { StringBuffer sbRet = new StringBuffer(); sbRet.append("JstormOnYarn\n"); sbRet.append("Instance Name:" + jstormMasterContext.instanceName + "\n"); sbRet.append("Jstorm's location on hdfs:" + jstormMasterContext.deployPath + "\n"); if (jstormMasterContext.user != null) { sbRet.append("Jstorm's data path:" + jstormMasterContext.nimbusDataDirPrefix + jstormMasterContext.instanceName + "\n"); sbRet.append("Cluster userName:" + jstormMasterContext.user + "\n"); } sbRet.append("Nimbus Count:" + jstormMasterContext.nimbusContainers.size() + "\n"); sbRet.append("Supervisor Count:" + jstormMasterContext.supervisorContainers.size() + "\n"); sbRet.append("detail :\n"); sbRet.append("Type \tContainerId \tHost \tContainerMemory\tContainerVCores\n"); for (Container container : jstormMasterContext.nimbusContainers) { sbRet.append("Nimbus \t" + container.getId().toString() + "\t" + container.getNodeId().getHost() + "\t" + container.getResource().getMemory() + "\t " + container.getResource().getVirtualCores() + "\n"); } for (Container container : jstormMasterContext.supervisorContainers) { sbRet.append("Supervisor\t" + container.getId().toString() + "\t" + container.getNodeId().getHost() + "\t" + container.getResource().getMemory() + "\t " + container.getResource().getVirtualCores() + "\n"); } LOG.info("info is: " + sbRet.toString()); return sbRet.toString(); }
/** * Utility method to display YARN container information in a useful way for * log messages. * * @param container * @return */ public static String describeContainer(Container container) { StringBuilder buf = new StringBuilder() .append("[id: ") .append(container.getId()) .append(", host: ") .append(container.getNodeId().getHost()) .append(", priority: ") .append(container.getPriority()) .append(", memory: ") .append(container.getResource().getMemory()) .append(" MB, vcores: ") .append(container.getResource().getVirtualCores()) .append("]"); return buf.toString(); }
/** {@inheritDoc} */ public synchronized void onNodesUpdated(List<NodeReport> updated) { for (NodeReport node : updated) { // If node unusable. if (node.getNodeState().isUnusable()) { for (IgniteContainer cont : containers.values()) { if (cont.nodeId().equals(node.getNodeId())) { containers.remove(cont.id()); log.log(Level.WARNING, "Node is unusable. Node: {0}, state: {1}.", new Object[]{node.getNodeId().getHost(), node.getNodeState()}); } } log.log(Level.WARNING, "Node is unusable. Node: {0}, state: {1}.", new Object[]{node.getNodeId().getHost(), node.getNodeState()}); } } }
instanceSet = serviceRegistry.getInstances().getByHost(containerNodeId.getHost()); } catch (IOException e) {
private void buildNodeMap() throws YarnFacadeException { List<NodeReport> nodes = yarn.getNodeReports(); for (NodeReport node : nodes) { String hostName = node.getNodeId().getHost(); nodeMap.put(hostName, node.getHttpAddress()); yarnNodes.put(hostName, node); } if (LOG.isInfoEnabled()) { LOG.info("YARN Node report"); for (NodeReport node : nodes) { LOG.info("Node: " + node.getHttpAddress() + ", Rack: " + node.getRackName() + " has " + node.getCapability().getMemory() + " MB, " + node.getCapability().getVirtualCores() + " vcores, labels: " + node.getNodeLabels()); } } }
+ " on host " + container.getNodeId().getHost(); LOG.info(message); sendInfoMessage(message);
/** * @param cont Container. * @return {@code True} if container satisfies requirements. */ private boolean checkContainer(Container cont) { // Check limit on running nodes. if (props.instances() <= containers.size()) return false; // Check host name if (props.hostnameConstraint() != null && props.hostnameConstraint().matcher(cont.getNodeId().getHost()).matches()) return false; // Check that slave satisfies min requirements. if (cont.getResource().getVirtualCores() < props.cpusPerNode() || cont.getResource().getMemory() < props.totalMemoryPerNode()) { log.log(Level.FINE, "Container resources not sufficient requirements. Host: {0}, cpu: {1}, mem: {2}", new Object[]{cont.getNodeId().getHost(), cont.getResource().getVirtualCores(), cont.getResource().getMemory()}); return false; } return true; }
@Override public synchronized void containersAllocated(List<Container> containers) { EventContext context = new EventContext(this); for (Container container : containers) { if (allocatedContainers.contains(container.getId())) { continue; } // We should never get a container on a node in the blacklist we // sent to YARN. If we do, something is wrong. Log the error and // reject the container. Else, bad things happen further along as // the tracking mechanisms assume one task per node. String host = container.getNodeId().getHost(); if (nodeInventory.isInUse(host)) { LOG.error( "Host is in use, but YARN allocated a container: " + DoYUtil.labelContainer(container) + " - container rejected." ); yarn.releaseContainer(container); continue; } // The container is fine. allocatedContainers.add(container.getId()); int priority = container.getPriority().getPriority(); int offset = priority - PRIORITY_OFFSET; if (offset < 0 || offset > prioritizedGroups.size()) { LOG.error("Container allocated with unknown priority " + DoYUtil.labelContainer(container)); continue; } context.setGroup(prioritizedGroups.get(offset)); context.group.containerAllocated(context, container); } }
container.getResource(), containerIdStr, container.getNodeId().getHost());
env.put("IGNITE_TCP_DISCOVERY_ADDRESSES", getAddress(c.getNodeId().getHost()));
LOG.info("Launching shell command on a new container." + ", containerId=" + allocatedContainer.getId() + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":" + allocatedContainer.getNodeId().getPort() + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress() String supervisorHost = allocatedContainer.getNodeId().getHost(); startType = STARTType.SUPERVISOR; String containerPath = RegistryUtils.componentPath( startType = STARTType.NIMBUS; jstormMasterContext.nimbusHost = allocatedContainer.getNodeId().getHost(); String path = RegistryUtils.serviceclassPath( JOYConstants.APP_TYPE, jstormMasterContext.instanceName);