@Override public void onContainerStarted(ContainerId containerId, Map<String, ByteBuffer> allServiceResponse) { if (LOG.isDebugEnabled()) { LOG.debug("Succeeded to start Container " + containerId); } Container container = containers.get(containerId); if (container != null) { applicationMaster.nmClientAsync.getContainerStatusAsync(containerId, container.getNodeId()); } if (applicationMaster.timelineClient != null) { JstormMaster.publishContainerStartEvent( applicationMaster.timelineClient, container, applicationMaster.jstormMasterContext.domainId, applicationMaster.appSubmitterUgi); } }
@Override public String apply(Container container) { return container.getNodeId().getHost(); } }));
public String getHostName() { if (container == null) { return null; } return container.getNodeId().getHost(); }
public String getHost( ) { if ( container == null ) { return ""; } return container.getNodeId().getHost(); }
public static String labelContainer(Container container) { StringBuilder buf = new StringBuilder() .append("[id: ") .append(container.getId()) .append(", host: ") .append(container.getNodeId().getHost()) .append(", priority: ") .append(container.getPriority()) .append("]"); return buf.toString(); }
@SuppressWarnings("unused") @Subscribe public void handleContainerShutdownRequest(ContainerShutdownRequest containerShutdownRequest) { for (Container container : containerShutdownRequest.getContainers()) { LOGGER.info(String.format("Stopping container %s running on %s", container.getId(), container.getNodeId())); this.nmClientAsync.stopContainerAsync(container.getId(), container.getNodeId()); } }
private void releaseYarnContainer(Container container) { LOG.info("Releasing YARN container {}", container.getId()); containersBeingReturned.put(container.getId(), container); // release the container on the node manager try { nodeManagerClient.stopContainer(container.getId(), container.getNodeId()); } catch (Throwable t) { // we only log this error. since the ResourceManager also gets the release // notification, the container should be eventually cleaned up LOG.error("Error while calling YARN Node Manager to release container", t); } // tell the master that the container is no longer needed resourceManagerClient.releaseAssignedContainer(container.getId()); }
@Override public void killContainer(Container container) { nodeMgr.stopContainerAsync(container.getId(), container.getNodeId()); }
public void reserve(Container container) { reserve(container.getNodeId().getHost()); }
@Override public void onContainersAllocated(List<Container> containers) { for (Container acquiredContainer : containers) { LOG.info("Acquired container " + acquiredContainer.getId() + " on host " + acquiredContainer.getNodeId().getHost() + " , with the resource " + acquiredContainer.getResource().toString()); String host = acquiredContainer.getNodeId().getHost(); if (!blackHosts.contains(host)) { if (workerContainersAllocating.get()) { acquiredWorkerContainers.add(acquiredContainer); acquiredWorkerContainersCount.incrementAndGet(); } else { acquiredPsContainers.add(acquiredContainer); acquiredPsContainersCount.incrementAndGet(); } } else { LOG.info("Add container " + acquiredContainer.getId() + " to cancel list"); cancelContainers.add(acquiredContainer); } } LOG.info("Current acquired worker container " + acquiredWorkerContainersCount.get() + " / " + neededWorkerContainersCount + " ps container " + acquiredPsContainersCount.get() + " / " + neededPsContainersCount); }
public void release(Container container) { release(container.getNodeId().getHost()); }
@Override public boolean stopWorker(final YarnWorkerNode workerNode) { final Container container = workerNode.getContainer(); log.info("Stopping container {}.", container.getId()); try { nodeManagerClient.stopContainer(container.getId(), container.getNodeId()); } catch (final Exception e) { log.warn("Error while calling YARN Node Manager to stop container", e); } resourceManagerClient.releaseAssignedContainer(container.getId()); workerNodeMap.remove(workerNode.getResourceID()); return true; }
@Override public String info() throws TException { StringBuffer sbRet = new StringBuffer(); sbRet.append("JstormOnYarn\n"); sbRet.append("Instance Name:" + jstormMasterContext.instanceName + "\n"); sbRet.append("Jstorm's location on hdfs:" + jstormMasterContext.deployPath + "\n"); if (jstormMasterContext.user != null) { sbRet.append("Jstorm's data path:" + jstormMasterContext.nimbusDataDirPrefix + jstormMasterContext.instanceName + "\n"); sbRet.append("Cluster userName:" + jstormMasterContext.user + "\n"); } sbRet.append("Nimbus Count:" + jstormMasterContext.nimbusContainers.size() + "\n"); sbRet.append("Supervisor Count:" + jstormMasterContext.supervisorContainers.size() + "\n"); sbRet.append("detail :\n"); sbRet.append("Type \tContainerId \tHost \tContainerMemory\tContainerVCores\n"); for (Container container : jstormMasterContext.nimbusContainers) { sbRet.append("Nimbus \t" + container.getId().toString() + "\t" + container.getNodeId().getHost() + "\t" + container.getResource().getMemory() + "\t " + container.getResource().getVirtualCores() + "\n"); } for (Container container : jstormMasterContext.supervisorContainers) { sbRet.append("Supervisor\t" + container.getId().toString() + "\t" + container.getNodeId().getHost() + "\t" + container.getResource().getMemory() + "\t " + container.getResource().getVirtualCores() + "\n"); } LOG.info("info is: " + sbRet.toString()); return sbRet.toString(); }
/** * Utility method to display YARN container information in a useful way for * log messages. * * @param container * @return */ public static String describeContainer(Container container) { StringBuilder buf = new StringBuilder() .append("[id: ") .append(container.getId()) .append(", host: ") .append(container.getNodeId().getHost()) .append(", priority: ") .append(container.getPriority()) .append(", memory: ") .append(container.getResource().getMemory()) .append(" MB, vcores: ") .append(container.getResource().getVirtualCores()) .append("]"); return buf.toString(); }
private static Container mockContainer(String host, int port, int containerId, Resource resource) { Container mockContainer = mock(Container.class); NodeId mockNodeId = NodeId.newInstance(host, port); ContainerId mockContainerId = ContainerId.newInstance( ApplicationAttemptId.newInstance( ApplicationId.newInstance(System.currentTimeMillis(), 1), 1 ), containerId ); when(mockContainer.getId()).thenReturn(mockContainerId); when(mockContainer.getNodeId()).thenReturn(mockNodeId); when(mockContainer.getResource()).thenReturn(resource); when(mockContainer.getPriority()).thenReturn(Priority.UNDEFINED); return mockContainer; }
+ " on host " + container.getNodeId().getHost(); LOG.info(message); sendInfoMessage(message);
/** * @param cont Container. * @return {@code True} if container satisfies requirements. */ private boolean checkContainer(Container cont) { // Check limit on running nodes. if (props.instances() <= containers.size()) return false; // Check host name if (props.hostnameConstraint() != null && props.hostnameConstraint().matcher(cont.getNodeId().getHost()).matches()) return false; // Check that slave satisfies min requirements. if (cont.getResource().getVirtualCores() < props.cpusPerNode() || cont.getResource().getMemory() < props.totalMemoryPerNode()) { log.log(Level.FINE, "Container resources not sufficient requirements. Host: {0}, cpu: {1}, mem: {2}", new Object[]{cont.getNodeId().getHost(), cont.getResource().getVirtualCores(), cont.getResource().getMemory()}); return false; } return true; }
@Override public synchronized void containersAllocated(List<Container> containers) { EventContext context = new EventContext(this); for (Container container : containers) { if (allocatedContainers.contains(container.getId())) { continue; } // We should never get a container on a node in the blacklist we // sent to YARN. If we do, something is wrong. Log the error and // reject the container. Else, bad things happen further along as // the tracking mechanisms assume one task per node. String host = container.getNodeId().getHost(); if (nodeInventory.isInUse(host)) { LOG.error( "Host is in use, but YARN allocated a container: " + DoYUtil.labelContainer(container) + " - container rejected." ); yarn.releaseContainer(container); continue; } // The container is fine. allocatedContainers.add(container.getId()); int priority = container.getPriority().getPriority(); int offset = priority - PRIORITY_OFFSET; if (offset < 0 || offset > prioritizedGroups.size()) { LOG.error("Container allocated with unknown priority " + DoYUtil.labelContainer(container)); continue; } context.setGroup(prioritizedGroups.get(offset)); context.group.containerAllocated(context, container); } }
container.getResource(), containerIdStr, container.getNodeId().getHost());
private static void publishContainerStartEvent( final TimelineClient timelineClient, Container container, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId(container.getId().toString()); entity.setEntityType(DSEntity.DS_CONTAINER.toString()); entity.setDomainId(domainId); entity.addPrimaryFilter(JOYConstants.USER, ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(DSEvent.DS_CONTAINER_START.toString()); event.addEventInfo(JOYConstants.NODE, container.getNodeId().toString()); event.addEventInfo(JOYConstants.RESOURCES, container.getResource().toString()); entity.addEvent(event); try { ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() { @Override public TimelinePutResponse run() throws Exception { return timelineClient.putEntities(entity); } }); } catch (Exception e) { LOG.error("Container start event could not be published for " + container.getId().toString(), e instanceof UndeclaredThrowableException ? e.getCause() : e); } }