public ContainerId getContainerId() { assert container != null; return container.getId(); }
private void releaseYarnContainer(Container container) { LOG.info("Releasing YARN container {}", container.getId()); containersBeingReturned.put(container.getId(), container); // release the container on the node manager try { nodeManagerClient.stopContainer(container.getId(), container.getNodeId()); } catch (Throwable t) { // we only log this error. since the ResourceManager also gets the release // notification, the container should be eventually cleaned up LOG.error("Error while calling YARN Node Manager to release container", t); } // tell the master that the container is no longer needed resourceManagerClient.releaseAssignedContainer(container.getId()); }
@Override public void removeSpecNimbus(String container_id) throws TException { if (jstormMasterContext.nimbusContainers.isEmpty()) return; for (Container container : jstormMasterContext.nimbusContainers) { if (container.getId().toString().trim().equals(container_id.trim())) { jstormMasterContext.nimbusContainers.remove(container); amRMClient.releaseAssignedContainer(container.getId()); } } }
@Override public void removeNimbus(int number) throws TException { if (jstormMasterContext.nimbusContainers.isEmpty()) return; for (int i = 0; i < number; i++) { Container container = jstormMasterContext.nimbusContainers.poll(); if (container != null) { amRMClient.releaseAssignedContainer(container.getId()); LOG.info("release nimbus container, id: " + container.getId().toString()); } } }
@Override public void removeSupervisors(int number) throws TException { if (jstormMaster.jstormMasterContext.supervisorContainers.isEmpty()) return; for (int i = 0; i < number; i++) { Container container = jstormMaster.jstormMasterContext.supervisorContainers.poll(); if (container != null) { amRMClient.releaseAssignedContainer(container.getId()); LOG.info("release supervisor's " + String.valueOf(number) + " container, id: " + container.getId().toString()); } } }
@Override public void stopNimbus() throws TException { if (jstormMasterContext.nimbusContainers.isEmpty()) return; int nimbusCount = jstormMasterContext.nimbusContainers.size(); for (int i = 0; i < nimbusCount; i++) { Container container = jstormMasterContext.nimbusContainers.poll(); if (container != null) { amRMClient.releaseAssignedContainer(container.getId()); LOG.info("release nimbus container, id: " + container.getId().toString()); } } }
@Override public void stopSupervisors() throws TException { if (jstormMaster.jstormMasterContext.supervisorContainers.isEmpty()) return; int supervisorCount = jstormMaster.jstormMasterContext.supervisorContainers.size(); for (int i = 0; i < supervisorCount; i++) { Container container = jstormMaster.jstormMasterContext.supervisorContainers.poll(); if (container != null) { amRMClient.releaseAssignedContainer(container.getId()); LOG.info("release all supervisor container, id: " + container.getId().toString()); } } }
@Override public boolean stopWorker(final YarnWorkerNode workerNode) { final Container container = workerNode.getContainer(); log.info("Stopping container {}.", container.getId()); try { nodeManagerClient.stopContainer(container.getId(), container.getNodeId()); } catch (final Exception e) { log.warn("Error while calling YARN Node Manager to stop container", e); } resourceManagerClient.releaseAssignedContainer(container.getId()); workerNodeMap.remove(workerNode.getResourceID()); return true; }
@Override public void releaseContainer(Container container) { resourceMgr.releaseAssignedContainer(container.getId()); }
@Override public void removeSpecSupervisors(String container_id) throws TException { LOG.info("remove spec supervisor: " + container_id); for (Container container : jstormMaster.jstormMasterContext.supervisorContainers) { if (container.getId().toString().trim().equals(container_id.trim())) { LOG.info("found one remove"); jstormMaster.jstormMasterContext.supervisorContainers.remove(container); amRMClient.releaseAssignedContainer(container.getId()); return; } } LOG.info("remove spec nimbus : " + container_id); for (Container container : jstormMaster.jstormMasterContext.nimbusContainers) { if (container.getId().toString().trim().equals(container_id.trim())) { LOG.info("found one remove"); jstormMaster.jstormMasterContext.nimbusContainers.remove(container); amRMClient.releaseAssignedContainer(container.getId()); } } }
/** * Extracts a unique ResourceID from the Yarn Container. * @param container The Yarn container * @return The ResourceID for the container */ static ResourceID extractResourceID(Container container) { return new ResourceID(container.getId().toString()); }
public static String labelContainer(Container container) { StringBuilder buf = new StringBuilder() .append("[id: ") .append(container.getId()) .append(", host: ") .append(container.getNodeId().getHost()) .append(", priority: ") .append(container.getPriority()) .append("]"); return buf.toString(); }
@SuppressWarnings("unused") @Subscribe public void handleContainerShutdownRequest(ContainerShutdownRequest containerShutdownRequest) { for (Container container : containerShutdownRequest.getContainers()) { LOGGER.info(String.format("Stopping container %s running on %s", container.getId(), container.getNodeId())); this.nmClientAsync.stopContainerAsync(container.getId(), container.getNodeId()); } }
private void getContainersFromPreviousAttempts(final RegisterApplicationMasterResponse registerApplicationMasterResponse) { final List<Container> containersFromPreviousAttempts = new RegisterApplicationMasterResponseReflector(log).getContainersFromPreviousAttempts(registerApplicationMasterResponse); log.info("Recovered {} containers from previous attempts ({}).", containersFromPreviousAttempts.size(), containersFromPreviousAttempts); for (final Container container : containersFromPreviousAttempts) { workerNodeMap.put(new ResourceID(container.getId().toString()), new YarnWorkerNode(container)); } }
public YarnWorkerNode(Container container) { Preconditions.checkNotNull(container); this.resourceID = new ResourceID(container.getId().toString()); this.container = container; }
@Override public void killContainer(Container container) { nodeMgr.stopContainerAsync(container.getId(), container.getNodeId()); }
@Override public String info() throws TException { StringBuffer sbRet = new StringBuffer(); sbRet.append("JstormOnYarn\n"); sbRet.append("Instance Name:" + jstormMasterContext.instanceName + "\n"); sbRet.append("Jstorm's location on hdfs:" + jstormMasterContext.deployPath + "\n"); if (jstormMasterContext.user != null) { sbRet.append("Jstorm's data path:" + jstormMasterContext.nimbusDataDirPrefix + jstormMasterContext.instanceName + "\n"); sbRet.append("Cluster userName:" + jstormMasterContext.user + "\n"); } sbRet.append("Nimbus Count:" + jstormMasterContext.nimbusContainers.size() + "\n"); sbRet.append("Supervisor Count:" + jstormMasterContext.supervisorContainers.size() + "\n"); sbRet.append("detail :\n"); sbRet.append("Type \tContainerId \tHost \tContainerMemory\tContainerVCores\n"); for (Container container : jstormMasterContext.nimbusContainers) { sbRet.append("Nimbus \t" + container.getId().toString() + "\t" + container.getNodeId().getHost() + "\t" + container.getResource().getMemory() + "\t " + container.getResource().getVirtualCores() + "\n"); } for (Container container : jstormMasterContext.supervisorContainers) { sbRet.append("Supervisor\t" + container.getId().toString() + "\t" + container.getNodeId().getHost() + "\t" + container.getResource().getMemory() + "\t " + container.getResource().getVirtualCores() + "\n"); } LOG.info("info is: " + sbRet.toString()); return sbRet.toString(); }
private static Container mockContainer(String host, int port, int containerId, Resource resource) { Container mockContainer = mock(Container.class); NodeId mockNodeId = NodeId.newInstance(host, port); ContainerId mockContainerId = ContainerId.newInstance( ApplicationAttemptId.newInstance( ApplicationId.newInstance(System.currentTimeMillis(), 1), 1 ), containerId ); when(mockContainer.getId()).thenReturn(mockContainerId); when(mockContainer.getNodeId()).thenReturn(mockNodeId); when(mockContainer.getResource()).thenReturn(resource); when(mockContainer.getPriority()).thenReturn(Priority.UNDEFINED); return mockContainer; }
ContainerStatus testingContainerStatus = mockContainerStatus(testingContainer.getId());