@Override public void onContainerStarted(ContainerId containerId, Map<String, ByteBuffer> allServiceResponse) { LOG.info("Container " + containerId.toString() + " started"); }
@Override public void onContainerStopped(ContainerId containerId) { LOG.info("Container " + containerId.toString() + " stopped"); }
@Override public void onStopContainerError(ContainerId containerId, Throwable t) { LOG.info("Container " + containerId.toString() + " failed to stop ", t); }
@Override public void onStartContainerError(ContainerId containerId, Throwable t) { LOG.info("Container " + containerId.toString() + " failed to start ", t); }
@Override public void onGetContainerStatusError(ContainerId containerId, Throwable t) { LOG.info("Container " + containerId.toString() + " get status error ", t); }
@Override public void onContainerStatusReceived(ContainerId containerId, ContainerStatus containerStatus) { LOG.info("Container " + containerId.toString() + " status " + containerStatus.toString() + " received"); }
private Path getHdfsLogDir(ContainerId containerId, FileSystem destFs, Path appWorkDir) throws IOException { Path logRootDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.APP_LOGS_DIR_NAME); if (!destFs.exists(logRootDir)) { destFs.mkdirs(logRootDir); } return new Path(logRootDir, containerId.toString()); } }
@Override public void onGetContainerStatusError(ContainerId containerId, Throwable t) { if (eventSubmitter.isPresent()) { eventSubmitter.get().submit(GobblinYarnEventConstants.EventNames.CONTAINER_GET_STATUS_ERROR, GobblinYarnMetricTagNames.CONTAINER_ID, containerId.toString(), GobblinYarnEventConstants.EventMetadata.ERROR_EXCEPTION, Throwables.getStackTraceAsString(t)); } LOGGER.error(String.format("Failed to get status for container %s due to error %s", containerId, t)); }
@Override public void onContainerStarted(ContainerId containerId, Map<String, ByteBuffer> allServiceResponse) { if (eventSubmitter.isPresent()) { eventSubmitter.get().submit(GobblinYarnEventConstants.EventNames.CONTAINER_STARTED, GobblinYarnMetricTagNames.CONTAINER_ID, containerId.toString()); } LOGGER.info(String.format("Container %s has been started", containerId)); }
@Override public void onStartContainerError(ContainerId containerId, Throwable t) { if (eventSubmitter.isPresent()) { eventSubmitter.get().submit(GobblinYarnEventConstants.EventNames.CONTAINER_START_ERROR, GobblinYarnMetricTagNames.CONTAINER_ID, containerId.toString(), GobblinYarnEventConstants.EventMetadata.ERROR_EXCEPTION, Throwables.getStackTraceAsString(t)); } LOGGER.error(String.format("Failed to start container %s due to error %s", containerId, t)); containerMap.remove(containerId); }
@Override public void onStopContainerError(ContainerId containerId, Throwable t) { if (eventSubmitter.isPresent()) { eventSubmitter.get().submit(GobblinYarnEventConstants.EventNames.CONTAINER_STOP_ERROR, GobblinYarnMetricTagNames.CONTAINER_ID, containerId.toString(), GobblinYarnEventConstants.EventMetadata.ERROR_EXCEPTION, Throwables.getStackTraceAsString(t)); } LOGGER.error(String.format("Failed to stop container %s due to error %s", containerId, t)); } }
@Override public void removeSpecNimbus(String container_id) throws TException { if (jstormMasterContext.nimbusContainers.isEmpty()) return; for (Container container : jstormMasterContext.nimbusContainers) { if (container.getId().toString().trim().equals(container_id.trim())) { jstormMasterContext.nimbusContainers.remove(container); amRMClient.releaseAssignedContainer(container.getId()); } } }
/** * Extracts a unique ResourceID from the Yarn Container. * @param container The Yarn container * @return The ResourceID for the container */ static ResourceID extractResourceID(Container container) { return new ResourceID(container.getId().toString()); }
@Override public void removeSupervisors(int number) throws TException { if (jstormMaster.jstormMasterContext.supervisorContainers.isEmpty()) return; for (int i = 0; i < number; i++) { Container container = jstormMaster.jstormMasterContext.supervisorContainers.poll(); if (container != null) { amRMClient.releaseAssignedContainer(container.getId()); LOG.info("release supervisor's " + String.valueOf(number) + " container, id: " + container.getId().toString()); } } }
@Override public void stopSupervisors() throws TException { if (jstormMaster.jstormMasterContext.supervisorContainers.isEmpty()) return; int supervisorCount = jstormMaster.jstormMasterContext.supervisorContainers.size(); for (int i = 0; i < supervisorCount; i++) { Container container = jstormMaster.jstormMasterContext.supervisorContainers.poll(); if (container != null) { amRMClient.releaseAssignedContainer(container.getId()); LOG.info("release all supervisor container, id: " + container.getId().toString()); } } }
@Override public void stopNimbus() throws TException { if (jstormMasterContext.nimbusContainers.isEmpty()) return; int nimbusCount = jstormMasterContext.nimbusContainers.size(); for (int i = 0; i < nimbusCount; i++) { Container container = jstormMasterContext.nimbusContainers.poll(); if (container != null) { amRMClient.releaseAssignedContainer(container.getId()); LOG.info("release nimbus container, id: " + container.getId().toString()); } } }
@Override public void removeNimbus(int number) throws TException { if (jstormMasterContext.nimbusContainers.isEmpty()) return; for (int i = 0; i < number; i++) { Container container = jstormMasterContext.nimbusContainers.poll(); if (container != null) { amRMClient.releaseAssignedContainer(container.getId()); LOG.info("release nimbus container, id: " + container.getId().toString()); } } }
private void getContainersFromPreviousAttempts(final RegisterApplicationMasterResponse registerApplicationMasterResponse) { final List<Container> containersFromPreviousAttempts = new RegisterApplicationMasterResponseReflector(log).getContainersFromPreviousAttempts(registerApplicationMasterResponse); log.info("Recovered {} containers from previous attempts ({}).", containersFromPreviousAttempts.size(), containersFromPreviousAttempts); for (final Container container : containersFromPreviousAttempts) { workerNodeMap.put(new ResourceID(container.getId().toString()), new YarnWorkerNode(container)); } }
public YarnWorkerNode(Container container) { Preconditions.checkNotNull(container); this.resourceID = new ResourceID(container.getId().toString()); this.container = container; }
@Override public void onContainersCompleted(final List<ContainerStatus> statuses) { runAsync(() -> { log.debug("YARN ResourceManager reported the following containers completed: {}.", statuses); for (final ContainerStatus containerStatus : statuses) { final ResourceID resourceId = new ResourceID(containerStatus.getContainerId().toString()); final YarnWorkerNode yarnWorkerNode = workerNodeMap.remove(resourceId); if (yarnWorkerNode != null) { // Container completed unexpectedly ~> start a new one requestYarnContainerIfRequired(); } // Eagerly close the connection with task manager. closeTaskManagerConnection(resourceId, new Exception(containerStatus.getDiagnostics())); } } ); }