/** * Stops the {@link SamzaContainer}. * @return true if {@link SamzaContainer} had shutdown within task.shutdown.ms. false otherwise. */ private boolean stopSamzaContainer() { boolean hasContainerShutdown = true; if (container != null) { try { container.shutdown(); LOGGER.info("Waiting {} ms for the container: {} to shutdown.", taskShutdownMs, container); hasContainerShutdown = containerShutdownLatch.await(taskShutdownMs, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { LOGGER.error("Exception occurred when shutting down the container: {}.", container, e); hasContainerShutdown = false; if (containerException != null) { containerException = e; } } LOGGER.info(String.format("Shutdown status of container: %s for stream processor: %s is: %b.", container, processorId, hasContainerShutdown)); } // We want to propagate TimeoutException when container shutdown times out. It is possible that the timeout exception // we propagate to the application runner maybe overwritten by container failure cause in case of interleaved execution. // It is acceptable since container exception is much more useful compared to timeout exception. // We can infer from the logs about the fact that container shutdown timed out or not for additional inference. if (!hasContainerShutdown && containerException == null) { containerException = new TimeoutException("Container shutdown timed out after " + taskShutdownMs + " ms."); } return hasContainerShutdown; }
/** * Stops the {@link SamzaContainer}. * @return true if {@link SamzaContainer} had shutdown within task.shutdown.ms. false otherwise. */ private boolean stopSamzaContainer() { boolean hasContainerShutdown = true; if (container != null) { try { container.shutdown(); LOGGER.info("Waiting {} ms for the container: {} to shutdown.", taskShutdownMs, container); hasContainerShutdown = containerShutdownLatch.await(taskShutdownMs, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { LOGGER.error("Exception occurred when shutting down the container: {}.", container, e); hasContainerShutdown = false; if (containerException != null) { containerException = e; } } LOGGER.info(String.format("Shutdown status of container: %s for stream processor: %s is: %b.", container, processorId, hasContainerShutdown)); } // We want to propagate TimeoutException when container shutdown times out. It is possible that the timeout exception // we propagate to the application runner maybe overwritten by container failure cause in case of interleaved execution. // It is acceptable since container exception is much more useful compared to timeout exception. // We can infer from the logs about the fact that container shutdown timed out or not for additional inference. if (!hasContainerShutdown && containerException == null) { containerException = new TimeoutException("Container shutdown timed out after " + taskShutdownMs + " ms."); } return hasContainerShutdown; }
/** * Stops the {@link SamzaContainer}. * @return true if {@link SamzaContainer} had shutdown within task.shutdown.ms. false otherwise. */ private boolean stopSamzaContainer() { boolean hasContainerShutdown = true; if (container != null) { try { container.shutdown(); LOGGER.info("Waiting {} ms for the container: {} to shutdown.", taskShutdownMs, container); hasContainerShutdown = containerShutdownLatch.await(taskShutdownMs, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { LOGGER.error("Exception occurred when shutting down the container: {}.", container, e); hasContainerShutdown = false; if (containerException != null) { containerException = e; } } LOGGER.info(String.format("Shutdown status of container: %s for stream processor: %s is: %b.", container, processorId, hasContainerShutdown)); } // We want to propagate TimeoutException when container shutdown times out. It is possible that the timeout exception // we propagate to the application runner maybe overwritten by container failure cause in case of interleaved execution. // It is acceptable since container exception is much more useful compared to timeout exception. // We can infer from the logs about the fact that container shutdown timed out or not for additional inference. if (!hasContainerShutdown && containerException == null) { containerException = new TimeoutException("Container shutdown timed out after " + taskShutdownMs + " ms."); } return hasContainerShutdown; }
/** * Stops the {@link SamzaContainer}. * @return true if {@link SamzaContainer} had shutdown within task.shutdown.ms. false otherwise. */ private boolean stopSamzaContainer() { boolean hasContainerShutdown = true; if (container != null) { try { container.shutdown(); LOGGER.info("Waiting {} ms for the container: {} to shutdown.", taskShutdownMs, container); hasContainerShutdown = containerShutdownLatch.await(taskShutdownMs, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { LOGGER.error("Exception occurred when shutting down the container: {}.", container, e); hasContainerShutdown = false; if (containerException != null) { containerException = e; } } LOGGER.info(String.format("Shutdown status of container: %s for stream processor: %s is: %b.", container, processorId, hasContainerShutdown)); } // We want to propagate TimeoutException when container shutdown times out. It is possible that the timeout exception // we propagate to the application runner maybe overwritten by container failure cause in case of interleaved execution. // It is acceptable since container exception is much more useful compared to timeout exception. // We can infer from the logs about the fact that container shutdown timed out or not for additional inference. if (!hasContainerShutdown && containerException == null) { containerException = new TimeoutException("Container shutdown timed out after " + taskShutdownMs + " ms."); } return hasContainerShutdown; }
/** * Stops the {@link SamzaContainer}. * @return true if {@link SamzaContainer} had shutdown within task.shutdown.ms. false otherwise. */ private boolean stopSamzaContainer() { boolean hasContainerShutdown = true; if (container != null) { try { container.shutdown(); LOGGER.info("Waiting {} ms for the container: {} to shutdown.", taskShutdownMs, container); hasContainerShutdown = containerShutdownLatch.await(taskShutdownMs, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { LOGGER.error("Exception occurred when shutting down the container: {}.", container, e); hasContainerShutdown = false; if (containerException != null) { containerException = e; } } LOGGER.info(String.format("Shutdown status of container: %s for stream processor: %s is: %b.", container, processorId, hasContainerShutdown)); } // We want to propagate TimeoutException when container shutdown times out. It is possible that the timeout exception // we propagate to the application runner maybe overwritten by container failure cause in case of interleaved execution. // It is acceptable since container exception is much more useful compared to timeout exception. // We can infer from the logs about the fact that container shutdown timed out or not for additional inference. if (!hasContainerShutdown && containerException == null) { containerException = new TimeoutException("Container shutdown timed out after " + taskShutdownMs + " ms."); } return hasContainerShutdown; }
/** * Creates a new container heartbeat monitor if possible. * @param container the container to monitor * @return a new {@link ContainerHeartbeatMonitor} instance, or null if could not create one */ private static ContainerHeartbeatMonitor createContainerHeartbeatMonitor(SamzaContainer container) { String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); String executionEnvContainerId = System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID()); if (executionEnvContainerId != null) { log.info("Got execution environment container id: {}", executionEnvContainerId); return new ContainerHeartbeatMonitor(() -> { try { container.shutdown(); containerRunnerException = new SamzaException("Container shutdown due to expired heartbeat"); } catch (Exception e) { log.error("Heartbeat monitor failed to shutdown the container gracefully. Exiting process.", e); System.exit(1); } }, new ContainerHeartbeatClient(coordinatorUrl, executionEnvContainerId)); } else { log.warn("Execution environment container id not set. Container heartbeat monitor will not be created"); return null; } } }
/** * Creates a new container heartbeat monitor if possible. * @param container the container to monitor * @return a new {@link ContainerHeartbeatMonitor} instance, or null if could not create one */ private static ContainerHeartbeatMonitor createContainerHeartbeatMonitor(SamzaContainer container) { String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); String executionEnvContainerId = System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID()); if (executionEnvContainerId != null) { log.info("Got execution environment container id: {}", executionEnvContainerId); return new ContainerHeartbeatMonitor(() -> { try { container.shutdown(); containerRunnerException = new SamzaException("Container shutdown due to expired heartbeat"); } catch (Exception e) { log.error("Heartbeat monitor failed to shutdown the container gracefully. Exiting process.", e); System.exit(1); } }, new ContainerHeartbeatClient(coordinatorUrl, executionEnvContainerId)); } else { log.warn("Execution environment container id not set. Container heartbeat monitor will not be created"); return null; } } }
/** * Creates a new container heartbeat monitor if possible. * @param container the container to monitor * @return a new {@link ContainerHeartbeatMonitor} instance, or null if could not create one */ private static ContainerHeartbeatMonitor createContainerHeartbeatMonitor(SamzaContainer container) { String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); String executionEnvContainerId = System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID()); if (executionEnvContainerId != null) { log.info("Got execution environment container id: {}", executionEnvContainerId); return new ContainerHeartbeatMonitor(() -> { try { container.shutdown(); containerRunnerException = new SamzaException("Container shutdown due to expired heartbeat"); } catch (Exception e) { log.error("Heartbeat monitor failed to shutdown the container gracefully. Exiting process.", e); System.exit(1); } }, new ContainerHeartbeatClient(coordinatorUrl, executionEnvContainerId)); } else { log.warn("Execution environment container id not set. Container heartbeat monitor will not be created"); return null; } } }
/** * Creates a new container heartbeat monitor if possible. * @param container the container to monitor * @return a new {@link ContainerHeartbeatMonitor} instance, or null if could not create one */ private static ContainerHeartbeatMonitor createContainerHeartbeatMonitor(SamzaContainer container) { String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); String executionEnvContainerId = System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID()); if (executionEnvContainerId != null) { log.info("Got execution environment container id: {}", executionEnvContainerId); return new ContainerHeartbeatMonitor(() -> { try { container.shutdown(); containerRunnerException = new SamzaException("Container shutdown due to expired heartbeat"); } catch (Exception e) { log.error("Heartbeat monitor failed to shutdown the container gracefully. Exiting process.", e); System.exit(1); } }, new ContainerHeartbeatClient(coordinatorUrl, executionEnvContainerId)); } else { log.warn("Execution environment container id not set. Container heartbeat monitor will not be created"); return null; } } }
/** * Creates a new container heartbeat monitor if possible. * @param container the container to monitor * @return a new {@link ContainerHeartbeatMonitor} instance, or null if could not create one */ private static ContainerHeartbeatMonitor createContainerHeartbeatMonitor(SamzaContainer container) { String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); String executionEnvContainerId = System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID()); if (executionEnvContainerId != null) { log.info("Got execution environment container id: {}", executionEnvContainerId); return new ContainerHeartbeatMonitor(() -> { try { container.shutdown(); containerRunnerException = new SamzaException("Container shutdown due to expired heartbeat"); } catch (Exception e) { log.error("Heartbeat monitor failed to shutdown the container gracefully. Exiting process.", e); System.exit(1); } }, new ContainerHeartbeatClient(coordinatorUrl, executionEnvContainerId)); } else { log.warn("Execution environment container id not set. Container heartbeat monitor will not be created"); return null; } } }
@Test public void testCoordinatorFailureShouldStopTheStreamProcessor() { JobCoordinator mockJobCoordinator = Mockito.mock(JobCoordinator.class); ProcessorLifecycleListener lifecycleListener = Mockito.mock(ProcessorLifecycleListener.class); SamzaContainer mockSamzaContainer = Mockito.mock(SamzaContainer.class); MapConfig config = new MapConfig(ImmutableMap.of("task.shutdown.ms", "0")); StreamProcessor streamProcessor = new StreamProcessor("TestProcessorId", config, new HashMap<>(), null, lifecycleListener, mockJobCoordinator); Exception failureException = new Exception("dummy exception"); streamProcessor.container = mockSamzaContainer; streamProcessor.state = State.RUNNING; streamProcessor.jobCoordinatorListener.onCoordinatorFailure(failureException); Mockito.doNothing().when(mockSamzaContainer).shutdown(); Mockito.when(mockSamzaContainer.hasStopped()).thenReturn(false); assertEquals(State.STOPPED, streamProcessor.state); Mockito.verify(lifecycleListener).afterFailure(failureException); Mockito.verify(mockSamzaContainer).shutdown(); }
@Test public void testStopShouldBeIdempotent() { JobCoordinator mockJobCoordinator = Mockito.mock(JobCoordinator.class); ProcessorLifecycleListener lifecycleListener = Mockito.mock(ProcessorLifecycleListener.class); SamzaContainer mockSamzaContainer = Mockito.mock(SamzaContainer.class); MapConfig config = new MapConfig(ImmutableMap.of("task.shutdown.ms", "0")); StreamProcessor streamProcessor = PowerMockito.spy(new StreamProcessor("TestProcessorId", config, new HashMap<>(), null, lifecycleListener, mockJobCoordinator)); Mockito.doNothing().when(mockJobCoordinator).stop(); Mockito.doNothing().when(mockSamzaContainer).shutdown(); Mockito.when(mockSamzaContainer.hasStopped()).thenReturn(false); Mockito.when(mockSamzaContainer.getStatus()) .thenReturn(SamzaContainerStatus.STARTED) .thenReturn(SamzaContainerStatus.STOPPED); streamProcessor.state = State.RUNNING; streamProcessor.stop(); assertEquals(State.STOPPING, streamProcessor.state); }