private void registerOrphanedJobManagerTerminationFuture(CompletableFuture<Void> jobManagerRunnerTerminationFuture) { orphanedJobManagerRunnersTerminationFuture = FutureUtils.completeAll(Arrays.asList( orphanedJobManagerRunnersTerminationFuture, jobManagerRunnerTerminationFuture)); }
private CompletableFuture<Void> terminateJobManagerRunnersAndGetTerminationFuture() { terminateJobManagerRunners(); final Collection<CompletableFuture<Void>> values = jobManagerTerminationFutures.values(); return FutureUtils.completeAll(values); }
private CompletableFuture<Void> terminateJobManagerRunnersAndGetTerminationFuture() { terminateJobManagerRunners(); final Collection<CompletableFuture<Void>> values = jobManagerTerminationFutures.values(); return FutureUtils.completeAll(values); }
/** * Terminate all currently running {@link JobManagerRunner}. * * @return Future which is completed once all {@link JobManagerRunner} have terminated */ private CompletableFuture<Void> terminateJobManagerRunners() { log.info("Stopping all currently running jobs of dispatcher {}.", getAddress()); final HashSet<JobID> jobsToRemove = new HashSet<>(jobManagerRunners.keySet()); final List<CompletableFuture<Void>> terminationFutures = jobsToRemove.stream() .map(jobId -> removeJob(jobId, false)) .collect(Collectors.toList()); return FutureUtils.completeAll(terminationFutures); }
private CompletableFuture<Void> closeMetricSystem() { synchronized (lock) { if (jobManagerMetricGroup != null) { jobManagerMetricGroup.close(); jobManagerMetricGroup = null; } final ArrayList<CompletableFuture<Void>> terminationFutures = new ArrayList<>(2); // metrics shutdown if (metricRegistry != null) { terminationFutures.add(metricRegistry.shutdown()); metricRegistry = null; } if (metricQueryServiceActorSystem != null) { terminationFutures.add(AkkaUtils.terminateActorSystem(metricQueryServiceActorSystem)); } return FutureUtils.completeAll(terminationFutures); } }
private CompletableFuture<Void> closeMetricSystem() { synchronized (lock) { if (jobManagerMetricGroup != null) { jobManagerMetricGroup.close(); jobManagerMetricGroup = null; } final ArrayList<CompletableFuture<Void>> terminationFutures = new ArrayList<>(2); // metrics shutdown if (metricRegistry != null) { terminationFutures.add(metricRegistry.shutdown()); metricRegistry = null; } if (metricQueryServiceActorSystem != null) { terminationFutures.add(AkkaUtils.terminateActorSystem(metricQueryServiceActorSystem)); } return FutureUtils.completeAll(terminationFutures); } }
@Nonnull private CompletionStage<Void> terminateRpcServices() { final int numRpcServices; if (miniClusterConfiguration.getRpcServiceSharing() == RpcServiceSharing.SHARED) { numRpcServices = 1; } else { numRpcServices = 1 + 2 + miniClusterConfiguration.getNumTaskManagers(); // common, JM, RM, TMs } final Collection<CompletableFuture<?>> rpcTerminationFutures = new ArrayList<>(numRpcServices); synchronized (lock) { rpcTerminationFutures.add(commonRpcService.stopService()); if (miniClusterConfiguration.getRpcServiceSharing() != RpcServiceSharing.SHARED) { rpcTerminationFutures.add(jobManagerRpcService.stopService()); rpcTerminationFutures.add(resourceManagerRpcService.stopService()); for (RpcService taskManagerRpcService : taskManagerRpcServices) { rpcTerminationFutures.add(taskManagerRpcService.stopService()); } } commonRpcService = null; jobManagerRpcService = null; taskManagerRpcServices = null; resourceManagerRpcService = null; } return FutureUtils.completeAll(rpcTerminationFutures); }
@Nonnull private CompletionStage<Void> terminateRpcServices() { final int numRpcServices; if (miniClusterConfiguration.getRpcServiceSharing() == RpcServiceSharing.SHARED) { numRpcServices = 1; } else { numRpcServices = 1 + 2 + miniClusterConfiguration.getNumTaskManagers(); // common, JM, RM, TMs } final Collection<CompletableFuture<?>> rpcTerminationFutures = new ArrayList<>(numRpcServices); synchronized (lock) { rpcTerminationFutures.add(commonRpcService.stopService()); if (miniClusterConfiguration.getRpcServiceSharing() != RpcServiceSharing.SHARED) { rpcTerminationFutures.add(jobManagerRpcService.stopService()); rpcTerminationFutures.add(resourceManagerRpcService.stopService()); for (RpcService taskManagerRpcService : taskManagerRpcServices) { rpcTerminationFutures.add(taskManagerRpcService.stopService()); } } commonRpcService = null; jobManagerRpcService = null; taskManagerRpcServices = null; resourceManagerRpcService = null; } return FutureUtils.completeAll(rpcTerminationFutures); }
@Nonnull private CompletionStage<Void> terminateRpcServices() { final int numRpcServices; if (miniClusterConfiguration.getRpcServiceSharing() == MiniClusterConfiguration.RpcServiceSharing.SHARED) { numRpcServices = 1; } else { numRpcServices = 1 + 2 + miniClusterConfiguration.getNumTaskManagers(); // common, JM, RM, TMs } final Collection<CompletableFuture<?>> rpcTerminationFutures = new ArrayList<>(numRpcServices); synchronized (lock) { rpcTerminationFutures.add(commonRpcService.stopService()); if (miniClusterConfiguration.getRpcServiceSharing() != MiniClusterConfiguration.RpcServiceSharing.SHARED) { rpcTerminationFutures.add(jobManagerRpcService.stopService()); rpcTerminationFutures.add(resourceManagerRpcService.stopService()); for (RpcService taskManagerRpcService : taskManagerRpcServices) { rpcTerminationFutures.add(taskManagerRpcService.stopService()); } } commonRpcService = null; jobManagerRpcService = null; taskManagerRpcServices = null; resourceManagerRpcService = null; } return FutureUtils.completeAll(rpcTerminationFutures); }
final CompletableFuture<Void> componentTerminationFuture = FutureUtils.completeAll(terminationFutures);
return FutureUtils.completeAll(terminationFutures);
final CompletableFuture<Void> componentTerminationFuture = FutureUtils.completeAll(terminationFutures);
return FutureUtils.completeAll(terminationFutures);
/** * Suspend the job and shutdown all other services including rpc. */ @Override public CompletableFuture<Void> postStop() { log.info("Stopping the JobMaster for job {}({}).", jobGraph.getName(), jobGraph.getJobID()); // disconnect from all registered TaskExecutors final Set<ResourceID> taskManagerResourceIds = new HashSet<>(registeredTaskManagers.keySet()); final FlinkException cause = new FlinkException("Stopping JobMaster for job " + jobGraph.getName() + '(' + jobGraph.getJobID() + ")."); for (ResourceID taskManagerResourceId : taskManagerResourceIds) { disconnectTaskManager(taskManagerResourceId, cause); } taskManagerHeartbeatManager.stop(); resourceManagerHeartbeatManager.stop(); // make sure there is a graceful exit suspendExecution(new FlinkException("JobManager is shutting down.")); // shut down will internally release all registered slots slotPool.shutDown(); final CompletableFuture<Void> disposeInternalSavepointFuture; if (lastInternalSavepoint != null) { disposeInternalSavepointFuture = CompletableFuture.runAsync(() -> disposeSavepoint(lastInternalSavepoint)); } else { disposeInternalSavepointFuture = CompletableFuture.completedFuture(null); } final CompletableFuture<Void> slotPoolTerminationFuture = slotPool.getTerminationFuture(); return FutureUtils.completeAll(Arrays.asList(disposeInternalSavepointFuture, slotPoolTerminationFuture)); }
private CompletableFuture<Void> shutDownServices() { synchronized (lock) { Collection<CompletableFuture<Void>> terminationFutures = new ArrayList<>(3); Exception exception = null; try { blobCacheService.close(); } catch (Exception e) { exception = ExceptionUtils.firstOrSuppressed(e, exception); } try { metricRegistry.shutdown(); } catch (Exception e) { exception = ExceptionUtils.firstOrSuppressed(e, exception); } if (metricQueryServiceActorSystem != null) { terminationFutures.add(AkkaUtils.terminateActorSystem(metricQueryServiceActorSystem)); } try { highAvailabilityServices.close(); } catch (Exception e) { exception = ExceptionUtils.firstOrSuppressed(e, exception); } terminationFutures.add(rpcService.stopService()); terminationFutures.add(ExecutorUtils.nonBlockingShutdown(timeout.toMilliseconds(), TimeUnit.MILLISECONDS, executor)); if (exception != null) { terminationFutures.add(FutureUtils.completedExceptionally(exception)); } return FutureUtils.completeAll(terminationFutures); } }
private CompletableFuture<Void> shutDownServices() { synchronized (lock) { Collection<CompletableFuture<Void>> terminationFutures = new ArrayList<>(3); Exception exception = null; try { blobCacheService.close(); } catch (Exception e) { exception = ExceptionUtils.firstOrSuppressed(e, exception); } try { metricRegistry.shutdown(); } catch (Exception e) { exception = ExceptionUtils.firstOrSuppressed(e, exception); } try { highAvailabilityServices.close(); } catch (Exception e) { exception = ExceptionUtils.firstOrSuppressed(e, exception); } terminationFutures.add(rpcService.stopService()); terminationFutures.add(ExecutorUtils.nonBlockingShutdown(timeout.toMilliseconds(), TimeUnit.MILLISECONDS, executor)); if (exception != null) { terminationFutures.add(FutureUtils.completedExceptionally(exception)); } return FutureUtils.completeAll(terminationFutures); } }
private CompletableFuture<Void> shutDownServices() { synchronized (lock) { Collection<CompletableFuture<Void>> terminationFutures = new ArrayList<>(3); Exception exception = null; try { blobCacheService.close(); } catch (Exception e) { exception = ExceptionUtils.firstOrSuppressed(e, exception); } try { metricRegistry.shutdown(); } catch (Exception e) { exception = ExceptionUtils.firstOrSuppressed(e, exception); } if (metricQueryServiceActorSystem != null) { terminationFutures.add(AkkaUtils.terminateActorSystem(metricQueryServiceActorSystem)); } try { highAvailabilityServices.close(); } catch (Exception e) { exception = ExceptionUtils.firstOrSuppressed(e, exception); } terminationFutures.add(rpcService.stopService()); terminationFutures.add(ExecutorUtils.nonBlockingShutdown(timeout.toMilliseconds(), TimeUnit.MILLISECONDS, executor)); if (exception != null) { terminationFutures.add(FutureUtils.completedExceptionally(exception)); } return FutureUtils.completeAll(terminationFutures); } }
/** * Suspend the job and shutdown all other services including rpc. */ @Override public CompletableFuture<Void> postStop() { log.info("Stopping the JobMaster for job {}({}).", jobGraph.getName(), jobGraph.getJobID()); // disconnect from all registered TaskExecutors final Set<ResourceID> taskManagerResourceIds = new HashSet<>(registeredTaskManagers.keySet()); final FlinkException cause = new FlinkException("Stopping JobMaster for job " + jobGraph.getName() + '(' + jobGraph.getJobID() + ")."); for (ResourceID taskManagerResourceId : taskManagerResourceIds) { disconnectTaskManager(taskManagerResourceId, cause); } taskManagerHeartbeatManager.stop(); resourceManagerHeartbeatManager.stop(); // make sure there is a graceful exit suspendExecution(new FlinkException("JobManager is shutting down.")); // shut down will internally release all registered slots slotPool.shutDown(); final CompletableFuture<Void> disposeInternalSavepointFuture; if (lastInternalSavepoint != null) { disposeInternalSavepointFuture = CompletableFuture.runAsync(() -> disposeSavepoint(lastInternalSavepoint)); } else { disposeInternalSavepointFuture = CompletableFuture.completedFuture(null); } final CompletableFuture<Void> slotPoolTerminationFuture = slotPool.getTerminationFuture(); return FutureUtils.completeAll(Arrays.asList(disposeInternalSavepointFuture, slotPoolTerminationFuture)); }
return FutureUtils.completeAll(Arrays.asList(disposeInternalSavepointFuture, slotPoolTerminationFuture));
final CompletableFuture<Void> allJobManagerRunnersTerminationFuture = FutureUtils.completeAll(Arrays.asList( jobManagerRunnersTerminationFuture, orphanedJobManagerRunnersTerminationFuture));