rmServicesConfiguration, highAvailabilityServices, rpcService.getScheduledExecutor());
/** * Helper method to wait until the {@link Dispatcher} has set its fencing token. * * @param rpcService to use to connect to the dispatcher * @param haServices high availability services to connect to the dispatcher * @return {@link DispatcherGateway} * @throws Exception if something goes wrong */ static DispatcherGateway retrieveDispatcherGateway(RpcService rpcService, HighAvailabilityServices haServices) throws Exception { final LeaderConnectionInfo leaderConnectionInfo = LeaderRetrievalUtils.retrieveLeaderConnectionInfo(haServices.getDispatcherLeaderRetriever(), Time.seconds(10L)); return rpcService.connect( leaderConnectionInfo.getAddress(), DispatcherId.fromUuid(leaderConnectionInfo.getLeaderSessionID()), DispatcherGateway.class).get(); }
/** * Fetches framework/worker information persisted by a prior incarnation of the RM. */ private CompletableFuture<List<MesosWorkerStore.Worker>> getWorkersAsync() { // if this resource manager is recovering from failure, // then some worker tasks are most likely still alive and we can re-obtain them return CompletableFuture.supplyAsync(() -> { try { final List<MesosWorkerStore.Worker> tasksFromPreviousAttempts = workerStore.recoverWorkers(); for (final MesosWorkerStore.Worker worker : tasksFromPreviousAttempts) { if (worker.state() == MesosWorkerStore.WorkerState.New) { // remove new workers because allocation requests are transient workerStore.removeWorker(worker.taskID()); } } return tasksFromPreviousAttempts; } catch (final Exception e) { throw new CompletionException(new ResourceManagerException(e)); } }, getRpcService().getExecutor()); }
protected void initializeServices(Configuration configuration) throws Exception { LOG.info("Initializing cluster services."); synchronized (lock) { final String bindAddress = configuration.getString(JobManagerOptions.ADDRESS); final String portRange = getRPCPortRange(configuration); commonRpcService = createRpcService(configuration, bindAddress, portRange); // update the configuration used to create the high availability services configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress()); configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort()); haServices = createHaServices(configuration, commonRpcService.getExecutor()); blobServer = new BlobServer(configuration, haServices.createBlobStore()); blobServer.start(); heartbeatServices = createHeartbeatServices(configuration); metricRegistry = createMetricRegistry(configuration); // TODO: This is a temporary hack until we have ported the MetricQueryService to the new RpcEndpoint // Start actor system for metric query service on any available port metricQueryServiceActorSystem = MetricUtils.startMetricsActorSystem(configuration, bindAddress, LOG); metricRegistry.startQueryService(metricQueryServiceActorSystem, null); archivedExecutionGraphStore = createSerializableExecutionGraphStore(configuration, commonRpcService.getScheduledExecutor()); transientBlobCache = new TransientBlobCache( configuration, new InetSocketAddress( commonRpcService.getAddress(), blobServer.getPort())); } }
InetAddress remoteAddress = InetAddress.getByName(rpcService.getAddress()); taskManagerServicesConfiguration, resourceID, rpcService.getExecutor(), // TODO replace this later with some dedicated executor for io. EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag(), EnvironmentInformation.getMaxJvmHeapMemory());
@Override protected CompletableFuture<T> createGateway(CompletableFuture<Tuple2<String, UUID>> leaderFuture) { return FutureUtils.retryWithDelay( () -> leaderFuture.thenCompose( (Tuple2<String, UUID> addressLeaderTuple) -> rpcService.connect( addressLeaderTuple.f0, fencingTokenMapper.apply(addressLeaderTuple.f1), gatewayType)), retries, retryDelay, rpcService.getScheduledExecutor()); } }
public void sendInfoMessage(final String message) { getRpcService().execute(new Runnable() { @Override public void run() { InfoMessage infoMessage = new InfoMessage(message); for (InfoMessageListenerRpcGateway listenerRpcGateway : infoMessageListeners.values()) { listenerRpcGateway .notifyInfoMessage(infoMessage); } } }); }
private static Throwable shutDownRpc(RpcService rpcService, Throwable priorException) { if (rpcService != null) { try { rpcService.stopService(); } catch (Throwable t) { return firstOrSuppressed(t, priorException); } } return priorException; }
try { Future<Gateway> resourceManagerFuture = rpcService.connect(targetAddress, targetType); register(result, 1, initialRegistrationTimeout); }, rpcService.getExecutor()); }, rpcService.getExecutor());
protected void runCluster(Configuration configuration) throws Exception { synchronized (lock) { initializeServices(configuration); // write host information into configuration configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress()); configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort()); startClusterComponents( configuration, commonRpcService, haServices, blobServer, heartbeatServices, metricRegistry); dispatcher.getTerminationFuture().whenComplete( (Void value, Throwable throwable) -> { if (throwable != null) { LOG.info("Could not properly terminate the Dispatcher.", throwable); } // This is the general shutdown path. If a separate more specific shutdown was // already triggered, this will do nothing shutDownAndTerminate( SUCCESS_RETURN_CODE, ApplicationStatus.SUCCEEDED, throwable != null ? throwable.getMessage() : null, true); }); } }
private void startRegistrationLater(final long delay) { rpcService.scheduleRunnable( this::startRegistration, delay, TimeUnit.MILLISECONDS); } }
/** * Initializes the RPC endpoint. * * @param rpcService The RPC server that dispatches calls to this RPC endpoint. * @param endpointId Unique identifier for this endpoint */ protected RpcEndpoint(final RpcService rpcService, final String endpointId) { this.rpcService = checkNotNull(rpcService, "rpcService"); this.endpointId = checkNotNull(endpointId, "endpointId"); this.rpcServer = rpcService.startServer(this); this.mainThreadExecutor = new MainThreadExecutor(rpcServer); }
/** * Triggers the shut down of the rpc endpoint. The shut down is executed asynchronously. * * <p>In order to wait on the completion of the shut down, obtain the termination future * via {@link #getTerminationFuture()}} and wait on its completion. */ public final void shutDown() { rpcService.stopServer(rpcServer); }
metricQueryServiceActorSystem = MetricUtils.startMetricsActorSystem(configuration, rpcService.getAddress(), LOG);
final int jobManagerPort = rpcService.getPort(); config.setInteger(JobManagerOptions.PORT, jobManagerPort);
protected void initializeServices(Configuration configuration) throws Exception { LOG.info("Initializing cluster services."); synchronized (lock) { final String bindAddress = configuration.getString(JobManagerOptions.ADDRESS); final String portRange = getRPCPortRange(configuration); commonRpcService = createRpcService(configuration, bindAddress, portRange); // update the configuration used to create the high availability services configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress()); configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort()); haServices = createHaServices(configuration, commonRpcService.getExecutor()); blobServer = new BlobServer(configuration, haServices.createBlobStore()); blobServer.start(); heartbeatServices = createHeartbeatServices(configuration); metricRegistry = createMetricRegistry(configuration); // TODO: This is a temporary hack until we have ported the MetricQueryService to the new RpcEndpoint // Start actor system for metric query service on any available port metricQueryServiceActorSystem = MetricUtils.startMetricsActorSystem(configuration, bindAddress, LOG); metricRegistry.startQueryService(metricQueryServiceActorSystem, null); archivedExecutionGraphStore = createSerializableExecutionGraphStore(configuration, commonRpcService.getScheduledExecutor()); transientBlobCache = new TransientBlobCache( configuration, new InetSocketAddress( commonRpcService.getAddress(), blobServer.getPort())); } }
checkNotNull(highAvailabilityServices); InetAddress remoteAddress = InetAddress.getByName(rpcService.getAddress()); taskManagerServicesConfiguration, resourceID, rpcService.getExecutor(), // TODO replace this later with some dedicated executor for io. EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag(), EnvironmentInformation.getMaxJvmHeapMemory());
@Override protected CompletableFuture<T> createGateway(CompletableFuture<Tuple2<String, UUID>> leaderFuture) { return FutureUtils.retryWithDelay( () -> leaderFuture.thenCompose( (Tuple2<String, UUID> addressLeaderTuple) -> rpcService.connect( addressLeaderTuple.f0, fencingTokenMapper.apply(addressLeaderTuple.f1), gatewayType)), retries, retryDelay, rpcService.getScheduledExecutor()); } }
public void sendInfoMessage(final String message) { getRpcService().execute(new Runnable() { @Override public void run() { InfoMessage infoMessage = new InfoMessage(message); for (InfoMessageListenerRpcGateway listenerRpcGateway : infoMessageListeners.values()) { listenerRpcGateway .notifyInfoMessage(infoMessage); } } }); }
private static Throwable shutDownRpc(RpcService rpcService, Throwable priorException) { if (rpcService != null) { try { rpcService.stopService().get(); } catch (Throwable t) { return ExceptionUtils.firstOrSuppressed(t, priorException); } } return priorException; }