TestingClusterClient( CompletableFuture<Tuple2<JobID, Integer>> rescaleJobFuture, Configuration configuration) { super(configuration, new TestingHighAvailabilityServices(), false); this.rescaleJobFuture = rescaleJobFuture; }
/** * Shuts down the client. This stops the internal actor system and actors. */ public void shutdown() throws Exception { synchronized (this) { actorSystemLoader.close(); if (!sharedHaServices && highAvailabilityServices != null) { highAvailabilityServices.close(); } } }
@Override public LeaderConnectionInfo getClusterConnectionInfo() throws LeaderRetrievalException { return LeaderRetrievalUtils.retrieveLeaderConnectionInfo( highAvailabilityServices.getDispatcherLeaderRetriever(), timeout); }
/** * Gets the current cluster connection info (may change in case of a HA setup). * * @return The the connection info to the leader component of the cluster * @throws LeaderRetrievalException if the leader could not be retrieved */ public LeaderConnectionInfo getClusterConnectionInfo() throws LeaderRetrievalException { return LeaderRetrievalUtils.retrieveLeaderConnectionInfo( highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID), timeout); }
dispatcherProcesses[0].startProcess(); highAvailabilityServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices( config, TestingUtils.defaultExecutor()); leaderRetrievalService = highAvailabilityServices.getDispatcherLeaderRetriever(); leaderRetrievalService.start(leaderListener); highAvailabilityServices.closeAndCleanupAllData();
/** * FLINK-6641 * * <p>Tests that the {@link ClusterClient} does not clean up HA data when being shut down. */ @Test public void testClusterClientShutdown() throws Exception { Configuration config = new Configuration(); HighAvailabilityServices highAvailabilityServices = mock(HighAvailabilityServices.class); StandaloneClusterClient clusterClient = new StandaloneClusterClient(config, highAvailabilityServices, false); clusterClient.shutdown(); // check that the client does not clean up HA data but closes the services verify(highAvailabilityServices, never()).closeAndCleanupAllData(); verify(highAvailabilityServices).close(); }
/** * Creates a instance that submits the programs to the JobManager defined in the * configuration. This method will try to resolve the JobManager hostname and throw an exception * if that is not possible. * * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer. * * @throws Exception we cannot create the high availability services */ public ClusterClient(Configuration flinkConfig) throws Exception { this( flinkConfig, HighAvailabilityServicesUtils.createHighAvailabilityServices( flinkConfig, Executors.directExecutor(), HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION), false); }
@VisibleForTesting RestClusterClient( Configuration configuration, @Nullable RestClient restClient, T clusterId, WaitStrategy waitStrategy, @Nullable LeaderRetrievalService webMonitorRetrievalService) throws Exception { super(configuration); this.restClusterClientConfiguration = RestClusterClientConfiguration.fromConfiguration(configuration); if (restClient != null) { this.restClient = restClient; } else { this.restClient = new RestClient(restClusterClientConfiguration.getRestClientConfiguration(), executorService); } this.waitStrategy = Preconditions.checkNotNull(waitStrategy); this.clusterId = Preconditions.checkNotNull(clusterId); if (webMonitorRetrievalService == null) { this.webMonitorRetrievalService = highAvailabilityServices.getWebMonitorLeaderRetriever(); } else { this.webMonitorRetrievalService = webMonitorRetrievalService; } this.dispatcherRetrievalService = highAvailabilityServices.getDispatcherLeaderRetriever(); this.retryExecutorService = Executors.newSingleThreadScheduledExecutor(new ExecutorThreadFactory("Flink-RestClusterClient-Retry")); startLeaderRetrievers(); }
this.dispatcher = Executors.newSingleThreadExecutor(new ServicesThreadFactory()); this.resourceManagerLeaderElectionService = new SingleLeaderElectionService(dispatcher, DEFAULT_LEADER_ID); this.dispatcherLeaderElectionService = new SingleLeaderElectionService(dispatcher, DEFAULT_LEADER_ID);
MockResourceManagerRuntimeServices() throws Exception { scheduledExecutor = mock(ScheduledExecutor.class); highAvailabilityServices = new TestingHighAvailabilityServices(); rmLeaderElectionService = new TestingLeaderElectionService(); highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService); heartbeatServices = new TestingHeartbeatServices(5L, 5L, scheduledExecutor); metricRegistry = NoOpMetricRegistry.INSTANCE; slotManager = new SlotManager( new ScheduledExecutorServiceAdapter(new DirectScheduledExecutorService()), Time.seconds(10), Time.seconds(10), Time.minutes(1)); jobLeaderIdService = new JobLeaderIdService( highAvailabilityServices, rpcService.getScheduledExecutor(), Time.minutes(5L)); }
public void testJobManagerRetrievalWithHAServices() throws Exception { final Configuration configuration = new Configuration(); final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices(); final ActorSystem actorSystem = AkkaUtils.createDefaultActorSystem(); ActorRef actorRef = null; highAvailabilityServices.setJobMasterLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID, settableLeaderRetrievalService);
@Override public RunningJobsRegistry getRunningJobsRegistry() throws IOException { enter(); try { // IMPORTANT: The registry must NOT place its data in a directory that is // cleaned up by these services. return new FsNegativeRunningJobsRegistry(flinkFileSystem, workingDirectory); } finally { exit(); } }
/** * Returns the {@link ActorGateway} of the current job manager leader using * the {@link LeaderRetrievalService}. * * @return ActorGateway of the current job manager leader * @throws Exception */ public ActorGateway getJobManagerGateway() throws FlinkException { log.debug("Looking up JobManager"); try { return LeaderRetrievalUtils.retrieveLeaderGateway( highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID), actorSystemLoader.get(), lookupTimeout); } catch (LeaderRetrievalException lre) { throw new FlinkException("Could not connect to the leading JobManager. Please check that the " + "JobManager is running.", lre); } }
@Override public LeaderConnectionInfo getClusterConnectionInfo() throws LeaderRetrievalException { return LeaderRetrievalUtils.retrieveLeaderConnectionInfo( highAvailabilityServices.getDispatcherLeaderRetriever(), timeout); }
TestClusterClient(Configuration config, ActorGateway jobmanagerGateway) throws Exception { super(config, new TestingHighAvailabilityServices(), false); this.jobmanagerGateway = jobmanagerGateway; }
try { ownHostname = LeaderRetrievalUtils.findConnectingAddress( highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID), timeout); } catch (LeaderRetrievalException lre) {
/** * Helper method to wait until the {@link Dispatcher} has set its fencing token. * * @param rpcService to use to connect to the dispatcher * @param haServices high availability services to connect to the dispatcher * @return {@link DispatcherGateway} * @throws Exception if something goes wrong */ static DispatcherGateway retrieveDispatcherGateway(RpcService rpcService, HighAvailabilityServices haServices) throws Exception { final LeaderConnectionInfo leaderConnectionInfo = LeaderRetrievalUtils.retrieveLeaderConnectionInfo(haServices.getDispatcherLeaderRetriever(), Time.seconds(10L)); return rpcService.connect( leaderConnectionInfo.getAddress(), DispatcherId.fromUuid(leaderConnectionInfo.getLeaderSessionID()), DispatcherGateway.class).get(); }
DisposeSavepointClusterClient(Function<String, CompletableFuture<Acknowledge>> disposeSavepointFunction, Configuration configuration) { super(configuration, new TestingHighAvailabilityServices(), false); this.disposeSavepointFunction = Preconditions.checkNotNull(disposeSavepointFunction); }
/** * Creates a new ApplicationClient actor or returns an existing one. May start an ActorSystem. * @return ActorSystem */ public ActorRef get() throws FlinkException { if (applicationClient == null) { // start application client LOG.info("Start application client."); final ActorSystem actorSystem; try { actorSystem = actorSystemLoader.get(); } catch (FlinkException fle) { throw new FlinkException("Could not start the ClusterClient's ActorSystem.", fle); } try { applicationClient = actorSystem.actorOf( Props.create( ApplicationClient.class, flinkConfig, highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID)), "applicationClient"); } catch (Exception e) { throw new FlinkException("Could not start the ApplicationClient.", e); } } return applicationClient; } }
@Test public void testTriggerSavepointFailureIllegalJobID() throws Exception { replaceStdOutAndStdErr(); try { CliFrontend frontend = new MockedCliFrontend(new StandaloneClusterClient( getConfiguration(), new TestingHighAvailabilityServices(), false)); String[] parameters = { "invalid job id" }; try { frontend.savepoint(parameters); fail("Should have failed."); } catch (CliArgsException e) { assertThat(e.getMessage(), Matchers.containsString("Cannot parse JobID")); } } finally { restoreStdOutAndStdErr(); } }