private static Configuration getConfiguration() { Configuration config = new Configuration(); config.setString(AkkaOptions.ASK_TIMEOUT, TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT()); config.setLong(HeartbeatManagerOptions.HEARTBEAT_INTERVAL, HEARTBEAT_INTERVAL); return config; }
private Collection<JobID> waitForRunningJobs(ClusterClient<?> clusterClient, Time timeout) throws ExecutionException, InterruptedException { return FutureUtils.retrySuccessfulWithDelay( CheckedSupplier.unchecked(clusterClient::listJobs), Time.milliseconds(50L), Deadline.fromNow(Duration.ofMillis(timeout.toMilliseconds())), jobs -> !jobs.isEmpty(), TestingUtils.defaultScheduledExecutor()) .get() .stream() .map(JobStatusMessage::getJobId) .collect(Collectors.toList()); }
static void init() throws Exception { jarDir = TMP.newFolder().toPath(); // properties are set property by surefire plugin final String parameterProgramJarName = System.getProperty("parameterJarName") + ".jar"; final String parameterProgramWithoutManifestJarName = System.getProperty("parameterJarWithoutManifestName") + ".jar"; final Path jarLocation = Paths.get(System.getProperty("targetDir")); jarWithManifest = Files.copy( jarLocation.resolve(parameterProgramJarName), jarDir.resolve("program-with-manifest.jar")); jarWithoutManifest = Files.copy( jarLocation.resolve(parameterProgramWithoutManifestJarName), jarDir.resolve("program-without-manifest.jar")); restfulGateway = new TestingDispatcherGateway.Builder() .setBlobServerPort(BLOB_SERVER_RESOURCE.getBlobServerPort()) .setSubmitFunction(jobGraph -> { LAST_SUBMITTED_JOB_GRAPH_REFERENCE.set(jobGraph); return CompletableFuture.completedFuture(Acknowledge.get()); }) .build(); gatewayRetriever = () -> CompletableFuture.completedFuture(restfulGateway); localAddressFuture = CompletableFuture.completedFuture("shazam://localhost:12345"); timeout = Time.seconds(10); responseHeaders = Collections.emptyMap(); executor = TestingUtils.defaultExecutor(); }
final TestingCluster cluster = new TestingCluster(configuration); cluster.start(); clientActorSystem = cluster.startJobClientActorSystem(graph.getJobID()); ActorGateway jm = cluster.getLeaderGateway(deadline.timeLeft()); cluster.waitForTaskManagersToBeRegisteredAtJobManager(jm.actor()); Future<Object> future = jm.ask(new WaitForAllVerticesToBeRunningOrFinished(graph.getJobID()), deadline.timeLeft()); cluster.clearLeader(); jm.tell(TestingJobManagerMessages.getDisablePostStop()); jm.tell(Kill.getInstance()); cluster.shutdownJobClientActorSystem(clientActorSystem); jobSubmission.finished = true; cluster.stop();
configuration.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, numTMs); TestingCluster cluster = new TestingCluster(configuration); cluster.start(); ActorGateway leadingJM = cluster.getLeaderGateway(timeout); cluster.waitForTaskManagersToBeRegisteredAtJobManager(leadingJM.actor()); cluster.clearLeader(); leadingJM.tell(PoisonPill.getInstance()); cluster.stop();
} finally { if (actorRef != null) { TestingUtils.stopActorGracefully(actorRef);
ActorGateway jmGateway = miniCluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
@BeforeClass public static void setup() throws Exception { init(); final GatewayRetriever<TestingDispatcherGateway> gatewayRetriever = () -> CompletableFuture.completedFuture(restfulGateway); final Time timeout = Time.seconds(10); final Map<String, String> responseHeaders = Collections.emptyMap(); final Executor executor = TestingUtils.defaultExecutor(); handler = new JarRunHandler( gatewayRetriever, timeout, responseHeaders, JarRunHeaders.getInstance(), jarDir, new Configuration(), executor); }
private void waitUntilAllSlotsAreUsed(DispatcherGateway dispatcherGateway, Time timeout) throws ExecutionException, InterruptedException { FutureUtils.retrySuccessfulWithDelay( () -> dispatcherGateway.requestClusterOverview(timeout), Time.milliseconds(50L), Deadline.fromNow(Duration.ofMillis(timeout.toMilliseconds())), clusterOverview -> clusterOverview.getNumTaskManagersConnected() >= 1 && clusterOverview.getNumSlotsAvailable() == 0 && clusterOverview.getNumSlotsTotal() == 2, TestingUtils.defaultScheduledExecutor()) .get(); }
private static Configuration getConfiguration() { verifyJvmOptions(); Configuration config = new Configuration(); config.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true); config.setString(AkkaOptions.ASK_TIMEOUT, TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT()); config.setString(TaskManagerOptions.MEMORY_SEGMENT_SIZE, "4096"); config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 2048); return config; }
/** * Test set user code ClassLoader before calling ProcessingTimeCallback. */ @Test public void testSetsUserCodeClassLoaderForTimerThreadFactory() throws Throwable { syncLatch = new OneShotLatch(); try (MockEnvironment mockEnvironment = new MockEnvironmentBuilder() .setUserCodeClassLoader(new TestUserCodeClassLoader()) .build()) { TimeServiceTask timerServiceTask = new TimeServiceTask(mockEnvironment); CompletableFuture<Void> invokeFuture = CompletableFuture.runAsync( () -> { try { timerServiceTask.invoke(); } catch (Exception e) { throw new CompletionException(e); } }, TestingUtils.defaultExecutor()); invokeFuture.get(); assertThat(timerServiceTask.getClassLoaders(), hasSize(greaterThanOrEqualTo(1))); assertThat(timerServiceTask.getClassLoaders(), everyItem(instanceOf(TestUserCodeClassLoader.class))); } }
@Override public void close() throws Exception { // Free cluster resources clusterClient.cancel(jobId); // cancel() is non-blocking so do this to make sure the job finished CompletableFuture<JobStatus> jobStatusFuture = FutureUtils.retrySuccessfulWithDelay( () -> clusterClient.getJobStatus(jobId), Time.milliseconds(50), deadline, (jobStatus) -> jobStatus.equals(JobStatus.CANCELED), TestingUtils.defaultScheduledExecutor()); assertEquals( JobStatus.CANCELED, jobStatusFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS)); } }
final Time timeout = Time.seconds(10); final Map<String, String> responseHeaders = Collections.emptyMap(); final Executor executor = TestingUtils.defaultExecutor();
private void restoreJob(ClassLoader classLoader, ClusterClient<?> clusterClient, Deadline deadline, String savepointPath) throws Exception { JobGraph jobToRestore = createJobGraph(ExecutionMode.RESTORE); jobToRestore.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath, allowNonRestoredState)); assertNotNull("Job doesn't have a JobID.", jobToRestore.getJobID()); clusterClient.submitJob(jobToRestore, classLoader); CompletableFuture<JobStatus> jobStatusFuture = FutureUtils.retrySuccessfulWithDelay( () -> clusterClient.getJobStatus(jobToRestore.getJobID()), Time.milliseconds(50), deadline, (jobStatus) -> jobStatus == JobStatus.FINISHED, TestingUtils.defaultScheduledExecutor()); assertEquals( JobStatus.FINISHED, jobStatusFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS)); }
RestClient client = new RestClient(RestClientConfiguration.fromConfiguration(clientConfig), TestingUtils.defaultExecutor());
&& accumulators.containsKey(ACCUMULATOR_NAME) && (int) accumulators.get(ACCUMULATOR_NAME).getUnchecked() == NUM_ITERATIONS, TestingUtils.defaultScheduledExecutor() ).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
TestingUtils.defaultExecutor());
deadline, (jobStatus) -> jobStatus == JobStatus.RUNNING, TestingUtils.defaultScheduledExecutor()); assertEquals( JobStatus.RUNNING, deadline, (jobStatus) -> jobStatus == JobStatus.CANCELED, TestingUtils.defaultScheduledExecutor()); assertEquals( JobStatus.CANCELED,
Deadline.now().plus(Duration.ofSeconds(30)), status -> status == JobStatus.CANCELED, TestingUtils.defaultScheduledExecutor() );
Time.seconds(10), (throwable) -> true, TestingUtils.defaultScheduledExecutor() );