private void waitUntilAllSlotsAreUsed(DispatcherGateway dispatcherGateway, Time timeout) throws ExecutionException, InterruptedException { FutureUtils.retrySuccessfulWithDelay( () -> dispatcherGateway.requestClusterOverview(timeout), Time.milliseconds(50L), Deadline.fromNow(Duration.ofMillis(timeout.toMilliseconds())), clusterOverview -> clusterOverview.getNumTaskManagersConnected() >= 1 && clusterOverview.getNumSlotsAvailable() == 0 && clusterOverview.getNumSlotsTotal() == 2, TestingUtils.defaultScheduledExecutor()) .get(); }
public static ClusterOverviewWithVersion fromStatusOverview(ClusterOverview statusOverview, String version, String commitId) { return new ClusterOverviewWithVersion( statusOverview.getNumTaskManagersConnected(), statusOverview.getNumSlotsTotal(), statusOverview.getNumSlotsAvailable(), statusOverview.getNumJobsRunningOrPending(), statusOverview.getNumJobsFinished(), statusOverview.getNumJobsCancelled(), statusOverview.getNumJobsFailed(), version, commitId); }
@Override public String toString() { return "StatusOverview {" + "numTaskManagersConnected=" + numTaskManagersConnected + ", numSlotsTotal=" + numSlotsTotal + ", numSlotsAvailable=" + numSlotsAvailable + ", numJobsRunningOrPending=" + getNumJobsRunningOrPending() + ", numJobsFinished=" + getNumJobsFinished() + ", numJobsCancelled=" + getNumJobsCancelled() + ", numJobsFailed=" + getNumJobsFailed() + '}'; } }
public static ClusterOverviewWithVersion fromStatusOverview(ClusterOverview statusOverview, String version, String commitId) { return new ClusterOverviewWithVersion( statusOverview.getNumTaskManagersConnected(), statusOverview.getNumSlotsTotal(), statusOverview.getNumSlotsAvailable(), statusOverview.getTotalResources(), statusOverview.getAvailableResources(), statusOverview.getNumJobsRunningOrPending(), statusOverview.getNumJobsFinished(), statusOverview.getNumJobsCancelled(), statusOverview.getNumJobsFailed(), version, commitId); }
@Override public int hashCode() { int result = super.hashCode(); result = 31 * result + (version != null ? version.hashCode() : 0); result = 31 * result + (commitId != null ? commitId.hashCode() : 0); return result; } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } if (!super.equals(o)) { return false; } ClusterOverviewWithVersion that = (ClusterOverviewWithVersion) o; return Objects.equals(version, that.getVersion()) && Objects.equals(commitId, that.getCommitId()); }
@Override public CompletableFuture<ClusterOverview> requestClusterOverview(Time timeout) { CompletableFuture<ResourceOverview> taskManagerOverviewFuture = resourceManagerGateway.requestResourceOverview(timeout); final List<CompletableFuture<Optional<JobStatus>>> optionalJobInformation = queryJobMastersForInformation( (JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobStatus(timeout)); CompletableFuture<Collection<Optional<JobStatus>>> allOptionalJobsFuture = FutureUtils.combineAll(optionalJobInformation); CompletableFuture<Collection<JobStatus>> allJobsFuture = allOptionalJobsFuture.thenApply(this::flattenOptionalCollection); final JobsOverview completedJobsOverview = archivedExecutionGraphStore.getStoredJobsOverview(); return allJobsFuture.thenCombine( taskManagerOverviewFuture, (Collection<JobStatus> runningJobsStatus, ResourceOverview resourceOverview) -> { final JobsOverview allJobsOverview = JobsOverview.create(runningJobsStatus).combine(completedJobsOverview); return new ClusterOverview(resourceOverview, allJobsOverview); }); }
private void waitForTaskManagers(int numberOfTaskManagers, DispatcherGateway dispatcherGateway, FiniteDuration timeLeft) throws ExecutionException, InterruptedException { FutureUtils.retrySuccessfulWithDelay( () -> dispatcherGateway.requestClusterOverview(Time.milliseconds(timeLeft.toMillis())), Time.milliseconds(50L), org.apache.flink.api.common.time.Deadline.fromNow(Duration.ofMillis(timeLeft.toMillis())), clusterOverview -> clusterOverview.getNumTaskManagersConnected() >= numberOfTaskManagers, new ScheduledExecutorServiceAdapter(Executors.newSingleThreadScheduledExecutor())) .get(); }
public static ClusterOverviewWithVersion fromStatusOverview(ClusterOverview statusOverview, String version, String commitId) { return new ClusterOverviewWithVersion( statusOverview.getNumTaskManagersConnected(), statusOverview.getNumSlotsTotal(), statusOverview.getNumSlotsAvailable(), statusOverview.getNumJobsRunningOrPending(), statusOverview.getNumJobsFinished(), statusOverview.getNumJobsCancelled(), statusOverview.getNumJobsFailed(), version, commitId); }
@Override public String toString() { return "StatusOverview {" + "numTaskManagersConnected=" + numTaskManagersConnected + ", numSlotsTotal=" + numSlotsTotal + ", numSlotsAvailable=" + numSlotsAvailable + ", numJobsRunningOrPending=" + getNumJobsRunningOrPending() + ", numJobsFinished=" + getNumJobsFinished() + ", numJobsCancelled=" + getNumJobsCancelled() + ", numJobsFailed=" + getNumJobsFailed() + '}'; } }
@Override public int hashCode() { int result = super.hashCode(); result = 31 * result + (version != null ? version.hashCode() : 0); result = 31 * result + (commitId != null ? commitId.hashCode() : 0); return result; } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } if (!super.equals(o)) { return false; } ClusterOverviewWithVersion that = (ClusterOverviewWithVersion) o; return Objects.equals(version, that.getVersion()) && Objects.equals(commitId, that.getCommitId()); }
@Override public CompletableFuture<ClusterOverview> requestClusterOverview(Time timeout) { CompletableFuture<ResourceOverview> taskManagerOverviewFuture = resourceManagerGateway.requestResourceOverview(timeout); final List<CompletableFuture<Optional<JobStatus>>> optionalJobInformation = queryJobMastersForInformation( (JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobStatus(timeout)); CompletableFuture<Collection<Optional<JobStatus>>> allOptionalJobsFuture = FutureUtils.combineAll(optionalJobInformation); CompletableFuture<Collection<JobStatus>> allJobsFuture = allOptionalJobsFuture.thenApply(this::flattenOptionalCollection); final JobsOverview completedJobsOverview = archivedExecutionGraphStore.getStoredJobsOverview(); return allJobsFuture.thenCombine( taskManagerOverviewFuture, (Collection<JobStatus> runningJobsStatus, ResourceOverview resourceOverview) -> { final JobsOverview allJobsOverview = JobsOverview.create(runningJobsStatus).combine(completedJobsOverview); return new ClusterOverview(resourceOverview, allJobsOverview); }); }
gen.writeNumberField(ClusterOverview.FIELD_NAME_TASKMANAGERS, overview.getNumTaskManagersConnected()); gen.writeNumberField(ClusterOverview.FIELD_NAME_SLOTS_TOTAL, overview.getNumSlotsTotal()); gen.writeNumberField(ClusterOverview.FIELD_NAME_SLOTS_AVAILABLE, overview.getNumSlotsAvailable()); gen.writeNumberField(JobsOverview.FIELD_NAME_JOBS_RUNNING, overview.getNumJobsRunningOrPending()); gen.writeNumberField(JobsOverview.FIELD_NAME_JOBS_FINISHED, overview.getNumJobsFinished()); gen.writeNumberField(JobsOverview.FIELD_NAME_JOBS_CANCELLED, overview.getNumJobsCancelled()); gen.writeNumberField(JobsOverview.FIELD_NAME_JOBS_FAILED, overview.getNumJobsFailed()); gen.writeStringField(ClusterOverviewWithVersion.FIELD_NAME_VERSION, version); if (!commitID.equals(EnvironmentInformation.UNKNOWN)) {
@Override public String toString() { return "StatusOverview {" + "numTaskManagersConnected=" + numTaskManagersConnected + ", numSlotsTotal=" + numSlotsTotal + ", numSlotsAvailable=" + numSlotsAvailable + ", numJobsRunningOrPending=" + getNumJobsRunningOrPending() + ", numJobsFinished=" + getNumJobsFinished() + ", numJobsCancelled=" + getNumJobsCancelled() + ", numJobsFailed=" + getNumJobsFailed() + '}'; } }
@Override public int hashCode() { int result = super.hashCode(); result = 31 * result + (version != null ? version.hashCode() : 0); result = 31 * result + (commitId != null ? commitId.hashCode() : 0); return result; } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } if (!super.equals(o)) { return false; } ClusterOverviewWithVersion that = (ClusterOverviewWithVersion) o; return Objects.equals(version, that.getVersion()) && Objects.equals(commitId, that.getCommitId()); }
@Override public CompletableFuture<ClusterOverview> requestClusterOverview(Time timeout) { CompletableFuture<ResourceOverview> taskManagerOverviewFuture = resourceManagerGateway.requestResourceOverview(timeout); final List<CompletableFuture<Optional<JobStatus>>> optionalJobInformation = queryJobMastersForInformation( (JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobStatus(timeout)); CompletableFuture<Collection<Optional<JobStatus>>> allOptionalJobsFuture = FutureUtils.combineAll(optionalJobInformation); CompletableFuture<Collection<JobStatus>> allJobsFuture = allOptionalJobsFuture.thenApply(this::flattenOptionalCollection); final JobsOverview completedJobsOverview = archivedExecutionGraphStore.getStoredJobsOverview(); return allJobsFuture.thenCombine( taskManagerOverviewFuture, (Collection<JobStatus> runningJobsStatus, ResourceOverview resourceOverview) -> { final JobsOverview allJobsOverview = JobsOverview.create(runningJobsStatus).combine(completedJobsOverview); return new ClusterOverview(resourceOverview, allJobsOverview); }); }
@Override public boolean equals(Object obj) { if (this == obj) { return true; } else if (obj instanceof ClusterOverview) { ClusterOverview that = (ClusterOverview) obj; return this.numTaskManagersConnected == that.numTaskManagersConnected && this.numSlotsTotal == that.numSlotsTotal && this.numSlotsAvailable == that.numSlotsAvailable && this.getNumJobsRunningOrPending() == that.getNumJobsRunningOrPending() && this.getNumJobsFinished() == that.getNumJobsFinished() && this.getNumJobsCancelled() == that.getNumJobsCancelled() && this.getNumJobsFailed() == that.getNumJobsFailed(); } else { return false; } }
@Override public boolean equals(Object obj) { if (this == obj) { return true; } else if (obj instanceof ClusterOverview) { ClusterOverview that = (ClusterOverview) obj; return this.numTaskManagersConnected == that.numTaskManagersConnected && this.numSlotsTotal == that.numSlotsTotal && this.numSlotsAvailable == that.numSlotsAvailable && this.getNumJobsRunningOrPending() == that.getNumJobsRunningOrPending() && this.getNumJobsFinished() == that.getNumJobsFinished() && this.getNumJobsCancelled() == that.getNumJobsCancelled() && this.getNumJobsFailed() == that.getNumJobsFailed(); } else { return false; } }