private static List<JobID> getRunningJobs(ClusterClient<?> client) throws Exception { Collection<JobStatusMessage> statusMessages = client.listJobs().get(); return statusMessages.stream() .filter(status -> !status.getJobState().isGloballyTerminalState()) .map(JobStatusMessage::getJobId) .collect(Collectors.toList()); }
@Override public void close() throws Exception { // Free cluster resources clusterClient.cancel(jobId); // cancel() is non-blocking so do this to make sure the job finished CompletableFuture<JobStatus> jobStatusFuture = FutureUtils.retrySuccessfulWithDelay( () -> clusterClient.getJobStatus(jobId), Time.milliseconds(50), deadline, (jobStatus) -> jobStatus.equals(JobStatus.CANCELED), TestingUtils.defaultScheduledExecutor()); assertEquals( JobStatus.CANCELED, jobStatusFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS)); } }
gen.writeStringField("jid", jobID.toString()); gen.writeStringField("name", "testjob"); gen.writeStringField("state", JobStatus.FINISHED.name());
gen.writeStringField("name", graph.getJobName()); gen.writeBooleanField("isStoppable", graph.isStoppable()); gen.writeStringField("state", graph.getState().name()); final long jobEndTime = graph.getState().isGloballyTerminalState() ? graph.getStatusTimestamp(graph.getState()) : -1L; gen.writeNumberField("start-time", jobStartTime); for (JobStatus status : JobStatus.values()) { gen.writeNumberField(status.name(), graph.getStatusTimestamp(status));
private static JobDetailsInfo createJobDetailsInfo(AccessExecutionGraph executionGraph, @Nullable MetricFetcher<?> metricFetcher) { final long now = System.currentTimeMillis(); final long startTime = executionGraph.getStatusTimestamp(JobStatus.CREATED); final long endTime = executionGraph.getState().isGloballyTerminalState() ? executionGraph.getStatusTimestamp(executionGraph.getState()) : -1L; final long duration = (endTime > 0L ? endTime : now) - startTime; final Map<JobStatus, Long> timestamps = new HashMap<>(JobStatus.values().length); for (JobStatus jobStatus : JobStatus.values()) { timestamps.put(jobStatus, executionGraph.getStatusTimestamp(jobStatus));
@Override public Long getValue() { final JobStatus status = eg.getState(); if (status == JobStatus.RUNNING) { // running right now - report the uptime final long runningTimestamp = eg.getStatusTimestamp(JobStatus.RUNNING); // we use 'Math.max' here to avoid negative timestamps when clocks change return Math.max(System.currentTimeMillis() - runningTimestamp, 0); } else if (status.isTerminalState()) { // not running any more -> finished or not on leader return NO_LONGER_RUNNING; } else { // not yet running or not up at the moment return 0L; } } }
private boolean transitionState(JobStatus current, JobStatus newState, Throwable error) { // consistency check if (current.isTerminalState()) { String message = "Job is trying to leave terminal state " + current; LOG.error(message); throw new IllegalStateException(message); } // now do the actual state transition if (STATE_UPDATER.compareAndSet(this, current, newState)) { LOG.info("Job {} ({}) switched from state {} to {}.", getJobName(), getJobID(), current, newState, error); stateTimestamps[newState.ordinal()] = System.currentTimeMillis(); notifyJobStatusChange(newState, error); return true; } else { return false; } }
private void assignExecutionGraph( ExecutionGraph newExecutionGraph, JobManagerJobMetricGroup newJobManagerJobMetricGroup) { checkState(executionGraph == null || JobStatus.CREATED.equals(executionGraph.getState()) || executionGraph.getState().isTerminalState(), "The job state is " + (executionGraph == null ? null : executionGraph.getState())); checkState(jobManagerJobMetricGroup == null); executionGraph = newExecutionGraph; jobManagerJobMetricGroup = newJobManagerJobMetricGroup; checkState(jobStatusListener == null); // register self as job status change listener jobStatusListener = new JobManagerJobStatusListener(); executionGraph.registerJobStatusListener(jobStatusListener); setupGraphManager(); }
@Override public long getStatusTimestamp(JobStatus status) { return this.stateTimestamps[status.ordinal()]; }
JobStatus state = JobStatus.valueOf(job.get("state").asText());
@Override public int hashCode() { int result = jobId.hashCode(); result = 31 * result + jobName.hashCode(); result = 31 * result + (int) (startTime ^ (startTime >>> 32)); result = 31 * result + (int) (endTime ^ (endTime >>> 32)); result = 31 * result + status.hashCode(); result = 31 * result + (int) (lastUpdateTime ^ (lastUpdateTime >>> 32)); result = 31 * result + Arrays.hashCode(tasksPerState); result = 31 * result + numTasks; return result; }
gen.writeStringField("name", graph.getJobName()); gen.writeBooleanField("isStoppable", graph.isStoppable()); gen.writeStringField("state", graph.getState().name()); final long jobEndTime = graph.getState().isGloballyTerminalState() ? graph.getStatusTimestamp(graph.getState()) : -1L; gen.writeNumberField("start-time", jobStartTime); for (JobStatus status : JobStatus.values()) { gen.writeNumberField(status.name(), graph.getStatusTimestamp(status));
private static JobDetailsInfo createJobDetailsInfo(AccessExecutionGraph executionGraph, @Nullable MetricFetcher<?> metricFetcher) { final long now = System.currentTimeMillis(); final long startTime = executionGraph.getStatusTimestamp(JobStatus.CREATED); final long endTime = executionGraph.getState().isGloballyTerminalState() ? executionGraph.getStatusTimestamp(executionGraph.getState()) : -1L; final long duration = (endTime > 0L ? endTime : now) - startTime; final Map<JobStatus, Long> timestamps = new HashMap<>(JobStatus.values().length); for (JobStatus jobStatus : JobStatus.values()) { timestamps.put(jobStatus, executionGraph.getStatusTimestamp(jobStatus));
@Override public Long getValue() { final JobStatus status = eg.getState(); if (status == JobStatus.RUNNING) { // running right now - report the uptime final long runningTimestamp = eg.getStatusTimestamp(JobStatus.RUNNING); // we use 'Math.max' here to avoid negative timestamps when clocks change return Math.max(System.currentTimeMillis() - runningTimestamp, 0); } else if (status.isTerminalState()) { // not running any more -> finished or not on leader return NO_LONGER_RUNNING; } else { // not yet running or not up at the moment return 0L; } } }
private boolean transitionState(JobStatus current, JobStatus newState, Throwable error) { // consistency check if (current.isTerminalState()) { String message = "Job is trying to leave terminal state " + current; LOG.error(message); throw new IllegalStateException(message); } // now do the actual state transition if (STATE_UPDATER.compareAndSet(this, current, newState)) { LOG.info("Job {} ({}) switched from state {} to {}.", getJobName(), getJobID(), current, newState, error); stateTimestamps[newState.ordinal()] = System.currentTimeMillis(); notifyJobStatusChange(newState, error); return true; } else { return false; } }
@Override public long getStatusTimestamp(JobStatus status) { return this.stateTimestamps[status.ordinal()]; }
@Override public JobDetails deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { JsonNode rootNode = jsonParser.readValueAsTree(); JobID jobId = JobID.fromHexString(rootNode.get(FIELD_NAME_JOB_ID).textValue()); String jobName = rootNode.get(FIELD_NAME_JOB_NAME).textValue(); long startTime = rootNode.get(FIELD_NAME_START_TIME).longValue(); long endTime = rootNode.get(FIELD_NAME_END_TIME).longValue(); long duration = rootNode.get(FIELD_NAME_DURATION).longValue(); JobStatus jobStatus = JobStatus.valueOf(rootNode.get(FIELD_NAME_STATUS).textValue()); long lastUpdateTime = rootNode.get(FIELD_NAME_LAST_MODIFICATION).longValue(); JsonNode tasksNode = rootNode.get("tasks"); int numTasks = tasksNode.get(FIELD_NAME_TOTAL_NUMBER_TASKS).intValue(); int[] numVerticesPerExecutionState = new int[ExecutionState.values().length]; for (ExecutionState executionState : ExecutionState.values()) { numVerticesPerExecutionState[executionState.ordinal()] = tasksNode.get(executionState.name().toLowerCase()).intValue(); } return new JobDetails( jobId, jobName, startTime, endTime, duration, jobStatus, lastUpdateTime, numVerticesPerExecutionState, numTasks); } }
@Override public int hashCode() { int result = jobId.hashCode(); result = 31 * result + jobName.hashCode(); result = 31 * result + (int) (startTime ^ (startTime >>> 32)); result = 31 * result + (int) (endTime ^ (endTime >>> 32)); result = 31 * result + status.hashCode(); result = 31 * result + (int) (lastUpdateTime ^ (lastUpdateTime >>> 32)); result = 31 * result + Arrays.hashCode(tasksPerState); result = 31 * result + numTasks; return result; }