@Override public Map<String, OptionalFailure<Object>> getAccumulators(JobID jobID, ClassLoader loader) throws Exception { AccessExecutionGraph executionGraph = miniCluster.getExecutionGraph(jobID).get(); Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorsSerialized = executionGraph.getAccumulatorsSerialized(); Map<String, OptionalFailure<Object>> result = new HashMap<>(accumulatorsSerialized.size()); for (Map.Entry<String, SerializedValue<OptionalFailure<Object>>> acc : accumulatorsSerialized.entrySet()) { result.put(acc.getKey(), acc.getValue().deserializeValue(loader)); } return result; }
@Override public Map<String, OptionalFailure<Object>> getAccumulators(JobID jobID, ClassLoader loader) throws Exception { AccessExecutionGraph executionGraph = miniCluster.getExecutionGraph(jobID).get(); Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorsSerialized = executionGraph.getAccumulatorsSerialized(); Map<String, OptionalFailure<Object>> result = new HashMap<>(accumulatorsSerialized.size()); for (Map.Entry<String, SerializedValue<OptionalFailure<Object>>> acc : accumulatorsSerialized.entrySet()) { result.put(acc.getKey(), acc.getValue().deserializeValue(loader)); } return result; }
@Override public Map<String, OptionalFailure<Object>> getAccumulators(JobID jobID, ClassLoader loader) throws Exception { AccessExecutionGraph executionGraph = miniCluster.getExecutionGraph(jobID).get(); Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorsSerialized = executionGraph.getAccumulatorsSerialized(); Map<String, OptionalFailure<Object>> result = new HashMap<>(accumulatorsSerialized.size()); for (Map.Entry<String, SerializedValue<OptionalFailure<Object>>> acc : accumulatorsSerialized.entrySet()) { result.put(acc.getKey(), acc.getValue().deserializeValue(loader)); } return result; }
private static JobAccumulatorsInfo createJobAccumulatorsInfo(AccessExecutionGraph graph, boolean includeSerializedValue) { StringifiedAccumulatorResult[] stringifiedAccs = graph.getAccumulatorResultsStringified(); List<JobAccumulatorsInfo.UserTaskAccumulator> userTaskAccumulators = new ArrayList<>(stringifiedAccs.length); for (StringifiedAccumulatorResult acc : stringifiedAccs) { userTaskAccumulators.add( new JobAccumulatorsInfo.UserTaskAccumulator( acc.getName(), acc.getType(), acc.getValue())); } JobAccumulatorsInfo accumulatorsInfo; if (includeSerializedValue) { Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserTaskAccumulators = graph.getAccumulatorsSerialized(); accumulatorsInfo = new JobAccumulatorsInfo(Collections.emptyList(), userTaskAccumulators, serializedUserTaskAccumulators); } else { accumulatorsInfo = new JobAccumulatorsInfo(Collections.emptyList(), userTaskAccumulators, Collections.emptyMap()); } return accumulatorsInfo; } }
private static JobAccumulatorsInfo createJobAccumulatorsInfo(AccessExecutionGraph graph, boolean includeSerializedValue) { StringifiedAccumulatorResult[] stringifiedAccs = graph.getAccumulatorResultsStringified(); List<JobAccumulatorsInfo.UserTaskAccumulator> userTaskAccumulators = new ArrayList<>(stringifiedAccs.length); for (StringifiedAccumulatorResult acc : stringifiedAccs) { userTaskAccumulators.add( new JobAccumulatorsInfo.UserTaskAccumulator( acc.getName(), acc.getType(), acc.getValue())); } JobAccumulatorsInfo accumulatorsInfo; if (includeSerializedValue) { Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserTaskAccumulators = graph.getAccumulatorsSerialized(); accumulatorsInfo = new JobAccumulatorsInfo(Collections.emptyList(), userTaskAccumulators, serializedUserTaskAccumulators); } else { accumulatorsInfo = new JobAccumulatorsInfo(Collections.emptyList(), userTaskAccumulators, Collections.emptyMap()); } return accumulatorsInfo; } }
private static JobAccumulatorsInfo createJobAccumulatorsInfo(AccessExecutionGraph graph, boolean includeSerializedValue) { StringifiedAccumulatorResult[] stringifiedAccs = graph.getAccumulatorResultsStringified(); List<JobAccumulatorsInfo.UserTaskAccumulator> userTaskAccumulators = new ArrayList<>(stringifiedAccs.length); for (StringifiedAccumulatorResult acc : stringifiedAccs) { userTaskAccumulators.add( new JobAccumulatorsInfo.UserTaskAccumulator( acc.getName(), acc.getType(), acc.getValue())); } JobAccumulatorsInfo accumulatorsInfo; if (includeSerializedValue) { Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserTaskAccumulators = graph.getAccumulatorsSerialized(); accumulatorsInfo = new JobAccumulatorsInfo(Collections.emptyList(), userTaskAccumulators, serializedUserTaskAccumulators); } else { accumulatorsInfo = new JobAccumulatorsInfo(Collections.emptyList(), userTaskAccumulators, Collections.emptyMap()); } return accumulatorsInfo; } }
/** * Creates the {@link JobResult} from the given {@link AccessExecutionGraph} which * must be in a globally terminal state. * * @param accessExecutionGraph to create the JobResult from * @return JobResult of the given AccessExecutionGraph */ public static JobResult createFrom(AccessExecutionGraph accessExecutionGraph) { final JobID jobId = accessExecutionGraph.getJobID(); final JobStatus jobStatus = accessExecutionGraph.getState(); checkArgument( jobStatus.isGloballyTerminalState(), "The job " + accessExecutionGraph.getJobName() + '(' + jobId + ") is not in a globally " + "terminal state. It is in state " + jobStatus + '.'); final JobResult.Builder builder = new JobResult.Builder(); builder.jobId(jobId); final long netRuntime = accessExecutionGraph.getStatusTimestamp(jobStatus) - accessExecutionGraph.getStatusTimestamp(JobStatus.CREATED); // guard against clock changes final long guardedNetRuntime = Math.max(netRuntime, 0L); builder.netRuntime(guardedNetRuntime); builder.accumulatorResults(accessExecutionGraph.getAccumulatorsSerialized()); if (jobStatus != JobStatus.FINISHED) { final ErrorInfo errorInfo = accessExecutionGraph.getFailureInfo(); if (errorInfo != null) { builder.serializedThrowable(errorInfo.getException()); } } return builder.build(); }
builder.accumulatorResults(accessExecutionGraph.getAccumulatorsSerialized());
builder.accumulatorResults(accessExecutionGraph.getAccumulatorsSerialized());