@Override public Map<String, OptionalFailure<Object>> getAccumulators(JobID jobID, ClassLoader loader) throws Exception { AccessExecutionGraph executionGraph = miniCluster.getExecutionGraph(jobID).get(); Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorsSerialized = executionGraph.getAccumulatorsSerialized(); Map<String, OptionalFailure<Object>> result = new HashMap<>(accumulatorsSerialized.size()); for (Map.Entry<String, SerializedValue<OptionalFailure<Object>>> acc : accumulatorsSerialized.entrySet()) { result.put(acc.getKey(), acc.getValue().deserializeValue(loader)); } return result; }
public StreamMockEnvironment( Configuration jobConfig, Configuration taskConfig, ExecutionConfig executionConfig, long memorySize, MockInputSplitProvider inputSplitProvider, int bufferSize, TaskStateManager taskStateManager) { this( new JobID(), new ExecutionAttemptID(0L, 0L), jobConfig, taskConfig, executionConfig, memorySize, inputSplitProvider, bufferSize, taskStateManager); }
final JobInformation jobInformation = new DummyJobInformation( jobId, jobName); ExecutionGraph eg = new ExecutionGraph( jobInformation, TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), AkkaUtils.getDefaultTimeout(), new NoRestartStrategy(), new RestartAllStrategy.Factory(), new TestingSlotProvider(ignored -> new CompletableFuture<>()), ExecutionGraph.class.getClassLoader(), VoidBlobWriter.getInstance(), AkkaUtils.getDefaultTimeout()); try { eg.attachJobGraph(jobVertices); ExecutionJobVertex execSourceVertex = eg.getJobVertex(sourceVertex.getID()); ExecutionJobVertex execMapVertex = eg.getJobVertex(mapVertex.getID()); ExecutionJobVertex execSinkVertex = eg.getJobVertex(sinkVertex.getID()); assertEquals(0, execSourceVertex.getInputs().size()); assertEquals(1, execMapVertex.getInputs().size()); assertEquals(4, execMapVertex.getParallelism()); ExecutionVertex[] mapTaskVertices = execMapVertex.getTaskVertices(); assertEquals(1, mapTaskVertex.getNumberOfInputs());
when(network.getTaskEventDispatcher()).thenReturn(taskEventDispatcher); JobInformation jobInformation = new JobInformation( new JobID(), "Job Name", Collections.emptyList()); TaskInformation taskInformation = new TaskInformation( new JobVertexID(), "Test Task", jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0,
public CompletableFuture<Void> scheduleForExecution() { final ExecutionGraph executionGraph = getVertex().getExecutionGraph(); final SlotProvider resourceProvider = executionGraph.getSlotProvider(); final boolean allowQueued = executionGraph.isQueuedSchedulingAllowed(); return scheduleForExecution( resourceProvider, allowQueued, LocationPreferenceConstraint.ANY); }
public void fail(Throwable t) { for (ExecutionVertex ev : getTaskVertices()) { ev.fail(t); } }
/** * Simply forward this notification */ void notifyStateTransition(Execution execution, ExecutionState newState, Throwable error) { // only forward this notification if the execution is still the current execution // otherwise we have an outdated execution if (currentExecution == execution) { getExecutionGraph().notifyExecutionChange(execution, newState, error); } }
/** * Cancels all currently running vertex executions. */ public void cancel() { for (ExecutionVertex ev : getTaskVertices()) { ev.cancel(); } }
/** * Gets the location where the latest completed/canceled/failed execution of the vertex's * task happened. * * @return The latest prior execution location, or null, if there is none, yet. */ public TaskManagerLocation getLatestPriorLocation() { ArchivedExecution latestPriorExecution = getLatestPriorExecution(); return latestPriorExecution != null ? latestPriorExecution.getAssignedResourceLocation() : null; }
executionConfig.setFailTaskOnCheckpointError(failOnCheckpointErrors); JobInformation jobInformation = new JobInformation( new JobID(), "test job name", Collections.emptyList()); TaskInformation taskInformation = new TaskInformation( new JobVertexID(), "test task name", jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0,
public void fail(Throwable t) { for (ExecutionVertex ev : getTaskVertices()) { ev.fail(t); } }
/** * Simply forward this notification. */ void notifyStateTransition(Execution execution, ExecutionState newState, Throwable error) { // only forward this notification if the execution is still the current execution // otherwise we have an outdated execution if (currentExecution == execution) { getExecutionGraph().notifyExecutionChange(execution, newState, error); } }
ExecutionAttemptID executionAttemptID = new ExecutionAttemptID(23L, 24L); TestCheckpointResponder checkpointResponderMock = new TestCheckpointResponder();
JobInformation jobInformation = new JobInformation( new JobID(), "test job name", Collections.emptyList()); TaskInformation taskInformation = new TaskInformation( jobVertexID, "test task name", jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0,
/** * Simply forward this notification. */ void notifyStateTransition(Execution execution, ExecutionState newState, Throwable error) { // only forward this notification if the execution is still the current execution // otherwise we have an outdated execution if (currentExecution == execution) { getExecutionGraph().notifyExecutionChange(execution, newState, error); } }
final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID(); final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(42L, 4711L); final CheckpointMetrics checkpointMetrics = new CheckpointMetrics();
final long checkpointTimestamp = 0L; final JobInformation jobInformation = new JobInformation( new JobID(), "Test Job", Collections.emptyList()); final TaskInformation taskInformation = new TaskInformation( new JobVertexID(), "Test Task", jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0,
new ExecutionAttemptID(1L, 2L), mock(TaskLocalStateStoreImpl.class), null,
new ExecutionAttemptID(1L, 2L), mock(TaskLocalStateStoreImpl.class), null,
new ExecutionAttemptID(), new TestTaskLocalStateStore(), jobManagerTaskRestore,