@Override public void onTaskCommitCompletion(Task task) { if (GobblinMetrics.isEnabled(task.getTaskState().getWorkunit())) { // Update record-level metrics after the task is done task.updateRecordMetrics(); task.updateByteMetrics(); } // Cancel the task state reporter associated with this task. The reporter might // not be found for the given task because the task fails before the task is // registered. So we need to make sure the reporter exists before calling cancel. if (this.scheduledReporters.containsKey(task.getTaskId())) { this.scheduledReporters.remove(task.getTaskId()).cancel(false); } LOGGER.info(String .format("Task %s completed in %dms with state %s", task.getTaskId(), task.getTaskState().getTaskDuration(), task.getTaskState().getWorkingState())); }
@Override public void onTaskRunCompletion(Task task) { try { // Check the task state and handle task retry if task failed and // it has not reached the maximum number of retries WorkUnitState.WorkingState state = task.getTaskState().getWorkingState(); if (state == WorkUnitState.WorkingState.FAILED && task.getRetryCount() < this.maxTaskRetries) { this.taskExecutor.retry(task); return; } } catch (Throwable t) { LOG.error("Failed to process a task completion callback", t); } // Mark the completion of this task task.markTaskCompletion(); }
/** * A method that shuts down all running tasks managed by this instance. * TODO: Call this from the right place. */ public void shutdownTasks() throws InterruptedException { log.info("Shutting down tasks"); for (Task task: this.tasks) { task.shutdown(); } for (Task task : this.tasks) { task.awaitShutdown(1000); } for (Task task : this.tasks) { if (task.cancel()) { log.info("Task {} cancelled.", task.getTaskId()); } else { log.info("Task {} could not be cancelled.", task.getTaskId()); } } }
protected void updateTaskMetrics() { if (GobblinMetrics.isEnabled(this.task.getTaskState().getWorkunit())) { this.task.updateRecordMetrics(); this.task.updateByteMetrics(); } } }
Task realTask = new Task(mockTaskContext, mockTaskStateTracker, taskExecutor, Optional.<CountDownLatch> absent()); Task task = spy(realTask); doNothing().when(task).submitTaskCommittedEvent(); task.run(); task.commit(); Assert.assertEquals(task.getTaskState().getWorkingState(), WorkUnitState.WorkingState.FAILED); task.run(); task.commit(); Assert.assertEquals(task.getTaskState().getWorkingState(), WorkUnitState.WorkingState.SUCCESSFUL);
@Test public void testControlMessages() throws Exception { MyExtractor extractor = new MyExtractor(new StreamEntity[]{new RecordEnvelope<>("a"), new BasicTestControlMessage("1"), new RecordEnvelope<>("b"), new BasicTestControlMessage("2")}); MyConverter converter = new MyConverter(); MyDataWriter writer = new MyDataWriter(); Task task = setupTask(extractor, writer, converter); task.run(); task.commit(); Assert.assertEquals(task.getTaskState().getWorkingState(), WorkUnitState.WorkingState.SUCCESSFUL); Assert.assertEquals(converter.records, Lists.newArrayList("a", "b")); Assert.assertEquals(converter.messages, Lists.newArrayList(new BasicTestControlMessage("1"), new BasicTestControlMessage("2"))); Assert.assertEquals(writer.records, Lists.newArrayList("a", "b")); Assert.assertEquals(writer.messages, Lists.newArrayList(new BasicTestControlMessage("1"), new BasicTestControlMessage("2"))); }
/** * Retry a failed {@link Task}. * * @param task failed {@link Task} to be retried */ public void retry(Task task) { if (GobblinMetrics.isEnabled(task.getTaskState().getWorkunit()) && task.getTaskState().contains(ConfigurationKeys.FORK_BRANCHES_KEY)) { // Adjust metrics to clean up numbers from the failed task task.getTaskState() .adjustJobMetricsOnRetry(task.getTaskState().getPropAsInt(ConfigurationKeys.FORK_BRANCHES_KEY)); } // Task retry interval increases linearly with number of retries long interval = task.getRetryCount() * this.retryIntervalInSeconds; // Schedule the retry of the failed task this.taskExecutor.schedule(new TrackingTask(task, interval, TimeUnit.SECONDS), interval, TimeUnit.SECONDS); LOG.info(String.format("Scheduled retry of failed task %s to run in %d seconds", task.getTaskId(), interval)); task.incrementRetryCount(); }
String taskId = task.getTaskId(); log.info("Writing task state for task " + task.getTaskId()); taskStateStore.put(task.getJobId(), task.getTaskId() + AbstractJobLauncher.TASK_STATE_STORE_TABLE_SUFFIX, task.getTaskState()); if (task.getTaskState().getWorkingState() == WorkUnitState.WorkingState.FAILED) { hasTaskFailure = true; if (task.getTaskState().contains(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)) { log.error(String.format("Task %s failed due to exception: %s", task.getTaskId(), task.getTaskState().getProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY))); if (task.getTaskState().getWorkingState() == WorkUnitState.WorkingState.SUCCESSFUL || task.getTaskState().getWorkingState() == WorkUnitState.WorkingState.COMMITTED) { taskStateStore.put(task.getJobId(), task.getTaskId() + TASK_STATE_STORE_SUCCESS_MARKER_SUFFIX, task.getTaskState());
Task task = new Task(mockTaskContext, mockTaskStateTracker, taskExecutor, Optional.<CountDownLatch>absent()); log.info("Observed committed watermark: {}", watermark); log.info("Task progress: {}", task.getProgress()); task.shutdown(); log.info("Shutting down task now"); boolean success = task.awaitShutdown(30000); Assert.assertTrue(success, "Task should shutdown in 3 seconds"); log.info("Task done waiting to shutdown {}", success); .getCommittedWatermarks(CheckpointableWatermark.class, ImmutableList.of("default")))); task.commit();
@Override public State getExecutionMetadata() { return getTaskState(); }
public TrackingTask(Task task, long interval, TimeUnit timeUnit) { long now = System.currentTimeMillis(); long timeToRun = now + timeUnit.toMillis(interval); LOG.debug(String.format("Task %s queued to run %s.", task.getTaskId(), timeToRun <= now ? "now" : "at " + timeToRun)); queuedTasks.putIfAbsent(task.getTaskId(), timeToRun); this.underlyingTask = task; }
private ArrayList<ArrayList<Object>> runTaskAndGetResults(TaskState taskState, int numRecords, int numForks, ForkOperator mockForkOperator) throws Exception { ArrayList<ArrayList<Object>> recordCollectors = new ArrayList<>(numForks); for (int i=0; i < numForks; ++i) { recordCollectors.add(new ArrayList<Object>()); } TaskContext mockTaskContext = getMockTaskContext(taskState, new StringExtractor(numRecords), recordCollectors, mockForkOperator); // Create a mock TaskStateTracker TaskStateTracker mockTaskStateTracker = mock(TaskStateTracker.class); // Create a TaskExecutor - a real TaskExecutor must be created so a Fork is run in a separate thread TaskExecutor taskExecutor = new TaskExecutor(new Properties()); // Create the Task Task task = new Task(mockTaskContext, mockTaskStateTracker, taskExecutor, Optional.<CountDownLatch>absent()); // Run and commit task.run(); task.commit(); return recordCollectors; }
private void onStart(long startTime) { Long queueTime = queuedTasks.remove(this.underlyingTask.getTaskId()); long workUnitCreationTime = this.underlyingTask.getTaskContext().getTaskState().getPropAsLong(ConfigurationKeys.WORK_UNIT_CREATION_TIME_IN_MILLIS, 0); long timeInQueue = startTime - queueTime; long timeSinceWorkUnitCreation = startTime - workUnitCreationTime; taskCreateAndRunTimer.update(timeSinceWorkUnitCreation, TimeUnit.MILLISECONDS); LOG.debug(String.format("Task %s started. Saving queued time of %d ms to history.", underlyingTask.getTaskId(), timeInQueue)); queuedTaskTimeHistorical.putIfAbsent(System.currentTimeMillis(), timeInQueue); runningTaskCount.inc(); } }
this.taskMode = getExecutionModel(this.taskState); this.recordsPulled = new AtomicLong(0); this.lastRecordPulledTimestampMillis = 0; if (isStreamingTask()) { Extractor underlyingExtractor = this.taskContext.getRawSourceExtractor(); if (!(underlyingExtractor instanceof StreamingExtractor)) {
@Override public void onTaskRunCompletion(Task task) { task.markTaskCompletion(); }
Task realTask = new Task(mockTaskContext, mockTaskStateTracker, taskExecutor, Optional.<CountDownLatch> absent()); Task task = spy(realTask); doNothing().when(task).submitTaskCommittedEvent();
if (Task.getExecutionModel(ConfigUtils.configToState(jobConfig)).equals(ExecutionModel.STREAMING)) { jobConfigBuilder.setRebalanceRunningTask(true);
private Task createTaskRunnable(WorkUnitState workUnitState, CountDownLatch countDownLatch) { Optional<TaskFactory> taskFactoryOpt = TaskUtils.getTaskFactory(workUnitState); if (taskFactoryOpt.isPresent()) { return new TaskIFaceWrapper(taskFactoryOpt.get().createTask(new TaskContext(workUnitState)), new TaskContext(workUnitState), countDownLatch, this.taskStateTracker); } else { return new Task(new TaskContext(workUnitState), this.taskStateTracker, this.taskExecutor, Optional.of(countDownLatch)); } }
@Override public void run() { long startTime = System.currentTimeMillis(); onStart(startTime); try { this.underlyingTask.run(); successfulTaskCount.mark();; } catch (Exception e) { failedTaskCount.mark(); throw e; } finally { runningTaskCount.dec(); } }
Task realTask = new Task(mockTaskContext, mockTaskStateTracker, taskExecutor, Optional.<CountDownLatch> absent()); Task task = spy(realTask); doNothing().when(task).submitTaskCommittedEvent(); task.run(); task.commit(); Assert.assertEquals(task.getTaskState().getWorkingState(), WorkUnitState.WorkingState.SUCCESSFUL);