/** * Subclasses can override this method to do whatever processing on the {@link JobState} and its * associated {@link TaskState}s, e.g., aggregate task-level metrics into job-level metrics. */ protected void postProcessJobState(JobState jobState) { postProcessTaskStates(jobState.getTaskStates()); }
public TaskIFaceWrapper(TaskIFace underlyingTask, TaskContext taskContext, CountDownLatch countDownLatch, TaskStateTracker taskStateTracker) { super(); this.underlyingTask = underlyingTask; this.taskContext = taskContext; this.jobId = taskContext.getTaskState().getJobId(); this.taskId = taskContext.getTaskState().getTaskId(); this.countDownLatch = countDownLatch; this.taskStateTracker = taskStateTracker; }
@Override public void onJobCancellation(JobContext jobContext) { this.jobExecutionEventSubmitter.submitJobExecutionEvents(jobContext.getJobState()); } }
protected void updateTaskMetrics() { if (GobblinMetrics.isEnabled(this.task.getTaskState().getWorkunit())) { this.task.updateRecordMetrics(); this.task.updateByteMetrics(); } } }
@Override public WorkUnitState.WorkingState getWorkingState() { return getTaskState().getWorkingState(); }
public SleepingTask(TaskContext taskContext) { super(taskContext); sleepTime = taskContext.getTaskState().getPropAsLong("data.publisher.sleep.time.in.seconds", 10L); }
private String createDatasetUrn(Map<String, DatasetState> datasetStatesByUrns, TaskState taskState) { String datasetUrn = taskState.getProp(ConfigurationKeys.DATASET_URN_KEY, ConfigurationKeys.DEFAULT_DATASET_URN); if (!datasetStatesByUrns.containsKey(datasetUrn)) { DatasetState datasetState = newDatasetState(false); datasetState.setDatasetUrn(datasetUrn); datasetStatesByUrns.put(datasetUrn, datasetState); } return datasetUrn; }
public JobState(State properties,String jobName, String jobId) { super(properties); this.jobName = jobName; this.jobId = jobId; this.setId(jobId); }
/** * Get the number of records written by all the writers * @return Sum of the writer records written count across all tasks */ public static long getProcessedCount(List<TaskState> taskStates) { long value = 0; for (TaskState taskState : taskStates) { value += taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN, 0); } return value; }
/** * Submits metadata about a given {@link JobState} and each of its {@link TaskState}s. This method will submit a * single event for the {@link JobState} called {@link #JOB_STATE_EVENT}. It will submit an event for each * {@link TaskState} called {@link #TASK_STATE_EVENT}. * * @param jobState is the {@link JobState} to emit events for */ public void submitJobExecutionEvents(JobState jobState) { submitJobStateEvent(jobState); submitTaskStateEvents(jobState); }
@Override public StreamEntity<T> buildClone() { return new BasicTestControlMessage(this.id); } }
/** * Get a pre-fork {@link RowLevelPolicyChecker} for executing row-level * {@link org.apache.gobblin.qualitychecker.row.RowLevelPolicy}. * * @return a {@link RowLevelPolicyChecker} */ public RowLevelPolicyChecker getRowLevelPolicyChecker() throws Exception { return getRowLevelPolicyChecker(-1); }
@Override public void write(DataOutput out) throws IOException { write(out, true, true); }
/** * Get the list of pre-fork {@link RecordStreamProcessor}s. * * @return list (possibly empty) of {@link RecordStreamProcessor}s */ public List<RecordStreamProcessor<?, ?, ?, ?>> getRecordStreamProcessors() { return getRecordStreamProcessors(-1, this.taskState); }
/** * Get a new {@link BoundedBlockingRecordQueue.Builder}. * * @param <T> record type * @return a new {@link BoundedBlockingRecordQueue.Builder} */ public static <T> Builder<T> newBuilder() { return new Builder<>(); }
public void addSkippedTaskStates(Collection<TaskState> taskStates) { for (TaskState taskState : taskStates) { addSkippedTaskState(taskState); } }
/** * Write a single {@link JobState} to json document. * * @param jsonWriter {@link com.google.gson.stream.JsonWriter} * @param jobState {@link JobState} to write to json document * @throws IOException */ private void writeJobState(JsonWriter jsonWriter, JobState jobState) throws IOException { jobState.toJson(jsonWriter, this.keepConfig); }
/** * Sets the {@link ConfigurationKeys#TASK_FAILURE_EXCEPTION_KEY} for each given {@link TaskState} to the given * {@link Throwable}. * * Make this method public as this exception catching routine can be reusable in other occasions as well. */ public static void setTaskFailureException(Collection<? extends WorkUnitState> taskStates, Throwable t) { for (WorkUnitState taskState : taskStates) { ((TaskState) taskState).setTaskFailureException(t); } }
@Override public DataWriter<Integer> build() throws IOException { return new DummyDataWriter(); } }
@Override public void onJobCompletion(JobContext jobContext) { this.jobExecutionEventSubmitter.submitJobExecutionEvents(jobContext.getJobState()); }