/** * If not already present, set the {@link ConfigurationKeys#TASK_FAILURE_EXCEPTION_KEY} to a {@link String} * representation of the given {@link Throwable}. */ public void setTaskFailureException(Throwable taskFailureException) { if (!this.contains(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)) { this.setProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY, Throwables.getStackTraceAsString(taskFailureException)); } }
/** * Append values for the given key from all {@link TaskState}s * @param sb a {@link StringBuffer} to hold the output * @param key the key of the values to retrieve */ private static void appendTaskStateValues(List<TaskState> taskStates, StringBuffer sb, String key) { // Add task failure messages in a group followed by task failure exceptions for (TaskState taskState : taskStates) { if (taskState.contains(key)) { if (sb.length() != 0) { sb.append(","); } sb.append(taskState.getProp(key)); } } } }
@SuppressWarnings("unchecked") private Class<? extends DataPublisher> getTaskPublisherClass() throws ReflectiveOperationException { if (this.taskState.contains(ConfigurationKeys.TASK_DATA_PUBLISHER_TYPE)) { return (Class<? extends DataPublisher>) Class .forName(this.taskState.getProp(ConfigurationKeys.TASK_DATA_PUBLISHER_TYPE)); } return (Class<? extends DataPublisher>) Class.forName( this.taskState.getProp(ConfigurationKeys.DATA_PUBLISHER_TYPE, ConfigurationKeys.DEFAULT_DATA_PUBLISHER_TYPE)); }
ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.CONVERTER_CLASSES_KEY, index); if (!this.taskState.contains(converterClassKey)) { return Collections.emptyList();
ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.RECORD_STREAM_PROCESSOR_CLASSES_KEY, index); if (!this.taskState.contains(streamProcessorClassKey)) { return Collections.emptyList();
@Override public DataPublisher createDataPublisher(JobState.DatasetState datasetState) { EventBus eventBus = null; if (datasetState.getTaskStates().get(0).contains(EVENTBUS_ID_KEY)) { eventBus = TestingEventBuses.getEventBus(datasetState.getTaskStates().get(0).getProp(EVENTBUS_ID_KEY)); } return new Publisher(datasetState, eventBus); }
/** * Retry a failed {@link Task}. * * @param task failed {@link Task} to be retried */ public void retry(Task task) { if (GobblinMetrics.isEnabled(task.getTaskState().getWorkunit()) && task.getTaskState().contains(ConfigurationKeys.FORK_BRANCHES_KEY)) { // Adjust metrics to clean up numbers from the failed task task.getTaskState() .adjustJobMetricsOnRetry(task.getTaskState().getPropAsInt(ConfigurationKeys.FORK_BRANCHES_KEY)); } // Task retry interval increases linearly with number of retries long interval = task.getRetryCount() * this.retryIntervalInSeconds; // Schedule the retry of the failed task this.taskExecutor.schedule(new TrackingTask(task, interval, TimeUnit.SECONDS), interval, TimeUnit.SECONDS); LOG.info(String.format("Scheduled retry of failed task %s to run in %d seconds", task.getTaskId(), interval)); task.incrementRetryCount(); }
if (task.getTaskState().contains(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)) { log.error(String.format("Task %s failed due to exception: %s", task.getTaskId(), task.getTaskState().getProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)));
@Override public List<WorkUnit> getWorkunits(SourceState state) { int numTasks = state.getPropAsInt(NUM_TASKS_KEY); String eventBusId = state.getProp(EVENTBUS_ID_KEY); EventBus eventBus = TestingEventBuses.getEventBus(eventBusId); Map<String, SourceState> previousStates = state.getPreviousDatasetStatesByUrns(); for (Map.Entry<String, SourceState> entry : previousStates.entrySet()) { JobState.DatasetState datasetState = (JobState.DatasetState) entry.getValue(); for (TaskState taskState : datasetState.getTaskStates()) { if (taskState.contains(Task.PERSISTENT_STATE) && eventBus != null) { eventBus.post(new Event(PREVIOUS_STATE_EVENT, taskState.getPropAsInt(Task.PERSISTENT_STATE))); } } } List<WorkUnit> workUnits = Lists.newArrayList(); for (int i = 0; i < numTasks; i++) { workUnits.add(createWorkUnit(i, eventBusId)); } return workUnits; }
@Override public TaskIFace createTask(TaskContext taskContext) { String taskId = taskContext.getTaskState().getProp(TASK_ID_KEY); EventBus eventBus = null; if (taskContext.getTaskState().contains(EVENTBUS_ID_KEY)) { String eventbusId = taskContext.getTaskState().getProp(EVENTBUS_ID_KEY); eventBus = TestingEventBuses.getEventBus(eventbusId); } return new Task(taskContext, taskId, eventBus); }
if (this.contains(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)) { taskExecutionInfo.setFailureException(this.getProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY));
/** * If not already present, set the {@link ConfigurationKeys#TASK_FAILURE_EXCEPTION_KEY} to a {@link String} * representation of the given {@link Throwable}. */ public void setTaskFailureException(Throwable taskFailureException) { if (!this.contains(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)) { this.setProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY, Throwables.getStackTraceAsString(taskFailureException)); } }
/** * Append values for the given key from all {@link TaskState}s * @param sb a {@link StringBuffer} to hold the output * @param key the key of the values to retrieve */ private static void appendTaskStateValues(List<TaskState> taskStates, StringBuffer sb, String key) { // Add task failure messages in a group followed by task failure exceptions for (TaskState taskState : taskStates) { if (taskState.contains(key)) { if (sb.length() != 0) { sb.append(","); } sb.append(taskState.getProp(key)); } } } }
@SuppressWarnings("unchecked") private Class<? extends DataPublisher> getTaskPublisherClass() throws ReflectiveOperationException { if (this.taskState.contains(ConfigurationKeys.TASK_DATA_PUBLISHER_TYPE)) { return (Class<? extends DataPublisher>) Class .forName(this.taskState.getProp(ConfigurationKeys.TASK_DATA_PUBLISHER_TYPE)); } return (Class<? extends DataPublisher>) Class.forName( this.taskState.getProp(ConfigurationKeys.DATA_PUBLISHER_TYPE, ConfigurationKeys.DEFAULT_DATA_PUBLISHER_TYPE)); }
ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.CONVERTER_CLASSES_KEY, index); if (!this.taskState.contains(converterClassKey)) { return Collections.emptyList();
ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.RECORD_STREAM_PROCESSOR_CLASSES_KEY, index); if (!this.taskState.contains(streamProcessorClassKey)) { return Collections.emptyList();
/** * Retry a failed {@link Task}. * * @param task failed {@link Task} to be retried */ public void retry(Task task) { if (GobblinMetrics.isEnabled(task.getTaskState().getWorkunit()) && task.getTaskState().contains(ConfigurationKeys.FORK_BRANCHES_KEY)) { // Adjust metrics to clean up numbers from the failed task task.getTaskState() .adjustJobMetricsOnRetry(task.getTaskState().getPropAsInt(ConfigurationKeys.FORK_BRANCHES_KEY)); } // Task retry interval increases linearly with number of retries long interval = task.getRetryCount() * this.retryIntervalInSeconds; // Schedule the retry of the failed task this.taskExecutor.schedule(new TrackingTask(task, interval, TimeUnit.SECONDS), interval, TimeUnit.SECONDS); LOG.info(String.format("Scheduled retry of failed task %s to run in %d seconds", task.getTaskId(), interval)); task.incrementRetryCount(); }
if (task.getTaskState().contains(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)) { log.error(String.format("Task %s failed due to exception: %s", task.getTaskId(), task.getTaskState().getProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)));
if (this.contains(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)) { taskExecutionInfo.setFailureException(this.getProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY));