/** * Compose meta data when limiter fails to acquire permit * The meta data key list is passed from source layer * A prefix matching is used because some work unit {@link org.apache.gobblin.source.workunit.MultiWorkUnit} have packing strategy, which * can append additional string after the key name * * @return String map representing all the meta data need to report. Return null if no meta data was found. */ private ImmutableMap<String, String> getLimiterStopMetadata() { WorkUnit workUnit = this.taskState.getWorkunit(); Properties properties = workUnit.getProperties(); String metadataKeyList = properties.getProperty(LimiterConfigurationKeys.LIMITER_REPORT_KEY_LIST, LimiterConfigurationKeys.DEFAULT_LIMITER_REPORT_KEY_LIST); List<String> keyList = Splitter.on(',').omitEmptyStrings().trimResults() .splitToList(metadataKeyList); if (keyList.isEmpty()) return ImmutableMap.of(); Set<String> names = properties.stringPropertyNames(); TreeMap<String, String> orderedProperties = new TreeMap<>(); for (String name : names) { orderedProperties.put(name, properties.getProperty(name)); } ImmutableMap.Builder builder = ImmutableMap.<String, String>builder(); for (String oldKey : keyList) { builder.putAll(orderedProperties.subMap(oldKey, oldKey + Character.MAX_VALUE)); } builder.put(LIMITER_STOP_CAUSE_KEY, LIMITER_STOP_CAUSE_VALUE); return builder.build(); }
/** * Get task states of {@link Task}s of this job as {@link WorkUnitState}s. * * @return a list of {@link WorkUnitState}s */ public List<WorkUnitState> getTaskStatesAsWorkUnitStates() { ImmutableList.Builder<WorkUnitState> builder = ImmutableList.builder(); for (TaskState taskState : this.taskStates.values()) { WorkUnitState workUnitState = new WorkUnitState(taskState.getWorkunit(), taskState.getJobState()); workUnitState.setId(taskState.getId()); workUnitState.addAll(taskState); builder.add(workUnitState); } return builder.build(); }
/** * Commit task data. */ private void commitData() throws IOException { if (this.writer.isPresent()) { // Not to catch the exception this may throw so it gets propagated this.writer.get().commit(); } try { if (GobblinMetrics.isEnabled(this.taskState.getWorkunit())) { // Update record-level and byte-level metrics after the writer commits updateRecordMetrics(); updateByteMetrics(); } } catch (IOException ioe) { // Trap the exception as failing to update metrics should not cause the task to fail this.logger.error("Failed to update byte-level metrics of task " + this.taskId); } }
private void getTaskStateWithCommonAndSpecWuProps(int numTaskStates, DataInput in) throws IOException { Properties commonWuProps = new Properties(); for (int i = 0; i < numTaskStates; i++) { TaskState taskState = new TaskState(); taskState.readFields(in); if (i == 0) { commonWuProps.putAll(taskState.getWorkunit().getProperties()); } else { Properties newCommonWuProps = new Properties(); newCommonWuProps .putAll(Maps.difference(commonWuProps, taskState.getWorkunit().getProperties()).entriesInCommon()); commonWuProps = newCommonWuProps; } this.taskStates.put(taskState.getTaskId().intern(), taskState); } ImmutableProperties immutableCommonProperties = new ImmutableProperties(commonWuProps); for (TaskState taskState : this.taskStates.values()) { Properties newSpecProps = new Properties(); newSpecProps.putAll( Maps.difference(immutableCommonProperties, taskState.getWorkunit().getProperties()).entriesOnlyOnRight()); taskState.setWuProperties(immutableCommonProperties, newSpecProps); } }
protected void updateTaskMetrics() { if (GobblinMetrics.isEnabled(this.task.getTaskState().getWorkunit())) { this.task.updateRecordMetrics(); this.task.updateByteMetrics(); } } }
@Override public void onTaskCommitCompletion(Task task) { if (GobblinMetrics.isEnabled(task.getTaskState().getWorkunit())) { // Update record-level metrics after the task is done task.updateRecordMetrics(); task.updateByteMetrics(); } // Cancel the task state reporter associated with this task. The reporter might // not be found for the given task because the task fails before the task is // registered. So we need to make sure the reporter exists before calling cancel. if (this.scheduledReporters.containsKey(task.getTaskId())) { this.scheduledReporters.remove(task.getTaskId()).cancel(false); } LOGGER.info(String .format("Task %s completed in %dms with state %s", task.getTaskId(), task.getTaskState().getTaskDuration(), task.getTaskState().getWorkingState())); }
TaskState taskState = mock (TaskState.class); WorkUnit workUnit = mock (WorkUnit.class); Mockito.when(taskState.getWorkunit()).thenReturn(workUnit); Mockito.when(taskState.getJobId()).thenReturn("123"); Mockito.when(taskState.getTaskAttemptId()).thenReturn(Optional.of("555"));
@Override protected void updateTaskMetrics() { super.updateTaskMetrics(); WorkUnit workUnit = this.task.getTaskState().getWorkunit(); if (GobblinMetrics.isEnabled(workUnit)) { if (workUnit.getPropAsBoolean(ConfigurationKeys.MR_REPORT_METRICS_AS_COUNTERS_KEY, ConfigurationKeys.DEFAULT_MR_REPORT_METRICS_AS_COUNTERS)) { updateCounters(this.task); } } // Tell the TaskTracker it's making progress this.context.progress(); } }
@Override public void onTaskCommitCompletion(Task task) { try { if (GobblinMetrics.isEnabled(task.getTaskState().getWorkunit())) {
TaskState taskState = mock (TaskState.class); WorkUnit workUnit = mock (WorkUnit.class); Mockito.when(taskState.getWorkunit()).thenReturn(workUnit); Mockito.when(taskState.getJobId()).thenReturn("123"); Mockito.when(taskState.getTaskAttemptId()).thenReturn(Optional.of("555"));
TaskState taskState = mock (TaskState.class); WorkUnit workUnit = mock (WorkUnit.class); Mockito.when(taskState.getWorkunit()).thenReturn(workUnit); Mockito.when(taskState.getJobId()).thenReturn("123"); Mockito.when(taskState.getTaskAttemptId()).thenReturn(Optional.of("555"));
/** * Retry a failed {@link Task}. * * @param task failed {@link Task} to be retried */ public void retry(Task task) { if (GobblinMetrics.isEnabled(task.getTaskState().getWorkunit()) && task.getTaskState().contains(ConfigurationKeys.FORK_BRANCHES_KEY)) { // Adjust metrics to clean up numbers from the failed task task.getTaskState() .adjustJobMetricsOnRetry(task.getTaskState().getPropAsInt(ConfigurationKeys.FORK_BRANCHES_KEY)); } // Task retry interval increases linearly with number of retries long interval = task.getRetryCount() * this.retryIntervalInSeconds; // Schedule the retry of the failed task this.taskExecutor.schedule(new TrackingTask(task, interval, TimeUnit.SECONDS), interval, TimeUnit.SECONDS); LOG.info(String.format("Scheduled retry of failed task %s to run in %d seconds", task.getTaskId(), interval)); task.incrementRetryCount(); }
@Override public void onTaskCommitCompletion(Task task) { WorkUnit workUnit = task.getTaskState().getWorkunit(); if (GobblinMetrics.isEnabled(workUnit)) { task.updateRecordMetrics(); task.updateByteMetrics(); if (workUnit.getPropAsBoolean(ConfigurationKeys.MR_REPORT_METRICS_AS_COUNTERS_KEY, ConfigurationKeys.DEFAULT_MR_REPORT_METRICS_AS_COUNTERS)) { updateCounters(task); } } LOG.info(String .format("Task %s completed running in %dms with state %s", task.getTaskId(), task.getTaskState().getTaskDuration(), task.getTaskState().getWorkingState())); }
public TaskState(TaskState taskState) { super(taskState.getWorkunit(), taskState.getJobState(), taskState.getTaskBrokerNullable()); addAll(taskState); this.jobId = taskState.getProp(ConfigurationKeys.JOB_ID_KEY); this.taskId = taskState.getProp(ConfigurationKeys.TASK_ID_KEY); this.taskAttemptId = taskState.getTaskAttemptId(); this.setId(this.taskId); }
Assert.assertEquals(taskState.getProp("foo"), "bar"); if (initial) { Assert.assertEquals(taskState.getWorkunit().getCommonProperties().size(), 0); Assert.assertEquals(taskState.getWorkunit().getSpecProperties().size(), 4); Assert.assertEquals(taskState.getProp("common1"), "1"); Assert.assertEquals(taskState.getProp("common2"), "2"); Assert.assertEquals(taskState.getProp("spec2"), "spec" + i); } else { Assert.assertEquals(taskState.getWorkunit().getCommonProperties().size(), 2); Assert.assertEquals(taskState.getWorkunit().getCommonProperties().getProperty("common1"), "1"); Assert.assertEquals(taskState.getWorkunit().getCommonProperties().getProperty("common2"), "2"); Assert.assertEquals(taskState.getWorkunit().getSpecProperties().size(), 2); Assert.assertEquals(taskState.getWorkunit().getSpecProperties().getProperty("spec1"), String.valueOf(i)); Assert.assertEquals(taskState.getWorkunit().getSpecProperties().getProperty("spec2"), "spec" + i); Assert.assertEquals(taskState.getProp("common1"), "1"); Assert.assertEquals(taskState.getProp("common2"), "2");
/** * Get task states of {@link Task}s of this job as {@link WorkUnitState}s. * * @return a list of {@link WorkUnitState}s */ public List<WorkUnitState> getTaskStatesAsWorkUnitStates() { ImmutableList.Builder<WorkUnitState> builder = ImmutableList.builder(); for (TaskState taskState : this.taskStates.values()) { WorkUnitState workUnitState = new WorkUnitState(taskState.getWorkunit(), taskState.getJobState()); workUnitState.setId(taskState.getId()); workUnitState.addAll(taskState); builder.add(workUnitState); } return builder.build(); }
protected void updateTaskMetrics() { if (GobblinMetrics.isEnabled(this.task.getTaskState().getWorkunit())) { this.task.updateRecordMetrics(); this.task.updateByteMetrics(); } } }
@Override protected void updateTaskMetrics() { super.updateTaskMetrics(); WorkUnit workUnit = this.task.getTaskState().getWorkunit(); if (GobblinMetrics.isEnabled(workUnit)) { if (workUnit.getPropAsBoolean(ConfigurationKeys.MR_REPORT_METRICS_AS_COUNTERS_KEY, ConfigurationKeys.DEFAULT_MR_REPORT_METRICS_AS_COUNTERS)) { updateCounters(this.task); } } // Tell the TaskTracker it's making progress this.context.progress(); } }
@Override public void onTaskCommitCompletion(Task task) { WorkUnit workUnit = task.getTaskState().getWorkunit(); if (GobblinMetrics.isEnabled(workUnit)) { task.updateRecordMetrics(); task.updateByteMetrics(); if (workUnit.getPropAsBoolean(ConfigurationKeys.MR_REPORT_METRICS_AS_COUNTERS_KEY, ConfigurationKeys.DEFAULT_MR_REPORT_METRICS_AS_COUNTERS)) { updateCounters(task); } } LOG.info(String .format("Task %s completed running in %dms with state %s", task.getTaskId(), task.getTaskState().getTaskDuration(), task.getTaskState().getWorkingState())); }
public TaskState(TaskState taskState) { super(taskState.getWorkunit(), taskState.getJobState(), taskState.getTaskBrokerNullable()); addAll(taskState); this.jobId = taskState.getProp(ConfigurationKeys.JOB_ID_KEY); this.taskId = taskState.getProp(ConfigurationKeys.TASK_ID_KEY); this.taskAttemptId = taskState.getTaskAttemptId(); this.setId(this.taskId); }