private void getLeafTasks(Task<? extends Serializable> task, Set<Task<? extends Serializable>> leaves) { if (task.getDependentTasks() == null) { if (!leaves.contains(task)) { leaves.add(task); } } else { getLeafTasks(task.getDependentTasks(), leaves); } }
private static void getSparkTasks(List<Task<? extends Serializable>> tasks, List<SparkTask> sparkTasks) { for (Task<? extends Serializable> task : tasks) { if (task instanceof SparkTask && !sparkTasks.contains(task)) { sparkTasks.add((SparkTask) task); } if (task.getDependentTasks() != null) { getSparkTasks(task.getDependentTasks(), sparkTasks); } } }
/** * Find the leaves of the task tree */ private void findLeaves(List<Task<?>> tasks, Set<Task<?>> leaves) { for (Task<?> t : tasks) { if (t.getDependentTasks() == null) { leaves.add(t); } else { findLeaves(t.getDependentTasks(), leaves); } } }
private static void getTezTasks(List<Task<? extends Serializable>> tasks, List<TezTask> tezTasks) { for (Task<? extends Serializable> task : tasks) { if (task instanceof TezTask && !tezTasks.contains(task)) { tezTasks.add((TezTask) task); } if (task.getDependentTasks() != null) { getTezTasks(task.getDependentTasks(), tezTasks); } } }
private static void getMRTasks(List<Task<? extends Serializable>> tasks, List<ExecDriver> mrTasks) { for (Task<? extends Serializable> task : tasks) { if (task instanceof ExecDriver && !mrTasks.contains(task)) { mrTasks.add((ExecDriver) task); } if (task.getDependentTasks() != null) { getMRTasks(task.getDependentTasks(), mrTasks); } } }
private void getLeafTasks(Task<? extends Serializable> task, Set<Task<? extends Serializable>> leaves) { if (task.getDependentTasks() == null) { if (!leaves.contains(task)) { leaves.add(task); } } else { getLeafTasks(task.getDependentTasks(), leaves); } }
private List<Task<?>> findStatsTasks( List<Task<?>> rootTasks, List<Task<?>> statsTasks) { for (Task<? extends Serializable> t : rootTasks) { if (t instanceof StatsTask) { if (statsTasks == null) { statsTasks = new ArrayList<>(); } statsTasks.add(t); } if (t.getDependentTasks() != null) { statsTasks = findStatsTasks(t.getDependentTasks(), statsTasks); } } return statsTasks; } }
private List<Task<? extends Serializable>> setQueryDisplays( List<Task<? extends Serializable>> tasks, Set<Task<? extends Serializable>> visited) { List<Task<? extends Serializable>> childTasks = new ArrayList<>(); for (Task<? extends Serializable> task : tasks) { if (visited.contains(task)) { continue; } task.setQueryDisplay(queryDisplay); if (task.getDependentTasks() != null) { childTasks.addAll(task.getDependentTasks()); } visited.add(task); } return childTasks; }
/** * Follows the task tree down from task and makes all leaves parents of mvTask * * @param mvTask * @param task * @param hconf * @param dependencyTask */ private static void linkMoveTask(Task<MoveWork> mvTask, Task<? extends Serializable> task, HiveConf hconf, DependencyCollectionTask dependencyTask) { if (task.getDependentTasks() == null || task.getDependentTasks().isEmpty()) { // If it's a leaf, add the move task as a child addDependentMoveTasks(mvTask, hconf, task, dependencyTask); } else { // Otherwise, for each child run this method recursively for (Task<? extends Serializable> childTask : task.getDependentTasks()) { linkMoveTask(mvTask, childTask, hconf, dependencyTask); } } }
private boolean ancestorOrSelf(Task<? extends Serializable> desc) { if (this == desc) { return true; } List<Task<? extends Serializable>> deps = getDependentTasks(); if (deps != null) { for (Task<? extends Serializable> d : deps) { if (d.ancestorOrSelf(desc)) { return true; } } } return false; }
private static <T> Collection<Task<? extends Serializable>> iterateTask(Collection<Task<?>> tasks, Class<T> clazz, Function<T> function, Set<Task> visited) { Collection<Task<?>> childTasks = new ArrayList<>(); for (Task<?> task : tasks) { if (!visited.add(task)) { continue; } if (clazz.isInstance(task)) { function.apply(clazz.cast(task)); } // this is for ConditionalTask if (task.getDependentTasks() != null) { childTasks.addAll(task.getDependentTasks()); } } return childTasks; }
private boolean ancestorOrSelf(Task<? extends Serializable> desc) { if (this == desc) { return true; } List<Task<? extends Serializable>> deps = getDependentTasks(); if (deps != null) { for (Task<? extends Serializable> d : deps) { if (d.ancestorOrSelf(desc)) { return true; } } } return false; }
private static <T> void iterateTask(Task<?> task, Class<T> clazz, Function<T> function, Set<Task> visited) { if (!visited.add(task)) { return; } if (clazz.isInstance(task)) { function.apply(clazz.cast(task)); } // this is for ConditionalTask if (task.getDependentTasks() != null) { for (Task<?> dependent : task.getDependentTasks()) { iterateTask(dependent, clazz, function, visited); } } }
/** * Follows the task tree down from task and makes all leaves parents of mvTask * * @param mvTask * @param task * @param hconf * @param dependencyTask */ public static void linkMoveTask(Task<MoveWork> mvTask, Task<? extends Serializable> task, HiveConf hconf, DependencyCollectionTask dependencyTask) { if (task.getDependentTasks() == null || task.getDependentTasks().isEmpty()) { // If it's a leaf, add the move task as a child addDependentMoveTasks(mvTask, hconf, task, dependencyTask); } else { // Otherwise, for each child run this method recursively for (Task<? extends Serializable> childTask : task.getDependentTasks()) { linkMoveTask(mvTask, childTask, hconf, dependencyTask); } } }
@Override public void process(Task<? extends Serializable> task) { if (task.getDependentTasks() == null || task.getDependentTasks().isEmpty()) { count++; } }
public static void traverse(List<Task<? extends Serializable>> tasks, Function function) { List<Task<? extends Serializable>> listOfTasks = new ArrayList<>(tasks); while (!listOfTasks.isEmpty()) { List<Task<? extends Serializable>> children = new ArrayList<>(); for (Task<? extends Serializable> task : listOfTasks) { // skip processing has to be done first before continuing if (function.skipProcessing(task)) { continue; } // Add list tasks from conditional tasks if (task instanceof ConditionalTask) { children.addAll(((ConditionalTask) task).getListTasks()); } if (task.getDependentTasks() != null) { children.addAll(task.getDependentTasks()); } function.process(task); } listOfTasks = children; } }
private void setQueryDisplays(List<Task<? extends Serializable>> tasks) { if (tasks != null) { for (Task<? extends Serializable> task : tasks) { task.setQueryDisplay(queryDisplay); setQueryDisplays(task.getDependentTasks()); } } }
private Task<? extends Serializable> linearTree(int numOfNodes) { Task<? extends Serializable> current = null, head = null; for (int i = 0; i < numOfNodes; i++) { @SuppressWarnings("unchecked") Task<? extends Serializable> task = mock(Task.class); if (current != null) { when(current.getDependentTasks()).thenReturn(Collections.singletonList(task)); } if (head == null) { head = task; } current = task; } return head; }
/** * Makes the exportTask run after all other tasks of the "insert into T ..." are done. */ private void addExportTask(List<Task<?>> rootTasks, Task<ExportWork> exportTask, Task<DDLWork> alterTable) { for (Task<? extends Serializable> t : rootTasks) { if (t.getNumChild() <= 0) { //todo: ConditionalTask#addDependentTask(Task) doesn't do the right thing: HIVE-18978 t.addDependentTask(alterTable); //this is a leaf so add exportTask to follow it alterTable.addDependentTask(exportTask); } else { addExportTask(t.getDependentTasks(), exportTask, alterTable); } } }
@Test public void shouldCountNumberOfLeafNodesCorrectly() { Task<? extends Serializable> taskWith5NodeTree = linearTree(5); Task<? extends Serializable> taskWith1NodeTree = linearTree(1); Task<? extends Serializable> taskWith3NodeTree = linearTree(3); @SuppressWarnings("unchecked") Task<? extends Serializable> rootTask = mock(Task.class); when(rootTask.getDependentTasks()) .thenReturn(Arrays.asList(taskWith1NodeTree, taskWith3NodeTree, taskWith5NodeTree)); CountLeafFunction function = new CountLeafFunction(); DAGTraversal.traverse(Collections.singletonList(rootTask), function); assertEquals(3, function.count); }