/** * The default dependent tasks are just child tasks, but different types could implement their own * (e.g. ConditionalTask will use the listTasks as dependents). * * @return a list of tasks that are dependent on this task. */ public List<Task<? extends Serializable>> getDependentTasks() { return getChildTasks(); }
@Override public List<? extends Node> getChildren() { return getChildTasks(); }
/** * The default dependent tasks are just child tasks, but different types could implement their own * (e.g. ConditionalTask will use the listTasks as dependents). * * @return a list of tasks that are dependent on this task. */ public List<Task<? extends Serializable>> getDependentTasks() { return getChildTasks(); }
@Override public List<? extends Node> getChildren() { return getChildTasks(); }
@Override public void process(Task<? extends Serializable> task) { if (task.getChildTasks() == null) { postDependencyCollectionTasks.forEach(task::addDependentTask); } }
private void updateTaskCount(Task<? extends Serializable> task, List <Task<? extends Serializable>> visited) { numberOfTasks += 1; visited.add(task); if (task.getChildTasks() != null) { for (Task<? extends Serializable> childTask : task.getChildTasks()) { if (visited.contains(childTask)) { continue; } updateTaskCount(childTask, visited); } } }
private void validate(Task<?> task, boolean reworkMapredWork) throws SemanticException { Utilities.reworkMapRedWork(task, reworkMapredWork, conf); if (task.getChildTasks() == null) { return; } for (Task<?> childTask : task.getChildTasks()) { validate(childTask, reworkMapredWork); } } /**
protected List<Task<?>> getChildTasks(Task<?> task) { if (task instanceof ConditionalTask) { return ((ConditionalTask) task).getListTasks(); } return task.getChildTasks(); } }
@SafeVarargs public static void makeChild(Task<?> ret, Task<? extends Serializable>... tasklist) { // Add the new task as child of each of the passed in tasks for (Task<? extends Serializable> tsk : tasklist) { List<Task<? extends Serializable>> children = tsk.getChildTasks(); if (children == null) { children = new ArrayList<Task<? extends Serializable>>(); } children.add(ret); tsk.setChildTasks(children); } }
/** * Remove the dependent task. * * @param dependent * the task to remove */ public void removeDependentTask(Task<? extends Serializable> dependent) { if ((getChildTasks() != null) && (getChildTasks().contains(dependent))) { getChildTasks().remove(dependent); if ((dependent.getParentTasks() != null) && (dependent.getParentTasks().contains(this))) { dependent.getParentTasks().remove(this); } } }
private void validate(Task<? extends Serializable> task, boolean reworkMapredWork) throws SemanticException { Utilities.reworkMapRedWork(task, reworkMapredWork, conf); if (task.getChildTasks() == null) { return; } for (Task<? extends Serializable> childTask : task.getChildTasks()) { validate(childTask, reworkMapredWork); } }
protected List<Task<?>> getChildTasks(Task<?> task) { if (task instanceof ConditionalTask) { return ((ConditionalTask) task).getListTasks(); } return task.getChildTasks(); } }
public static void makeChild(Task<?> ret, Task<? extends Serializable>... tasklist) { // Add the new task as child of each of the passed in tasks for (Task<? extends Serializable> tsk : tasklist) { List<Task<? extends Serializable>> children = tsk.getChildTasks(); if (children == null) { children = new ArrayList<Task<? extends Serializable>>(); } children.add(ret); tsk.setChildTasks(children); } }
/** * Remove the dependent task. * * @param dependent * the task to remove */ public void removeDependentTask(Task<? extends Serializable> dependent) { if ((getChildTasks() != null) && (getChildTasks().contains(dependent))) { getChildTasks().remove(dependent); if ((dependent.getParentTasks() != null) && (dependent.getParentTasks().contains(this))) { dependent.getParentTasks().remove(this); } } } public void setStarted() {
/** * Recursively go through the children of the given {@link Task} and check if any child {@link SparkTask} contains * the specified {@link MapWork} object. */ private boolean taskContainsDependentMapWork(Task<? extends Serializable> task, MapWork work) throws SemanticException { if (task == null || task.getChildTasks() == null) { return false; } for (Task<? extends Serializable> childTask : task.getChildTasks()) { if (childTask != null && childTask instanceof SparkTask && childTask.getMapWork().contains(work)) { return true; } else if (taskContainsDependentMapWork(childTask, work)) { return true; } } return false; } }
/** * Add a dependent task on the current task. Return if the dependency already existed or is this a * new one * * @return true if the task got added false if it already existed */ public boolean addDependentTask(Task<? extends Serializable> dependent) { boolean ret = false; if (getChildTasks() == null) { setChildTasks(new ArrayList<Task<? extends Serializable>>()); } if (!getChildTasks().contains(dependent)) { ret = true; getChildTasks().add(dependent); if (dependent.getParentTasks() == null) { dependent.setParentTasks(new ArrayList<Task<? extends Serializable>>()); } if (!dependent.getParentTasks().contains(this)) { dependent.getParentTasks().add(this); } } return ret; }
public void removeFromChildrenTasks() { List<Task<? extends Serializable>> childrenTasks = this.getChildTasks(); if (childrenTasks == null) { return; } for (Task<? extends Serializable> childTsk : childrenTasks) { // remove this task from its children tasks childTsk.getParentTasks().remove(this); // recursively remove non-parent task from its children List<Task<? extends Serializable>> siblingTasks = childTsk.getParentTasks(); if (siblingTasks == null || siblingTasks.size() == 0) { childTsk.removeFromChildrenTasks(); } } }
public void removeFromChildrenTasks() { List<Task<? extends Serializable>> childrenTasks = this.getChildTasks(); if (childrenTasks == null) { return; } for (Task<? extends Serializable> childTsk : childrenTasks) { // remove this task from its children tasks childTsk.getParentTasks().remove(this); // recursively remove non-parent task from its children List<Task<? extends Serializable>> siblingTasks = childTsk.getParentTasks(); if (siblingTasks == null || siblingTasks.size() == 0) { childTsk.removeFromChildrenTasks(); } } }
private static boolean supportRuntimeSkewJoin(JoinOperator joinOp, ReduceWork reduceWork, Task<? extends Serializable> currTask, HiveConf hiveConf) { if (currTask instanceof SparkTask && GenMRSkewJoinProcessor.skewJoinEnabled(hiveConf, joinOp)) { SparkWork sparkWork = ((SparkTask) currTask).getWork(); List<Task<? extends Serializable>> children = currTask.getChildTasks(); return !joinOp.getConf().isFixedAsSorted() && sparkWork.contains(reduceWork) && (children == null || children.size() <= 1) && OperatorUtils.getOp(reduceWork, CommonJoinOperator.class).size() == 1; } return false; } }
private static boolean supportRuntimeSkewJoin(JoinOperator joinOp, ReduceWork reduceWork, Task<? extends Serializable> currTask, HiveConf hiveConf) { if (currTask instanceof SparkTask && GenMRSkewJoinProcessor.skewJoinEnabled(hiveConf, joinOp)) { SparkWork sparkWork = ((SparkTask) currTask).getWork(); List<Task<? extends Serializable>> children = currTask.getChildTasks(); return !joinOp.getConf().isFixedAsSorted() && sparkWork.contains(reduceWork) && (children == null || children.size() <= 1) && SparkMapJoinResolver.getOp(reduceWork, CommonJoinOperator.class).size() == 1; } return false; } }