public void cleanUpInputFileChangedOp() throws HiveException { Path fpath = new Path((new Path(this.getExecContext().getCurrentInputFile())) .toUri().getPath()); for (String onefile : conf.getPathToAliases().keySet()) { Path onepath = new Path(new Path(onefile).toUri().getPath()); // check for the operators who will process rows coming to this Map // Operator if (!onepath.toUri().relativize(fpath.toUri()).equals(fpath.toUri())) { String onealias = conf.getPathToAliases().get(onefile).get(0); Operator<? extends Serializable> op = conf.getAliasToWork().get(onealias); LOG.info("Processing alias " + onealias + " for file " + onefile); MapInputPath inp = new MapInputPath(onefile, onealias, op); setInspectorInput(inp); break; } } }
final Operator<? extends Serializable> op = mrwork.getAliasToWork().get( alias); if (op != null && op instanceof TableScanOperator) {
public static void initUnionPlan(GenMRProcContext opProcCtx, Task<? extends Serializable> currTask, boolean local) { MapredWork plan = (MapredWork) currTask.getWork(); UnionOperator currUnionOp = opProcCtx.getCurrUnionOp(); assert currUnionOp != null; GenMRUnionCtx uCtx = opProcCtx.getUnionTask(currUnionOp); assert uCtx != null; List<String> taskTmpDirLst = uCtx.getTaskTmpDir(); List<TableDesc> tt_descLst = uCtx.getTTDesc(); assert !taskTmpDirLst.isEmpty() && !tt_descLst.isEmpty(); assert taskTmpDirLst.size() == tt_descLst.size(); int size = taskTmpDirLst.size(); assert local == false; for (int pos = 0; pos < size; pos++) { String taskTmpDir = taskTmpDirLst.get(pos); TableDesc tt_desc = tt_descLst.get(pos); if (plan.getPathToAliases().get(taskTmpDir) == null) { plan.getPathToAliases().put(taskTmpDir, new ArrayList<String>()); plan.getPathToAliases().get(taskTmpDir).add(taskTmpDir); plan.getPathToPartitionInfo().put(taskTmpDir, new PartitionDesc(tt_desc, null)); plan.getAliasToWork().put(taskTmpDir, currUnionOp); } } }
for (String oneAlias : work.getAliasToWork().keySet()) { LOG.info("Processing alias " + oneAlias); List<String> emptyPaths = new ArrayList<String>();
final Operator<? extends Serializable> op = mrwork.getAliasToWork().get( alias); if (op != null && op instanceof TableScanOperator) {
/** * Update counters relevant to this task. */ private void updateCounters(Counters ctrs, RunningJob rj) throws IOException { mapProgress = Math.round(rj.mapProgress() * 100); reduceProgress = Math.round(rj.reduceProgress() * 100); taskCounters.put("CNTR_NAME_" + getId() + "_MAP_PROGRESS", Long.valueOf(mapProgress)); taskCounters.put("CNTR_NAME_" + getId() + "_REDUCE_PROGRESS", Long.valueOf(reduceProgress)); if (ctrs == null) { // hadoop might return null if it cannot locate the job. // we may still be able to retrieve the job status - so ignore return; } for (Operator<? extends Serializable> op : work.getAliasToWork().values()) { op.updateCounters(ctrs); } if (work.getReducer() != null) { work.getReducer().updateCounters(ctrs); } }
Operator<? extends Serializable> op = this.mrwork.getAliasToWork().get( alias); if (op != null && op instanceof TableScanOperator) {
Operator<? extends Serializable> op = conf.getAliasToWork().get( onealias); LOG.info("Adding alias " + onealias + " to work list for file "
mrwork.getPathToAliases().get(dir.toUri().toString()); if ((aliases != null) && (aliases.size() == 1)) { Operator op = mrwork.getAliasToWork().get(aliases.get(0)); if ((op != null) && (op instanceof TableScanOperator)) { TableScanOperator tableScan = (TableScanOperator) op;
/** * Create a MapredWork based on input path, the top operator and the input * table descriptor. * @param conf * @param topOp the table scan operator that is the root of the MapReduce task. * @param fsDesc the file sink descriptor that serves as the input to this merge task. * @param parentMR the parent MapReduce work * @param parentFS the last FileSinkOperator in the parent MapReduce work * @return the MapredWork */ private MapredWork createMergeTask(HiveConf conf, Operator<? extends Serializable> topOp, FileSinkDesc fsDesc) { ArrayList<String> aliases = new ArrayList<String>(); String inputDir = fsDesc.getDirName(); TableDesc tblDesc = fsDesc.getTableInfo(); aliases.add(inputDir); // dummy alias: just use the input path // constructing the default MapredWork MapredWork cplan = GenMapRedUtils.getMapRedWork(conf); cplan.getPathToAliases().put(inputDir, aliases); cplan.getPathToPartitionInfo().put(inputDir, new PartitionDesc(tblDesc, null)); cplan.setNumReduceTasks(0); cplan.getAliasToWork().put(inputDir, topOp); cplan.setMapperCannotSpanPartns(true); return cplan; } /**
plan.getAliasToWork().put(alias, topOp); } else {
for (Operator<? extends Serializable> op : work.getAliasToWork().values()) { if (op.checkFatalErrors(ctrs, errMsg)) { return true;
private void breakTaskTree(Task<? extends Serializable> task) { if (task instanceof ExecDriver) { HashMap<String, Operator<? extends Serializable>> opMap = ((MapredWork) task .getWork()).getAliasToWork(); if (!opMap.isEmpty()) { for (Operator<? extends Serializable> op : opMap.values()) { breakOperatorTree(op); } } } else if (task instanceof ConditionalTask) { List<Task<? extends Serializable>> listTasks = ((ConditionalTask) task) .getListTasks(); for (Task<? extends Serializable> tsk : listTasks) { breakTaskTree(tsk); } } if (task.getChildTasks() == null) { return; } for (Task<? extends Serializable> childTask : task.getChildTasks()) { breakTaskTree(childTask); } }
private void setKeyDescTaskTree(Task<? extends Serializable> task) { if (task instanceof ExecDriver) { MapredWork work = (MapredWork) task.getWork(); work.deriveExplainAttributes(); HashMap<String, Operator<? extends Serializable>> opMap = work .getAliasToWork(); if (!opMap.isEmpty()) { for (Operator<? extends Serializable> op : opMap.values()) { GenMapRedUtils.setKeyAndValueDesc(work, op); } } } else if (task instanceof ConditionalTask) { List<Task<? extends Serializable>> listTasks = ((ConditionalTask) task) .getListTasks(); for (Task<? extends Serializable> tsk : listTasks) { setKeyDescTaskTree(tsk); } } if (task.getChildTasks() == null) { return; } for (Task<? extends Serializable> childTask : task.getChildTasks()) { setKeyDescTaskTree(childTask); } }
extractOperatorCounters(mrTask.getWork().getAliasToWork().values(), task.getId() + "_MAP"); if (mrTask.mapStarted()) {
private void generateCountersTask(Task<? extends Serializable> task) { if (task instanceof ExecDriver) { HashMap<String, Operator<? extends Serializable>> opMap = ((MapredWork) task .getWork()).getAliasToWork(); if (!opMap.isEmpty()) { for (Operator<? extends Serializable> op : opMap.values()) {
for (String a : ls) { ArrayList<Operator<? extends Serializable>> opList = new ArrayList<Operator<? extends Serializable>>(); opList.add(work.getAliasToWork().get(a));
plan.getPathToPartitionInfo().put(taskTmpDir, new PartitionDesc(tt_desc, null)); plan.getAliasToWork().put(taskTmpDir, mjCtx.getRootMapJoinOp()); return dest;
uPlan.getPathToPartitionInfo().put(taskTmpDir, new PartitionDesc(mjCtx.getTTDesc(), null)); uPlan.getAliasToWork().put(taskTmpDir, mjCtx.getRootMapJoinOp());
mapTask.setTaskType(TaskType.MAP); stage.addToTaskList(mapTask); populateOperatorGraph(mapTask, mrTask.getWork().getAliasToWork() .values());