public void cleanUpInputFileChangedOp() throws HiveException { Path fpath = new Path((new Path(this.getExecContext().getCurrentInputFile())) .toUri().getPath()); for (String onefile : conf.getPathToAliases().keySet()) { Path onepath = new Path(new Path(onefile).toUri().getPath()); // check for the operators who will process rows coming to this Map // Operator if (!onepath.toUri().relativize(fpath.toUri()).equals(fpath.toUri())) { String onealias = conf.getPathToAliases().get(onefile).get(0); Operator<? extends Serializable> op = conf.getAliasToWork().get(onealias); LOG.info("Processing alias " + onealias + " for file " + onefile); MapInputPath inp = new MapInputPath(onefile, onealias, op); setInspectorInput(inp); break; } } }
public static void initUnionPlan(GenMRProcContext opProcCtx, Task<? extends Serializable> currTask, boolean local) { MapredWork plan = (MapredWork) currTask.getWork(); UnionOperator currUnionOp = opProcCtx.getCurrUnionOp(); assert currUnionOp != null; GenMRUnionCtx uCtx = opProcCtx.getUnionTask(currUnionOp); assert uCtx != null; List<String> taskTmpDirLst = uCtx.getTaskTmpDir(); List<TableDesc> tt_descLst = uCtx.getTTDesc(); assert !taskTmpDirLst.isEmpty() && !tt_descLst.isEmpty(); assert taskTmpDirLst.size() == tt_descLst.size(); int size = taskTmpDirLst.size(); assert local == false; for (int pos = 0; pos < size; pos++) { String taskTmpDir = taskTmpDirLst.get(pos); TableDesc tt_desc = tt_descLst.get(pos); if (plan.getPathToAliases().get(taskTmpDir) == null) { plan.getPathToAliases().put(taskTmpDir, new ArrayList<String>()); plan.getPathToAliases().get(taskTmpDir).add(taskTmpDir); plan.getPathToPartitionInfo().put(taskTmpDir, new PartitionDesc(tt_desc, null)); plan.getAliasToWork().put(taskTmpDir, currUnionOp); } } }
LOG.debug("Not pushing projections and filters because MapredWork is null"); return; } else if (mrwork.getPathToAliases() == null) { LOG.debug("Not pushing projections and filters because pathToAliases is null"); return; final Iterator<Entry<String, ArrayList<String>>> iterator = mrwork.getPathToAliases().entrySet().iterator();
for (String onefile : work.getPathToAliases().keySet()) { List<String> aliases = work.getPathToAliases().get(onefile); if (aliases.contains(oneAlias)) { path = onefile;
LOG.debug("Not pushing projections and filters because MapredWork is null"); return; } else if (mrwork.getPathToAliases() == null) { LOG.debug("Not pushing projections and filters because pathToAliases is null"); return; final Iterator<Entry<String, ArrayList<String>>> iterator = mrwork.getPathToAliases().entrySet().iterator();
List<String> toRemovePaths = new ArrayList<String>(); Map<String, PartitionDesc> toAddPathToPart = new HashMap<String, PartitionDesc>(); Map<String, ArrayList<String>> pathToAliases = work.getPathToAliases();
for (String onefile : conf.getPathToAliases().keySet()) { MapOpCtx opCtx = initObjectInspector(conf, hconf, onefile); Path onepath = new Path(new Path(onefile).toUri().getPath()); List<String> aliases = conf.getPathToAliases().get(onefile);
if (plan.getPathToAliases().get(path) == null) { plan.getPathToAliases().put(path, new ArrayList<String>()); plan.getPathToAliases().get(path).add(alias); plan.getPathToPartitionInfo().put(path, new PartitionDesc(tt_desc, null)); plan.getAliasToWork().put(alias, topOp);
.getPathToAliases().entrySet().iterator();
mrwork.getPathToAliases().get(dir.toUri().toString()); if ((aliases != null) && (aliases.size() == 1)) { Operator op = mrwork.getAliasToWork().get(aliases.get(0));
/** * Create a MapredWork based on input path, the top operator and the input * table descriptor. * @param conf * @param topOp the table scan operator that is the root of the MapReduce task. * @param fsDesc the file sink descriptor that serves as the input to this merge task. * @param parentMR the parent MapReduce work * @param parentFS the last FileSinkOperator in the parent MapReduce work * @return the MapredWork */ private MapredWork createMergeTask(HiveConf conf, Operator<? extends Serializable> topOp, FileSinkDesc fsDesc) { ArrayList<String> aliases = new ArrayList<String>(); String inputDir = fsDesc.getDirName(); TableDesc tblDesc = fsDesc.getTableInfo(); aliases.add(inputDir); // dummy alias: just use the input path // constructing the default MapredWork MapredWork cplan = GenMapRedUtils.getMapRedWork(conf); cplan.getPathToAliases().put(inputDir, aliases); cplan.getPathToPartitionInfo().put(inputDir, new PartitionDesc(tblDesc, null)); cplan.setNumReduceTasks(0); cplan.getAliasToWork().put(inputDir, topOp); cplan.setMapperCannotSpanPartns(true); return cplan; } /**
LinkedHashMap<String, ArrayList<String>> pathToAliases = work.getPathToAliases();
LinkedHashMap<String, ArrayList<String>> pathToAliases = newWork.getPathToAliases();
ctx.localizeKeys((Map<String, Object>) ((Object) work.getPathToAliases())); ctx.localizeKeys((Map<String, Object>) ((Object) work.getPathToPartitionInfo())); Map<String, ArrayList<String>> pa = work.getPathToAliases(); if (pa != null) { for (List<String> ls : pa.values()) {
assert plan.getPathToAliases().get(taskTmpDir) == null; plan.getPathToAliases().put(taskTmpDir, new ArrayList<String>()); plan.getPathToAliases().get(taskTmpDir).add(taskTmpDir); plan.getPathToPartitionInfo().put(taskTmpDir, new PartitionDesc(tt_desc, null));
for (String path : work.getPathToAliases().keySet()) { try { Path p = new Path(path);
Map<String, ArrayList<String>> pathToAliases = mrwork.getPathToAliases(); Map<String, Operator<? extends Serializable>> aliasToWork = mrwork.getAliasToWork();
HashMap<String, ArrayList<String>> pathToAliases = currTask.getWork().getPathToAliases();
GenMRMapJoinCtx mjCtx = ctx.getMapJoinCtx(ctx.getCurrMapJoinOp()); String taskTmpDir = mjCtx.getTaskTmpDir(); if (uPlan.getPathToAliases().get(taskTmpDir) == null) { uPlan.getPathToAliases().put(taskTmpDir, new ArrayList<String>()); uPlan.getPathToAliases().get(taskTmpDir).add(taskTmpDir); uPlan.getPathToPartitionInfo().put(taskTmpDir, new PartitionDesc(mjCtx.getTTDesc(), null));
assert plan.getPathToAliases().get(taskTmpDir) == null; plan.getPathToAliases().put(taskTmpDir, new ArrayList<String>()); plan.getPathToAliases().get(taskTmpDir).add(taskTmpDir); plan.getPathToPartitionInfo().put(taskTmpDir, new PartitionDesc(tt_desc, null));