/** * Create a MapredWork based on input path, the top operator and the input * table descriptor. * @param conf * @param topOp the table scan operator that is the root of the MapReduce task. * @param fsDesc the file sink descriptor that serves as the input to this merge task. * @param parentMR the parent MapReduce work * @param parentFS the last FileSinkOperator in the parent MapReduce work * @return the MapredWork */ private MapredWork createMergeTask(HiveConf conf, Operator<? extends Serializable> topOp, FileSinkDesc fsDesc) { ArrayList<String> aliases = new ArrayList<String>(); String inputDir = fsDesc.getDirName(); TableDesc tblDesc = fsDesc.getTableInfo(); aliases.add(inputDir); // dummy alias: just use the input path // constructing the default MapredWork MapredWork cplan = GenMapRedUtils.getMapRedWork(conf); cplan.getPathToAliases().put(inputDir, aliases); cplan.getPathToPartitionInfo().put(inputDir, new PartitionDesc(tblDesc, null)); cplan.setNumReduceTasks(0); cplan.getAliasToWork().put(inputDir, topOp); cplan.setMapperCannotSpanPartns(true); return cplan; } /**
/** * create a new plan and return. * * @return the new plan */ public static MapredWork getMapRedWork(HiveConf conf) { MapredWork work = new MapredWork(); // This code has been only added for testing boolean mapperCannotSpanPartns = conf.getBoolVar( HiveConf.ConfVars.HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS); work.setMapperCannotSpanPartns(mapperCannotSpanPartns); work.setPathToAliases(new LinkedHashMap<String, ArrayList<String>>()); work.setPathToPartitionInfo(new LinkedHashMap<String, PartitionDesc>()); work.setAliasToWork(new LinkedHashMap<String, Operator<? extends Serializable>>()); work.setTagToValueDesc(new ArrayList<TableDesc>()); work.setReducer(null); work.setHadoopSupportsSplittable( conf.getBoolVar(HiveConf.ConfVars.HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE)); return work; }
parseCtx.getConf().getBoolVar( HiveConf.ConfVars.HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS); newPlan.setMapperCannotSpanPartns(mapperCannotSpanPartns);