protected void init(JobConf job) { mrwork = Utilities.getMapRedWork(job); pathToPartitionInfo = mrwork.getPathToPartitionInfo(); }
/** * Initialize the mrwork variable in order to get all the partition and start to update the jobconf * * @param job */ private void init(final JobConf job) { final String plan = HiveConf.getVar(job, HiveConf.ConfVars.PLAN); if (mrwork == null && plan != null && plan.length() > 0) { mrwork = Utilities.getMapRedWork(job); pathToPartitionInfo.clear(); for (final Map.Entry<String, PartitionDesc> entry : mrwork.getPathToPartitionInfo().entrySet()) { pathToPartitionInfo.put(new Path(entry.getKey()).toUri().getPath().toString(), entry.getValue()); } } }
/** * Initialize the mrwork variable in order to get all the partition and start to update the jobconf * * @param job */ private void init(final JobConf job) { final String plan = HiveConf.getVar(job, HiveConf.ConfVars.PLAN); if (mrwork == null && plan != null && plan.length() > 0) { mrwork = Utilities.getMapRedWork(job); pathToPartitionInfo.clear(); for (final Map.Entry<String, PartitionDesc> entry : mrwork.getPathToPartitionInfo().entrySet()) { pathToPartitionInfo.put(new Path(entry.getKey()).toUri().getPath().toString(), entry.getValue()); } } }
public void rework(HiveConf job, MapredWork work) throws IOException { Map<String, PartitionDesc> pathToParts = work.getPathToPartitionInfo(); List<String> toRemovePaths = new ArrayList<String>(); Map<String, PartitionDesc> toAddPathToPart = new HashMap<String, PartitionDesc>();
public static void initUnionPlan(GenMRProcContext opProcCtx, Task<? extends Serializable> currTask, boolean local) { MapredWork plan = (MapredWork) currTask.getWork(); UnionOperator currUnionOp = opProcCtx.getCurrUnionOp(); assert currUnionOp != null; GenMRUnionCtx uCtx = opProcCtx.getUnionTask(currUnionOp); assert uCtx != null; List<String> taskTmpDirLst = uCtx.getTaskTmpDir(); List<TableDesc> tt_descLst = uCtx.getTTDesc(); assert !taskTmpDirLst.isEmpty() && !tt_descLst.isEmpty(); assert taskTmpDirLst.size() == tt_descLst.size(); int size = taskTmpDirLst.size(); assert local == false; for (int pos = 0; pos < size; pos++) { String taskTmpDir = taskTmpDirLst.get(pos); TableDesc tt_desc = tt_descLst.get(pos); if (plan.getPathToAliases().get(taskTmpDir) == null) { plan.getPathToAliases().put(taskTmpDir, new ArrayList<String>()); plan.getPathToAliases().get(taskTmpDir).add(taskTmpDir); plan.getPathToPartitionInfo().put(taskTmpDir, new PartitionDesc(tt_desc, null)); plan.getAliasToWork().put(taskTmpDir, currUnionOp); } } }
MapredWork mapredWork = ((MapRedTask) task).getWork(); Set<Class<? extends InputFormat>> reworkInputFormats = new HashSet<Class<? extends InputFormat>>(); for (PartitionDesc part : mapredWork.getPathToPartitionInfo().values()) { Class<? extends InputFormat> inputFormatCls = part .getInputFileFormatClass();
for (Map.Entry<String,PartitionDesc> pathsAndParts: mapRedWork.getPathToPartitionInfo().entrySet()){ String partitionPath = pathsAndParts.getKey(); if(pathIsInPartition(split.getPath(), partitionPath)) {
Utilities.getMapRedWork(job).getPathToPartitionInfo();
Properties props; if (isEmptyPath) { PartitionDesc partDesc = work.getPathToPartitionInfo().get(path); props = partDesc.getProperties(); outFileFormat = partDesc.getOutputFileFormatClass(); LinkedHashMap<String, PartitionDesc> pathToPartitionInfo = work.getPathToPartitionInfo(); if (isEmptyPath) { pathToPartitionInfo.put(newPath.toUri().toString(), pathToPartitionInfo.get(path));
/** * Create a MapredWork based on input path, the top operator and the input * table descriptor. * @param conf * @param topOp the table scan operator that is the root of the MapReduce task. * @param fsDesc the file sink descriptor that serves as the input to this merge task. * @param parentMR the parent MapReduce work * @param parentFS the last FileSinkOperator in the parent MapReduce work * @return the MapredWork */ private MapredWork createMergeTask(HiveConf conf, Operator<? extends Serializable> topOp, FileSinkDesc fsDesc) { ArrayList<String> aliases = new ArrayList<String>(); String inputDir = fsDesc.getDirName(); TableDesc tblDesc = fsDesc.getTableInfo(); aliases.add(inputDir); // dummy alias: just use the input path // constructing the default MapredWork MapredWork cplan = GenMapRedUtils.getMapRedWork(conf); cplan.getPathToAliases().put(inputDir, aliases); cplan.getPathToPartitionInfo().put(inputDir, new PartitionDesc(tblDesc, null)); cplan.setNumReduceTasks(0); cplan.getAliasToWork().put(inputDir, topOp); cplan.setMapperCannotSpanPartns(true); return cplan; } /**
plan.getPathToPartitionInfo().put(path, new PartitionDesc(tt_desc, null)); plan.getAliasToWork().put(alias, topOp); } else {
ClassNotFoundException, InstantiationException, IllegalAccessException, SerDeException { PartitionDesc td = conf.getPathToPartitionInfo().get(onefile); LinkedHashMap<String, String> partSpec = td.getPartSpec(); Properties tblProps = td.getProperties();
PartitionDesc partitionDesc = newWork.getPathToPartitionInfo().get(tablePath);
public CombineHiveInputSplit(JobConf job, InputSplitShim inputSplitShim) throws IOException { this.inputSplitShim = inputSplitShim; if (job != null) { Map<String, PartitionDesc> pathToPartitionInfo = Utilities .getMapRedWork(job).getPathToPartitionInfo(); // extract all the inputFormatClass names for each chunk in the // CombinedSplit. Path[] ipaths = inputSplitShim.getPaths(); if (ipaths.length > 0) { PartitionDesc part = HiveFileFormatUtils .getPartitionDescFromPathRecursively(pathToPartitionInfo, ipaths[0], IOPrepareCache.get().getPartitionDescMap()); inputFormatClassName = part.getInputFileFormatClass().getName(); } } }
Map<String, PartitionDesc> ptpi = work.getPathToPartitionInfo(); assert ptpi.size() == 1; String path = ptpi.keySet().iterator().next(); Warehouse.makeSpecFromName(fullPartSpec, status[i].getPath()); PartitionDesc pDesc = new PartitionDesc(tblDesc, (LinkedHashMap) fullPartSpec); work.getPathToPartitionInfo().put(status[i].getPath().toString(), pDesc); } else { toMove.add(status[i].getPath().toString());
/** * Writable interface. */ public void write(DataOutput out) throws IOException { inputSplitShim.write(out); if (inputFormatClassName == null) { Map<String, PartitionDesc> pathToPartitionInfo = Utilities .getMapRedWork(getJob()).getPathToPartitionInfo(); // extract all the inputFormatClass names for each chunk in the // CombinedSplit. PartitionDesc part = HiveFileFormatUtils.getPartitionDescFromPathRecursively(pathToPartitionInfo, inputSplitShim.getPath(0), IOPrepareCache.get().getPartitionDescMap()); // create a new InputFormat instance if this is the first time to see // this class inputFormatClassName = part.getInputFileFormatClass().getName(); } out.writeUTF(inputFormatClassName); } }
if (cs == null) { JobConf jobConf = new JobConf(ctx.getConf()); PartitionDesc partDesc = work.getPathToPartitionInfo().get( p.toString()); Class<? extends InputFormat> inputFormatCls = partDesc
ctx.localizeKeys((Map<String, Object>) ((Object) work.getPathToPartitionInfo()));
plan.getPathToAliases().put(taskTmpDir, new ArrayList<String>()); plan.getPathToAliases().get(taskTmpDir).add(taskTmpDir); plan.getPathToPartitionInfo().put(taskTmpDir, new PartitionDesc(tt_desc, null)); plan.getAliasToWork().put(taskTmpDir, mjCtx.getRootMapJoinOp());
uPlan.getPathToAliases().put(taskTmpDir, new ArrayList<String>()); uPlan.getPathToAliases().get(taskTmpDir).add(taskTmpDir); uPlan.getPathToPartitionInfo().put(taskTmpDir, new PartitionDesc(mjCtx.getTTDesc(), null)); uPlan.getAliasToWork().put(taskTmpDir, mjCtx.getRootMapJoinOp());