@Override public Collection<MapWork> getMapWork() { return Collections.<MapWork>singleton(getWork().getMapWork()); }
@Override public Collection<MapWork> getMapWork() { return Collections.<MapWork>singleton(getWork().getMapWork()); }
@Override public Collection<Operator<? extends OperatorDesc>> getTopOperators() { return getWork().getMapWork().getAliasToWork().values(); }
public static void genLocalWorkForMapJoin(MapredWork newWork, MapJoinOperator newMapJoinOp, int mapJoinPos) throws SemanticException { try { // generate the local work for the big table alias MapJoinProcessor.genMapJoinLocalWork(newWork, newMapJoinOp, mapJoinPos); // clean up the mapred work newWork.getMapWork().setLeftInputJoin(false); newWork.getMapWork().setBaseSrc(null); newWork.getMapWork().setMapAliases(null); } catch (Exception e) { throw new SemanticException("Failed to generate new mapJoin operator " + "by exception : ", e); } }
@Override public Collection<Operator<? extends OperatorDesc>> getTopOperators() { return getWork().getMapWork().getAliasToWork().values(); }
public static void setMapRedWork(Configuration conf, MapredWork w, Path hiveScratchDir) { String useName = conf.get(INPUT_NAME); if (useName == null) { useName = "mapreduce:" + hiveScratchDir; } conf.set(INPUT_NAME, useName); setMapWork(conf, w.getMapWork(), hiveScratchDir, true); if (w.getReduceWork() != null) { conf.set(INPUT_NAME, useName); setReduceWork(conf, w.getReduceWork(), hiveScratchDir, true); } }
public static void setMapRedWork(Configuration conf, MapredWork w, Path hiveScratchDir) { String useName = conf.get(INPUT_NAME); if (useName == null) { useName = "mapreduce"; } conf.set(INPUT_NAME, useName); setMapWork(conf, w.getMapWork(), hiveScratchDir, true); if (w.getReduceWork() != null) { conf.set(INPUT_NAME, useName); setReduceWork(conf, w.getReduceWork(), hiveScratchDir, true); } }
private MapredWork convertSMBWorkToJoinWork(MapredWork currWork, SMBMapJoinOperator oldSMBJoinOp) throws SemanticException { try { // deep copy a new mapred work MapredWork currJoinWork = SerializationUtilities.clonePlan(currWork); SMBMapJoinOperator newSMBJoinOp = getSMBMapJoinOp(currJoinWork); // change the newly created map-red plan as if it was a join operator genSMBJoinWork(currJoinWork.getMapWork(), newSMBJoinOp); return currJoinWork; } catch (Exception e) { e.printStackTrace(); throw new SemanticException("Generate Map Join Task Error: " + e.getMessage()); } }
private MapredWork convertSMBWorkToJoinWork(MapredWork currWork, SMBMapJoinOperator oldSMBJoinOp) throws SemanticException { try { // deep copy a new mapred work MapredWork currJoinWork = SerializationUtilities.clonePlan(currWork); SMBMapJoinOperator newSMBJoinOp = getSMBMapJoinOp(currJoinWork); // change the newly created map-red plan as if it was a join operator genSMBJoinWork(currJoinWork.getMapWork(), newSMBJoinOp); return currJoinWork; } catch (Exception e) { e.printStackTrace(); throw new SemanticException("Generate Map Join Task Error: " + e.getMessage()); } }
private SMBMapJoinOperator getSMBMapJoinOp(MapredWork work) throws SemanticException { if (work != null && work.getReduceWork() != null) { Operator<? extends OperatorDesc> reducer = work.getReduceWork().getReducer(); for (Operator<? extends OperatorDesc> op : work.getMapWork().getAliasToWork().values()) { SMBMapJoinOperator smbMapJoinOp = getSMBMapJoinOp(op, reducer); if (smbMapJoinOp != null) { return smbMapJoinOp; } } } return null; }
/** * create a new plan and return. The pan won't contain the name to split * sample information in parse context. * * @return the new plan */ public static MapredWork getMapRedWorkFromConf(HiveConf conf) { MapredWork mrWork = new MapredWork(); MapWork work = mrWork.getMapWork(); boolean mapperCannotSpanPartns = conf.getBoolVar( HiveConf.ConfVars.HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS); work.setMapperCannotSpanPartns(mapperCannotSpanPartns); work.setPathToAliases(new LinkedHashMap<Path, ArrayList<String>>()); work.setPathToPartitionInfo(new LinkedHashMap<Path, PartitionDesc>()); work.setAliasToWork(new LinkedHashMap<String, Operator<? extends OperatorDesc>>()); return mrWork; }
/** * create a new plan and return. * * @return the new plan */ public static MapredWork getMapRedWork(ParseContext parseCtx) { MapredWork work = getMapRedWorkFromConf(parseCtx.getConf()); work.getMapWork().setNameToSplitSample(parseCtx.getNameToSplitSample()); return work; }
private SMBMapJoinOperator getSMBMapJoinOp(MapredWork work) throws SemanticException { if (work != null && work.getReduceWork() != null) { Operator<? extends OperatorDesc> reducer = work.getReduceWork().getReducer(); for (Operator<? extends OperatorDesc> op : work.getMapWork().getAliasToWork().values()) { SMBMapJoinOperator smbMapJoinOp = getSMBMapJoinOp(op, reducer); if (smbMapJoinOp != null) { return smbMapJoinOp; } } } return null; }
/** * create a new plan and return. * * @return the new plan */ public static MapredWork getMapRedWork(ParseContext parseCtx) { MapredWork work = getMapRedWorkFromConf(parseCtx.getConf()); work.getMapWork().setNameToSplitSample(parseCtx.getNameToSplitSample()); return work; }
private void checkMRReducer(String taskName, MapredWork mrWrk) throws SemanticException { ReduceWork rWrk = mrWrk.getReduceWork(); if ( rWrk == null) { return; } Operator<? extends OperatorDesc> reducer = rWrk.getReducer(); if ( reducer instanceof JoinOperator|| reducer instanceof CommonMergeJoinOperator ) { BaseWork parentWork = mrWrk.getMapWork(); checkForCrossProduct(taskName, reducer, new ExtractReduceSinkInfo(null).analyze(parentWork)); } }
/** * Set hive input format, and input format file if necessary. */ protected void setInputAttributes(Configuration conf) { MapWork mWork = work.getMapWork(); if (mWork.getInputformat() != null) { HiveConf.setVar(conf, ConfVars.HIVEINPUTFORMAT, mWork.getInputformat()); } // Intentionally overwrites anything the user may have put here conf.setBoolean("hive.input.format.sorted", mWork.isInputFormatSorted()); if (HiveConf.getVar(conf, ConfVars.HIVE_CURRENT_DATABASE, (String)null) == null) { HiveConf.setVar(conf, ConfVars.HIVE_CURRENT_DATABASE, getCurrentDB()); } }
private void checkMRReducer(String taskName, MapredWork mrWrk) throws SemanticException { ReduceWork rWrk = mrWrk.getReduceWork(); if ( rWrk == null) { return; } Operator<? extends OperatorDesc> reducer = rWrk.getReducer(); if ( reducer instanceof JoinOperator|| reducer instanceof CommonMergeJoinOperator ) { BaseWork prntWork = mrWrk.getMapWork(); checkForCrossProduct(taskName, reducer, new ExtractReduceSinkInfo(null).analyze(prntWork)); } }
@Override public Operator<? extends OperatorDesc> getReducer(MapWork mapWork) { if (getWork().getMapWork() == mapWork) { return getWork().getReduceWork() == null ? null : getWork().getReduceWork().getReducer(); } return null; }
@Override public Operator<? extends OperatorDesc> getReducer(MapWork mapWork) { if (getWork().getMapWork() == mapWork) { return getWork().getReduceWork() == null ? null : getWork().getReduceWork().getReducer(); } return null; }
public static void addMapWork(MapredWork mr, Table tbl, String alias, Operator<?> work) { mr.getMapWork().addMapWork(tbl.getDataLocation(), alias, work, new PartitionDesc( Utilities.getTableDesc(tbl), null)); }