private void evaluateMergeWork(MergeJoinWork w) throws SemanticException { for (BaseWork baseWork : w.getBaseWorkList()) { evaluateOperators(baseWork, pctx); } }
private JobConf initializeVertexConf(JobConf conf, Context context, MergeJoinWork work) { if (work.getMainWork() instanceof MapWork) { return initializeVertexConf(conf, context, (MapWork) (work.getMainWork())); } else { return initializeVertexConf(conf, context, (ReduceWork) (work.getMainWork())); } }
setName(work.getName()); setReduceSinkOutputName(connectWork, leafOperatorToFollowingWork, bigTableWork.getName()); setReduceSinkOutputName(mergeWork, leafOperatorToFollowingWork, work.getName());
} else { mergeJoinWork = new MergeJoinWork(); mergeJoinWork.setMergeJoinOperator(context.currentMergeJoinOperator); tezWork.add(mergeJoinWork); context.opMergeJoinWorkMap.put(context.currentMergeJoinOperator, mergeJoinWork); mergeJoinWork.addMergedWork(work, null, context.leafOperatorToFollowingWork); Operator<? extends OperatorDesc> parentOp = getParentFromStack(context.currentMergeJoinOperator, stack); CommonMergeJoinOperator mergeJoinOp = mergeJoinWork.getMergeJoinOperator(); work.setTag(mergeJoinOp.getTagForOperator(operator)); mergeJoinWork.addMergedWork(null, work, context.leafOperatorToFollowingWork); tezWork.setVertexType(mergeJoinWork, VertexType.MULTI_INPUT_UNINITIALIZED_EDGES); for (BaseWork parentWork : tezWork.getParents(work)) { if (followingWork instanceof MergeJoinWork) { MergeJoinWork mergeJoinWork = (MergeJoinWork) followingWork; rWork = (ReduceWork) mergeJoinWork.getMainWork(); } else if (followingWork instanceof UnionWork) { rWork = (ReduceWork) mergeJoinWork.getMainWork(); } else { rWork = (ReduceWork) baseWork;
Map<String, LocalResource> localResources) throws Exception { Utilities.setMergeWork(conf, mergeJoinWork, mrScratchDir, false); if (mergeJoinWork.getMainWork() instanceof MapWork) { List<BaseWork> mapWorkList = mergeJoinWork.getBaseWorkList(); MapWork mapWork = (MapWork) (mergeJoinWork.getMainWork()); Vertex mergeVx = createVertex( conf, mapWork, fs, mrScratchDir, ctx, vertexType, localResources); if (mergeJoinWork.getMergeJoinOperator().getParentOperators().size() == 1 && mergeJoinWork.getMergeJoinOperator().getOpTraits() != null) { new CustomVertexConfiguration(mergeJoinWork.getMergeJoinOperator().getConf() .getNumBuckets(), vertexType, mergeJoinWork.getBigTableAlias(), mapWorkList.size() + 1, inputToBucketMap); DataOutputBuffer dob = new DataOutputBuffer(); } else { return createVertex(conf, (ReduceWork) mergeJoinWork.getMainWork(), fs, mrScratchDir, ctx, localResources);
mergeWork = new MergeJoinWork(); tezWork.add(mergeWork); context.opMergeJoinWorkMap.put(mergeJoinOp, mergeWork); mergeWork.setTag(mergeJoinOp.getTagForOperator(parentOp)); return null; mergeWork.addMergedWork(null, parentWork, context.leafOperatorToFollowingWork); mergeWork.setMergeJoinOperator(mergeJoinOp); tezWork.setVertexType(mergeWork, VertexType.MULTI_INPUT_UNINITIALIZED_EDGES);
mergeWork = new MergeJoinWork(); tezWork.add(mergeWork); context.opMergeJoinWorkMap.put(mergeJoinOp, mergeWork); mergeWork.addMergedWork(null, parentWork, context.leafOperatorToFollowingWork); mergeWork.setMergeJoinOperator(mergeJoinOp); tezWork.setVertexType(mergeWork, VertexType.MULTI_INPUT_UNINITIALIZED_EDGES);
} else { mergeJoinWork = new MergeJoinWork(); mergeJoinWork.setMergeJoinOperator(context.currentMergeJoinOperator); tezWork.add(mergeJoinWork); context.opMergeJoinWorkMap.put(context.currentMergeJoinOperator, mergeJoinWork); mergeJoinWork.addMergedWork(work, null, context.leafOperatorToFollowingWork); Operator<? extends OperatorDesc> parentOp = getParentFromStack(context.currentMergeJoinOperator, stack); CommonMergeJoinOperator mergeJoinOp = mergeJoinWork.getMergeJoinOperator(); work.setTag(mergeJoinOp.getTagForOperator(operator)); mergeJoinWork.addMergedWork(null, work, context.leafOperatorToFollowingWork); tezWork.setVertexType(mergeJoinWork, VertexType.MULTI_INPUT_UNINITIALIZED_EDGES); for (BaseWork parentWork : tezWork.getParents(work)) { if (followingWork instanceof MergeJoinWork) { MergeJoinWork mergeJoinWork = (MergeJoinWork) followingWork; rWork = (ReduceWork) mergeJoinWork.getMainWork(); } else if (followingWork instanceof UnionWork) { rWork = (ReduceWork) mergeJoinWork.getMainWork(); } else { rWork = (ReduceWork) baseWork;
throws Exception { Utilities.setMergeWork(conf, mergeJoinWork, mrScratchDir, false); if (mergeJoinWork.getMainWork() instanceof MapWork) { List<BaseWork> mapWorkList = mergeJoinWork.getBaseWorkList(); MapWork mapWork = (MapWork) (mergeJoinWork.getMainWork()); Vertex mergeVx = createVertex(conf, mapWork, appJarLr, additionalLr, fs, mrScratchDir, ctx, vertexType); new CustomVertexConfiguration(mergeJoinWork.getMergeJoinOperator().getConf() .getNumBuckets(), vertexType, mergeJoinWork.getBigTableAlias(), mapWorkList.size() + 1); DataOutputBuffer dob = new DataOutputBuffer(); } else { Vertex mergeVx = createVertex(conf, (ReduceWork) mergeJoinWork.getMainWork(), appJarLr, additionalLr, fs, mrScratchDir, ctx); return mergeVx;
mergeWork = new MergeJoinWork(); tezWork.add(mergeWork); context.opMergeJoinWorkMap.put(mergeJoinOp, mergeWork); mergeWork.addMergedWork(null, parentWork, context.leafOperatorToFollowingWork); mergeWork.setMergeJoinOperator(mergeJoinOp); tezWork.setVertexType(mergeWork, VertexType.MULTI_INPUT_UNINITIALIZED_EDGES);
} else { mergeJoinWork = new MergeJoinWork(); mergeJoinWork.setMergeJoinOperator(context.currentMergeJoinOperator); tezWork.add(mergeJoinWork); context.opMergeJoinWorkMap.put(context.currentMergeJoinOperator, mergeJoinWork); mergeJoinWork.addMergedWork(work, null, context.leafOperatorToFollowingWork); Operator<? extends OperatorDesc> parentOp = getParentFromStack(context.currentMergeJoinOperator, stack); CommonMergeJoinOperator mergeJoinOp = mergeJoinWork.getMergeJoinOperator(); work.setTag(mergeJoinOp.getTagForOperator(operator)); mergeJoinWork.addMergedWork(null, work, context.leafOperatorToFollowingWork); tezWork.setVertexType(mergeJoinWork, VertexType.MULTI_INPUT_UNINITIALIZED_EDGES); for (BaseWork parentWork : tezWork.getParents(work)) { if (followingWork instanceof MergeJoinWork) { MergeJoinWork mergeJoinWork = (MergeJoinWork) followingWork; rWork = (ReduceWork) mergeJoinWork.getMainWork(); } else if (followingWork instanceof UnionWork) { rWork = (ReduceWork) mergeJoinWork.getMainWork(); } else { rWork = (ReduceWork) baseWork;
throws Exception { Utilities.setMergeWork(conf, mergeJoinWork, mrScratchDir, false); if (mergeJoinWork.getMainWork() instanceof MapWork) { List<BaseWork> mapWorkList = mergeJoinWork.getBaseWorkList(); MapWork mapWork = (MapWork) (mergeJoinWork.getMainWork()); Vertex mergeVx = createVertex(conf, mapWork, appJarLr, additionalLr, fs, mrScratchDir, ctx, vertexType); new CustomVertexConfiguration(mergeJoinWork.getMergeJoinOperator().getConf() .getNumBuckets(), vertexType, mergeJoinWork.getBigTableAlias(), mapWorkList.size() + 1); DataOutputBuffer dob = new DataOutputBuffer(); } else { Vertex mergeVx = createVertex(conf, (ReduceWork) mergeJoinWork.getMainWork(), appJarLr, additionalLr, fs, mrScratchDir, ctx); return mergeVx;
private JobConf initializeVertexConf(JobConf conf, Context context, MergeJoinWork work) { if (work.getMainWork() instanceof MapWork) { return initializeVertexConf(conf, context, (MapWork) (work.getMainWork())); } else { return initializeVertexConf(conf, context, (ReduceWork) (work.getMainWork())); } }
setName(work.getName()); setReduceSinkOutputName(connectWork, leafOperatorToFollowingWork, bigTableWork.getName()); setReduceSinkOutputName(mergeWork, leafOperatorToFollowingWork, work.getName());
private void evaluateMergeWork(MergeJoinWork w) throws SemanticException { for (BaseWork baseWork : w.getBaseWorkList()) { evaluateOperators(baseWork, pctx); } }
@Override public void setUberMode(boolean uberMode) { getMainWork().setUberMode(uberMode); }
setName(work.getName()); setReduceSinkOutputName(connectWork, leafOperatorToFollowingWork, bigTableWork.getName()); setReduceSinkOutputName(mergeWork, leafOperatorToFollowingWork, work.getName());
private void evaluateMergeWork(MergeJoinWork w) throws SemanticException { for (BaseWork baseWork : w.getBaseWorkList()) { evaluateOperators(baseWork, pctx); } }
@Override public Operator<?> getAnyRootOperator() { return getMainWork().getAnyRootOperator(); }
private void evaluateMergeWork(MergeJoinWork w) throws SemanticException { for (BaseWork baseWork : w.getBaseWorkList()) { evaluateOperators(baseWork, pctx); } }