@Override public LogicalNode visitJoin(Object context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode joinNode, Stack<LogicalNode> stack) throws TajoException { if (joinNode.hasJoinQual()) { joinNode.setJoinQual(evalRewriter.visit(null, joinNode.getJoinQual(), new Stack<EvalNode>())); } return null; } }
@Override public LogicalNode visitJoin(Object context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode joinNode, Stack<LogicalNode> stack) throws TajoException { if (joinNode.hasJoinQual()) { joinNode.setJoinQual(evalRewriter.visit(null, joinNode.getJoinQual(), new Stack<>())); } return null; } }
@Override public LogicalNode visitJoin(PlanShapeFixerContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode node, Stack<LogicalNode> stack) throws TajoException { super.visitJoin(context, plan, block, node, stack); int rightChildNum = context.childNumbers.pop(); int leftChildNum = context.childNumbers.pop(); if (node.hasTargets()) { node.setTargets(sortTargets(node.getTargets())); } if (node.hasJoinQual()) { node.setJoinQual(sortQual(node.getJoinQual())); } context.childNumbers.push(rightChildNum + leftChildNum); return null; }
@Override public LogicalNode visitJoin(PlanShapeFixerContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode node, Stack<LogicalNode> stack) throws TajoException { super.visitJoin(context, plan, block, node, stack); int rightChildNum = context.childNumbers.pop(); int leftChildNum = context.childNumbers.pop(); if (node.hasTargets()) { node.setTargets(sortTargets(node.getTargets())); } if (node.hasJoinQual()) { node.setJoinQual(sortQual(node.getJoinQual())); } context.childNumbers.push(rightChildNum + leftChildNum); return null; }
@Override public LogicalNode visitJoin(CompilationContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode node, Stack<LogicalNode> stack) throws TajoException { super.visitJoin(context, plan, block, node, stack); compileProjectableNode(context, node.getInSchema(), node); if (node.hasJoinQual()) { compileIfAbsent(context, node.getInSchema(), node.getJoinQual()); } return node; }
@Override public LogicalNode visitJoin(CompilationContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode node, Stack<LogicalNode> stack) throws TajoException { super.visitJoin(context, plan, block, node, stack); compileProjectableNode(context, node.getInSchema(), node); if (node.hasJoinQual()) { compileIfAbsent(context, node.getInSchema(), node.getJoinQual()); } return node; }
@Override public LogicalNode visitJoin(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode node, Stack<LogicalNode> stack) throws TajoException { visit(context, plan, block, node.getLeftChild(), stack); visit(context, plan, block, node.getRightChild(), stack); if (node.hasJoinQual()) { ExprsVerifier.verify(context.state, node, node.getJoinQual()); } verifyProjectableOutputSchema(context, node); return node; }
@Override public LogicalNode visitJoin(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode node, Stack<LogicalNode> stack) throws TajoException { visit(context, plan, block, node.getLeftChild(), stack); visit(context, plan, block, node.getRightChild(), stack); if (node.hasJoinQual()) { ExprsVerifier.verify(context.state, node, node.getJoinQual()); } verifyProjectableOutputSchema(context, node); return node; }
@Override public LogicalNode visitJoin(CostContext joinGraphContext, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode joinNode, Stack<LogicalNode> stack) throws TajoException { super.visitJoin(joinGraphContext, plan, block, joinNode, stack); double filterFactor = 1; if (joinNode.hasJoinQual()) { EvalNode [] quals = AlgebraicUtil.toConjunctiveNormalFormArray(joinNode.getJoinQual()); filterFactor = Math.pow(GreedyHeuristicJoinOrderAlgorithm.DEFAULT_SELECTION_FACTOR, quals.length); } if (joinNode.getLeftChild() instanceof RelationNode) { joinGraphContext.accumulatedCost = getCost(joinNode.getLeftChild()) * getCost(joinNode.getRightChild()) * filterFactor; } else { joinGraphContext.accumulatedCost = joinGraphContext.accumulatedCost + (joinGraphContext.accumulatedCost * getCost(joinNode.getRightChild()) * filterFactor); } return joinNode; } }
public RightOuterMergeJoinExec(TaskAttemptContext context, JoinNode plan, PhysicalExec outer, PhysicalExec inner, SortSpec[] outerSortKey, SortSpec[] innerSortKey) { super(context, plan, outer, inner); Preconditions.checkArgument(plan.hasJoinQual(), "Sort-merge join is only used for the equi-join, " + "but there is no join condition"); final int INITIAL_TUPLE_SLOT = context.getQueryContext().getInt(SessionVars.JOIN_HASH_TABLE_SIZE); this.leftTupleSlots = new TupleList(INITIAL_TUPLE_SLOT); this.innerTupleSlots = new TupleList(INITIAL_TUPLE_SLOT); SortSpec[][] sortSpecs = new SortSpec[2][]; sortSpecs[0] = outerSortKey; sortSpecs[1] = innerSortKey; this.joinComparator = new JoinTupleComparator(outer.getSchema(), inner.getSchema(), sortSpecs); this.tupleComparator = PhysicalPlanUtil.getComparatorsFromJoinQual( plan.getJoinQual(), outer.getSchema(), inner.getSchema()); leftNumCols = outer.getSchema().size(); nullPaddedTuple = NullTuple.create(leftNumCols); }
public RightOuterMergeJoinExec(TaskAttemptContext context, JoinNode plan, PhysicalExec outer, PhysicalExec inner, SortSpec[] outerSortKey, SortSpec[] innerSortKey) { super(context, plan, outer, inner); Preconditions.checkArgument(plan.hasJoinQual(), "Sort-merge join is only used for the equi-join, " + "but there is no join condition"); final int INITIAL_TUPLE_SLOT = context.getQueryContext().getInt(SessionVars.JOIN_HASH_TABLE_SIZE); this.leftTupleSlots = new TupleList(INITIAL_TUPLE_SLOT); this.innerTupleSlots = new TupleList(INITIAL_TUPLE_SLOT); SortSpec[][] sortSpecs = new SortSpec[2][]; sortSpecs[0] = outerSortKey; sortSpecs[1] = innerSortKey; this.joinComparator = new JoinTupleComparator(outer.getSchema(), inner.getSchema(), sortSpecs); this.tupleComparator = PhysicalPlanUtil.getComparatorsFromJoinQual( plan.getJoinQual(), outer.getSchema(), inner.getSchema()); leftNumCols = outer.getSchema().size(); nullPaddedTuple = NullTuple.create(leftNumCols); }
private DataChannel createDataChannelFromJoin(ExecutionBlock leftBlock, ExecutionBlock rightBlock, ExecutionBlock parent, JoinNode join, boolean leftTable) { ExecutionBlock childBlock = leftTable ? leftBlock : rightBlock; DataChannel channel = new DataChannel(childBlock, parent, HASH_SHUFFLE, 32); channel.setDataFormat(dataFormat); if (join.getJoinType() != JoinType.CROSS) { // ShuffleKeys need to not have thea-join condition because Tajo supports only equi-join. Column [][] joinColumns = PlannerUtil.joinJoinKeyForEachTable(join.getJoinQual(), leftBlock.getPlan().getOutSchema(), rightBlock.getPlan().getOutSchema(), false); if (leftTable) { channel.setShuffleKeys(joinColumns[0]); } else { channel.setShuffleKeys(joinColumns[1]); } } return channel; }
private DataChannel createDataChannelFromJoin(ExecutionBlock leftBlock, ExecutionBlock rightBlock, ExecutionBlock parent, JoinNode join, boolean leftTable) { ExecutionBlock childBlock = leftTable ? leftBlock : rightBlock; DataChannel channel = new DataChannel(childBlock, parent, HASH_SHUFFLE, 32); channel.setDataFormat(dataFormat); if (join.getJoinType() != JoinType.CROSS) { // ShuffleKeys need to not have thea-join condition because Tajo supports only equi-join. Column [][] joinColumns = PlannerUtil.joinJoinKeyForEachTable(join.getJoinQual(), leftBlock.getPlan().getOutSchema(), rightBlock.getPlan().getOutSchema(), false); if (leftTable) { channel.setShuffleKeys(joinColumns[0]); } else { channel.setShuffleKeys(joinColumns[1]); } } return channel; }
private PhysicalExec createRightOuterMergeJoinPlan(TaskAttemptContext context, JoinNode plan, PhysicalExec leftExec, PhysicalExec rightExec) throws IOException { //the left operand is too large, so opt for merge join implementation LOG.info("Right Outer Join (" + plan.getPID() +") chooses [Merge Join]."); SortSpec[][] sortSpecs2 = PlannerUtil.getSortKeysFromJoinQual( plan.getJoinQual(), leftExec.getSchema(), rightExec.getSchema()); SortNode leftSortNode2 = LogicalPlan.createNodeWithoutPID(SortNode.class); leftSortNode2.setSortSpecs(sortSpecs2[0]); leftSortNode2.setInSchema(leftExec.getSchema()); leftSortNode2.setOutSchema(leftExec.getSchema()); ExternalSortExec outerSort2 = new ExternalSortExec(context, leftSortNode2, leftExec); SortNode rightSortNode2 = LogicalPlan.createNodeWithoutPID(SortNode.class); rightSortNode2.setSortSpecs(sortSpecs2[1]); rightSortNode2.setInSchema(rightExec.getSchema()); rightSortNode2.setOutSchema(rightExec.getSchema()); ExternalSortExec innerSort2 = new ExternalSortExec(context, rightSortNode2, rightExec); return new RightOuterMergeJoinExec(context, plan, outerSort2, innerSort2, sortSpecs2[0], sortSpecs2[1]); }
private PhysicalExec createRightOuterMergeJoinPlan(TaskAttemptContext context, JoinNode plan, PhysicalExec leftExec, PhysicalExec rightExec) throws IOException { //the left operand is too large, so opt for merge join implementation LOG.info("Right Outer Join (" + plan.getPID() +") chooses [Merge Join]."); SortSpec[][] sortSpecs2 = PlannerUtil.getSortKeysFromJoinQual( plan.getJoinQual(), leftExec.getSchema(), rightExec.getSchema()); SortNode leftSortNode2 = LogicalPlan.createNodeWithoutPID(SortNode.class); leftSortNode2.setSortSpecs(sortSpecs2[0]); leftSortNode2.setInSchema(leftExec.getSchema()); leftSortNode2.setOutSchema(leftExec.getSchema()); ExternalSortExec outerSort2 = new ExternalSortExec(context, leftSortNode2, leftExec); SortNode rightSortNode2 = LogicalPlan.createNodeWithoutPID(SortNode.class); rightSortNode2.setSortSpecs(sortSpecs2[1]); rightSortNode2.setInSchema(rightExec.getSchema()); rightSortNode2.setOutSchema(rightExec.getSchema()); ExternalSortExec innerSort2 = new ExternalSortExec(context, rightSortNode2, rightExec); return new RightOuterMergeJoinExec(context, plan, outerSort2, innerSort2, sortSpecs2[0], sortSpecs2[1]); }
private MergeJoinExec createMergeInnerJoin(TaskAttemptContext context, JoinNode plan, PhysicalExec leftExec, PhysicalExec rightExec) throws IOException { SortSpec[][] sortSpecs = PlannerUtil.getSortKeysFromJoinQual( plan.getJoinQual(), leftExec.getSchema(), rightExec.getSchema()); SortNode leftSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); leftSortNode.setSortSpecs(sortSpecs[0]); leftSortNode.setInSchema(leftExec.getSchema()); leftSortNode.setOutSchema(leftExec.getSchema()); ExternalSortExec outerSort = new ExternalSortExec(context, leftSortNode, leftExec); SortNode rightSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); rightSortNode.setSortSpecs(sortSpecs[1]); rightSortNode.setInSchema(rightExec.getSchema()); rightSortNode.setOutSchema(rightExec.getSchema()); ExternalSortExec innerSort = new ExternalSortExec(context, rightSortNode, rightExec); LOG.info("Join (" + plan.getPID() +") chooses [Merge Join]"); return new MergeJoinExec(context, plan, outerSort, innerSort, sortSpecs[0], sortSpecs[1]); }
private MergeJoinExec createMergeInnerJoin(TaskAttemptContext context, JoinNode plan, PhysicalExec leftExec, PhysicalExec rightExec) throws IOException { SortSpec[][] sortSpecs = PlannerUtil.getSortKeysFromJoinQual( plan.getJoinQual(), leftExec.getSchema(), rightExec.getSchema()); SortNode leftSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); leftSortNode.setSortSpecs(sortSpecs[0]); leftSortNode.setInSchema(leftExec.getSchema()); leftSortNode.setOutSchema(leftExec.getSchema()); ExternalSortExec outerSort = new ExternalSortExec(context, leftSortNode, leftExec); SortNode rightSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); rightSortNode.setSortSpecs(sortSpecs[1]); rightSortNode.setInSchema(rightExec.getSchema()); rightSortNode.setOutSchema(rightExec.getSchema()); ExternalSortExec innerSort = new ExternalSortExec(context, rightSortNode, rightExec); LOG.info("Join (" + plan.getPID() +") chooses [Merge Join]"); return new MergeJoinExec(context, plan, outerSort, innerSort, sortSpecs[0], sortSpecs[1]); }
private MergeFullOuterJoinExec createFullOuterMergeJoinPlan(TaskAttemptContext context, JoinNode plan, PhysicalExec leftExec, PhysicalExec rightExec) throws IOException { // if size too large, full outer merge join implementation LOG.info("Full Outer Join (" + plan.getPID() +") chooses [Merge Join]"); SortSpec[][] sortSpecs3 = PlannerUtil.getSortKeysFromJoinQual(plan.getJoinQual(), leftExec.getSchema(), rightExec.getSchema()); SortNode leftSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); leftSortNode.setSortSpecs(sortSpecs3[0]); leftSortNode.setInSchema(leftExec.getSchema()); leftSortNode.setOutSchema(leftExec.getSchema()); ExternalSortExec outerSort3 = new ExternalSortExec(context, leftSortNode, leftExec); SortNode rightSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); rightSortNode.setSortSpecs(sortSpecs3[1]); rightSortNode.setInSchema(rightExec.getSchema()); rightSortNode.setOutSchema(rightExec.getSchema()); ExternalSortExec innerSort3 = new ExternalSortExec(context, rightSortNode, rightExec); return new MergeFullOuterJoinExec(context, plan, outerSort3, innerSort3, sortSpecs3[0], sortSpecs3[1]); }
private MergeFullOuterJoinExec createFullOuterMergeJoinPlan(TaskAttemptContext context, JoinNode plan, PhysicalExec leftExec, PhysicalExec rightExec) throws IOException { // if size too large, full outer merge join implementation LOG.info("Full Outer Join (" + plan.getPID() +") chooses [Merge Join]"); SortSpec[][] sortSpecs3 = PlannerUtil.getSortKeysFromJoinQual(plan.getJoinQual(), leftExec.getSchema(), rightExec.getSchema()); SortNode leftSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); leftSortNode.setSortSpecs(sortSpecs3[0]); leftSortNode.setInSchema(leftExec.getSchema()); leftSortNode.setOutSchema(leftExec.getSchema()); ExternalSortExec outerSort3 = new ExternalSortExec(context, leftSortNode, leftExec); SortNode rightSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); rightSortNode.setSortSpecs(sortSpecs3[1]); rightSortNode.setInSchema(rightExec.getSchema()); rightSortNode.setOutSchema(rightExec.getSchema()); ExternalSortExec innerSort3 = new ExternalSortExec(context, rightSortNode, rightExec); return new MergeFullOuterJoinExec(context, plan, outerSort3, innerSort3, sortSpecs3[0], sortSpecs3[1]); }
public MergeJoinExec(TaskAttemptContext context, JoinNode plan, PhysicalExec outer, PhysicalExec inner, SortSpec[] outerSortKey, SortSpec[] innerSortKey) { super(context, plan, outer, inner); Preconditions.checkArgument(plan.hasJoinQual(), "Sort-merge join is only used for the equi-join, " + "but there is no join condition"); final int INITIAL_TUPLE_SLOT = context.getQueryContext().getInt(SessionVars.JOIN_HASH_TABLE_SIZE); this.outerTupleSlots = new TupleList(INITIAL_TUPLE_SLOT); this.innerTupleSlots = new TupleList(INITIAL_TUPLE_SLOT); SortSpec[][] sortSpecs = new SortSpec[2][]; sortSpecs[0] = outerSortKey; sortSpecs[1] = innerSortKey; this.joincomparator = new JoinTupleComparator(outer.getSchema(), inner.getSchema(), sortSpecs); this.tupleComparator = PhysicalPlanUtil.getComparatorsFromJoinQual( plan.getJoinQual(), outer.getSchema(), inner.getSchema()); this.outerIterator = outerTupleSlots.iterator(); this.innerIterator = innerTupleSlots.iterator(); prevOuterTuple = new VTuple(leftChild.getSchema().size()); prevInnerTuple = new VTuple(rightChild.getSchema().size()); }