@Override public LogicalNode visitSort(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Sort expr) throws TajoException { stack.push(expr); LogicalNode child = visit(ctx, stack, expr.getChild()); stack.pop(); SortNode sortNode = ctx.getPlan().createNode(SortNode.class); sortNode.setInSchema(child.getOutSchema()); sortNode.setOutSchema(child.getOutSchema()); return sortNode; }
@Override public LogicalNode visitSort(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Sort expr) throws TajoException { stack.push(expr); LogicalNode child = visit(ctx, stack, expr.getChild()); stack.pop(); SortNode sortNode = ctx.getPlan().createNode(SortNode.class); sortNode.setInSchema(child.getOutSchema()); sortNode.setOutSchema(child.getOutSchema()); return sortNode; }
private SortExec createSortExecForDistinctGroupby(TaskAttemptContext context, DistinctGroupbyNode distinctNode, PhysicalExec subOp, int phase) throws IOException { SortNode sortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); //2 phase: seq, groupby columns, distinct1 keys, distinct2 keys, //3 phase: groupby columns, seq, distinct1 keys, distinct2 keys, List<SortSpec> sortSpecs = new ArrayList<>(); if (phase == 2) { sortSpecs.add(new SortSpec(distinctNode.getTargets().get(0).getNamedColumn())); } for (Column eachColumn: distinctNode.getGroupingColumns()) { sortSpecs.add(new SortSpec(eachColumn)); } if (phase == 3) { sortSpecs.add(new SortSpec(distinctNode.getTargets().get(0).getNamedColumn())); } for (GroupbyNode eachGroupbyNode: distinctNode.getSubPlans()) { for (Column eachColumn: eachGroupbyNode.getGroupingColumns()) { sortSpecs.add(new SortSpec(eachColumn)); } } sortNode.setSortSpecs(sortSpecs.toArray(new SortSpec[sortSpecs.size()])); sortNode.setInSchema(distinctNode.getInSchema()); sortNode.setOutSchema(distinctNode.getInSchema()); ExternalSortExec sortExec = new ExternalSortExec(context, sortNode, subOp); return sortExec; }
private SortExec createSortExecForDistinctGroupby(TaskAttemptContext context, DistinctGroupbyNode distinctNode, PhysicalExec subOp, int phase) throws IOException { SortNode sortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); //2 phase: seq, groupby columns, distinct1 keys, distinct2 keys, //3 phase: groupby columns, seq, distinct1 keys, distinct2 keys, List<SortSpec> sortSpecs = new ArrayList<SortSpec>(); if (phase == 2) { sortSpecs.add(new SortSpec(distinctNode.getTargets()[0].getNamedColumn())); } for (Column eachColumn: distinctNode.getGroupingColumns()) { sortSpecs.add(new SortSpec(eachColumn)); } if (phase == 3) { sortSpecs.add(new SortSpec(distinctNode.getTargets()[0].getNamedColumn())); } for (GroupbyNode eachGroupbyNode: distinctNode.getSubPlans()) { for (Column eachColumn: eachGroupbyNode.getGroupingColumns()) { sortSpecs.add(new SortSpec(eachColumn)); } } sortNode.setSortSpecs(sortSpecs.toArray(new SortSpec[]{})); sortNode.setInSchema(distinctNode.getInSchema()); sortNode.setOutSchema(distinctNode.getInSchema()); ExternalSortExec sortExec = new ExternalSortExec(context, sortNode, subOp); return sortExec; }
private PhysicalExec createSortAggregationDistinctGroupbyExec(TaskAttemptContext ctx, DistinctGroupbyNode distinctGroupbyNode, PhysicalExec subOp, DistinctGroupbyEnforcer enforcer) throws IOException { List<GroupbyNode> groupbyNodes = distinctGroupbyNode.getSubPlans(); SortAggregateExec[] sortAggregateExec = new SortAggregateExec[groupbyNodes.size()]; List<SortSpecArray> sortSpecArrays = enforcer.getSortSpecArraysList(); int index = 0; for (GroupbyNode eachGroupbyNode: groupbyNodes) { SortSpecArray sortSpecArray = sortSpecArrays.get(index); SortSpec[] sortSpecs = new SortSpec[sortSpecArray.getSortSpecsList().size()]; int sortIndex = 0; for (SortSpecProto eachProto: sortSpecArray.getSortSpecsList()) { sortSpecs[sortIndex++] = new SortSpec(eachProto); } SortNode sortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); sortNode.setSortSpecs(sortSpecs); sortNode.setInSchema(subOp.getSchema()); sortNode.setOutSchema(eachGroupbyNode.getInSchema()); ExternalSortExec sortExec = new ExternalSortExec(ctx, sortNode, subOp); sortAggregateExec[index++] = new SortAggregateExec(ctx, eachGroupbyNode, sortExec); } return new DistinctGroupbySortAggregationExec(ctx, distinctGroupbyNode, sortAggregateExec); }
private PhysicalExec createSortAggregationDistinctGroupbyExec(TaskAttemptContext ctx, DistinctGroupbyNode distinctGroupbyNode, PhysicalExec subOp, DistinctGroupbyEnforcer enforcer) throws IOException { List<GroupbyNode> groupbyNodes = distinctGroupbyNode.getSubPlans(); SortAggregateExec[] sortAggregateExec = new SortAggregateExec[groupbyNodes.size()]; List<SortSpecArray> sortSpecArrays = enforcer.getSortSpecArraysList(); int index = 0; for (GroupbyNode eachGroupbyNode: groupbyNodes) { SortSpecArray sortSpecArray = sortSpecArrays.get(index); SortSpec[] sortSpecs = new SortSpec[sortSpecArray.getSortSpecsList().size()]; int sortIndex = 0; for (SortSpecProto eachProto: sortSpecArray.getSortSpecsList()) { sortSpecs[sortIndex++] = new SortSpec(eachProto); } SortNode sortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); sortNode.setSortSpecs(sortSpecs); sortNode.setInSchema(subOp.getSchema()); sortNode.setOutSchema(eachGroupbyNode.getInSchema()); ExternalSortExec sortExec = new ExternalSortExec(ctx, sortNode, subOp); sortAggregateExec[index++] = new SortAggregateExec(ctx, eachGroupbyNode, sortExec); } return new DistinctGroupbySortAggregationExec(ctx, distinctGroupbyNode, sortAggregateExec); }
private static SortNode convertSort(Map<Integer, LogicalNode> nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.SortNode sortProto = protoNode.getSort(); SortNode sortNode = new SortNode(protoNode.getNodeId()); sortNode.setChild(nodeMap.get(sortProto.getChildSeq())); sortNode.setInSchema(convertSchema(protoNode.getInSchema())); sortNode.setOutSchema(convertSchema(protoNode.getOutSchema())); sortNode.setSortSpecs(convertSortSpecs(sortProto.getSortSpecsList())); return sortNode; }
private MergeJoinExec createMergeInnerJoin(TaskAttemptContext context, JoinNode plan, PhysicalExec leftExec, PhysicalExec rightExec) throws IOException { SortSpec[][] sortSpecs = PlannerUtil.getSortKeysFromJoinQual( plan.getJoinQual(), leftExec.getSchema(), rightExec.getSchema()); SortNode leftSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); leftSortNode.setSortSpecs(sortSpecs[0]); leftSortNode.setInSchema(leftExec.getSchema()); leftSortNode.setOutSchema(leftExec.getSchema()); ExternalSortExec outerSort = new ExternalSortExec(context, leftSortNode, leftExec); SortNode rightSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); rightSortNode.setSortSpecs(sortSpecs[1]); rightSortNode.setInSchema(rightExec.getSchema()); rightSortNode.setOutSchema(rightExec.getSchema()); ExternalSortExec innerSort = new ExternalSortExec(context, rightSortNode, rightExec); LOG.info("Join (" + plan.getPID() +") chooses [Merge Join]"); return new MergeJoinExec(context, plan, outerSort, innerSort, sortSpecs[0], sortSpecs[1]); }
private PhysicalExec createRightOuterMergeJoinPlan(TaskAttemptContext context, JoinNode plan, PhysicalExec leftExec, PhysicalExec rightExec) throws IOException { //the left operand is too large, so opt for merge join implementation LOG.info("Right Outer Join (" + plan.getPID() +") chooses [Merge Join]."); SortSpec[][] sortSpecs2 = PlannerUtil.getSortKeysFromJoinQual( plan.getJoinQual(), leftExec.getSchema(), rightExec.getSchema()); SortNode leftSortNode2 = LogicalPlan.createNodeWithoutPID(SortNode.class); leftSortNode2.setSortSpecs(sortSpecs2[0]); leftSortNode2.setInSchema(leftExec.getSchema()); leftSortNode2.setOutSchema(leftExec.getSchema()); ExternalSortExec outerSort2 = new ExternalSortExec(context, leftSortNode2, leftExec); SortNode rightSortNode2 = LogicalPlan.createNodeWithoutPID(SortNode.class); rightSortNode2.setSortSpecs(sortSpecs2[1]); rightSortNode2.setInSchema(rightExec.getSchema()); rightSortNode2.setOutSchema(rightExec.getSchema()); ExternalSortExec innerSort2 = new ExternalSortExec(context, rightSortNode2, rightExec); return new RightOuterMergeJoinExec(context, plan, outerSort2, innerSort2, sortSpecs2[0], sortSpecs2[1]); }
private MergeJoinExec createMergeInnerJoin(TaskAttemptContext context, JoinNode plan, PhysicalExec leftExec, PhysicalExec rightExec) throws IOException { SortSpec[][] sortSpecs = PlannerUtil.getSortKeysFromJoinQual( plan.getJoinQual(), leftExec.getSchema(), rightExec.getSchema()); SortNode leftSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); leftSortNode.setSortSpecs(sortSpecs[0]); leftSortNode.setInSchema(leftExec.getSchema()); leftSortNode.setOutSchema(leftExec.getSchema()); ExternalSortExec outerSort = new ExternalSortExec(context, leftSortNode, leftExec); SortNode rightSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); rightSortNode.setSortSpecs(sortSpecs[1]); rightSortNode.setInSchema(rightExec.getSchema()); rightSortNode.setOutSchema(rightExec.getSchema()); ExternalSortExec innerSort = new ExternalSortExec(context, rightSortNode, rightExec); LOG.info("Join (" + plan.getPID() +") chooses [Merge Join]"); return new MergeJoinExec(context, plan, outerSort, innerSort, sortSpecs[0], sortSpecs[1]); }
@Override public LogicalNode visitSort(ProcessorContext ctx, Stack<Expr> stack, Sort expr) throws TajoException { super.visitSort(ctx, stack, expr); SortNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr); LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getChild()); node.setInSchema(child.getOutSchema()); node.setOutSchema(node.getInSchema()); return node; }
private PhysicalExec createRightOuterMergeJoinPlan(TaskAttemptContext context, JoinNode plan, PhysicalExec leftExec, PhysicalExec rightExec) throws IOException { //the left operand is too large, so opt for merge join implementation LOG.info("Right Outer Join (" + plan.getPID() +") chooses [Merge Join]."); SortSpec[][] sortSpecs2 = PlannerUtil.getSortKeysFromJoinQual( plan.getJoinQual(), leftExec.getSchema(), rightExec.getSchema()); SortNode leftSortNode2 = LogicalPlan.createNodeWithoutPID(SortNode.class); leftSortNode2.setSortSpecs(sortSpecs2[0]); leftSortNode2.setInSchema(leftExec.getSchema()); leftSortNode2.setOutSchema(leftExec.getSchema()); ExternalSortExec outerSort2 = new ExternalSortExec(context, leftSortNode2, leftExec); SortNode rightSortNode2 = LogicalPlan.createNodeWithoutPID(SortNode.class); rightSortNode2.setSortSpecs(sortSpecs2[1]); rightSortNode2.setInSchema(rightExec.getSchema()); rightSortNode2.setOutSchema(rightExec.getSchema()); ExternalSortExec innerSort2 = new ExternalSortExec(context, rightSortNode2, rightExec); return new RightOuterMergeJoinExec(context, plan, outerSort2, innerSort2, sortSpecs2[0], sortSpecs2[1]); }
private MergeFullOuterJoinExec createFullOuterMergeJoinPlan(TaskAttemptContext context, JoinNode plan, PhysicalExec leftExec, PhysicalExec rightExec) throws IOException { // if size too large, full outer merge join implementation LOG.info("Full Outer Join (" + plan.getPID() +") chooses [Merge Join]"); SortSpec[][] sortSpecs3 = PlannerUtil.getSortKeysFromJoinQual(plan.getJoinQual(), leftExec.getSchema(), rightExec.getSchema()); SortNode leftSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); leftSortNode.setSortSpecs(sortSpecs3[0]); leftSortNode.setInSchema(leftExec.getSchema()); leftSortNode.setOutSchema(leftExec.getSchema()); ExternalSortExec outerSort3 = new ExternalSortExec(context, leftSortNode, leftExec); SortNode rightSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); rightSortNode.setSortSpecs(sortSpecs3[1]); rightSortNode.setInSchema(rightExec.getSchema()); rightSortNode.setOutSchema(rightExec.getSchema()); ExternalSortExec innerSort3 = new ExternalSortExec(context, rightSortNode, rightExec); return new MergeFullOuterJoinExec(context, plan, outerSort3, innerSort3, sortSpecs3[0], sortSpecs3[1]); }
private static SortNode convertSort(Map<Integer, LogicalNode> nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.SortNode sortProto = protoNode.getSort(); SortNode sortNode = new SortNode(protoNode.getNodeId()); sortNode.setChild(nodeMap.get(sortProto.getChildSeq())); sortNode.setInSchema(convertSchema(protoNode.getInSchema())); sortNode.setOutSchema(convertSchema(protoNode.getOutSchema())); sortNode.setSortSpecs(convertSortSpecs(sortProto.getSortSpecsList())); return sortNode; }
public PhysicalExec createWindowAgg(TaskAttemptContext context,WindowAggNode windowAggNode, PhysicalExec subOp) throws IOException { PhysicalExec child = subOp; if (windowAggNode.hasPartitionKeys()) { Column[] grpColumns = windowAggNode.getPartitionKeys(); SortSpec[] sortSpecs = new SortSpec[grpColumns.length]; for (int i = 0; i < grpColumns.length; i++) { sortSpecs[i] = new SortSpec(grpColumns[i], true, false); } SortNode sortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); sortNode.setSortSpecs(sortSpecs); sortNode.setInSchema(subOp.getSchema()); sortNode.setOutSchema(subOp.getSchema()); child = new ExternalSortExec(context, sortNode, subOp); LOG.info("The planner chooses [Sort Aggregation] in (" + StringUtils.join(sortSpecs) + ")"); } return new WindowAggExec(context, windowAggNode, child); }
private MergeFullOuterJoinExec createFullOuterMergeJoinPlan(TaskAttemptContext context, JoinNode plan, PhysicalExec leftExec, PhysicalExec rightExec) throws IOException { // if size too large, full outer merge join implementation LOG.info("Full Outer Join (" + plan.getPID() +") chooses [Merge Join]"); SortSpec[][] sortSpecs3 = PlannerUtil.getSortKeysFromJoinQual(plan.getJoinQual(), leftExec.getSchema(), rightExec.getSchema()); SortNode leftSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); leftSortNode.setSortSpecs(sortSpecs3[0]); leftSortNode.setInSchema(leftExec.getSchema()); leftSortNode.setOutSchema(leftExec.getSchema()); ExternalSortExec outerSort3 = new ExternalSortExec(context, leftSortNode, leftExec); SortNode rightSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); rightSortNode.setSortSpecs(sortSpecs3[1]); rightSortNode.setInSchema(rightExec.getSchema()); rightSortNode.setOutSchema(rightExec.getSchema()); ExternalSortExec innerSort3 = new ExternalSortExec(context, rightSortNode, rightExec); return new MergeFullOuterJoinExec(context, plan, outerSort3, innerSort3, sortSpecs3[0], sortSpecs3[1]); }
public PhysicalExec createWindowAgg(TaskAttemptContext context,WindowAggNode windowAggNode, PhysicalExec subOp) throws IOException { PhysicalExec child = subOp; if (windowAggNode.hasPartitionKeys()) { Column[] grpColumns = windowAggNode.getPartitionKeys(); SortSpec[] sortSpecs = new SortSpec[grpColumns.length]; for (int i = 0; i < grpColumns.length; i++) { sortSpecs[i] = new SortSpec(grpColumns[i], true, false); } SortNode sortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); sortNode.setSortSpecs(sortSpecs); sortNode.setInSchema(subOp.getSchema()); sortNode.setOutSchema(subOp.getSchema()); child = new ExternalSortExec(context, sortNode, subOp); LOG.info("The planner chooses [Sort Aggregation] in (" + StringUtils.join(sortSpecs) + ")"); } return new WindowAggExec(context, windowAggNode, child); }
@Override public LogicalNode visitSort(ProcessorContext ctx, Stack<Expr> stack, Sort expr) throws TajoException { for (Sort.SortSpec sortSpec : expr.getSortSpecs()) { Set<ColumnReferenceExpr> columns = ExprFinder.finds(sortSpec.getKey(), OpType.Column); for (ColumnReferenceExpr col : columns) { if (!ctx.aliasSet.contains(col.getName())) { NameRefInSelectListNormalizer.normalize(ctx.planContext, col); TUtil.putToNestedList(ctx.projectColumns, col.getQualifier(), col); } } } super.visitSort(ctx, stack, expr); SortNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr); LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getChild()); node.setInSchema(child.getOutSchema()); node.setOutSchema(node.getInSchema()); return node; }
sortNode.setInSchema(child.getSchema()); sortNode.setOutSchema(child.getSchema());
sortNode.setInSchema(dupRemoval.getOutSchema()); sortNode.setOutSchema(dupRemoval.getOutSchema()); projectionNode.setInSchema(sortNode.getOutSchema());