private MergeJoinExec createMergeInnerJoin(TaskAttemptContext context, JoinNode plan, PhysicalExec leftExec, PhysicalExec rightExec) throws IOException { SortSpec[][] sortSpecs = PlannerUtil.getSortKeysFromJoinQual( plan.getJoinQual(), leftExec.getSchema(), rightExec.getSchema()); SortNode leftSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); leftSortNode.setSortSpecs(sortSpecs[0]); leftSortNode.setInSchema(leftExec.getSchema()); leftSortNode.setOutSchema(leftExec.getSchema()); ExternalSortExec outerSort = new ExternalSortExec(context, leftSortNode, leftExec); SortNode rightSortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); rightSortNode.setSortSpecs(sortSpecs[1]); rightSortNode.setInSchema(rightExec.getSchema()); rightSortNode.setOutSchema(rightExec.getSchema()); ExternalSortExec innerSort = new ExternalSortExec(context, rightSortNode, rightExec); LOG.info("Join (" + plan.getPID() +") chooses [Merge Join]"); return new MergeJoinExec(context, plan, outerSort, innerSort, sortSpecs[0], sortSpecs[1]); }
public String toString() { StringBuilder sb = new StringBuilder("Sort [key= "); for (int i = 0; i < sortKeys.length; i++) { sb.append(sortKeys[i].toString()); if(i < sortKeys.length - 1) { sb.append(","); } } sb.append("]"); sb.append("\n\"out schema: " + getOutSchema() + "\n\"in schema: " + getInSchema()); return sb.toString()+"\n" + getChild().toString(); }
private static SortNode convertSort(Map<Integer, LogicalNode> nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.SortNode sortProto = protoNode.getSort(); SortNode sortNode = new SortNode(protoNode.getNodeId()); sortNode.setChild(nodeMap.get(sortProto.getChildSeq())); sortNode.setInSchema(convertSchema(protoNode.getInSchema())); sortNode.setOutSchema(convertSchema(protoNode.getOutSchema())); sortNode.setSortSpecs(convertSortSpecs(sortProto.getSortSpecsList())); return sortNode; }
@Override public LogicalNode visitSort(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Sort expr) throws TajoException { stack.push(expr); LogicalNode child = visit(ctx, stack, expr.getChild()); stack.pop(); SortNode sortNode = ctx.getPlan().createNode(SortNode.class); sortNode.setInSchema(child.getOutSchema()); sortNode.setOutSchema(child.getOutSchema()); return sortNode; }
dupRemoval.setChild(sortNode.getChild()); dupRemoval.setInSchema(sortNode.getInSchema()); dupRemoval.setTargets(PlannerUtil.schemaToTargets(sortNode.getInSchema())); dupRemoval.setGroupingColumns(sortNode.getInSchema().toArray()); sortNode.setChild(dupRemoval); sortNode.setInSchema(dupRemoval.getOutSchema()); sortNode.setOutSchema(dupRemoval.getOutSchema()); projectionNode.setInSchema(sortNode.getOutSchema()); projectionNode.setChild(sortNode);
if (firstSortNode.getChild().getType() == NodeType.TABLE_SUBQUERY && ((TableSubQueryNode)firstSortNode.getChild()).getSubQuery().getType() == NodeType.UNION) { channel.setShuffle(RANGE_SHUFFLE, PlannerUtil.sortSpecsToSchema(currentNode.getSortKeys()).toArray(), 32); channel.setSchema(firstSortNode.getOutSchema()); s1.setChild(subBlock.getPlan()); subBlock.setPlan(s1); currentNode.setChild(secondScan); currentNode.setInSchema(secondScan.getOutSchema()); currentBlock.setPlan(currentNode); currentBlock.getEnforcer().addSortedInput(secondScan.getTableName(), currentNode.getSortKeys()); firstSortNode.setChild(childBlockPlan); firstSortNode.setInSchema(childBlockPlan.getOutSchema()); firstSortNode.setOutSchema(childBlockPlan.getOutSchema()); childBlock.setPlan(firstSortNode); channel.setShuffleKeys(PlannerUtil.sortSpecsToSchema(currentNode.getSortKeys()).toArray()); channel.setSchema(firstSortNode.getOutSchema()); currentNode.setChild(secondScan); currentNode.setInSchema(secondScan.getOutSchema()); currentBlock.setPlan(currentNode); currentBlock.getEnforcer().addSortedInput(secondScan.getTableName(), currentNode.getSortKeys()); masterPlan.addConnect(channel);
Context newContext = new Context(context); final int sortKeyNum = node.getSortKeys().length; String [] keyNames = new String[sortKeyNum]; for (int i = 0; i < sortKeyNum; i++) { SortSpec sortSpec = node.getSortKeys()[i]; keyNames[i] = newContext.addExpr(new FieldEval(sortSpec.getSortKey())); if (context.targetListMgr.isEvaluated(sortKey)) { Column c = target.getNamedColumn(); SortSpec sortSpec = new SortSpec(c, node.getSortKeys()[i].isAscending(), node.getSortKeys()[i].isNullsFirst()); if (!sortSpecs.contains(sortSpec)) { sortSpecs.add(sortSpec); if (target.getEvalTree().getType() == EvalType.FIELD) { Column c = ((FieldEval)target.getEvalTree()).getColumnRef(); SortSpec sortSpec = new SortSpec(c, node.getSortKeys()[i].isAscending(), node.getSortKeys()[i].isNullsFirst()); if (!sortSpecs.contains(sortSpec)) { sortSpecs.add(sortSpec); node.setSortSpecs(sortSpecs.toArray(new SortSpec[sortSpecs.size()])); node.setInSchema(child.getOutSchema()); node.setOutSchema(child.getOutSchema()); return node;
} else if (child.getType() == NodeType.SORT) { SortNode sortNode = (SortNode) child; windowAggNode.setChild(sortNode.getChild()); windowAggNode.setInSchema(sortNode.getChild().getOutSchema()); sortNode.setChild(windowAggNode); } else { windowAggNode.setChild(child); } else if (child.getType() == NodeType.SORT) { SortNode sortNode = (SortNode) child; sortNode.setInSchema(windowAggNode.getOutSchema()); sortNode.setOutSchema(windowAggNode.getOutSchema()); return null; } else {
@Override public LogicalNode visitSort(ProcessorContext ctx, Stack<Expr> stack, Sort expr) throws TajoException { super.visitSort(ctx, stack, expr); SortNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr); LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getChild()); node.setInSchema(child.getOutSchema()); node.setOutSchema(node.getInSchema()); return node; }
private ExternalSortExec(final TaskAttemptContext context, final SortNode plan) throws PhysicalPlanningException { super(context, plan.getInSchema(), plan.getOutSchema(), null, plan.getSortKeys()); this.plan = plan; this.defaultFanout = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT); if (defaultFanout < 2) { throw new PhysicalPlanningException(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT.varname + " cannot be lower than 2"); } // TODO - sort buffer and core num should be changed to use the allocated container resource. this.sortBufferBytesNum = context.getQueryContext().getInt(SessionVars.EXTSORT_BUFFER_SIZE) * StorageUnit.MB; this.allocatedCoreNum = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_THREAD_NUM); this.localDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); this.localFS = new RawLocalFileSystem(); this.intermediateMeta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW, context.getConf()); this.inputStats = new TableStats(); }
SortSpec [] sortSpecs = sortNode.getSortKeys(); Schema sortSchema = new Schema(channel.getShuffleKeys()); if (sortNode.getSortPurpose() == SortPurpose.STORAGE_SPECIFIED) { String dataFormat = PlannerUtil.getDataFormat(masterPlan.getLogicalPlan()); CatalogService catalog = stage.getContext().getQueryMasterContext().getWorkerContext().getCatalog(); stage.getContext().getQueryContext(), tableDesc, sortNode.getInSchema(), sortSpecs, mergedRange);
@Override public RESULT visitSort(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, SortNode node, Stack<LogicalNode> stack) throws TajoException { stack.push(node); RESULT result = visit(context, plan, block, node.getChild(), stack); stack.pop(); return result; }
public PhysicalExec createSortPlan(TaskAttemptContext context, SortNode sortNode, PhysicalExec child) throws IOException { // check if it is a distributed merge sort // If so, it does need to create a sort executor because // the sort executor is created at the scan planning if (child instanceof SortExec) { SortExec childSortExec = (SortExec) child; if (TUtil.checkEquals(sortNode.getSortKeys(), childSortExec.getSortSpecs())) { return child; } } return new ExternalSortExec(context, sortNode, child); }
dupRemoval.setChild(sortNode.getChild()); dupRemoval.setInSchema(sortNode.getInSchema()); dupRemoval.setTargets(PlannerUtil.schemaToTargets(sortNode.getInSchema())); dupRemoval.setGroupingColumns(sortNode.getInSchema().toArray()); sortNode.setChild(dupRemoval); sortNode.setInSchema(dupRemoval.getOutSchema()); sortNode.setOutSchema(dupRemoval.getOutSchema()); projectionNode.setInSchema(sortNode.getOutSchema()); projectionNode.setChild(sortNode);
if (firstSortNode.getChild().getType() == NodeType.TABLE_SUBQUERY && ((TableSubQueryNode)firstSortNode.getChild()).getSubQuery().getType() == NodeType.UNION) { channel.setShuffle(RANGE_SHUFFLE, PlannerUtil.sortSpecsToSchema(currentNode.getSortKeys()).toArray(), 32); channel.setSchema(firstSortNode.getOutSchema()); s1.setChild(subBlock.getPlan()); subBlock.setPlan(s1); currentNode.setChild(secondScan); currentNode.setInSchema(secondScan.getOutSchema()); currentBlock.setPlan(currentNode); currentBlock.getEnforcer().addSortedInput(secondScan.getTableName(), currentNode.getSortKeys()); firstSortNode.setChild(childBlockPlan); firstSortNode.setInSchema(childBlockPlan.getOutSchema()); firstSortNode.setOutSchema(childBlockPlan.getOutSchema()); childBlock.setPlan(firstSortNode); channel.setShuffleKeys(PlannerUtil.sortSpecsToSchema(currentNode.getSortKeys()).toArray()); channel.setSchema(firstSortNode.getOutSchema()); currentNode.setChild(secondScan); currentNode.setInSchema(secondScan.getOutSchema()); currentBlock.setPlan(currentNode); currentBlock.getEnforcer().addSortedInput(secondScan.getTableName(), currentNode.getSortKeys()); masterPlan.addConnect(channel);
Context newContext = new Context(context); final int sortKeyNum = node.getSortKeys().length; String [] keyNames = new String[sortKeyNum]; for (int i = 0; i < sortKeyNum; i++) { SortSpec sortSpec = node.getSortKeys()[i]; keyNames[i] = newContext.addExpr(new FieldEval(sortSpec.getSortKey())); if (context.targetListMgr.isEvaluated(sortKey)) { Column c = target.getNamedColumn(); SortSpec sortSpec = new SortSpec(c, node.getSortKeys()[i].isAscending(), node.getSortKeys()[i].isNullsFirst()); if (!sortSpecs.contains(sortSpec)) { sortSpecs.add(sortSpec); if (target.getEvalTree().getType() == EvalType.FIELD) { Column c = ((FieldEval)target.getEvalTree()).getColumnRef(); SortSpec sortSpec = new SortSpec(c, node.getSortKeys()[i].isAscending(), node.getSortKeys()[i].isNullsFirst()); if (!sortSpecs.contains(sortSpec)) { sortSpecs.add(sortSpec); node.setSortSpecs(sortSpecs.toArray(new SortSpec[sortSpecs.size()])); node.setInSchema(child.getOutSchema()); node.setOutSchema(child.getOutSchema()); return node;
} else if (child.getType() == NodeType.SORT) { SortNode sortNode = (SortNode) child; windowAggNode.setChild(sortNode.getChild()); windowAggNode.setInSchema(sortNode.getChild().getOutSchema()); sortNode.setChild(windowAggNode); } else { windowAggNode.setChild(child); } else if (child.getType() == NodeType.SORT) { SortNode sortNode = (SortNode) child; sortNode.setInSchema(windowAggNode.getOutSchema()); sortNode.setOutSchema(windowAggNode.getOutSchema()); return null; } else {
@Override public LogicalNode visitSort(ProcessorContext ctx, Stack<Expr> stack, Sort expr) throws TajoException { for (Sort.SortSpec sortSpec : expr.getSortSpecs()) { Set<ColumnReferenceExpr> columns = ExprFinder.finds(sortSpec.getKey(), OpType.Column); for (ColumnReferenceExpr col : columns) { if (!ctx.aliasSet.contains(col.getName())) { NameRefInSelectListNormalizer.normalize(ctx.planContext, col); TUtil.putToNestedList(ctx.projectColumns, col.getQualifier(), col); } } } super.visitSort(ctx, stack, expr); SortNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr); LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getChild()); node.setInSchema(child.getOutSchema()); node.setOutSchema(node.getInSchema()); return node; }
@Override public LogicalNode visitSort(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, Sort expr) throws TajoException { stack.push(expr); LogicalNode child = visit(ctx, stack, expr.getChild()); stack.pop(); SortNode sortNode = ctx.getPlan().createNode(SortNode.class); sortNode.setInSchema(child.getOutSchema()); sortNode.setOutSchema(child.getOutSchema()); return sortNode; }
private ExternalSortExec(final TaskAttemptContext context, final SortNode plan) throws PhysicalPlanningException { super(context, plan.getInSchema(), plan.getOutSchema(), null, plan.getSortKeys()); this.plan = plan; this.defaultFanout = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT); if (defaultFanout < 2) { throw new PhysicalPlanningException(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT.varname + " cannot be lower than 2"); } // TODO - sort buffer and core num should be changed to use the allocated container resource. this.sortBufferBytesNum = context.getQueryContext().getInt(SessionVars.EXTSORT_BUFFER_SIZE) * StorageUnit.MB; this.allocatedCoreNum = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_THREAD_NUM); this.localDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); this.localFS = new RawLocalFileSystem(); this.intermediateMeta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW, context.getConf()); this.inputStats = new TableStats(); this.sortAlgorithm = getSortAlgorithm(context.getQueryContext(), sortSpecs); LOG.info(sortAlgorithm.name() + " sort is selected"); }