public PhysicalExec createSortPlan(TaskAttemptContext context, SortNode sortNode, PhysicalExec child) throws IOException { // check if it is a distributed merge sort // If so, it does need to create a sort executor because // the sort executor is created at the scan planning if (child instanceof SortExec) { SortExec childSortExec = (SortExec) child; if (TUtil.checkEquals(sortNode.getSortKeys(), childSortExec.getSortSpecs())) { return child; } } return new ExternalSortExec(context, sortNode, child); }
public PhysicalExec createSortPlan(TaskAttemptContext context, SortNode sortNode, PhysicalExec child) throws IOException { // check if it is a distributed merge sort // If so, it does need to create a sort executor because // the sort executor is created at the scan planning if (child instanceof SortExec) { SortExec childSortExec = (SortExec) child; if (TUtil.checkEquals(sortNode.getSortKeys(), childSortExec.getSortSpecs())) { return child; } } return new ExternalSortExec(context, sortNode, child); }
private boolean checkIfSortEquivalance(TaskAttemptContext ctx, ScanNode scanNode, Stack<LogicalNode> node) { Enforcer enforcer = ctx.getEnforcer(); List<EnforceProperty> property = enforcer.getEnforceProperties(EnforceType.SORTED_INPUT); if (property != null && property.size() > 0 && node.peek().getType() == NodeType.SORT) { SortNode sortNode = (SortNode) node.peek(); SortedInputEnforce sortEnforcer = property.get(0).getSortedInput(); boolean condition = scanNode.getTableName().equals(sortEnforcer.getTableName()); SortSpec [] sortSpecs = LogicalNodeDeserializer.convertSortSpecs(sortEnforcer.getSortSpecsList()); return condition && TUtil.checkEquals(sortNode.getSortKeys(), sortSpecs); } else { return false; } }
private boolean checkIfSortEquivalance(TaskAttemptContext ctx, ScanNode scanNode, Stack<LogicalNode> node) { Enforcer enforcer = ctx.getEnforcer(); List<EnforceProperty> property = enforcer.getEnforceProperties(EnforceType.SORTED_INPUT); if (property != null && property.size() > 0 && node.peek().getType() == NodeType.SORT) { SortNode sortNode = (SortNode) node.peek(); SortedInputEnforce sortEnforcer = property.get(0).getSortedInput(); boolean condition = scanNode.getTableName().equals(sortEnforcer.getTableName()); SortSpec [] sortSpecs = LogicalNodeDeserializer.convertSortSpecs(sortEnforcer.getSortSpecsList()); return condition && TUtil.checkEquals(sortNode.getSortKeys(), sortSpecs); } else { return false; } }
@Override public LogicalNode visitSort(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, SortNode sort, Stack<LogicalNode> stack) throws TajoException { super.visitSort(context, plan, block, sort, stack); int [] childIds = registerGetChildIds(context, sort); PlanProto.SortNode.Builder sortBuilder = PlanProto.SortNode.newBuilder(); sortBuilder.setChildSeq(childIds[0]); for (int i = 0; i < sort.getSortKeys().length; i++) { sortBuilder.addSortSpecs(sort.getSortKeys()[i].getProto()); } PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, sort); nodeBuilder.setSort(sortBuilder); context.treeBuilder.addNodes(nodeBuilder); return sort; }
Context newContext = new Context(context); final int sortKeyNum = node.getSortKeys().length; String [] keyNames = new String[sortKeyNum]; for (int i = 0; i < sortKeyNum; i++) { SortSpec sortSpec = node.getSortKeys()[i]; keyNames[i] = newContext.addExpr(new FieldEval(sortSpec.getSortKey())); if (context.targetListMgr.isEvaluated(sortKey)) { Column c = target.getNamedColumn(); SortSpec sortSpec = new SortSpec(c, node.getSortKeys()[i].isAscending(), node.getSortKeys()[i].isNullsFirst()); if (!sortSpecs.contains(sortSpec)) { sortSpecs.add(sortSpec); if (target.getEvalTree().getType() == EvalType.FIELD) { Column c = ((FieldEval)target.getEvalTree()).getColumnRef(); SortSpec sortSpec = new SortSpec(c, node.getSortKeys()[i].isAscending(), node.getSortKeys()[i].isNullsFirst()); if (!sortSpecs.contains(sortSpec)) { sortSpecs.add(sortSpec);
Context newContext = new Context(context); final int sortKeyNum = node.getSortKeys().length; String [] keyNames = new String[sortKeyNum]; for (int i = 0; i < sortKeyNum; i++) { SortSpec sortSpec = node.getSortKeys()[i]; keyNames[i] = newContext.addExpr(new FieldEval(sortSpec.getSortKey())); if (context.targetListMgr.isEvaluated(sortKey)) { Column c = target.getNamedColumn(); SortSpec sortSpec = new SortSpec(c, node.getSortKeys()[i].isAscending(), node.getSortKeys()[i].isNullsFirst()); if (!sortSpecs.contains(sortSpec)) { sortSpecs.add(sortSpec); if (target.getEvalTree().getType() == EvalType.FIELD) { Column c = ((FieldEval)target.getEvalTree()).getColumnRef(); SortSpec sortSpec = new SortSpec(c, node.getSortKeys()[i].isAscending(), node.getSortKeys()[i].isNullsFirst()); if (!sortSpecs.contains(sortSpec)) { sortSpecs.add(sortSpec);
@Override public LogicalNode visitSort(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, SortNode sort, Stack<LogicalNode> stack) throws TajoException { super.visitSort(context, plan, block, sort, stack); int [] childIds = registerGetChildIds(context, sort); PlanProto.SortNode.Builder sortBuilder = PlanProto.SortNode.newBuilder(); sortBuilder.setChildSeq(childIds[0]); for (int i = 0; i < sort.getSortKeys().length; i++) { sortBuilder.addSortSpecs(sort.getSortKeys()[i].getProto()); } PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, sort); nodeBuilder.setSort(sortBuilder); context.treeBuilder.addNodes(nodeBuilder); return sort; }
this.finalSchema = PlannerUtil.sortSpecsToSchema(sortNode.getSortKeys()); this.sortComp = new BaseTupleComparator(finalSchema, sortNode.getSortKeys());
this.finalSchema = PlannerUtil.sortSpecsToSchema(sortNode.getSortKeys()); this.sortComp = new BaseTupleComparator(finalSchema, sortNode.getSortKeys());
private ExternalSortExec(final TaskAttemptContext context, final SortNode plan) throws PhysicalPlanningException { super(context, plan.getInSchema(), plan.getOutSchema(), null, plan.getSortKeys()); this.plan = plan; this.defaultFanout = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT); if (defaultFanout < 2) { throw new PhysicalPlanningException(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT.varname + " cannot be lower than 2"); } // TODO - sort buffer and core num should be changed to use the allocated container resource. this.sortBufferBytesNum = context.getQueryContext().getInt(SessionVars.EXTSORT_BUFFER_SIZE) * StorageUnit.MB; this.allocatedCoreNum = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_THREAD_NUM); this.localDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); this.localFS = new RawLocalFileSystem(); this.intermediateMeta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW, context.getConf()); this.inputStats = new TableStats(); this.sortAlgorithm = getSortAlgorithm(context.getQueryContext(), sortSpecs); LOG.info(sortAlgorithm.name() + " sort is selected"); }
SortSpec [] sortSpecs = sortNode.getSortKeys(); Schema sortSchema = new Schema(channel.getShuffleKeys());
SortSpec [] sortSpecs = sortNode.getSortKeys(); Schema sortSchema = SchemaBuilder.builder().addAll(channel.getShuffleKeys()).build();
private ExternalSortExec(final TaskAttemptContext context, final SortNode plan) throws PhysicalPlanningException { super(context, plan.getInSchema(), plan.getOutSchema(), null, plan.getSortKeys()); this.plan = plan; this.defaultFanout = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT); if (defaultFanout < 2) { throw new PhysicalPlanningException(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT.varname + " cannot be lower than 2"); } // TODO - sort buffer and core num should be changed to use the allocated container resource. this.sortBufferBytesNum = context.getQueryContext().getInt(SessionVars.EXTSORT_BUFFER_SIZE) * StorageUnit.MB; this.allocatedCoreNum = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_THREAD_NUM); this.localDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); this.localFS = new RawLocalFileSystem(); this.intermediateMeta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW, context.getConf()); this.inputStats = new TableStats(); }
if (execBlock.getPlan().getType() == NodeType.SORT) { SortNode sort = (SortNode) execBlock.getPlan(); keys = new Column[sort.getSortKeys().length]; for (int i = 0; i < keys.length; i++) { keys[i] = sort.getSortKeys()[i].getSortKey();
if (execBlock.getPlan().getType() == NodeType.SORT) { SortNode sort = (SortNode) execBlock.getPlan(); keys = new Column[sort.getSortKeys().length]; for (int i = 0; i < keys.length; i++) { keys[i] = sort.getSortKeys()[i].getSortKey();
channel.setShuffle(RANGE_SHUFFLE, PlannerUtil.sortSpecsToSchema(currentNode.getSortKeys()).toArray(), 32); channel.setSchema(firstSortNode.getOutSchema()); currentNode.setInSchema(secondScan.getOutSchema()); currentBlock.setPlan(currentNode); currentBlock.getEnforcer().addSortedInput(secondScan.getTableName(), currentNode.getSortKeys()); channel.setShuffleKeys(PlannerUtil.sortSpecsToSchema(currentNode.getSortKeys()).toArray()); channel.setSchema(firstSortNode.getOutSchema()); currentNode.setInSchema(secondScan.getOutSchema()); currentBlock.setPlan(currentNode); currentBlock.getEnforcer().addSortedInput(secondScan.getTableName(), currentNode.getSortKeys()); masterPlan.addConnect(channel);
channel.setShuffle(RANGE_SHUFFLE, PlannerUtil.sortSpecsToSchema(currentNode.getSortKeys()).toArray(), 32); channel.setSchema(firstSortNode.getOutSchema()); currentNode.setInSchema(secondScan.getOutSchema()); currentBlock.setPlan(currentNode); currentBlock.getEnforcer().addSortedInput(secondScan.getTableName(), currentNode.getSortKeys()); channel.setShuffleKeys(PlannerUtil.sortSpecsToSchema(currentNode.getSortKeys()).toArray()); channel.setSchema(firstSortNode.getOutSchema()); currentNode.setInSchema(secondScan.getOutSchema()); currentBlock.setPlan(currentNode); currentBlock.getEnforcer().addSortedInput(secondScan.getTableName(), currentNode.getSortKeys()); masterPlan.addConnect(channel);