/** * Generate new ORDER operator that uses TopKSort module and replaces the old ORDER operator. */ private boolean pushLimitIntoOrder(Mutable<ILogicalOperator> opRef, Mutable<ILogicalOperator> opRef2, IOptimizationContext context) throws AlgebricksException { PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig(); LimitOperator limitOp = (LimitOperator) opRef.getValue(); OrderOperator orderOp = (OrderOperator) opRef2.getValue(); // We don't push-down LIMIT into in-memory sort. if (orderOp.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.STABLE_SORT) { return false; } Integer topK = getOutputLimit(limitOp); if (topK == null) { return false; } // Create the new ORDER operator, set the topK value, and replace the current one. OrderOperator newOrderOp = new OrderOperator(orderOp.getOrderExpressions(), topK); newOrderOp.setSourceLocation(orderOp.getSourceLocation()); newOrderOp.setPhysicalOperator( new StableSortPOperator(physicalOptimizationConfig.getMaxFramesExternalSort(), newOrderOp.getTopK())); newOrderOp.getInputs().addAll(orderOp.getInputs()); newOrderOp.setExecutionMode(orderOp.getExecutionMode()); newOrderOp.recomputeSchema(); newOrderOp.computeDeliveredPhysicalProperties(context); opRef2.setValue(newOrderOp); context.computeAndSetTypeEnvironmentForOperator(newOrderOp); context.addToDontApplySet(this, limitOp); return true; }
@Override public Void visitOrderOperator(OrderOperator op, IOptimizationContext arg) throws AlgebricksException { Object annot1 = op.getAnnotations().get(OperatorAnnotations.CARDINALITY); if (annot1 == null) { return null; } Integer m = (Integer) annot1; LogicalPropertiesVectorImpl v = new LogicalPropertiesVectorImpl(); v.setNumberOfTuples(m); Object annot2 = op.getAnnotations().get(OperatorAnnotations.MAX_NUMBER_FRAMES); if (annot2 != null) { Integer f = (Integer) annot2; v.setMaxOutputFrames(f); } arg.putLogicalPropertiesVector(op, v); return null; }
@Override public ILogicalOperator visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException { return new OrderOperator(this.deepCopyOrderAndExpression(op.getOrderExpressions())); }
@Override public Void visitOrderOperator(OrderOperator op, Integer indent) throws AlgebricksException { addIndent(indent).append("order "); if (op.getTopK() != -1) { buffer.append("(topK: " + op.getTopK() + ") "); } pprintOrderList(op.getOrderExpressions(), indent); return null; }
@Override public ILogicalOperator visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException { boolean underJoin = hasJoinAncestor; visitSingleInputOperator(op); if (!rewritten || !underJoin) { return op; } // Adjust the ordering if its input operator pipeline has been rewritten. List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExprList = new ArrayList<>(); // Adds keyVars to the prefix of sorting columns. for (LogicalVariable liveVar : liveVarsFromSubplanInput) { VariableReferenceExpression liveVarRef = new VariableReferenceExpression(liveVar); liveVarRef.setSourceLocation(op.getSourceLocation()); orderExprList.add(new Pair<IOrder, Mutable<ILogicalExpression>>(OrderOperator.ASC_ORDER, new MutableObject<ILogicalExpression>(liveVarRef))); } orderExprList.addAll(op.getOrderExpressions()); // Creates an order operator with the new expression list. OrderOperator orderOp = new OrderOperator(orderExprList); orderOp.setSourceLocation(op.getSourceLocation()); orderOp.getInputs().addAll(op.getInputs()); context.computeAndSetTypeEnvironmentForOperator(orderOp); return orderOp; }
order = new OrderOperator(); order.setSourceLocation(sourceLoc); for (LogicalVariable pkVar : primaryKeyVars) { VariableReferenceExpression pkVarRef = new VariableReferenceExpression(pkVar); pkVarRef.setSourceLocation(sourceLoc); Mutable<ILogicalExpression> vRef = new MutableObject<>(pkVarRef); order.getOrderExpressions().add(new Pair<>(OrderOperator.ASC_ORDER, vRef)); order.getInputs().add(new MutableObject<>(inputOp)); order.setExecutionMode(ExecutionMode.LOCAL); context.computeAndSetTypeEnvironmentForOperator(order);
private Mutable<ILogicalOperator> enforceOrderProperties(List<LocalOrderProperty> oList, Mutable<ILogicalOperator> topOp, boolean isMicroOp, IOptimizationContext context) throws AlgebricksException { SourceLocation sourceLoc = topOp.getValue().getSourceLocation(); List<Pair<IOrder, Mutable<ILogicalExpression>>> oe = new LinkedList<>(); for (LocalOrderProperty orderProperty : oList) { for (OrderColumn oc : orderProperty.getOrderColumns()) { IOrder ordType = (oc.getOrder() == OrderKind.ASC) ? OrderOperator.ASC_ORDER : OrderOperator.DESC_ORDER; VariableReferenceExpression ocColumnRef = new VariableReferenceExpression(oc.getColumn()); ocColumnRef.setSourceLocation(sourceLoc); Pair<IOrder, Mutable<ILogicalExpression>> pair = new Pair<>(ordType, new MutableObject<ILogicalExpression>(ocColumnRef)); oe.add(pair); } } OrderOperator oo = new OrderOperator(oe); oo.setSourceLocation(sourceLoc); oo.setExecutionMode(AbstractLogicalOperator.ExecutionMode.LOCAL); if (isMicroOp) { oo.setPhysicalOperator(new InMemoryStableSortPOperator()); } else { oo.setPhysicalOperator(new StableSortPOperator(physicalOptimizationConfig.getMaxFramesExternalSort())); } oo.getInputs().add(topOp); context.computeAndSetTypeEnvironmentForOperator(oo); if (AlgebricksConfig.ALGEBRICKS_LOGGER.isTraceEnabled()) { AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">>>> Added sort enforcer " + oo.getPhysicalOperator() + ".\n"); } return new MutableObject<ILogicalOperator>(oo); }
@Override public Pair<ILogicalOperator, LogicalVariable> visit(OrderbyClause oc, Mutable<ILogicalOperator> tupSource) throws CompilationException { SourceLocation sourceLoc = oc.getSourceLocation(); OrderOperator ord = new OrderOperator(); ord.setSourceLocation(sourceLoc); Iterator<OrderModifier> modifIter = oc.getModifierList().iterator(); Mutable<ILogicalOperator> topOp = tupSource; for (Expression e : oc.getOrderbyList()) { Pair<ILogicalExpression, Mutable<ILogicalOperator>> p = langExprToAlgExpression(e, topOp); OrderModifier m = modifIter.next(); OrderOperator.IOrder comp = translateOrderModifier(m); ord.getOrderExpressions().add(new Pair<>(comp, new MutableObject<>(p.first))); topOp = p.second; } ord.getInputs().add(topOp); if (oc.getNumTuples() > 0) { ord.getAnnotations().put(OperatorAnnotations.CARDINALITY, oc.getNumTuples()); } if (oc.getNumFrames() > 0) { ord.getAnnotations().put(OperatorAnnotations.MAX_NUMBER_FRAMES, oc.getNumFrames()); } if (oc.getRangeMap() != null) { Iterator<OrderModifier> orderModifIter = oc.getModifierList().iterator(); boolean ascending = orderModifIter.next() == OrderModifier.ASC; RangeMapBuilder.verifyRangeOrder(oc.getRangeMap(), ascending); ord.getAnnotations().put(OperatorAnnotations.USE_STATIC_RANGE, oc.getRangeMap()); } return new Pair<>(ord, null); }
replicateOperator.getInputs().add(new MutableObject<>(scanOperator)); OrderOperator order1 = new OrderOperator(); order1.setExecutionMode(PARTITIONED); order1.setPhysicalOperator(new OneToOneExchangePOperator()); order1.getInputs().add(new MutableObject<>(replicateOperator)); OrderOperator order2 = new OrderOperator(); order2.setExecutionMode(PARTITIONED); order2.setPhysicalOperator(new OneToOneExchangePOperator()); order2.getInputs().add(new MutableObject<>(replicateOperator));
groupByOperator.getInputs().add(new MutableObject<>(exchange)); OrderOperator orderOperator = new OrderOperator(); orderOperator.setExecutionMode(PARTITIONED); orderOperator.getInputs().add(new MutableObject<>(groupByOperator));
List<Pair<IOrder, Mutable<ILogicalExpression>>> orderKeys = orderOperator.getOrderExpressions(); Set<LogicalVariable> orderUsedVars = new HashSet<LogicalVariable>(); for (Pair<IOrder, Mutable<ILogicalExpression>> orderKey : orderKeys) { orderKey.second.getValue().getUsedVariables(orderUsedVars); Mutable<ILogicalOperator> inputOpRef = orderOperator.getInputs().get(0); ILogicalOperator inputOperator = inputOpRef.getValue();
@Override public Boolean visitOrderOperator(OrderOperator op, ILogicalOperator arg) throws AlgebricksException { AbstractLogicalOperator aop = (AbstractLogicalOperator) arg; if (aop.getOperatorTag() != LogicalOperatorTag.ORDER) { return Boolean.FALSE; } OrderOperator orderOpArg = (OrderOperator) copyAndSubstituteVar(op, arg); boolean isomorphic = compareIOrderAndExpressions(op.getOrderExpressions(), orderOpArg.getOrderExpressions()); return isomorphic; }
o, mutable(oExpr))); OrderOperator order = new OrderOperator(oExprs); order.getInputs().add(mutable(tCtx.op)); tCtx.op = order; break;
ILogicalOperator currentOp = op; if (!orderingExprs.isEmpty()) { OrderOperator orderOp = new OrderOperator(OperatorManipulationUtil.cloneOrderExpressions(orderingExprs)); orderOp.setSourceLocation(sourceLoc); op.getInputs().add(new MutableObject<>(orderOp)); currentOp = orderOp;
@SuppressWarnings("unused") private OrderOperator getOrderOperator(Mutable<ILogicalExpression> variableRef) { List<Pair<IOrder, Mutable<ILogicalExpression>>> orderArgs = new ArrayList<Pair<IOrder, Mutable<ILogicalExpression>>>(); orderArgs.add(new Pair<IOrder, Mutable<ILogicalExpression>>(OrderOperator.ASC_ORDER, variableRef)); OrderOperator oo = new OrderOperator(orderArgs); return oo; }
@Override public Long visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException { return op.getInputs().get(0).getValue().accept(this, arg); }
@Override public ILogicalOperator visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException { visitSingleInputOperator(op); if (correlatedKeyVars.isEmpty()) { return op; } orderingExprs.clear(); orderingExprs.addAll(OperatorManipulationUtil.cloneOrderExpressions(op.getOrderExpressions())); List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExprList = new ArrayList<>(); // Adds keyVars to the prefix of sorting columns. for (LogicalVariable keyVar : correlatedKeyVars) { VariableReferenceExpression keyVarRef = new VariableReferenceExpression(keyVar); keyVarRef.setSourceLocation(op.getSourceLocation()); orderExprList.add(new Pair<>(OrderOperator.ASC_ORDER, new MutableObject<>(keyVarRef))); } orderExprList.addAll(op.getOrderExpressions()); // Creates an order operator with the new expression list. OrderOperator orderOp = new OrderOperator(orderExprList); orderOp.setSourceLocation(op.getSourceLocation()); orderOp.getInputs().addAll(op.getInputs()); context.computeAndSetTypeEnvironmentForOperator(orderOp); return orderOp; }
metaRecordType, inputOp, secondaryIndex, SecondaryUnnestMapOutputVarType.PRIMARY_KEY); OrderOperator order = new OrderOperator(); order.setSourceLocation(sourceLoc); for (LogicalVariable pkVar : primaryKeyVars) { VariableReferenceExpression pkVarRef = new VariableReferenceExpression(pkVar); pkVarRef.setSourceLocation(sourceLoc); Mutable<ILogicalExpression> vRef = new MutableObject<>(pkVarRef); order.getOrderExpressions().add(new Pair<>(OrderOperator.ASC_ORDER, vRef)); order.getInputs().add(new MutableObject<>(inputOp)); order.setExecutionMode(ExecutionMode.LOCAL); context.computeAndSetTypeEnvironmentForOperator(order); List<Mutable<ILogicalExpression>> externalLookupArgs = new ArrayList<>();
@Override public Void visitOrderOperator(OrderOperator op, Integer indent) throws AlgebricksException { addIndent(indent).append("\"operator\": \"order\""); int topK = op.getTopK(); List<Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>>> orderExpressions = op.getOrderExpressions(); pprintOrderExprList(orderExpressions, topK, indent); return null; }
@Override public ILogicalOperator visitOrderOperator(OrderOperator op, ILogicalOperator arg) throws AlgebricksException { OrderOperator opCopy = new OrderOperator(deepCopyOrderExpressionReferencePairList(op.getOrderExpressions())); deepCopyInputsAnnotationsAndExecutionMode(op, arg, opCopy); return opCopy; }