@Override public Long visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException { if (op.isGroupAll()) { return ONE; } ILogicalOperator inputOp = op.getInputs().get(0).getValue(); long inputCardinality = inputOp.accept(this, arg); List<LogicalVariable> gbyVar = op.getGbyVarList(); if (inputCardinality == ONE_GROUP && keyVariables.containsAll(gbyVar)) { keyVariables.clear(); return ONE; } if (inputCardinality == ZERO_OR_ONE_GROUP && keyVariables.containsAll(gbyVar)) { keyVariables.clear(); return ZERO_OR_ONE; } // ZERO_OR_ONE, ONE, ZERO_OR_ONE_GROUP, ONE_GROUP, OR UNKNOWN return inputCardinality; }
SourceLocation sourceLoc = gbyOp.getSourceLocation(); Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0); ILogicalOperator op3 = opRef3.getValue(); GroupByOperator newGbyOp = new GroupByOperator(); newGbyOp.setSourceLocation(sourceLoc); newGbyOp.getInputs().add(new MutableObject<ILogicalOperator>(op3));
nestedTupleSourceRef.setValue(groupOp.getInputs().get(0).getValue()); return true;
@Override public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) { List<ILocalStructuralProperty> propsLocal = new ArrayList<>(); GroupByOperator gby = (GroupByOperator) op; ILogicalOperator op2 = gby.getInputs().get(0).getValue(); IPhysicalPropertiesVector childProp = op2.getDeliveredPhysicalProperties(); IPartitioningProperty pp = childProp.getPartitioningProperty(); Map<LogicalVariable, LogicalVariable> ppSubstMap = computePartitioningPropertySubstitutionMap(gby, pp); if (ppSubstMap != null) { // We cannot modify pp directly, since it is owned by the input operator. // Otherwise, the partitioning property would be modified even before this group by operator, // which will be undesirable. pp = pp.clonePartitioningProperty(); pp.substituteColumnVars(ppSubstMap); } List<ILocalStructuralProperty> childLocals = childProp.getLocalProperties(); if (childLocals == null) { deliveredProperties = new StructuralPropertiesVector(pp, propsLocal); return; } for (ILocalStructuralProperty lsp : childLocals) { ILocalStructuralProperty propagatedLsp = getPropagatedProperty(lsp, gby); if (propagatedLsp != null) { propsLocal.add(propagatedLsp); } } deliveredProperties = new StructuralPropertiesVector(pp, propsLocal); }
private Map<LogicalVariable, LogicalVariable> buildVarExprList(Collection<LogicalVariable> vars, IOptimizationContext context, GroupByOperator g, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException { SourceLocation sourceLoc = g.getSourceLocation(); Map<LogicalVariable, LogicalVariable> m = new HashMap<LogicalVariable, LogicalVariable>(); for (LogicalVariable ov : vars) { LogicalVariable newVar = context.newVar(); ILogicalExpression varExpr = new VariableReferenceExpression(newVar); ((VariableReferenceExpression) varExpr).setSourceLocation(sourceLoc); outVeList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(ov, new MutableObject<ILogicalExpression>(varExpr))); for (ILogicalPlan p : g.getNestedPlans()) { for (Mutable<ILogicalOperator> r : p.getRoots()) { OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true, context); } } AbstractLogicalOperator opUnder = (AbstractLogicalOperator) g.getInputs().get(0).getValue(); OperatorManipulationUtil.substituteVarRec(opUnder, ov, newVar, true, context); m.put(ov, newVar); } return m; } }
ILogicalOperator op2 = op1.getInputs().get(0).getValue(); op1Ref.setValue(op2); Mutable<ILogicalOperator> opUnderRef = gby.getInputs().get(0); ILogicalOperator opUnder = opUnderRef.getValue(); subplan.getInputs().clear();
@Override public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { ILogicalOperator op1 = opRef.getValue(); // Checks the plan pattern. if (!checkOperatorPattern(op1)) { return false; } LeftOuterUnnestOperator outerUnnest = (LeftOuterUnnestOperator) op1; GroupByOperator gbyOperator = (GroupByOperator) outerUnnest.getInputs().get(0).getValue(); LeftOuterJoinOperator lojOperator = (LeftOuterJoinOperator) gbyOperator.getInputs().get(0).getValue(); // Checks whether the left outer unnest and the group-by operator are qualified for rewriting. Triple<Boolean, ILogicalExpression, ILogicalExpression> checkGbyResult = checkUnnestAndGby(outerUnnest, gbyOperator); // The argument for listify and not(is-missing(...)) check should be variables. if (!isVariableReference(checkGbyResult.second) || !isVariableReference(checkGbyResult.third)) { return false; } // Checks whether both the listify variable and the condition test variable are from the right input // branch of the left outer join. LogicalVariable listifyVar = ((VariableReferenceExpression) checkGbyResult.second).getVariableReference(); LogicalVariable conditionTestVar = ((VariableReferenceExpression) checkGbyResult.third).getVariableReference(); if (!checkListifyAndConditionVar(lojOperator, listifyVar, conditionTestVar)) { return false; } // Does the rewrite. removeGroupByAndOuterUnnest(opRef, context, outerUnnest, gbyOperator, lojOperator, listifyVar); return true; }
Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0); opRef3.setValue(newGbyOp); typeGby(newGbyOp, context);
gbyOp.getInputs().add(new MutableObject<>(inputOp));
nestedRoots.add(aggOpRef); nestedPlans.add(new ALogicalPlanImpl(nestedRoots)); groupbyOp.getInputs().add(new MutableObject<>(leftOuterJoinOp));
groupByDecorList.add(new Pair<>(null, new MutableObject<>(liveVarRef))); groupbyOp.getInputs().add(new MutableObject<>(topJoinRef.getValue()));
@Test public void testParallelGroupBy() throws AlgebricksException { IClusterCapacity clusterCapacity = new ClusterCapacity(); RequiredCapacityVisitor visitor = makeComputationCapacityVisitor(PARALLELISM, clusterCapacity); // Constructs a parallel group-by query plan. GroupByOperator globalGby = makeGroupByOperator(AbstractLogicalOperator.ExecutionMode.PARTITIONED); ExchangeOperator exchange = new ExchangeOperator(); exchange.setPhysicalOperator(new HashPartitionExchangePOperator(Collections.emptyList(), null)); GroupByOperator localGby = makeGroupByOperator(AbstractLogicalOperator.ExecutionMode.LOCAL); globalGby.getInputs().add(new MutableObject<>(exchange)); exchange.getInputs().add(new MutableObject<>(localGby)); // Verifies the calculated cluster capacity requirement for the test quer plan. globalGby.accept(visitor, null); Assert.assertTrue(clusterCapacity.getAggregatedCores() == PARALLELISM); Assert.assertTrue(clusterCapacity.getAggregatedMemoryByteSize() == 2 * MEMORY_BUDGET * PARALLELISM + 2 * FRAME_SIZE * PARALLELISM * PARALLELISM); }
ILogicalExpression nodeIdKeyVre = new VariableReferenceExpression(nodeIdKeyVariable); GroupByOperator groupByOperator = getGroupByOperator(groupByVariable, nodeIdKeyVre); groupByOperator.getInputs().add(nextOperatorRef); nextOperatorRef = new MutableObject<ILogicalOperator>(groupByOperator);
@Test public void testUnPartitionedGroupBy() throws AlgebricksException { IClusterCapacity clusterCapacity = new ClusterCapacity(); RequiredCapacityVisitor visitor = makeComputationCapacityVisitor(PARALLELISM, clusterCapacity); // Constructs a parallel group-by query plan. GroupByOperator globalGby = makeGroupByOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED); ExchangeOperator exchange = new ExchangeOperator(); exchange.setPhysicalOperator(new OneToOneExchangePOperator()); exchange.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED); GroupByOperator localGby = makeGroupByOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED); globalGby.getInputs().add(new MutableObject<>(exchange)); exchange.getInputs().add(new MutableObject<>(localGby)); // Verifies the calculated cluster capacity requirement for the test quer plan. globalGby.accept(visitor, null); Assert.assertTrue(clusterCapacity.getAggregatedCores() == 1); Assert.assertTrue(clusterCapacity.getAggregatedMemoryByteSize() == 2 * MEMORY_BUDGET + FRAME_SIZE); }
join.getInputs().add(new MutableObject<>(rightChildExchange)); rightChildExchange.getInputs().add(new MutableObject<>(rightChild)); rightChild.getInputs().add(new MutableObject<>(ets));
groupByOperator .setPhysicalOperator(new PreclusteredGroupByPOperator(Collections.emptyList(), true, FRAME_LIMIT)); groupByOperator.getInputs().add(new MutableObject<>(exchange));
gOp.getInputs().add(topOp);
@Test public void testUnPartitionedJoin() throws AlgebricksException { IClusterCapacity clusterCapacity = new ClusterCapacity(); RequiredCapacityVisitor visitor = makeComputationCapacityVisitor(PARALLELISM, clusterCapacity); // Constructs a join query plan. InnerJoinOperator join = makeJoinOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED); // Left child plan of the join. ExchangeOperator leftChildExchange = new ExchangeOperator(); leftChildExchange.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED); leftChildExchange.setPhysicalOperator(new OneToOneExchangePOperator()); InnerJoinOperator leftChild = makeJoinOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED); join.getInputs().add(new MutableObject<>(leftChildExchange)); leftChildExchange.getInputs().add(new MutableObject<>(leftChild)); EmptyTupleSourceOperator ets = new EmptyTupleSourceOperator(); ets.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED); leftChild.getInputs().add(new MutableObject<>(ets)); leftChild.getInputs().add(new MutableObject<>(ets)); // Right child plan of the join. ExchangeOperator rightChildExchange = new ExchangeOperator(); rightChildExchange.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED); rightChildExchange.setPhysicalOperator(new OneToOneExchangePOperator()); GroupByOperator rightChild = makeGroupByOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED); join.getInputs().add(new MutableObject<>(rightChildExchange)); rightChildExchange.getInputs().add(new MutableObject<>(rightChild)); rightChild.getInputs().add(new MutableObject<>(ets)); // Verifies the calculated cluster capacity requirement for the test quer plan. join.accept(visitor, null); Assert.assertTrue(clusterCapacity.getAggregatedCores() == 1); Assert.assertTrue(clusterCapacity.getAggregatedMemoryByteSize() == 3 * MEMORY_BUDGET + 5L * FRAME_SIZE); }
.setPhysicalOperator(new ExternalGroupByPOperator(Collections.emptyList(), FRAME_LIMIT, FRAME_LIMIT)); groupByOperator.setExecutionMode(LOCAL); groupByOperator.getInputs().add(new MutableObject<>(ets3));