private static Pair<LogicalVariable, Mutable<ILogicalExpression>> getGbyPairByRhsVar(GroupByOperator gby, LogicalVariable var) { for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gby.getGroupByList()) { if (ve.first == var) { return ve; } } return null; }
@Override public Void visitGroupByOperator(GroupByOperator op, IOptimizationContext ctx) throws AlgebricksException { List<LogicalVariable> header = new ArrayList<>(); for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gbyTerm : op.getGroupByList()) { header.add(gbyTerm.first); } List<LogicalVariable> liveVars = new ArrayList<>(); VariableUtilities.getSubplanLocalLiveVariables(op, liveVars); ctx.addPrimaryKey(new FunctionalDependency(header, liveVars)); return null; }
@Override public String visitGroupByOperator(GroupByOperator op, Boolean showDetails) throws AlgebricksException { stringBuilder.setLength(0); stringBuilder.append("group by").append(op.isGroupAll() ? " (all)" : "").append(" ("); printVariableAndExprList(op.getGroupByList()); stringBuilder.append(") decor ("); printVariableAndExprList(op.getDecorList()); stringBuilder.append(")"); appendSchema(op, showDetails); appendAnnotations(op, showDetails); appendPhysicalOperatorInfo(op, showDetails); return stringBuilder.toString(); }
private Map<LogicalVariable, LogicalVariable> computePartitioningPropertySubstitutionMap(GroupByOperator gbyOp, IPartitioningProperty childpp) { Set<LogicalVariable> childPartitioningColumns = new HashSet<>(); childpp.getColumns(childPartitioningColumns); List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = gbyOp.getGroupByList(); if (groupByList.size() != childPartitioningColumns.size()) { return null; } Map<LogicalVariable, LogicalVariable> substMap = null; for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : groupByList) { ILogicalExpression expr = ve.second.getValue(); if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) { return null; } VariableReferenceExpression varRefExpr = (VariableReferenceExpression) expr; LogicalVariable var = varRefExpr.getVariableReference(); if (!childPartitioningColumns.remove(var)) { return null; } if (substMap == null) { substMap = new HashMap<>(); } substMap.put(var, ve.first); } return childPartitioningColumns.isEmpty() ? substMap : null; }
private LogicalVariable getNewGbyVar(GroupByOperator g, LogicalVariable v) { for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : g.getGroupByList()) { ILogicalExpression e = p.second.getValue(); if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) { LogicalVariable v2 = ((VariableReferenceExpression) e).getVariableReference(); if (v2 == v) { return p.first; } } } return null; }
private void mapVariablesForGroupBy(ILogicalOperator left, ILogicalOperator right) { if (left.getOperatorTag() != right.getOperatorTag()) { return; } GroupByOperator leftOp = (GroupByOperator) left; GroupByOperator rightOp = (GroupByOperator) right; List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> leftPairs = leftOp.getGroupByList(); List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> rightPairs = rightOp.getGroupByList(); mapVarExprPairList(leftPairs, rightPairs); leftPairs = leftOp.getDecorList(); rightPairs = rightOp.getDecorList(); mapVarExprPairList(leftPairs, rightPairs); }
private static LogicalVariable getLhsGbyVar(GroupByOperator gby, LogicalVariable var) { for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gby.getGroupByList()) { ILogicalExpression e = ve.second.getValue(); if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) { throw new IllegalStateException( "Right hand side of group by assignment should have been normalized to a variable reference."); } LogicalVariable v = ((VariableReferenceExpression) e).getVariableReference(); if (v.equals(var)) { return ve.first; } } return null; }
@Override public ILogicalOperator visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException { List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = new ArrayList<>(); List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decoList = new ArrayList<>(); ArrayList<ILogicalPlan> newSubplans = new ArrayList<>(); for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : op.getGroupByList()) { groupByList.add(new Pair<>(pair.first, deepCopyExpressionRef(pair.second))); } for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : op.getDecorList()) { decoList.add(new Pair<>(pair.first, deepCopyExpressionRef(pair.second))); } GroupByOperator gbyOp = new GroupByOperator(groupByList, decoList, newSubplans, op.isGroupAll()); for (ILogicalPlan plan : op.getNestedPlans()) { newSubplans.add(OperatorManipulationUtil.deepCopy(plan, gbyOp)); } return gbyOp; }
@Override public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); if (op.getOperatorTag() != LogicalOperatorTag.GROUP) { return false; } GroupByOperator gby = (GroupByOperator) op; Map<LogicalVariable, LogicalVariable> varRhsToLhs = new HashMap<LogicalVariable, LogicalVariable>(); boolean gvChanged = factorRedundantRhsVars(gby.getGroupByList(), opRef, varRhsToLhs, context); boolean dvChanged = factorRedundantRhsVars(gby.getDecorList(), opRef, varRhsToLhs, context); return gvChanged || dvChanged; }
@Override public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException { for (ILogicalPlan p : op.getNestedPlans()) { for (Mutable<ILogicalOperator> r : p.getRoots()) { VariableUtilities.getLiveVariables(r.getValue(), producedVariables); } } for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) { if (p.first != null) { producedVariables.add(p.first); } } for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) { if (p.first != null) { producedVariables.add(p.first); } } return null; }
private void removeGroupByAndOuterUnnest(Mutable<ILogicalOperator> opRef, IOptimizationContext context, LeftOuterUnnestOperator outerUnnest, GroupByOperator gbyOperator, LeftOuterJoinOperator lojOperator, LogicalVariable listifyVar) throws AlgebricksException { List<LogicalVariable> lhs = new ArrayList<>(); List<Mutable<ILogicalExpression>> rhs = new ArrayList<>(); lhs.add(outerUnnest.getVariable()); VariableReferenceExpression listifyVarRef = new VariableReferenceExpression(listifyVar); listifyVarRef.setSourceLocation(gbyOperator.getSourceLocation()); rhs.add(new MutableObject<ILogicalExpression>(listifyVarRef)); List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList = gbyOperator.getGroupByList(); for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gbyPair : gbyList) { lhs.add(gbyPair.first); rhs.add(gbyPair.second); } AssignOperator assignOp = new AssignOperator(lhs, rhs); assignOp.setSourceLocation(outerUnnest.getSourceLocation()); assignOp.getInputs().add(new MutableObject<ILogicalOperator>(lojOperator)); context.computeAndSetTypeEnvironmentForOperator(assignOp); opRef.setValue(assignOp); }
@Override public Void visitGroupByOperator(GroupByOperator op, Pair<LogicalVariable, LogicalVariable> pair) throws AlgebricksException { subst(pair.first, pair.second, op.getGroupByList()); subst(pair.first, pair.second, op.getDecorList()); substInNestedPlans(pair.first, pair.second, op); substVarTypes(op, pair); return null; }
@Override public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException { for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) { sweepExpression(p.second.getValue()); } for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) { sweepExpression(p.second.getValue()); } return null; }
@Override public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException { for (ILogicalPlan p : op.getNestedPlans()) { for (Mutable<ILogicalOperator> r : p.getRoots()) { VariableUtilities.getLiveVariables(r.getValue(), schemaVariables); } } for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) { if (p.first != null) { schemaVariables.add(p.first); } } for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) { if (p.first != null) { schemaVariables.add(p.first); } else { ILogicalExpression e = p.second.getValue(); if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) { schemaVariables.add(((VariableReferenceExpression) e).getVariableReference()); } } } return null; }
@Override public ILogicalOperator visitGroupByOperator(GroupByOperator op, ILogicalOperator arg) throws AlgebricksException { List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByListCopy = deepCopyVariableExpressionReferencePairList(op.getGroupByList()); List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorListCopy = deepCopyVariableExpressionReferencePairList(op.getDecorList()); List<ILogicalPlan> nestedPlansCopy = new ArrayList<ILogicalPlan>(); GroupByOperator opCopy = new GroupByOperator(groupByListCopy, decorListCopy, nestedPlansCopy, op.isGroupAll()); deepCopyInputsAnnotationsAndExecutionMode(op, arg, opCopy); deepCopyPlanList(op.getNestedPlans(), nestedPlansCopy, opCopy); return opCopy; }
@Override public Void visitGroupByOperator(GroupByOperator op, Integer indent) throws AlgebricksException { addIndent(indent).append("group by" + (op.isGroupAll() ? " (all)" : "") + " ("); pprintVeList(op.getGroupByList(), indent); buffer.append(") decor ("); pprintVeList(op.getDecorList(), indent); buffer.append(") {"); printNestedPlans(op, indent); return null; }
@Override public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException { visitNestedPlans(op); for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : op.getGroupByList()) { g.second.getValue().getUsedVariables(usedVariables); } for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : op.getDecorList()) { g.second.getValue().getUsedVariables(usedVariables); } return null; }
@Override public Void visitGroupByOperator(GroupByOperator op, Integer indent) throws AlgebricksException { addIndent(indent).append("\"operator\": \"group-by\""); if (op.isGroupAll()) { buffer.append(",\n"); addIndent(indent).append("\"option\": \"all\""); } if (!op.getGroupByList().isEmpty()) { buffer.append(",\n"); addIndent(indent).append("\"group-by-list\": "); pprintVeList(op.getGroupByList(), indent); } if (!op.getDecorList().isEmpty()) { buffer.append(",\n"); addIndent(indent).append("\"decor-list\": "); pprintVeList(op.getDecorList(), indent); } if (!op.getNestedPlans().isEmpty()) { buffer.append(",\n"); addIndent(indent).append("\"subplan\": "); printNestedPlans(op, indent); } return null; }
@Override public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue(); if (op1.getOperatorTag() != LogicalOperatorTag.GROUP) { return false; } if (context.checkIfInDontApplySet(this, op1)) { return false; } context.addToDontApplySet(this, op1); GroupByOperator g = (GroupByOperator) op1; boolean r1 = extractComplexExpressions(g, g.getGroupByList(), context); boolean r2 = extractComplexExpressions(g, g.getDecorList(), context); boolean fired = r1 || r2; if (fired) { context.computeAndSetTypeEnvironmentForOperator(g); } return fired; }
private void optimizeUsingConstraintsAndEquivClasses(AbstractLogicalOperator op) { IPhysicalOperator pOp = op.getPhysicalOperator(); switch (pOp.getOperatorTag()) { case HASH_GROUP_BY: case EXTERNAL_GROUP_BY: { GroupByOperator gby = (GroupByOperator) op; ExternalGroupByPOperator hgbyOp = (ExternalGroupByPOperator) pOp; hgbyOp.computeColumnSet(gby.getGroupByList()); break; } case PRE_CLUSTERED_GROUP_BY: case MICRO_PRE_CLUSTERED_GROUP_BY: { GroupByOperator gby = (GroupByOperator) op; AbstractPreclusteredGroupByPOperator preSortedGby = (AbstractPreclusteredGroupByPOperator) pOp; preSortedGby.setGbyColumns(gby.getGbyVarList()); break; } case PRE_SORTED_DISTINCT_BY: case MICRO_PRE_SORTED_DISTINCT_BY: { DistinctOperator d = (DistinctOperator) op; AbstractPreSortedDistinctByPOperator preSortedDistinct = (AbstractPreSortedDistinctByPOperator) pOp; preSortedDistinct.setDistinctByColumns(d.getDistinctByVarList()); break; } default: } }