public static ScalarFunctionCallExpression findLOJIsMissingFuncInGroupBy(GroupByOperator lojGroupbyOp, OptimizableOperatorSubTree rightSubTree) throws AlgebricksException { //find IS_MISSING function of which argument has the nullPlaceholder variable in the nested plan of groupby. ALogicalPlanImpl subPlan = (ALogicalPlanImpl) lojGroupbyOp.getNestedPlans().get(0); Mutable<ILogicalOperator> subPlanRootOpRef = subPlan.getRoots().get(0); AbstractLogicalOperator subPlanRootOp = (AbstractLogicalOperator) subPlanRootOpRef.getValue(); ScalarFunctionCallExpression isMissingFuncExpr = findIsMissingInSubplan(subPlanRootOp, rightSubTree); if (isMissingFuncExpr == null) { throw CompilationException.create(ErrorCode.CANNOT_FIND_NON_MISSING_SELECT_OPERATOR, lojGroupbyOp.getSourceLocation()); } return isMissingFuncExpr; }
private void removeGroupByAndOuterUnnest(Mutable<ILogicalOperator> opRef, IOptimizationContext context, LeftOuterUnnestOperator outerUnnest, GroupByOperator gbyOperator, LeftOuterJoinOperator lojOperator, LogicalVariable listifyVar) throws AlgebricksException { List<LogicalVariable> lhs = new ArrayList<>(); List<Mutable<ILogicalExpression>> rhs = new ArrayList<>(); lhs.add(outerUnnest.getVariable()); VariableReferenceExpression listifyVarRef = new VariableReferenceExpression(listifyVar); listifyVarRef.setSourceLocation(gbyOperator.getSourceLocation()); rhs.add(new MutableObject<ILogicalExpression>(listifyVarRef)); List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList = gbyOperator.getGroupByList(); for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gbyPair : gbyList) { lhs.add(gbyPair.first); rhs.add(gbyPair.second); } AssignOperator assignOp = new AssignOperator(lhs, rhs); assignOp.setSourceLocation(outerUnnest.getSourceLocation()); assignOp.getInputs().add(new MutableObject<ILogicalOperator>(lojOperator)); context.computeAndSetTypeEnvironmentForOperator(assignOp); opRef.setValue(assignOp); }
LogicalVariable newVar = context.newVar(); VariableReferenceExpression keyVarRef = new VariableReferenceExpression(keyVar); keyVarRef.setSourceLocation(op.getSourceLocation()); op.getGroupByList().add(new Pair<>(newVar, new MutableObject<>(keyVarRef))); addedGroupKeyMapping.put(keyVar, newVar);
private static void generateMergeAggregationExpressions(GroupByOperator gby, IOptimizationContext context) throws AlgebricksException { if (gby.getNestedPlans().size() != 1) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, gby.getSourceLocation(), "External group-by currently works only for one nested plan with one root containing" + "an aggregate and a nested-tuple-source."); throw new CompilationException(ErrorCode.COMPILATION_ERROR, gby.getSourceLocation(), "External group-by currently works only for one nested plan with one root containing" + "an aggregate and a nested-tuple-source."); AbstractLogicalOperator r0Logical = (AbstractLogicalOperator) r0.getValue(); if (r0Logical.getOperatorTag() != LogicalOperatorTag.AGGREGATE) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, gby.getSourceLocation(), "The merge aggregation expression generation should not process a " + r0Logical.getOperatorTag() + " operator.");
private GroupByOperator opToPush(GroupByOperator gbyOp, BookkeepingInfo bi, IOptimizationContext context) throws AlgebricksException { SourceLocation sourceLoc = gbyOp.getSourceLocation();
private Map<LogicalVariable, LogicalVariable> buildVarExprList(Collection<LogicalVariable> vars, IOptimizationContext context, GroupByOperator g, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException { SourceLocation sourceLoc = g.getSourceLocation(); Map<LogicalVariable, LogicalVariable> m = new HashMap<LogicalVariable, LogicalVariable>(); for (LogicalVariable ov : vars) { LogicalVariable newVar = context.newVar(); ILogicalExpression varExpr = new VariableReferenceExpression(newVar); ((VariableReferenceExpression) varExpr).setSourceLocation(sourceLoc); outVeList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(ov, new MutableObject<ILogicalExpression>(varExpr))); for (ILogicalPlan p : g.getNestedPlans()) { for (Mutable<ILogicalOperator> r : p.getRoots()) { OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true, context); } } AbstractLogicalOperator opUnder = (AbstractLogicalOperator) g.getInputs().get(0).getValue(); OperatorManipulationUtil.substituteVarRec(opUnder, ov, newVar, true, context); m.put(ov, newVar); } return m; } }
protected void buildVarExprList(Collection<LogicalVariable> vars, IOptimizationContext context, GroupByOperator g, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException { SourceLocation sourceLoc = g.getSourceLocation(); for (LogicalVariable ov : vars) { LogicalVariable newVar = context.newVar(); VariableReferenceExpression varExpr = new VariableReferenceExpression(newVar); varExpr.setSourceLocation(sourceLoc); outVeList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(ov, new MutableObject<ILogicalExpression>(varExpr))); for (ILogicalPlan p : g.getNestedPlans()) { for (Mutable<ILogicalOperator> r : p.getRoots()) { OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true, context); } } } }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchemas[0]); GroupByOperator gby = (GroupByOperator) op; int fdColumns[] = getFdColumns(gby, inputSchemas[0]); // compile subplans and set the gby op. schema accordingly AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], gby, opSchema, context); AbstractAggregatorDescriptorFactory aggregatorFactory; List<ILogicalPlan> nestedPlans = gby.getNestedPlans(); if (!nestedPlans.isEmpty() && nestedPlans.get(0).getRoots().get(0).getValue() .getOperatorTag() == LogicalOperatorTag.RUNNINGAGGREGATE) { aggregatorFactory = new NestedPlansRunningAggregatorFactory(subplans, keys, fdColumns); } else { aggregatorFactory = new NestedPlansAccumulatingAggregatorFactory(subplans, keys, fdColumns); } aggregatorFactory.setSourceLocation(gby.getSourceLocation()); IOperatorDescriptorRegistry spec = builder.getJobSpec(); IBinaryComparatorFactory[] comparatorFactories = JobGenHelper .variablesToAscBinaryComparatorFactories(columnList, context.getTypeEnvironment(op), context); RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); PreclusteredGroupOperatorDescriptor opDesc = new PreclusteredGroupOperatorDescriptor(spec, keys, comparatorFactories, aggregatorFactory, recordDescriptor, groupAll, framesLimit); opDesc.setSourceLocation(gby.getSourceLocation()); contributeOpDesc(builder, gby, opDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }
LogicalVariable newDecorVar = context.newVar(); VariableReferenceExpression varRef = new VariableReferenceExpression(var); varRef.setSourceLocation(gbyOp.getSourceLocation()); newGbyOp.addDecorExpression(newDecorVar, varRef); VariableUtilities.substituteVariables(gbyOp.getNestedPlans().get(0).getRoots().get(0).getValue(), var,
assign.getInputs().add(aggInputOpRef); AssignOperator gbyKeyAssign = new AssignOperator(gbyKeyAssgnVars, gbyKeyAssgnExprs); gbyKeyAssign.setSourceLocation(gby.getSourceLocation()); gbyKeyAssign.getInputs().add(gby.getInputs().get(0)); bottomOpRef.setValue(gbyKeyAssign);
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchemas[0]); GroupByOperator gby = (GroupByOperator) op; IVariableTypeEnvironment env = context.getTypeEnvironment(op); int fdColumns[] = getFdColumns(gby, inputSchemas[0]); // compile subplans and set the gby op. schema accordingly AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], gby, opSchema, context); IAggregatorDescriptorFactory aggregatorFactory = new NestedPlansAccumulatingAggregatorFactory(subplans, keys, fdColumns); IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(columnList, env, context); RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor( context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context); MicroPreClusteredGroupRuntimeFactory runtime = new MicroPreClusteredGroupRuntimeFactory(keys, comparatorFactories, aggregatorFactory, inputRecordDesc, recordDescriptor, null); runtime.setSourceLocation(gby.getSourceLocation()); builder.contributeMicroOperator(gby, runtime, recordDescriptor); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }
aggregatorFactory.setSourceLocation(gby.getSourceLocation()); AbstractAggregatorDescriptorFactory mergeFactory = new SerializableAggregatorDescriptorFactory(merges); mergeFactory.setSourceLocation(gby.getSourceLocation()); keyAndDecFields, frameLimit, comparatorFactories, normalizedKeyFactory, aggregatorFactory, mergeFactory, recordDescriptor, recordDescriptor, new HashSpillableTableFactory(hashFunctionFactories)); gbyOpDesc.setSourceLocation(gby.getSourceLocation()); contributeOpDesc(builder, gby, gbyOpDesc); ILogicalOperator src = op.getInputs().get(0).getValue();
aggregatorFactory.setSourceLocation(gby.getSourceLocation()); AbstractAggregatorDescriptorFactory mergeFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(merges, keyAndDecFields); mergeFactory.setSourceLocation(gby.getSourceLocation()); keyAndDecFields, normalizedKeyFactory, compFactories, aggregatorFactory, mergeFactory, partialAggRecordDescriptor, recordDescriptor, false); gbyOpDesc.setSourceLocation(gby.getSourceLocation());
LogicalVariable oldVar = assignOp.getVariables().get(0); VariableReferenceExpression v2Ref = new VariableReferenceExpression(v2); v2Ref.setSourceLocation(g.getSourceLocation()); g.getDecorList().add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(oldVar, new MutableObject<ILogicalExpression>(v2Ref)));
protected void processNullTest(IOptimizationContext context, GroupByOperator nestedGby, List<LogicalVariable> aggregateVarsProducedByCombiner) { SourceLocation sourceLoc = nestedGby.getSourceLocation(); IFunctionInfo finfoEq = context.getMetadataProvider().lookupFunction(BuiltinFunctions.IS_SYSTEM_NULL); SelectOperator selectNonSystemNull;
context.getPhysicalOptimizationConfig().getMaxFramesForGroupBy())); } else { throw new CompilationException(ErrorCode.COMPILATION_ERROR, gby.getSourceLocation(), "Unsupported nested operator within a group-by: " + r0.getValue().getOperatorTag().name());