for (int i = 0; i < evalNum; i++) { try { firstPhaseEvals.add((AggregationFunctionCallEval) secondPhaseEvals.get(i).clone()); } catch (CloneNotSupportedException e) { throw new RuntimeException(e); firstPhaseEvals.get(i).setFirstPhase(); firstPhaseEvalNames[i] = plan.generateUniqueColumnName(firstPhaseEvals.get(i)); FieldEval param = new FieldEval(firstPhaseEvalNames[i], firstPhaseEvals.get(i).getValueType()); secondPhaseEvals.get(i).setLastPhase(); secondPhaseEvals.get(i).setArgs(new EvalNode[]{param});
private AggregationFunctionCallEval createCountRowFunction(EvalNode[] args) throws TajoException { FunctionDesc functionDesc = getCatalog().getFunction("count", CatalogProtos.FunctionType.AGGREGATION, new TajoDataTypes.DataType[]{}); return new AggregationFunctionCallEval(functionDesc, args); }
private NonDistinctHashAggregator(GroupbyNode groupbyNode) throws IOException { nonDistinctAggrDatas = new TupleMap<>(); if (groupbyNode.hasAggFunctions()) { aggFunctions = groupbyNode.getAggFunctions(); aggFunctionsNum = aggFunctions.size(); } else { aggFunctions = new ArrayList<>(); aggFunctionsNum = 0; } for (AggregationFunctionCallEval eachFunction: aggFunctions) { eachFunction.bind(context.getEvalContext(), inSchema); eachFunction.setFirstPhase(); } outTuple = new VTuple(aggFunctionsNum); dummyTuple = NullTuple.create(aggFunctionsNum); tupleLength = aggFunctionsNum; }
public void merge(FunctionContext context, Tuple tuple) { if (!isBound) { throw new IllegalStateException("bind() must be called before merge()"); } mergeParam(context, evalParams(tuple)); }
public DistinctFinalAggregator(int seq, int inTupleIndex, int outTupleIndex, GroupbyNode groupbyNode) { this.seq = seq; this.inTupleIndex = inTupleIndex; this.outTupleIndex = outTupleIndex; aggrFunctions = groupbyNode.getAggFunctions(); if (aggrFunctions != null) { for (AggregationFunctionCallEval eachFunction: aggrFunctions) { eachFunction.bind(context.getEvalContext(), inSchema); eachFunction.setLastPhase(); } } newFunctionContext(); }
if (type == EvalType.AGG_FUNCTION) { AggregationFunctionCallEval aggFunc = new AggregationFunctionCallEval(new FunctionDesc(funcProto.getFuncion()), params); aggFunc.setFirstAndLastPhase(); } else if (aggFunctionProto.getFirstPhase()) { aggFunc.setFirstPhase(); } else if (aggFunctionProto.getLastPhase()) { aggFunc.setLastPhase(); } else { aggFunc.setIntermediatePhase(); aggFunc.setAlias(aggFunctionProto.getAlias()); aggFunc.isFirstPhase(), aggFunc.isLastPhase()));
aggFunctions.add((AggregationFunctionCallEval) groupbyNode.getAggFunctions().get(i).clone()); aggFunctions.get(i).setFirstPhase(); aggFunctions.get(i).setAlias(groupbyNode.getOutSchema().getColumn(i).getQualifiedName()); int index = 0; for (AggregationFunctionCallEval aggrFunction: lastGroupbyNode.getAggFunctions()) { aggrFunction.setIntermediatePhase(); aggrFunction.setArgs(new EvalNode[]{new FieldEval(lastGroupbyNode.getTargets().get(index).getNamedColumn())}); index++; int index = 0; for (AggregationFunctionCallEval aggrFunction: lastGroupbyNode.getAggFunctions()) { aggrFunction.setFirstPhase(); aggrFunction.setArgs(new EvalNode[]{new FieldEval(lastGroupbyNode.getTargets().get(index).getNamedColumn())}); index++;
for (int aggIdx = 0; aggIdx < aggFunctions.size(); aggIdx++) { AggregationFunctionCallEval aggFunction = aggFunctions.get(aggIdx); aggFunction.setFirstPhase(); Target originAggFunctionTarget = groupbyNode.getTargets().get(originalGroupingColumns.size() + aggIdx); Target aggFunctionTarget = new Target(new FieldEval(originAggFunctionTarget.getEvalTree().getName(), aggFunction.getValueType())); if (aggFunction.isDistinct()) {
if (function.getName().equalsIgnoreCase("count")) { rewritten = new RewrittenFunctions(1); if (function.getArgs().length == 0) { rewritten.firstStageEvals[0] = createCountRowFunction(function.getArgs()); } else { rewritten.firstStageEvals[0] = createCountFunction(function.getArgs()); FieldEval fieldEval = new FieldEval(referenceName, rewritten.firstStageEvals[0].getValueType()); rewritten.firstStageTargets[0] = new Target(fieldEval); rewritten.secondStageEvals = createSumFunction(new EvalNode[]{fieldEval}); } else if (function.getName().equalsIgnoreCase("sum")) { rewritten = new RewrittenFunctions(1); rewritten.firstStageEvals[0] = createSumFunction(function.getArgs()); String referenceName = plan.generateUniqueColumnName(rewritten.firstStageEvals[0]); FieldEval fieldEval = new FieldEval(referenceName, rewritten.firstStageEvals[0].getValueType()); rewritten.firstStageTargets[0] = new Target(fieldEval); rewritten.secondStageEvals = createSumFunction(new EvalNode[]{fieldEval}); } else if (function.getName().equals("max")) { rewritten = new RewrittenFunctions(1); rewritten.firstStageEvals[0] = createMaxFunction(function.getArgs()); String referenceName = plan.generateUniqueColumnName(rewritten.firstStageEvals[0]); FieldEval fieldEval = new FieldEval(referenceName, rewritten.firstStageEvals[0].getValueType()); rewritten.firstStageTargets[0] = new Target(fieldEval); rewritten.secondStageEvals = createMaxFunction(new EvalNode[]{fieldEval});
@Test public final void testCountFunction() throws IOException, TajoException { FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getUri()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testCountFunction"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newTaskAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[9]); LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); // Set all aggregation functions to the first phase mode GroupbyNode groupbyNode = PlannerUtil.findTopNode(rootNode, NodeType.GROUP_BY); for (AggregationFunctionCallEval function : groupbyNode.getAggFunctions()) { function.setFirstPhase(); } PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); Tuple tuple = exec.next(); assertEquals(30, tuple.getInt8(0)); assertNull(exec.next()); exec.close(); }
if (nonDistinctAggrFunctions != null) { for (AggregationFunctionCallEval eachFunction: nonDistinctAggrFunctions) { eachFunction.bind(context.getEvalContext(), inSchema); eachFunction.setIntermediatePhase();
Target aggFunctionTarget = groupbyNode.getTargets().get(originalGroupingColumns.size() + aggIdx); if (aggFunction.isDistinct()) { buildInfo.addAggFunctionTarget(aggFunctionTarget); } else { aggFunction.setLastPhase(); otherAggregationFunctionCallEvals.add(aggFunction); otherAggregationFunctionTargets.add(aggFunctionTarget);
@Override public Object clone() throws CloneNotSupportedException { GroupbyNode grp = (GroupbyNode) super.clone(); if (groupingKeys != null) { grp.groupingKeys = new Column[groupingKeys.length]; System.arraycopy(groupingKeys, 0, grp.groupingKeys, 0, groupingKeys.length); } if (aggrFunctions != null) { grp.aggrFunctions = new ArrayList<>(); for (int i = 0; i < aggrFunctions.size(); i++) { grp.aggrFunctions.add((AggregationFunctionCallEval) aggrFunctions.get(i).clone()); } } if (targets != null) { grp.targets = new ArrayList<>(); for (Target t : targets) { grp.targets.add((Target) t.clone()); } } return grp; }
@Test public final void testFindDistinctAggFunctions() throws TajoException { String query = "select sum(score) + max(age) from people"; Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); GroupbyNode groupByNode = plan.getRootBlock().getNode(NodeType.GROUP_BY); List aggEvals = groupByNode.getAggFunctions(); List<AggregationFunctionCallEval> list = new ArrayList<>(); for (Object aggEval : aggEvals) { list.addAll(EvalTreeUtil.findDistinctAggFunction((EvalNode) aggEval)); } assertEquals(2, list.size()); Set<String> result = Sets.newHashSet("max", "sum"); for (AggregationFunctionCallEval eval : list) { assertTrue(result.contains(eval.getName())); } }
if (type == EvalType.AGG_FUNCTION) { AggregationFunctionCallEval aggFunc = new AggregationFunctionCallEval(new FunctionDesc(funcProto.getFuncion()), params); aggFunc.setFirstAndLastPhase(); } else if (aggFunctionProto.getFirstPhase()) { aggFunc.setFirstPhase(); } else if (aggFunctionProto.getLastPhase()) { aggFunc.setLastPhase(); } else { aggFunc.setIntermediatePhase(); aggFunc.setAlias(aggFunctionProto.getAlias()); aggFunc.isFirstPhase(), aggFunc.isLastPhase()));
aggFunctions[i] = (AggregationFunctionCallEval) groupbyNode.getAggFunctions()[i].clone(); aggFunctions[i].setFirstPhase(); aggFunctions[i].setAlias(groupbyNode.getOutSchema().getColumn(i).getQualifiedName()); int index = 0; for (AggregationFunctionCallEval aggrFunction: lastGroupbyNode.getAggFunctions()) { aggrFunction.setIntermediatePhase(); aggrFunction.setArgs(new EvalNode[]{new FieldEval(lastGroupbyNode.getTargets()[index].getNamedColumn())}); index++; int index = 0; for (AggregationFunctionCallEval aggrFunction: lastGroupbyNode.getAggFunctions()) { aggrFunction.setFirstPhase(); aggrFunction.setArgs(new EvalNode[]{new FieldEval(lastGroupbyNode.getTargets()[index].getNamedColumn())}); index++;
for (int aggIdx = 0; aggIdx < aggFunctions.length; aggIdx++) { AggregationFunctionCallEval aggFunction = aggFunctions[aggIdx]; aggFunction.setFirstPhase(); Target originAggFunctionTarget = groupbyNode.getTargets()[originalGroupingColumns.size() + aggIdx]; Target aggFunctionTarget = new Target(new FieldEval(originAggFunctionTarget.getEvalTree().getName(), aggFunction.getValueType())); if (aggFunction.isDistinct()) {
if (function.getName().equalsIgnoreCase("count")) { rewritten = new RewrittenFunctions(1); if (function.getArgs().length == 0) { rewritten.firstStageEvals[0] = createCountRowFunction(function.getArgs()); } else { rewritten.firstStageEvals[0] = createCountFunction(function.getArgs()); FieldEval fieldEval = new FieldEval(referenceName, rewritten.firstStageEvals[0].getValueType()); rewritten.firstStageTargets.add(0, new Target(fieldEval)); rewritten.secondStageEvals = createSumFunction(new EvalNode[]{fieldEval}); } else if (function.getName().equalsIgnoreCase("sum")) { rewritten = new RewrittenFunctions(1); rewritten.firstStageEvals[0] = createSumFunction(function.getArgs()); String referenceName = plan.generateUniqueColumnName(rewritten.firstStageEvals[0]); FieldEval fieldEval = new FieldEval(referenceName, rewritten.firstStageEvals[0].getValueType()); rewritten.firstStageTargets.add(0, new Target(fieldEval)); rewritten.secondStageEvals = createSumFunction(new EvalNode[]{fieldEval}); } else if (function.getName().equals("max")) { rewritten = new RewrittenFunctions(1); rewritten.firstStageEvals[0] = createMaxFunction(function.getArgs()); String referenceName = plan.generateUniqueColumnName(rewritten.firstStageEvals[0]); FieldEval fieldEval = new FieldEval(referenceName, rewritten.firstStageEvals[0].getValueType()); rewritten.firstStageTargets.add(0, new Target(fieldEval)); rewritten.secondStageEvals = createMaxFunction(new EvalNode[]{fieldEval});
public DistinctFinalAggregator(int seq, int inTupleIndex, int outTupleIndex, GroupbyNode groupbyNode) { this.seq = seq; this.inTupleIndex = inTupleIndex; this.outTupleIndex = outTupleIndex; aggrFunctions = groupbyNode.getAggFunctions(); if (aggrFunctions != null) { for (AggregationFunctionCallEval eachFunction: aggrFunctions) { eachFunction.bind(context.getEvalContext(), inSchema); eachFunction.setLastPhase(); } } newFunctionContext(); }
@Test public final void testAggregationFunction() throws IOException, TajoException { FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getUri()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testAggregationFunction"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newTaskAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[8]); LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); // Set all aggregation functions to the first phase mode GroupbyNode groupbyNode = PlannerUtil.findTopNode(rootNode, NodeType.GROUP_BY); for (AggregationFunctionCallEval function : groupbyNode.getAggFunctions()) { function.setFirstPhase(); } PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); Tuple tuple = exec.next(); assertEquals(30, tuple.getInt8(0)); assertEquals(3, tuple.getInt4(1)); assertEquals(1, tuple.getInt4(2)); assertNull(exec.next()); exec.close(); }