public AggregationExec(final TaskAttemptContext context, GroupbyNode plan, PhysicalExec child) throws IOException { super(context, plan.getInSchema(), plan.getOutSchema(), child); final Column [] keyColumns = plan.getGroupingColumns(); groupingKeyNum = keyColumns.length; if (plan.hasAggFunctions()) { aggFunctions = plan.getAggFunctions(); aggFunctionsNum = aggFunctions.size(); } else { aggFunctions = new ArrayList<>(); aggFunctionsNum = 0; } }
public HashAggregator(GroupbyNode groupbyNode, Schema schema) throws IOException { hashTable = new TupleMap<>(context.getQueryContext().getInt(SessionVars.AGG_HASH_TABLE_SIZE)); List<Column> groupingKeyColumnList = new ArrayList<>(distinctGroupingKeyColumnSet); Column[] keyColumns = groupbyNode.getGroupingColumns(); for (Column keyColumn : keyColumns) { if (!distinctGroupingKeyColumnSet.contains(keyColumn)) { groupingKeyColumnList.add(keyColumn); } } Column[] groupingKeyColumns = groupingKeyColumnList.toArray(new Column[groupingKeyColumnList.size()]); if (groupbyNode.hasAggFunctions()) { aggFunctions = groupbyNode.getAggFunctions(); aggFunctionsNum = aggFunctions.size(); } else { aggFunctions = new ArrayList<>(); aggFunctionsNum = 0; } for (EvalNode aggFunction : aggFunctions) { aggFunction.bind(context.getEvalContext(), schema); } tupleSize = groupingKeyColumns.length + aggFunctionsNum; aggregatedTuple = new VTuple(groupingKeyColumns.length + aggFunctionsNum); innerKeyProjector = new KeyProjector(inSchema, groupingKeyColumns); }
public DistinctFinalAggregator(int seq, int inTupleIndex, int outTupleIndex, GroupbyNode groupbyNode) { this.seq = seq; this.inTupleIndex = inTupleIndex; this.outTupleIndex = outTupleIndex; aggrFunctions = groupbyNode.getAggFunctions(); if (aggrFunctions != null) { for (AggregationFunctionCallEval eachFunction: aggrFunctions) { eachFunction.bind(context.getEvalContext(), inSchema); eachFunction.setLastPhase(); } } newFunctionContext(); }
public AggregationExec(final TaskAttemptContext context, GroupbyNode plan, PhysicalExec child) throws IOException { super(context, plan.getInSchema(), plan.getOutSchema(), child); final Column [] keyColumns = plan.getGroupingColumns(); groupingKeyNum = keyColumns.length; if (plan.hasAggFunctions()) { aggFunctions = plan.getAggFunctions(); aggFunctionsNum = aggFunctions.length; } else { aggFunctions = new AggregationFunctionCallEval[0]; aggFunctionsNum = 0; } }
public DistinctFinalAggregator(int seq, int inTupleIndex, int outTupleIndex, GroupbyNode groupbyNode) { this.seq = seq; this.inTupleIndex = inTupleIndex; this.outTupleIndex = outTupleIndex; aggrFunctions = groupbyNode.getAggFunctions(); if (aggrFunctions != null) { for (AggregationFunctionCallEval eachFunction: aggrFunctions) { eachFunction.bind(context.getEvalContext(), inSchema); eachFunction.setLastPhase(); } } newFunctionContext(); }
public static List<Target> buildGroupByTarget(GroupbyNode groupbyNode, @Nullable List<Target> groupingKeyTargets, String [] aggEvalNames) { final int groupingKeyNum = groupingKeyTargets == null ? groupbyNode.getGroupingColumns().length : groupingKeyTargets.size(); final int aggrFuncNum = aggEvalNames != null ? aggEvalNames.length : 0; List<EvalNode> aggEvalNodes = (List<EvalNode>)(List<?>) groupbyNode.getAggFunctions(); List<Target> targets = new ArrayList<>(); if (groupingKeyTargets != null) { for (int groupingKeyIdx = 0; groupingKeyIdx < groupingKeyNum; groupingKeyIdx++) { targets.add(groupingKeyTargets.get(groupingKeyIdx)); } } else { for (int groupingKeyIdx = 0; groupingKeyIdx < groupingKeyNum; groupingKeyIdx++) { targets.add(new Target(new FieldEval(groupbyNode.getGroupingColumns()[groupingKeyIdx]))); } } if (aggEvalNames != null) { for (int aggrFuncIdx = 0, targetIdx = groupingKeyNum; aggrFuncIdx < aggrFuncNum; aggrFuncIdx++, targetIdx++) { targets.add(new Target(new FieldEval(aggEvalNames[aggrFuncIdx], aggEvalNodes.get(aggrFuncIdx).getValueType()))); } } return targets; }
int evalNum = secondPhaseGroupBy.getAggFunctions().size(); List<AggregationFunctionCallEval> secondPhaseEvals = secondPhaseGroupBy.getAggFunctions(); List<AggregationFunctionCallEval> firstPhaseEvals = new ArrayList<>();
public HashAggregator(GroupbyNode groupbyNode, Schema schema) throws IOException { hashTable = new TupleMap<>(context.getQueryContext().getInt(SessionVars.AGG_HASH_TABLE_SIZE)); List<Column> groupingKeyColumnList = new ArrayList<>(distinctGroupingKeyColumnSet); Column[] keyColumns = groupbyNode.getGroupingColumns(); Column col; for (int idx = 0; idx < keyColumns.length; idx++) { col = keyColumns[idx]; if (!distinctGroupingKeyColumnSet.contains(col)) { groupingKeyColumnList.add(col); } } Column[] groupingKeyColumns = groupingKeyColumnList.toArray(new Column[groupingKeyColumnList.size()]); if (groupbyNode.hasAggFunctions()) { aggFunctions = groupbyNode.getAggFunctions(); aggFunctionsNum = aggFunctions.length; } else { aggFunctions = new AggregationFunctionCallEval[0]; aggFunctionsNum = 0; } for (EvalNode aggFunction : aggFunctions) { aggFunction.bind(context.getEvalContext(), schema); } tupleSize = groupingKeyColumns.length + aggFunctionsNum; aggregatedTuple = new VTuple(groupingKeyColumns.length + aggFunctionsNum); innerKeyProjector = new KeyProjector(inSchema, groupingKeyColumns); }
private NonDistinctHashAggregator(GroupbyNode groupbyNode) throws IOException { nonDistinctAggrDatas = new TupleMap<>(); if (groupbyNode.hasAggFunctions()) { aggFunctions = groupbyNode.getAggFunctions(); aggFunctionsNum = aggFunctions.size(); } else { aggFunctions = new ArrayList<>(); aggFunctionsNum = 0; } for (AggregationFunctionCallEval eachFunction: aggFunctions) { eachFunction.bind(context.getEvalContext(), inSchema); eachFunction.setFirstPhase(); } outTuple = new VTuple(aggFunctionsNum); dummyTuple = NullTuple.create(aggFunctionsNum); tupleLength = aggFunctionsNum; }
public SortAggregateExec(TaskAttemptContext context, GroupbyNode plan, PhysicalExec child) throws IOException { super(context, plan, child); contexts = new FunctionContext[plan.getAggFunctions() == null ? 0 : plan.getAggFunctions().size()]; final Column [] keyColumns = plan.getGroupingColumns(); groupingKeyIds = new int[groupingKeyNum]; Column col; for (int idx = 0; idx < plan.getGroupingColumns().length; idx++) { col = keyColumns[idx]; if (col.hasQualifier()) { groupingKeyIds[idx] = inSchema.getColumnId(col.getQualifiedName()); } else { groupingKeyIds[idx] = inSchema.getColumnIdByName(col.getSimpleName()); } } currentKey = new VTuple(groupingKeyNum); outTuple = new VTuple(outSchema.size()); }
for (GroupbyNode eachNode: subGroupbyPlan) { if (eachNode.hasAggFunctions()) { List<AggregationFunctionCallEval> aggrFunctions = eachNode.getAggFunctions(); for (AggregationFunctionCallEval aggrFunction : aggrFunctions) { sb.append(prefix).append(aggrFunction);
public SortAggregateExec(TaskAttemptContext context, GroupbyNode plan, PhysicalExec child) throws IOException { super(context, plan, child); contexts = new FunctionContext[plan.getAggFunctions() == null ? 0 : plan.getAggFunctions().length]; final Column [] keyColumns = plan.getGroupingColumns(); groupingKeyIds = new int[groupingKeyNum]; Column col; for (int idx = 0; idx < plan.getGroupingColumns().length; idx++) { col = keyColumns[idx]; if (col.hasQualifier()) { groupingKeyIds[idx] = inSchema.getColumnId(col.getQualifiedName()); } else { groupingKeyIds[idx] = inSchema.getColumnIdByName(col.getSimpleName()); } } currentKey = new VTuple(groupingKeyNum); outTuple = new VTuple(outSchema.size()); }
private PlanProto.LogicalNode.Builder buildGroupby(SerializeContext context, GroupbyNode node) throws TajoException { int [] childIds = registerGetChildIds(context, node); PlanProto.GroupbyNode.Builder groupbyBuilder = PlanProto.GroupbyNode.newBuilder(); groupbyBuilder.setChildSeq(childIds[0]); groupbyBuilder.setDistinct(node.isDistinct()); if (node.groupingKeyNum() > 0) { groupbyBuilder.addAllGroupingKeys( ProtoUtil.<CatalogProtos.ColumnProto>toProtoObjects(node.getGroupingColumns())); } if (node.hasAggFunctions()) { groupbyBuilder.addAllAggFunctions( ProtoUtil.<PlanProto.EvalNodeTree>toProtoObjects(node.getAggFunctions().toArray(new ProtoObject[node.getAggFunctions().size()]))); } if (node.hasTargets()) { groupbyBuilder.addAllTargets(ProtoUtil.<PlanProto.Target>toProtoObjects(node.getTargets().toArray(new ProtoObject[node.getTargets().size()]))); } PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); nodeBuilder.setGroupby(groupbyBuilder); return nodeBuilder; }
public static Target [] buildGroupByTarget(GroupbyNode groupbyNode, @Nullable List<Target> groupingKeyTargets, String [] aggEvalNames) { final int groupingKeyNum = groupingKeyTargets == null ? groupbyNode.getGroupingColumns().length : groupingKeyTargets.size(); final int aggrFuncNum = aggEvalNames != null ? aggEvalNames.length : 0; EvalNode [] aggEvalNodes = groupbyNode.getAggFunctions(); Target [] targets = new Target[groupingKeyNum + aggrFuncNum]; if (groupingKeyTargets != null) { for (int groupingKeyIdx = 0; groupingKeyIdx < groupingKeyNum; groupingKeyIdx++) { targets[groupingKeyIdx] = groupingKeyTargets.get(groupingKeyIdx); } } else { for (int groupingKeyIdx = 0; groupingKeyIdx < groupingKeyNum; groupingKeyIdx++) { targets[groupingKeyIdx] = new Target(new FieldEval(groupbyNode.getGroupingColumns()[groupingKeyIdx])); } } if (aggEvalNames != null) { for (int aggrFuncIdx = 0, targetIdx = groupingKeyNum; aggrFuncIdx < aggrFuncNum; aggrFuncIdx++, targetIdx++) { targets[targetIdx] = new Target(new FieldEval(aggEvalNames[aggrFuncIdx], aggEvalNodes[aggrFuncIdx].getValueType())); } } return targets; }
int evalNum = secondPhaseGroupBy.getAggFunctions().length; AggregationFunctionCallEval [] secondPhaseEvals = secondPhaseGroupBy.getAggFunctions(); AggregationFunctionCallEval [] firstPhaseEvals = new AggregationFunctionCallEval[evalNum];
private NonDistinctHashAggregator(GroupbyNode groupbyNode) throws IOException { nonDistinctAggrDatas = new TupleMap<FunctionContext[]>(); if (groupbyNode.hasAggFunctions()) { aggFunctions = groupbyNode.getAggFunctions(); aggFunctionsNum = aggFunctions.length; } else { aggFunctions = new AggregationFunctionCallEval[0]; aggFunctionsNum = 0; } for (AggregationFunctionCallEval eachFunction: aggFunctions) { eachFunction.bind(context.getEvalContext(), inSchema); eachFunction.setFirstPhase(); } outTuple = new VTuple(aggFunctionsNum); dummyTuple = NullTuple.create(aggFunctionsNum); tupleLength = aggFunctionsNum; }
@Test public final void testFindDistinctAggFunctions() throws TajoException { String query = "select sum(score) + max(age) from people"; Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); GroupbyNode groupByNode = plan.getRootBlock().getNode(NodeType.GROUP_BY); List aggEvals = groupByNode.getAggFunctions(); List<AggregationFunctionCallEval> list = new ArrayList<>(); for (Object aggEval : aggEvals) { list.addAll(EvalTreeUtil.findDistinctAggFunction((EvalNode) aggEval)); } assertEquals(2, list.size()); Set<String> result = Sets.newHashSet("max", "sum"); for (AggregationFunctionCallEval eval : list) { assertTrue(result.contains(eval.getName())); } }
private PlanProto.LogicalNode.Builder buildGroupby(SerializeContext context, GroupbyNode node) throws TajoException { int [] childIds = registerGetChildIds(context, node); PlanProto.GroupbyNode.Builder groupbyBuilder = PlanProto.GroupbyNode.newBuilder(); groupbyBuilder.setChildSeq(childIds[0]); groupbyBuilder.setDistinct(node.isDistinct()); if (node.groupingKeyNum() > 0) { groupbyBuilder.addAllGroupingKeys( ProtoUtil.<CatalogProtos.ColumnProto>toProtoObjects(node.getGroupingColumns())); } if (node.hasAggFunctions()) { groupbyBuilder.addAllAggFunctions( ProtoUtil.<PlanProto.EvalNodeTree>toProtoObjects(node.getAggFunctions())); } if (node.hasTargets()) { groupbyBuilder.addAllTargets(ProtoUtil.<PlanProto.Target>toProtoObjects(node.getTargets())); } PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); nodeBuilder.setGroupby(groupbyBuilder); return nodeBuilder; }
@Test public final void testCountFunction() throws IOException, TajoException { FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getUri()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testCountFunction"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newTaskAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[9]); LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); // Set all aggregation functions to the first phase mode GroupbyNode groupbyNode = PlannerUtil.findTopNode(rootNode, NodeType.GROUP_BY); for (AggregationFunctionCallEval function : groupbyNode.getAggFunctions()) { function.setFirstPhase(); } PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); Tuple tuple = exec.next(); assertEquals(30, tuple.getInt8(0)); assertNull(exec.next()); exec.close(); }
@Test public final void testAggregationFunction() throws IOException, TajoException { FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getUri()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testAggregationFunction"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newTaskAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[8]); LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); // Set all aggregation functions to the first phase mode GroupbyNode groupbyNode = PlannerUtil.findTopNode(rootNode, NodeType.GROUP_BY); for (AggregationFunctionCallEval function : groupbyNode.getAggFunctions()) { function.setFirstPhase(); } PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); Tuple tuple = exec.next(); assertEquals(30, tuple.getInt8(0)); assertEquals(3, tuple.getInt4(1)); assertEquals(1, tuple.getInt4(2)); assertNull(exec.next()); exec.close(); }