private Aggregator(AccumulatorFactory accumulatorFactory, AggregationNode.Step step, Optional<Integer> overwriteIntermediateChannel) { if (step.isInputRaw()) { this.intermediateChannel = -1; this.aggregation = accumulatorFactory.createGroupedAccumulator(); } else if (overwriteIntermediateChannel.isPresent()) { this.intermediateChannel = overwriteIntermediateChannel.get(); this.aggregation = accumulatorFactory.createGroupedIntermediateAccumulator(); } else { checkArgument(accumulatorFactory.getInputChannels().size() == 1, "expected 1 input channel for intermediate aggregation"); this.intermediateChannel = accumulatorFactory.getInputChannels().get(0); this.aggregation = accumulatorFactory.createGroupedIntermediateAccumulator(); } this.step = step; }
updateMemory); this.operatorContext = operatorContext; this.partial = step.isOutputPartial(); this.maxPartialMemory = maxPartialMemory.map(dataSize -> OptionalLong.of(dataSize.toBytes())).orElseGet(OptionalLong::empty); this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(InMemoryHashAggregationBuilder.class.getSimpleName());
updateMemory); this.operatorContext = operatorContext; this.partial = step.isOutputPartial(); this.maxPartialMemory = maxPartialMemory.map(dataSize -> OptionalLong.of(dataSize.toBytes())).orElseGet(OptionalLong::empty); this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(InMemoryHashAggregationBuilder.class.getSimpleName());
if (aggregationNode.getStep().equals(SINGLE) && aggregationNode.hasEmptyGroupingSet() && aggregationNode.hasNonEmptyGroupingSet() &&
if (aggregationNode.getStep().equals(SINGLE) && aggregationNode.hasEmptyGroupingSet() && aggregationNode.hasNonEmptyGroupingSet() &&
if (step.isOutputPartial() || !spillEnabled || hasOrderBy() || hasDistinct()) { aggregationBuilder = new InMemoryHashAggregationBuilder( accumulatorFactories,
if (step.isOutputPartial() || !spillEnabled || hasOrderBy() || hasDistinct()) { aggregationBuilder = new InMemoryHashAggregationBuilder( accumulatorFactories,
if (step.isOutputPartial()) { accumulators.get(j).evaluateIntermediate(output.getBlockBuilder(channel));
this.step = AggregationNode.Step.partialInput(step); this.expectedGroups = expectedGroups; this.groupByPartialChannels = groupByPartialChannels.build();
if (step.isOutputPartial()) { accumulators.get(j).evaluateIntermediate(output.getBlockBuilder(channel));
this.step = AggregationNode.Step.partialInput(step); this.expectedGroups = expectedGroups; this.groupByPartialChannels = groupByPartialChannels.build();
@Override public Void visitAggregation(AggregationNode node, Integer indent) { String type = ""; if (node.getStep() != AggregationNode.Step.SINGLE) { type = format("(%s)", node.getStep().toString()); } if (node.isStreamable()) { type = format("%s(STREAMING)", type); } String key = ""; if (!node.getGroupingKeys().isEmpty()) { key = node.getGroupingKeys().toString(); } print(indent, "- Aggregate%s%s%s => [%s]", type, key, formatHash(node.getHashSymbol()), formatOutputs(node.getOutputSymbols())); printPlanNodesStatsAndCost(indent + 2, node); printStats(indent + 2, node.getId()); for (Map.Entry<Symbol, Aggregation> entry : node.getAggregations().entrySet()) { if (entry.getValue().getMask().isPresent()) { print(indent + 2, "%s := %s (mask = %s)", entry.getKey(), entry.getValue().getCall(), entry.getValue().getMask().get()); } else { print(indent + 2, "%s := %s", entry.getKey(), entry.getValue().getCall()); } } return processChildren(node, indent + 1); }
@Override public Void visitAggregation(AggregationNode node, Integer indent) { String type = ""; if (node.getStep() != AggregationNode.Step.SINGLE) { type = format("(%s)", node.getStep().toString()); } if (node.isStreamable()) { type = format("%s(STREAMING)", type); } String key = ""; if (!node.getGroupingKeys().isEmpty()) { key = node.getGroupingKeys().toString(); } print(indent, "- Aggregate%s%s%s => [%s]", type, key, formatHash(node.getHashSymbol()), formatOutputs(node.getOutputSymbols())); printPlanNodesStatsAndCost(indent + 2, node); printStats(indent + 2, node.getId()); for (Map.Entry<Symbol, Aggregation> entry : node.getAggregations().entrySet()) { if (entry.getValue().getMask().isPresent()) { print(indent + 2, "%s := %s (mask = %s)", entry.getKey(), entry.getValue().getCall(), entry.getValue().getMask().get()); } else { print(indent + 2, "%s := %s", entry.getKey(), entry.getValue().getCall()); } } return processChildren(node, indent + 1); }
private void updateMemoryUsage() { long memorySize = pageBuilder.getRetainedSizeInBytes(); for (Page output : outputPages) { memorySize += output.getRetainedSizeInBytes(); } for (Aggregator aggregator : aggregates) { memorySize += aggregator.getEstimatedSize(); } if (currentGroup != null) { memorySize += currentGroup.getRetainedSizeInBytes(); } if (step.isOutputPartial()) { systemMemoryContext.setBytes(memorySize); } else { userMemoryContext.setBytes(memorySize); } }
private void updateMemoryUsage() { long memorySize = pageBuilder.getRetainedSizeInBytes(); for (Page output : outputPages) { memorySize += output.getRetainedSizeInBytes(); } for (Aggregator aggregator : aggregates) { memorySize += aggregator.getEstimatedSize(); } if (currentGroup != null) { memorySize += currentGroup.getRetainedSizeInBytes(); } if (step.isOutputPartial()) { systemMemoryContext.setBytes(memorySize); } else { userMemoryContext.setBytes(memorySize); } }
private Aggregator(AccumulatorFactory accumulatorFactory, AggregationNode.Step step, Optional<Integer> overwriteIntermediateChannel) { if (step.isInputRaw()) { this.intermediateChannel = -1; this.aggregation = accumulatorFactory.createGroupedAccumulator(); } else if (overwriteIntermediateChannel.isPresent()) { this.intermediateChannel = overwriteIntermediateChannel.get(); this.aggregation = accumulatorFactory.createGroupedIntermediateAccumulator(); } else { checkArgument(accumulatorFactory.getInputChannels().size() == 1, "expected 1 input channel for intermediate aggregation"); this.intermediateChannel = accumulatorFactory.getInputChannels().get(0); this.aggregation = accumulatorFactory.createGroupedIntermediateAccumulator(); } this.step = step; }
private PhysicalOperation planGlobalAggregation(AggregationNode node, PhysicalOperation source, LocalExecutionPlanContext context) { ImmutableMap.Builder<Symbol, Integer> outputMappings = ImmutableMap.builder(); AggregationOperatorFactory operatorFactory = createAggregationOperatorFactory( node.getId(), node.getAggregations(), node.getStep(), 0, outputMappings, source, context, node.getStep().isOutputPartial()); return new PhysicalOperation(operatorFactory, outputMappings.build(), context, source); }
private PhysicalOperation planGlobalAggregation(AggregationNode node, PhysicalOperation source, LocalExecutionPlanContext context) { ImmutableMap.Builder<Symbol, Integer> outputMappings = ImmutableMap.builder(); AggregationOperatorFactory operatorFactory = createAggregationOperatorFactory( node.getId(), node.getAggregations(), node.getStep(), 0, outputMappings, source, context, node.getStep().isOutputPartial()); return new PhysicalOperation(operatorFactory, outputMappings.build(), context, source); }