private void resetAccumulator() { if (currentStart >= 0) { accumulator = accumulatorFactory.createAccumulator(); currentStart = -1; currentEnd = -1; } }
Aggregator(AccumulatorFactory accumulatorFactory, AggregationNode.Step step) { if (step.isInputRaw()) { intermediateChannel = -1; aggregation = accumulatorFactory.createAccumulator(); } else { checkArgument(accumulatorFactory.getInputChannels().size() == 1, "expected 1 input channel for intermediate aggregation"); intermediateChannel = accumulatorFactory.getInputChannels().get(0); aggregation = accumulatorFactory.createIntermediateAccumulator(); } this.step = step; }
@Test public void testNull() { Accumulator accumulator = factory.createAccumulator(); Block result = getFinalBlock(accumulator); assertTrue(result.getPositionCount() == 1); assertTrue(result.isNull(0)); }
@Test public void testNull() { Accumulator accumulator = factory.createAccumulator(); Block result = getFinalBlock(accumulator); assertTrue(result.getPositionCount() == 1); assertTrue(result.isNull(0)); }
@Test(expectedExceptions = PrestoException.class) public void testBadNumberOfBuckets() { Accumulator singleStep = factory.createAccumulator(); singleStep.addInput(makeInput(0)); getFinalBlock(singleStep); }
@Test(expectedExceptions = PrestoException.class) public void testBadNumberOfBuckets() { Accumulator singleStep = factory.createAccumulator(); singleStep.addInput(makeInput(0)); getFinalBlock(singleStep); }
@Test public void test() { Accumulator singleStep = factory.createAccumulator(); singleStep.addInput(input); Block expected = getFinalBlock(singleStep); Accumulator partialStep = factory.createAccumulator(); partialStep.addInput(input); Block partialBlock = getIntermediateBlock(partialStep); Accumulator finalStep = factory.createAccumulator(); finalStep.addIntermediate(partialBlock); Block actual = getFinalBlock(finalStep); assertEquals(extractSingleValue(actual), extractSingleValue(expected)); }
@Test public void test() { Accumulator singleStep = factory.createAccumulator(); singleStep.addInput(input); Block expected = getFinalBlock(singleStep); Accumulator partialStep = factory.createAccumulator(); partialStep.addInput(input); Block partialBlock = getIntermediateBlock(partialStep); Accumulator finalStep = factory.createAccumulator(); finalStep.addIntermediate(partialBlock); Block actual = getFinalBlock(finalStep); assertEquals(extractSingleValue(actual), extractSingleValue(expected)); }
@Test public void testMerge() { Accumulator singleStep = factory.createAccumulator(); singleStep.addInput(input); Block singleStepResult = getFinalBlock(singleStep); Accumulator partialStep = factory.createAccumulator(); partialStep.addInput(input); Block intermediate = getIntermediateBlock(partialStep); Accumulator finalStep = factory.createAccumulator(); finalStep.addIntermediate(intermediate); finalStep.addIntermediate(intermediate); Block actual = getFinalBlock(finalStep); Map<Double, Double> expected = Maps.transformValues(extractSingleValue(singleStepResult), value -> value * 2); assertEquals(extractSingleValue(actual), expected); }
@Test public void testMerge() { Accumulator singleStep = factory.createAccumulator(); singleStep.addInput(input); Block singleStepResult = getFinalBlock(singleStep); Accumulator partialStep = factory.createAccumulator(); partialStep.addInput(input); Block intermediate = getIntermediateBlock(partialStep); Accumulator finalStep = factory.createAccumulator(); finalStep.addIntermediate(intermediate); finalStep.addIntermediate(intermediate); Block actual = getFinalBlock(finalStep); Map<Float, Float> expected = Maps.transformValues(extractSingleValue(singleStepResult), value -> value * 2); assertEquals(extractSingleValue(actual), expected); }
@Test public void testLearn() { Type mapType = typeManager.getParameterizedType("map", ImmutableList.of(TypeSignatureParameter.of(parseTypeSignature(StandardTypes.BIGINT)), TypeSignatureParameter.of(parseTypeSignature(StandardTypes.DOUBLE)))); InternalAggregationFunction aggregation = generateInternalAggregationFunction(LearnClassifierAggregation.class, ClassifierType.BIGINT_CLASSIFIER.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature(), mapType.getTypeSignature()), typeManager); assertLearnClassifer(aggregation.bind(ImmutableList.of(0, 1), Optional.empty()).createAccumulator()); }
@Test public void testEvaluateClassifierPredictions() { metadata.addFunctions(extractFunctions(new MLPlugin().getFunctions())); InternalAggregationFunction aggregation = metadata.getFunctionRegistry().getAggregateFunctionImplementation( new Signature("evaluate_classifier_predictions", AGGREGATE, parseTypeSignature(StandardTypes.VARCHAR), parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.BIGINT))); Accumulator accumulator = aggregation.bind(ImmutableList.of(0, 1), Optional.empty()).createAccumulator(); accumulator.addInput(getPage()); BlockBuilder finalOut = accumulator.getFinalType().createBlockBuilder(null, 1); accumulator.evaluateFinal(finalOut); Block block = finalOut.build(); String output = VARCHAR.getSlice(block, 0).toStringUtf8(); List<String> parts = ImmutableList.copyOf(Splitter.on('\n').omitEmptyStrings().split(output)); assertEquals(parts.size(), 7, output); assertEquals(parts.get(0), "Accuracy: 1/2 (50.00%)"); }
@Test public void testLearnLibSvm() { Type mapType = typeManager.getParameterizedType("map", ImmutableList.of(TypeSignatureParameter.of(parseTypeSignature(StandardTypes.BIGINT)), TypeSignatureParameter.of(parseTypeSignature(StandardTypes.DOUBLE)))); InternalAggregationFunction aggregation = AggregationFromAnnotationsParser.parseFunctionDefinitionWithTypesConstraint( LearnLibSvmClassifierAggregation.class, ClassifierType.BIGINT_CLASSIFIER.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature(), mapType.getTypeSignature(), VarcharType.getParametrizedVarcharSignature("x")) ).specialize(BoundVariables.builder().setLongVariable("x", (long) Integer.MAX_VALUE).build(), 3, typeManager); assertLearnClassifer(aggregation.bind(ImmutableList.of(0, 1, 2), Optional.empty()).createAccumulator()); }
public static Object partialAggregation(InternalAggregationFunction function, int[] args, Page... pages) { AccumulatorFactory factory = function.bind(Ints.asList(args), Optional.empty()); Accumulator finalAggregation = factory.createIntermediateAccumulator(); // Test handling of empty intermediate blocks Accumulator emptyAggregation = factory.createAccumulator(); Block emptyBlock = getIntermediateBlock(emptyAggregation); finalAggregation.addIntermediate(emptyBlock); for (Page page : pages) { Accumulator partialAggregation = factory.createAccumulator(); if (page.getPositionCount() > 0) { partialAggregation.addInput(page); } Block partialBlock = getIntermediateBlock(partialAggregation); finalAggregation.addIntermediate(partialBlock); } finalAggregation.addIntermediate(emptyBlock); Block finalBlock = getFinalBlock(finalAggregation); return BlockAssertions.getOnlyValue(finalAggregation.getFinalType(), finalBlock); }
private static Object aggregation(InternalAggregationFunction function, int[] args, Optional<Integer> maskChannel, Page... pages) { Accumulator aggregation = function.bind(Ints.asList(args), maskChannel).createAccumulator(); for (Page page : pages) { if (page.getPositionCount() > 0) { aggregation.addInput(page); } } Block block = getFinalBlock(aggregation); return BlockAssertions.getOnlyValue(aggregation.getFinalType(), block); }
@Setup(Invocation) public void setup() { MetadataManager metadata = MetadataManager.createTestMetadataManager(); Block block; Type elementType; switch (type) { case "BIGINT": elementType = BIGINT; break; case "VARCHAR": elementType = VARCHAR; break; case "DOUBLE": elementType = DOUBLE; break; case "BOOLEAN": elementType = BOOLEAN; break; default: throw new UnsupportedOperationException(); } ArrayType arrayType = new ArrayType(elementType); Signature signature = new Signature(name, AGGREGATE, arrayType.getTypeSignature(), elementType.getTypeSignature()); InternalAggregationFunction function = metadata.getFunctionRegistry().getAggregateFunctionImplementation(signature); accumulator = function.bind(ImmutableList.of(0), Optional.empty()).createAccumulator(); block = createChannel(ARRAY_SIZE, elementType); page = new Page(block); }
@Test(dataProvider = "partitionCount") public void test(int partitionCount) { InternalAggregationFunction function = getFunction(); List<OGCGeometry> geometries = makeGeometries(); Block geometryBlock = makeGeometryBlock(geometries); Block partitionCountBlock = BlockAssertions.createRLEBlock(partitionCount, geometries.size()); Rectangle expectedExtent = new Rectangle(-10, -10, Math.nextUp(10.0), Math.nextUp(10.0)); String expectedValue = getSpatialPartitioning(expectedExtent, geometries, partitionCount); AccumulatorFactory accumulatorFactory = function.bind(Ints.asList(0, 1, 2), Optional.empty()); Page page = new Page(geometryBlock, partitionCountBlock); Accumulator accumulator = accumulatorFactory.createAccumulator(); accumulator.addInput(page); String aggregation = (String) BlockAssertions.getOnlyValue(accumulator.getFinalType(), getFinalBlock(accumulator)); assertEquals(aggregation, expectedValue); GroupedAccumulator groupedAggregation = accumulatorFactory.createGroupedAccumulator(); groupedAggregation.addInput(createGroupByIdBlock(0, page.getPositionCount()), page); String groupValue = (String) getGroupValue(groupedAggregation, 0); assertEquals(groupValue, expectedValue); }
@Test public void testNull() throws Exception { Accumulator accumulator = factory.createAccumulator(); Block result = getFinalBlock(accumulator); assertTrue(result.getPositionCount() == 1); assertTrue(result.isNull(0)); }
@Test(expectedExceptions = PrestoException.class) public void testBadNumberOfBuckets() { Accumulator singleStep = factory.createAccumulator(); singleStep.addInput(makeInput(0)); getFinalBlock(singleStep); }
public static boolean approximateAggregationWithinErrorBound(InternalAggregationFunction function, int sampleWeightChannel, double confidence, Double expectedValue, Page... pages) { Accumulator accumulator = function.bind(ImmutableList.of(0), Optional.empty(), Optional.of(sampleWeightChannel), confidence).createAccumulator(); for (Page page : pages) { accumulator.addInput(page); } Block result = getFinalBlock(accumulator); if (expectedValue == null) { return BlockAssertions.toValues(function.getFinalType(), result).get(0) == null; } return withinErrorBound(BlockAssertions.toValues(function.getFinalType(), result).get(0).toString(), expectedValue); }