public void processPage(Page page) { if (step.isInputRaw()) { aggregation.addInput(page); } else { aggregation.addIntermediate(page.getBlock(intermediateChannel)); } }
public Type getType() { if (step.isOutputPartial()) { return aggregation.getIntermediateType(); } else { return aggregation.getFinalType(); } }
public void evaluate(BlockBuilder blockBuilder) { if (step.isOutputPartial()) { aggregation.evaluateIntermediate(blockBuilder); } else { aggregation.evaluateFinal(blockBuilder); } }
private static void assertLearnClassifer(Accumulator accumulator) { accumulator.addInput(getPage()); BlockBuilder finalOut = accumulator.getFinalType().createBlockBuilder(null, 1); accumulator.evaluateFinal(finalOut); Block block = finalOut.build(); Slice slice = accumulator.getFinalType().getSlice(block, 0); Model deserialized = ModelUtils.deserialize(slice); assertNotNull(deserialized, "deserialization failed"); assertTrue(deserialized instanceof Classifier, "deserialized model is not a classifier"); }
public static Object partialAggregation(InternalAggregationFunction function, int[] args, Page... pages) { AccumulatorFactory factory = function.bind(Ints.asList(args), Optional.empty()); Accumulator finalAggregation = factory.createIntermediateAccumulator(); // Test handling of empty intermediate blocks Accumulator emptyAggregation = factory.createAccumulator(); Block emptyBlock = getIntermediateBlock(emptyAggregation); finalAggregation.addIntermediate(emptyBlock); for (Page page : pages) { Accumulator partialAggregation = factory.createAccumulator(); if (page.getPositionCount() > 0) { partialAggregation.addInput(page); } Block partialBlock = getIntermediateBlock(partialAggregation); finalAggregation.addIntermediate(partialBlock); } finalAggregation.addIntermediate(emptyBlock); Block finalBlock = getFinalBlock(finalAggregation); return BlockAssertions.getOnlyValue(finalAggregation.getFinalType(), finalBlock); }
private static Object aggregation(InternalAggregationFunction function, int[] args, Optional<Integer> maskChannel, Page... pages) { Accumulator aggregation = function.bind(Ints.asList(args), maskChannel).createAccumulator(); for (Page page : pages) { if (page.getPositionCount() > 0) { aggregation.addInput(page); } } Block block = getFinalBlock(aggregation); return BlockAssertions.getOnlyValue(aggregation.getFinalType(), block); }
private void accumulate(int start, int end) { accumulator.addInput(windowIndex, argumentChannels, start, end); }
public static Block getFinalBlock(Accumulator accumulator) { BlockBuilder blockBuilder = accumulator.getFinalType().createBlockBuilder(null, 1000); accumulator.evaluateFinal(blockBuilder); return blockBuilder.build(); }
@Override public Type getFinalType() { return accumulator.getFinalType(); }
@Override public void evaluateFinal(BlockBuilder blockBuilder) { accumulator.evaluateFinal(blockBuilder); } }
public static Block getIntermediateBlock(Accumulator accumulator) { BlockBuilder blockBuilder = accumulator.getIntermediateType().createBlockBuilder(null, 1000); accumulator.evaluateIntermediate(blockBuilder); return blockBuilder.build(); }
public long getEstimatedSize() { return aggregation.getEstimatedSize(); } }
@Test public void testEvaluateClassifierPredictions() { metadata.addFunctions(extractFunctions(new MLPlugin().getFunctions())); InternalAggregationFunction aggregation = metadata.getFunctionRegistry().getAggregateFunctionImplementation( new Signature("evaluate_classifier_predictions", AGGREGATE, parseTypeSignature(StandardTypes.VARCHAR), parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.BIGINT))); Accumulator accumulator = aggregation.bind(ImmutableList.of(0, 1), Optional.empty()).createAccumulator(); accumulator.addInput(getPage()); BlockBuilder finalOut = accumulator.getFinalType().createBlockBuilder(null, 1); accumulator.evaluateFinal(finalOut); Block block = finalOut.build(); String output = VARCHAR.getSlice(block, 0).toStringUtf8(); List<String> parts = ImmutableList.copyOf(Splitter.on('\n').omitEmptyStrings().split(output)); assertEquals(parts.size(), 7, output); assertEquals(parts.get(0), "Accuracy: 1/2 (50.00%)"); }
public static Object partialAggregation(InternalAggregationFunction function, int[] args, Page... pages) { AccumulatorFactory factory = function.bind(Ints.asList(args), Optional.empty()); Accumulator finalAggregation = factory.createIntermediateAccumulator(); // Test handling of empty intermediate blocks Accumulator emptyAggregation = factory.createAccumulator(); Block emptyBlock = getIntermediateBlock(emptyAggregation); finalAggregation.addIntermediate(emptyBlock); for (Page page : pages) { Accumulator partialAggregation = factory.createAccumulator(); if (page.getPositionCount() > 0) { partialAggregation.addInput(page); } Block partialBlock = getIntermediateBlock(partialAggregation); finalAggregation.addIntermediate(partialBlock); } finalAggregation.addIntermediate(emptyBlock); Block finalBlock = getFinalBlock(finalAggregation); return BlockAssertions.getOnlyValue(finalAggregation.getFinalType(), finalBlock); }
private static Object aggregation(InternalAggregationFunction function, int[] args, Optional<Integer> maskChannel, Page... pages) { Accumulator aggregation = function.bind(Ints.asList(args), maskChannel).createAccumulator(); for (Page page : pages) { if (page.getPositionCount() > 0) { aggregation.addInput(page); } } Block block = getFinalBlock(aggregation); return BlockAssertions.getOnlyValue(aggregation.getFinalType(), block); }
private void accumulate(int start, int end) { accumulator.addInput(windowIndex, argumentChannels, start, end); }
public static Block getFinalBlock(Accumulator accumulator) { BlockBuilder blockBuilder = accumulator.getFinalType().createBlockBuilder(null, 1000); accumulator.evaluateFinal(blockBuilder); return blockBuilder.build(); }
@Override public Type getFinalType() { return accumulator.getFinalType(); }
@Override public void evaluateFinal(BlockBuilder blockBuilder) { accumulator.evaluateFinal(blockBuilder); } }
public static Block getIntermediateBlock(Accumulator accumulator) { BlockBuilder blockBuilder = accumulator.getIntermediateType().createBlockBuilder(null, 1000); accumulator.evaluateIntermediate(blockBuilder); return blockBuilder.build(); }