@CombineFunction public static void combine(@AggregationState DigestAndPercentileState state, DigestAndPercentileState otherState) { QuantileDigest input = otherState.getDigest(); QuantileDigest previous = state.getDigest(); if (previous == null) { state.setDigest(input); state.addMemoryUsage(input.estimatedInMemorySizeInBytes()); } else { state.addMemoryUsage(-previous.estimatedInMemorySizeInBytes()); previous.merge(input); state.addMemoryUsage(previous.estimatedInMemorySizeInBytes()); } state.setPercentile(otherState.getPercentile()); }
@OutputFunction(StandardTypes.BIGINT) public static void output(@AggregationState DigestAndPercentileState state, BlockBuilder out) { QuantileDigest digest = state.getDigest(); double percentile = state.getPercentile(); if (digest == null || digest.getCount() == 0.0) { out.appendNull(); } else { checkState(percentile != -1.0, "Percentile is missing"); checkCondition(0 <= percentile && percentile <= 1, INVALID_FUNCTION_ARGUMENT, "Percentile must be between 0 and 1"); BIGINT.writeLong(out, digest.getQuantile(percentile)); } }
@OutputFunction(StandardTypes.REAL) public static void output(@AggregationState DigestAndPercentileState state, BlockBuilder out) { QuantileDigest digest = state.getDigest(); double percentile = state.getPercentile(); if (digest == null || digest.getCount() == 0.0) { out.appendNull(); } else { checkState(percentile != -1.0, "Percentile is missing"); checkCondition(0 <= percentile && percentile <= 1, INVALID_FUNCTION_ARGUMENT, "Percentile must be between 0 and 1"); REAL.writeLong(out, floatToRawIntBits(sortableIntToFloat((int) digest.getQuantile(percentile)))); } } }
@Override public void deserialize(Block block, int index, DigestAndPercentileState state) { SliceInput input = VARBINARY.getSlice(block, index).getInput(); // read percentile state.setPercentile(input.readDouble()); // read digest int length = input.readInt(); QuantileDigest digest = new QuantileDigest(input.readSlice(length)); state.setDigest(digest); state.addMemoryUsage(state.getDigest().estimatedInMemorySizeInBytes()); } }
@OutputFunction(StandardTypes.DOUBLE) public static void output(@AggregationState DigestAndPercentileState state, BlockBuilder out) { QuantileDigest digest = state.getDigest(); double percentile = state.getPercentile(); if (digest == null || digest.getCount() == 0.0) { out.appendNull(); } else { checkState(percentile != -1.0, "Percentile is missing"); checkCondition(0 <= percentile && percentile <= 1, INVALID_FUNCTION_ARGUMENT, "Percentile must be between 0 and 1"); DOUBLE.writeDouble(out, sortableLongToDouble(digest.getQuantile(percentile))); } } }
@InputFunction public static void input(@AggregationState DigestAndPercentileState state, @SqlType(StandardTypes.BIGINT) long value, @SqlType(StandardTypes.DOUBLE) double percentile) { QuantileDigest digest = state.getDigest(); if (digest == null) { digest = new QuantileDigest(0.01); state.setDigest(digest); state.addMemoryUsage(digest.estimatedInMemorySizeInBytes()); } state.addMemoryUsage(-digest.estimatedInMemorySizeInBytes()); digest.add(value); state.addMemoryUsage(digest.estimatedInMemorySizeInBytes()); // use last percentile state.setPercentile(percentile); }
@Override public void serialize(DigestAndPercentileState state, BlockBuilder out) { if (state.getDigest() == null) { out.appendNull(); } else { Slice serialized = state.getDigest().serialize(); SliceOutput output = Slices.allocate(SIZE_OF_DOUBLE + SIZE_OF_INT + serialized.length()).getOutput(); output.appendDouble(state.getPercentile()); output.appendInt(serialized.length()); output.appendBytes(serialized); VARBINARY.writeSlice(out, output.slice()); } }
@InputFunction public static void weightedInput(@AggregationState DigestAndPercentileState state, @SqlType(StandardTypes.BIGINT) long value, @SqlType(StandardTypes.BIGINT) long weight, @SqlType(StandardTypes.DOUBLE) double percentile) { checkWeight(weight); QuantileDigest digest = state.getDigest(); if (digest == null) { digest = new QuantileDigest(0.01); state.setDigest(digest); state.addMemoryUsage(digest.estimatedInMemorySizeInBytes()); } state.addMemoryUsage(-digest.estimatedInMemorySizeInBytes()); digest.add(value, weight); state.addMemoryUsage(digest.estimatedInMemorySizeInBytes()); // use last percentile state.setPercentile(percentile); }
@Override public void serialize(DigestAndPercentileState state, BlockBuilder out) { if (state.getDigest() == null) { out.appendNull(); } else { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(state.getDigest().estimatedSerializedSizeInBytes() + SIZE_OF_DOUBLE); // write digest state.getDigest().serialize(sliceOutput); // write percentile sliceOutput.appendDouble(state.getPercentile()); Slice slice = sliceOutput.slice(); VARCHAR.writeSlice(out, slice); } }
@InputFunction public static void weightedInput(@AggregationState DigestAndPercentileState state, @SqlType(StandardTypes.BIGINT) long value, @SqlType(StandardTypes.BIGINT) long weight, @SqlType(StandardTypes.DOUBLE) double percentile, @SqlType(StandardTypes.DOUBLE) double accuracy) { checkWeight(weight); QuantileDigest digest = state.getDigest(); if (digest == null) { if (accuracy > 0 && accuracy < 1) { digest = new QuantileDigest(accuracy); } else { throw new IllegalArgumentException("Percentile accuracy must be strictly between 0 and 1"); } state.setDigest(digest); state.addMemoryUsage(digest.estimatedInMemorySizeInBytes()); } state.addMemoryUsage(-digest.estimatedInMemorySizeInBytes()); digest.add(value, weight); state.addMemoryUsage(digest.estimatedInMemorySizeInBytes()); // use last percentile state.setPercentile(percentile); }
@OutputFunction(StandardTypes.BIGINT) public static void output(DigestAndPercentileState state, BlockBuilder out) { QuantileDigest digest = state.getDigest(); double percentile = state.getPercentile(); if (digest == null || digest.getCount() == 0.0) { out.appendNull(); } else { checkState(percentile != -1.0, "Percentile is missing"); checkCondition(0 <= percentile && percentile <= 1, INVALID_FUNCTION_ARGUMENT, "Percentile must be between 0 and 1"); BIGINT.writeLong(out, digest.getQuantile(percentile)); } }
@CombineFunction public static void combine(DigestAndPercentileState state, DigestAndPercentileState otherState) { QuantileDigest input = otherState.getDigest(); QuantileDigest previous = state.getDigest(); if (previous == null) { state.setDigest(input); state.addMemoryUsage(input.estimatedInMemorySizeInBytes()); } else { state.addMemoryUsage(-previous.estimatedInMemorySizeInBytes()); previous.merge(input); state.addMemoryUsage(previous.estimatedInMemorySizeInBytes()); } state.setPercentile(otherState.getPercentile()); }
@OutputFunction(StandardTypes.DOUBLE) public static void output(DigestAndPercentileState state, BlockBuilder out) { QuantileDigest digest = state.getDigest(); double percentile = state.getPercentile(); if (digest == null || digest.getCount() == 0.0) { out.appendNull(); } else { checkState(percentile != -1.0, "Percentile is missing"); checkCondition(0 <= percentile && percentile <= 1, INVALID_FUNCTION_ARGUMENT, "Percentile must be between 0 and 1"); DOUBLE.writeDouble(out, sortableLongToDouble(digest.getQuantile(percentile))); } } }
@InputFunction public static void input(DigestAndPercentileState state, @SqlType(StandardTypes.BIGINT) long value, @SqlType(StandardTypes.DOUBLE) double percentile) { QuantileDigest digest = state.getDigest(); if (digest == null) { digest = new QuantileDigest(0.01); state.setDigest(digest); state.addMemoryUsage(digest.estimatedInMemorySizeInBytes()); } state.addMemoryUsage(-digest.estimatedInMemorySizeInBytes()); digest.add(value); state.addMemoryUsage(digest.estimatedInMemorySizeInBytes()); // use last percentile state.setPercentile(percentile); }
@Override public void deserialize(Block block, int index, DigestAndPercentileState state) { if (!block.isNull(index)) { SliceInput input = VARCHAR.getSlice(block, index).getInput(); // read digest state.setDigest(QuantileDigest.deserialize(input)); state.addMemoryUsage(state.getDigest().estimatedInMemorySizeInBytes()); // read percentile state.setPercentile(input.readDouble()); } } }
@InputFunction public static void weightedInput(DigestAndPercentileState state, @SqlType(StandardTypes.BIGINT) long value, @SqlType(StandardTypes.BIGINT) long weight, @SqlType(StandardTypes.DOUBLE) double percentile) { checkWeight(weight); QuantileDigest digest = state.getDigest(); if (digest == null) { digest = new QuantileDigest(0.01); state.setDigest(digest); state.addMemoryUsage(digest.estimatedInMemorySizeInBytes()); } state.addMemoryUsage(-digest.estimatedInMemorySizeInBytes()); digest.add(value, weight); state.addMemoryUsage(digest.estimatedInMemorySizeInBytes()); // use last percentile state.setPercentile(percentile); }
@InputFunction public static void weightedInput(DigestAndPercentileState state, @SqlType(StandardTypes.BIGINT) long value, @SqlType(StandardTypes.BIGINT) long weight, @SqlType(StandardTypes.DOUBLE) double percentile, @SqlType(StandardTypes.DOUBLE) double accuracy) { checkWeight(weight); QuantileDigest digest = state.getDigest(); if (digest == null) { if (accuracy > 0 && accuracy < 1) { digest = new QuantileDigest(accuracy); } else { throw new IllegalArgumentException("Percentile accuracy must be strictly between 0 and 1"); } state.setDigest(digest); state.addMemoryUsage(digest.estimatedInMemorySizeInBytes()); } state.addMemoryUsage(-digest.estimatedInMemorySizeInBytes()); digest.add(value, weight); state.addMemoryUsage(digest.estimatedInMemorySizeInBytes()); // use last percentile state.setPercentile(percentile); }