private static HyperLogLog getOrCreateHyperLogLog(@AggregationState HyperLogLogState state) { HyperLogLog hll = state.getHyperLogLog(); if (hll == null) { hll = newHyperLogLog(); state.setHyperLogLog(hll); state.addMemoryUsage(hll.estimatedInMemorySize()); } return hll; }
@InputFunction public static void input(@AggregationState HyperLogLogState state, @SqlType(StandardTypes.DOUBLE) double value) { HyperLogLog hll = getOrCreateHyperLogLog(state); state.addMemoryUsage(-hll.estimatedInMemorySize()); hll.add(Double.doubleToLongBits(value)); state.addMemoryUsage(hll.estimatedInMemorySize()); }
@Override public void serialize(HyperLogLogState state, BlockBuilder out) { if (state.getHyperLogLog() == null) { out.appendNull(); } else { HYPER_LOG_LOG.writeSlice(out, state.getHyperLogLog().serialize()); } }
@Override public void deserialize(Block block, int index, HyperLogLogState state) { state.setHyperLogLog(HyperLogLog.newInstance(HYPER_LOG_LOG.getSlice(block, index))); } }
@InputFunction public static void input(@AggregationState HyperLogLogState state, @SqlType(StandardTypes.BIGINT) long value) { HyperLogLog hll = getOrCreateHyperLogLog(state); state.addMemoryUsage(-hll.estimatedInMemorySize()); hll.add(value); state.addMemoryUsage(hll.estimatedInMemorySize()); }
@CombineFunction public static void combine(@AggregationState HyperLogLogState state, @AggregationState HyperLogLogState otherState) { merge(state, otherState.getHyperLogLog()); }
@Override public void deserialize(Block block, int index, HyperLogLogState state) { if (!block.isNull(index)) { state.setHyperLogLog(HyperLogLog.newInstance(HYPER_LOG_LOG.getSlice(block, index))); } } }
private static HyperLogLog getOrCreateHyperLogLog(HyperLogLogState state, double maxStandardError) { HyperLogLog hll = state.getHyperLogLog(); if (hll == null) { hll = HyperLogLog.newInstance(standardErrorToBuckets(maxStandardError)); state.setHyperLogLog(hll); state.addMemoryUsage(hll.estimatedInMemorySize()); } return hll; }
@InputFunction @LiteralParameters("x") public static void input(@AggregationState HyperLogLogState state, @SqlType("varchar(x)") Slice value) { HyperLogLog hll = getOrCreateHyperLogLog(state); state.addMemoryUsage(-hll.estimatedInMemorySize()); hll.add(value); state.addMemoryUsage(hll.estimatedInMemorySize()); }
@OutputFunction(StandardTypes.BIGINT) public static void evaluateFinal(@AggregationState HyperLogLogState state, BlockBuilder out) { HyperLogLog hyperLogLog = state.getHyperLogLog(); if (hyperLogLog == null) { BIGINT.writeLong(out, 0); } else { BIGINT.writeLong(out, hyperLogLog.cardinality()); } }
private static void merge(@AggregationState HyperLogLogState state, HyperLogLog input) { HyperLogLog previous = state.getHyperLogLog(); if (previous == null) { state.setHyperLogLog(input); state.addMemoryUsage(input.estimatedInMemorySize()); } else { state.addMemoryUsage(-previous.estimatedInMemorySize()); previous.mergeWith(input); state.addMemoryUsage(previous.estimatedInMemorySize()); } }
@InputFunction @TypeParameter("T") public static void input( @OperatorDependency(operator = XX_HASH_64, returnType = StandardTypes.BIGINT, argumentTypes = {"T"}) MethodHandle methodHandle, @AggregationState HyperLogLogState state, @SqlType("T") double value, @SqlType(StandardTypes.DOUBLE) double maxStandardError) { HyperLogLog hll = getOrCreateHyperLogLog(state, maxStandardError); state.addMemoryUsage(-hll.estimatedInMemorySize()); long hash; try { hash = (long) methodHandle.invokeExact(value); } catch (Throwable t) { throw internalError(t); } hll.addHash(hash); state.addMemoryUsage(hll.estimatedInMemorySize()); }
@Override public void serialize(HyperLogLogState state, BlockBuilder out) { if (state.getHyperLogLog() == null) { out.appendNull(); } else { HYPER_LOG_LOG.writeSlice(out, state.getHyperLogLog().serialize()); } }
@CombineFunction public static void combineState(@AggregationState HyperLogLogState state, @AggregationState HyperLogLogState otherState) { HyperLogLog input = otherState.getHyperLogLog(); HyperLogLog previous = state.getHyperLogLog(); if (previous == null) { state.setHyperLogLog(input); state.addMemoryUsage(input.estimatedInMemorySize()); } else { state.addMemoryUsage(-previous.estimatedInMemorySize()); previous.mergeWith(input); state.addMemoryUsage(previous.estimatedInMemorySize()); } }
@InputFunction @TypeParameter("T") public static void input( @OperatorDependency(operator = XX_HASH_64, returnType = StandardTypes.BIGINT, argumentTypes = {"T"}) MethodHandle methodHandle, @AggregationState HyperLogLogState state, @SqlType("T") Slice value, @SqlType(StandardTypes.DOUBLE) double maxStandardError) { HyperLogLog hll = getOrCreateHyperLogLog(state, maxStandardError); state.addMemoryUsage(-hll.estimatedInMemorySize()); long hash; try { hash = (long) methodHandle.invokeExact(value); } catch (Throwable t) { throw internalError(t); } hll.addHash(hash); state.addMemoryUsage(hll.estimatedInMemorySize()); }
@OutputFunction(StandardTypes.BIGINT) public static void evaluateFinal(HyperLogLogState state, BlockBuilder out) { HyperLogLog hyperLogLog = state.getHyperLogLog(); if (hyperLogLog == null) { BIGINT.writeLong(out, 0); } else { BIGINT.writeLong(out, hyperLogLog.cardinality()); } } }
@CombineFunction public static void combineState(@AggregationState HyperLogLogState state, @AggregationState HyperLogLogState otherState) { HyperLogLog input = otherState.getHyperLogLog(); HyperLogLog previous = state.getHyperLogLog(); if (previous == null) { state.setHyperLogLog(input); state.addMemoryUsage(input.estimatedInMemorySize()); } else { state.addMemoryUsage(-previous.estimatedInMemorySize()); previous.mergeWith(input); state.addMemoryUsage(previous.estimatedInMemorySize()); } }
@InputFunction @TypeParameter("T") public static void input( @OperatorDependency(operator = XX_HASH_64, returnType = StandardTypes.BIGINT, argumentTypes = {"T"}) MethodHandle methodHandle, @AggregationState HyperLogLogState state, @SqlType("T") long value, @SqlType(StandardTypes.DOUBLE) double maxStandardError) { HyperLogLog hll = getOrCreateHyperLogLog(state, maxStandardError); state.addMemoryUsage(-hll.estimatedInMemorySize()); long hash; try { hash = (long) methodHandle.invokeExact(value); } catch (Throwable t) { throw internalError(t); } hll.addHash(hash); state.addMemoryUsage(hll.estimatedInMemorySize()); }
private static HyperLogLog getOrCreateHyperLogLog(HyperLogLogState state) { HyperLogLog hll = state.getHyperLogLog(); if (hll == null) { hll = HyperLogLog.newInstance(NUMBER_OF_BUCKETS); state.setHyperLogLog(hll); state.addMemoryUsage(hll.estimatedInMemorySize()); } return hll; }
@InputFunction public static void input(HyperLogLogState state, @SqlType(StandardTypes.DOUBLE) double value) { HyperLogLog hll = getOrCreateHyperLogLog(state); state.addMemoryUsage(-hll.estimatedInMemorySize()); hll.add(Double.doubleToLongBits(value)); state.addMemoryUsage(hll.estimatedInMemorySize()); }