@Override public void aggregate(PercentileCounter value) { if (sum == null) sum = new PercentileCounter(value); else sum.merge(value); }
public static double result(PercentileCounter counter) { return counter == null ? 0L : counter.getResultEstimate(); }
@Override public PercentileCounter deserialize(ByteBuffer in) { PercentileCounter counter = new PercentileCounter(compression); counter.readRegisters(in); return counter; } }
@Test public void testBasic() { int times = 1; int compression = 100; for (int t = 0; t < times; t++) { PercentileCounter counter = new PercentileCounter(compression, 0.5); Random random = new Random(); int dataSize = 10000; List<Double> dataset = Lists.newArrayListWithCapacity(dataSize); for (int i = 0; i < dataSize; i++) { double d = random.nextDouble(); counter.add(d); dataset.add(d); } Collections.sort(dataset); double actualResult = counter.getResultEstimate(); double expectedResult = MathUtil.findMedianInSortedList(dataset); assertEquals(expectedResult, actualResult, 0.001); } }
@Test public void testSerialization() { Kryo kryo = new Kryo(); kryo.register(PercentileCounter.class, new PercentileCounterSerializer()); double compression = 100; double quantile = 0.8; PercentileCounter origin_counter = new PercentileCounter(compression, quantile); for (int i = 1; i < 10; i++) { origin_counter.add(i); } byte[] buffer = serialize(kryo, origin_counter); PercentileCounter deserialized_counter = deserialize(kryo, buffer, PercentileCounter.class); Assert.assertEquals("Compression Error", origin_counter.getCompression(), deserialized_counter.getCompression(), 0.00000001); Assert.assertEquals("QuantileRatio Error", origin_counter.getQuantileRatio(), deserialized_counter.getQuantileRatio(), 0.00000001); Assert.assertEquals("Estimation Error", origin_counter.getResultEstimate(), deserialized_counter.getResultEstimate(), 0.00000001); }
@Test public void testBasic() { PercentileSerializer serializer = new PercentileSerializer(DataType.getType("percentile(100)")); PercentileCounter counter = new PercentileCounter(100, 0.5); Random random = new Random(); for (int i = 0; i < 1000; i++) { counter.add(random.nextDouble()); } double markResult = counter.getResultEstimate(); ByteBuffer buffer = ByteBuffer.allocateDirect(serializer.getStorageBytesEstimate()); serializer.serialize(counter, buffer); buffer.flip(); counter = serializer.deserialize(buffer); PercentileCounter counter1 = new PercentileCounter(100, 0.5); counter1.merge(counter); assertEquals(markResult, counter1.getResultEstimate(), 0.01); }
private PercentileCounter current() { if (current == null) { current = new ThreadLocal<>(); } PercentileCounter counter = current.get(); if (counter == null) { counter = new PercentileCounter(compression); current.set(counter); } return counter; }
@Override public void write(Kryo kryo, Output output, PercentileCounter counter) { int length = counter.getRegisters().byteSize(); ByteBuffer buffer = ByteBuffer.allocate(length); counter.getRegisters().asSmallBytes(buffer); output.writeDouble(counter.getCompression()); output.writeDouble(counter.getQuantileRatio()); output.writeInt(buffer.position()); output.write(buffer.array(), 0, buffer.position()); }
private double getEstimateSize(int numBefore, int numAfter, int compression) { return new PercentileCounter(compression == 0 ? DEFAULT_COMPRESSION : compression) .getBytesEstimate(numBefore * 1.0 / numAfter); }
private PercentileAggregator createPercentileAggreator(int sumNums, Integer sqrtNum, Integer compression) { compression = compression == null ? DEFAULT_COMPRESSION : compression; PercentileAggregator aggregator = new PercentileAggregator(compression); Random random = new Random(); for (int i = 0; i < sumNums; i++) { double d = 0; if (sqrtNum == null) d = random.nextInt(1000000000); else d = Math.sqrt(sqrtNum.intValue()) * random.nextGaussian(); PercentileCounter c = new PercentileCounter(compression, 0.5); c.add(d); aggregator.aggregate(c); } return aggregator; }
@Override public int getStorageBytesEstimate() { return current().getBytesEstimate(); }
public PercentileCounter(PercentileCounter another) { this(another.compression, another.quantileRatio); merge(another); }
@Override public void serialize(PercentileCounter value, ByteBuffer out) { value.writeRegisters(out); }
private void testPercentileSize(int sumNums, Integer sqrtNum, Integer compresion) throws Exception { compresion = compresion == null ? DEFAULT_COMPRESSION : compresion; PercentileAggregator aggregator = createPercentileAggreator(sumNums, sqrtNum, compresion); double actual = getActualSize(aggregator); double estimate = getEstimateSize((int) aggregator.getState().getRegisters().size(), 1, compresion); assertTrue(Math.abs(actual - estimate) / actual < 0.3); aggregator.reset(); }
@Test public void testTDigest() { double compression = 100; double quantile = 0.5; PercentileCounter counter = new PercentileCounter(compression, quantile); TDigest tDigest = TDigest.createAvlTreeDigest(compression); Random random = new Random(); int dataSize = 10000; List<Double> dataset = Lists.newArrayListWithCapacity(dataSize); for (int i = 0; i < dataSize; i++) { double d = random.nextDouble(); counter.add(d); tDigest.add(d); } double actualResult = counter.getResultEstimate(); Collections.sort(dataset); double expectedResult = tDigest.quantile(quantile); assertEquals(expectedResult, actualResult, 0); }
@Override public PercentileCounter read(Kryo kryo, Input input, Class type) { double compression = input.readDouble(); double quantileRatio = input.readDouble(); int length = input.readInt(); byte[] buffer = new byte[length]; input.read(buffer); PercentileCounter counter = new PercentileCounter(compression, quantileRatio); counter.readRegisters(ByteBuffer.wrap(buffer)); return counter; } }
@Override public void reset() { current = new PercentileCounter(dataType.getPrecision()); } };
@Override public void write(Kryo kryo, Output output, PercentileCounter counter) { int length = counter.getRegisters().byteSize(); ByteBuffer buffer = ByteBuffer.allocate(length); counter.getRegisters().asSmallBytes(buffer); output.writeDouble(counter.getCompression()); output.writeDouble(counter.getQuantileRatio()); output.writeInt(buffer.position()); output.write(buffer.array(), 0, buffer.position()); }
@Override protected double getStorageBytesEstimate(double count) { return current().getBytesEstimate(count); }