@Override public Object compute(final Map<String, Object> combinedAggregators) { final DoublesSketch sketch = (DoublesSketch) field.compute(combinedAggregators); return sketch.getQuantile(fraction); }
@Test public void checkEmpty() { int k = PreambleUtil.DEFAULT_K; DoublesSketch qs1 = buildAndLoadQS(k, 0); byte[] byteArr = qs1.toByteArray(); Memory mem = Memory.wrap(byteArr); DoublesSketch qs2 = DoublesSketch.heapify(mem); assertTrue(qs2.isEmpty()); final int expectedSizeBytes = 8; //COMBINED_BUFFER + ((2 * MIN_K) << 3); assertEquals(byteArr.length, expectedSizeBytes); assertTrue(Double.isNaN(qs2.getQuantile(0.0))); assertTrue(Double.isNaN(qs2.getQuantile(1.0))); assertTrue(Double.isNaN(qs2.getQuantile(0.5))); double[] quantiles = qs2.getQuantiles(new double[] {0.0, 0.5, 1.0}); assertNull(quantiles); assertTrue(Double.isNaN(qs2.getRank(0))); //println(qs1.toString(true, true)); }
private void testSerialiser(final DoublesSketch sketch) { final double quantile1 = sketch.getQuantile(0.5D); final byte[] sketchSerialised; try { sketchSerialised = SERIALISER.serialise(sketch); } catch (final SerialisationException exception) { fail("A SerialisationException occurred"); return; } final DoublesSketch sketchDeserialised; try { sketchDeserialised = SERIALISER.deserialise(sketchSerialised); } catch (final SerialisationException exception) { fail("A SerialisationException occurred"); return; } assertEquals(quantile1, sketchDeserialised.getQuantile(0.5D), DELTA); }
@Test public void testAggregate() { final DoublesSketchAggregator sketchAggregator = new DoublesSketchAggregator(); DoublesSketch currentState = sketch1; assertEquals(3L, currentState.getN()); assertEquals(2.0D, currentState.getQuantile(0.5D), DELTA); currentState = sketchAggregator.apply(currentState, sketch2); assertEquals(7L, currentState.getN()); assertEquals(4.0D, currentState.getQuantile(0.5D), DELTA); }
/** * Gets the lower bound of the value interval in which the true quantile of the given rank * exists with a confidence of at least 99%. * @param fraction the given normalized rank as a fraction * @return the lower bound of the value interval in which the true quantile of the given rank * exists with a confidence of at least 99%. Returns NaN if the sketch is empty. */ public double getQuantileLowerBound(final double fraction) { return getQuantile(max(0, fraction - Util.getNormalizedRankError(k_, false))); }
/** * Gets the upper bound of the value interval in which the true quantile of the given rank * exists with a confidence of at least 99%. * @param fraction the given normalized rank as a fraction * @return the upper bound of the value interval in which the true quantile of the given rank * exists with a confidence of at least 99%. Returns NaN if the sketch is empty. */ public double getQuantileUpperBound(final double fraction) { return getQuantile(min(1.0, fraction + Util.getNormalizedRankError(k_, false))); }
@Test(expectedExceptions = SketchesArgumentException.class) public void checkGetQuantile() { int k = PreambleUtil.DEFAULT_K; int n = 1000000; DoublesSketch qs = buildAndLoadQS(k, n); double frac = -0.5; //negative not allowed qs.getQuantile(frac); }
@Test public void checkMerge() { int k = PreambleUtil.DEFAULT_K; int n = 1000000; DoublesSketch qs1 = buildAndLoadQS(k,n,0); DoublesSketch qs2 = buildAndLoadQS(k,0,0); //empty DoublesUnion union = DoublesUnion.heapify(qs2); union.update(qs1); DoublesSketch result = union.getResult(); double med1 = qs1.getQuantile(0.5); double med2 = result.getQuantile(0.5); assertEquals(med1, med2, 0.0); //println(med1+","+med2); }
private void getAndCheck(String ver, int n, double quantile) { DoublesSketch.rand.setSeed(131); //make deterministic //create fileName int k = 128; double nf = 0.5; String fileName = String.format("Qk%d_n%d_v%s.bin", k, n, ver); println("fullName: "+ fileName); println("Old Median: " + quantile); //create & Read File File file = new File(getClass().getClassLoader().getResource(fileName).getFile()); byte[] byteArr2 = readFile(file); Memory srcMem = Memory.wrap(byteArr2); // heapify as update sketch DoublesSketch qs2 = UpdateDoublesSketch.heapify(srcMem); //Test the quantile double q2 = qs2.getQuantile(nf); println("New Median: " + q2); Assert.assertEquals(q2, quantile, 0.0); // same thing with compact sketch qs2 = CompactDoublesSketch.heapify(srcMem); //Test the quantile q2 = qs2.getQuantile(nf); println("New Median: " + q2); Assert.assertEquals(q2, quantile, 0.0); }
private static void checkToFromByteArray2(int k, int n) { DoublesSketch qs = buildAndLoadQS(k, n); byte[] byteArr; Memory mem; DoublesSketch qs2; // from compact byteArr = qs.toByteArray(true); mem = Memory.wrap(byteArr); qs2 = UpdateDoublesSketch.heapify(mem); for (double f = 0.1; f < 0.95; f += 0.1) { assertEquals(qs.getQuantile(f), qs2.getQuantile(f), 0.0); } // ordered, non-compact byteArr = qs.toByteArray(false); mem = Memory.wrap(byteArr); qs2 = DoublesSketch.heapify(mem); final DoublesSketchAccessor dsa = DoublesSketchAccessor.wrap(qs2); dsa.sort(); for (double f = 0.1; f < 0.95; f += 0.1) { assertEquals(qs.getQuantile(f), qs2.getQuantile(f), 0.0); } // not ordered, not compact byteArr = qs.toByteArray(false); mem = Memory.wrap(byteArr); qs2 = DoublesSketch.heapify(mem); for (double f = 0.1; f < 0.95; f += 0.1) { assertEquals(qs.getQuantile(f), qs2.getQuantile(f), 0.0); } }
@Test public void heapToDirect() { UpdateDoublesSketch heapSketch = DoublesSketch.builder().build(); for (int i = 0; i < 1000; i++) { heapSketch.update(i); } DoublesSketch directSketch = DoublesSketch.wrap(WritableMemory.wrap(heapSketch.toByteArray(false))); assertEquals(directSketch.getMinValue(), 0.0); assertEquals(directSketch.getMaxValue(), 999.0); assertEquals(directSketch.getQuantile(0.5), 500.0, 4.0); }
public double getQuantileValue(boolean success, double quantile) { DoublesSketch s = success ? successResult : failResult; return s != null ? s.getQuantile(quantile) : Double.NaN; }
@Test public void mergeTest() { DoublesSketch dqs1 = buildAndLoadDQS(128, 256); DoublesSketch dqs2 = buildAndLoadDQS(128, 256, 256); DoublesUnion union = DoublesUnion.builder().setMaxK(128).build(); union.update(dqs1); union.update(dqs2); DoublesSketch result = union.getResult(); double median = result.getQuantile(0.5); println("Median: " + median); assertEquals(median, 258.0, .05 * 258); }
@Test public void serializeDeserialize() { int sizeBytes = DoublesSketch.getUpdatableStorageBytes(128, 2000); WritableMemory mem = WritableMemory.wrap(new byte[sizeBytes]); UpdateDoublesSketch sketch1 = DoublesSketch.builder().build(mem); for (int i = 0; i < 1000; i++) { sketch1.update(i); } UpdateDoublesSketch sketch2 = UpdateDoublesSketch.wrap(mem); for (int i = 0; i < 1000; i++) { sketch2.update(i + 1000); } assertEquals(sketch2.getMinValue(), 0.0); assertEquals(sketch2.getMaxValue(), 1999.0); assertEquals(sketch2.getQuantile(0.5), 1000.0, 10.0); byte[] arr2 = sketch2.toByteArray(false); assertEquals(arr2.length, sketch2.getStorageBytes()); DoublesSketch sketch3 = DoublesSketch.wrap(WritableMemory.wrap(arr2)); assertEquals(sketch3.getMinValue(), 0.0); assertEquals(sketch3.getMaxValue(), 1999.0); assertEquals(sketch3.getQuantile(0.5), 1000.0, 10.0); }
@Test public void checkWrapInstance() { final int k = 128; final int n = 1000; final UpdateDoublesSketch sketch = DoublesSketch.builder().setK(k).build(); for (int i = 1; i <= n; i++) { sketch.update(i); } final double skMedian = sketch.getQuantile(.5); Assert.assertEquals(skMedian, 500, 10); final byte[] byteArr = sketch.toByteArray(false); final WritableMemory mem = WritableMemory.wrap(byteArr); final DoublesUnion union = DoublesUnion.wrap(mem); Assert.assertFalse(union.isEmpty()); assertTrue(union.isDirect()); final DoublesSketch sketch2 = union.getResult(); final double uMedian = sketch2.getQuantile(0.5); Assert.assertEquals(skMedian, uMedian, 0.0); // check serializing again final byte[] bytesOut = union.toByteArray(); assertEquals(bytesOut.length, byteArr.length); assertEquals(bytesOut, byteArr); // wrapped, so should be exact }
@Test public void checkDirectInstance() { final int k = 128; final int n = 1000; final DoublesUnionBuilder bldr = DoublesUnion.builder(); bldr.setMaxK(k); Assert.assertEquals(bldr.getMaxK(), k); final int bytes = DoublesSketch.getUpdatableStorageBytes(k, n); final byte[] byteArr = new byte[bytes]; final WritableMemory mem = WritableMemory.wrap(byteArr); final DoublesUnion union = bldr.build(mem); assertTrue(union.isEmpty()); assertTrue(union.isDirect()); for (int i = 1; i <= n; i++) { union.update(i); } Assert.assertFalse(union.isEmpty()); final DoublesSketch res = union.getResult(); final double median = res.getQuantile(.5); Assert.assertEquals(median, 500, 10); println(union.toString()); }
@Test public void checkUnionQuantiles() { final int k = 128; final int n1 = k * 13; final int n2 = (k * 8) + (k / 2); final int n = n1 + n2; final double errorTolerance = 0.0175 * n; // assuming k = 128 final UpdateDoublesSketch sketch1 = buildAndLoadQS(k, n1); final CompactDoublesSketch sketch2 = buildAndLoadQS(k, n2, n1).compact(); final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(); //virgin 256 union.update(sketch2); union.update(sketch1); final Memory mem = Memory.wrap(union.getResult().toByteArray(true)); final DoublesSketch result = DoublesSketch.wrap(mem); assertEquals(result.getN(), n1 + n2); assertEquals(result.getK(), k); for (double fraction = 0.05; fraction < 1.0; fraction += 0.05) { assertEquals(result.getQuantile(fraction), fraction * n, errorTolerance); } }
/** * Gets the upper bound of the value interval in which the true quantile of the given rank * exists with a confidence of at least 99%. * @param fraction the given normalized rank as a fraction * @return the upper bound of the value interval in which the true quantile of the given rank * exists with a confidence of at least 99%. Returns NaN if the sketch is empty. */ public double getQuantileUpperBound(final double fraction) { return getQuantile(min(1.0, fraction + Util.getNormalizedRankError(k_, false))); }
@Override public Object compute(final Map<String, Object> combinedAggregators) { final DoublesSketch sketch = (DoublesSketch) field.compute(combinedAggregators); return sketch.getQuantile(fraction); }
/** * Gets the lower bound of the value interval in which the true quantile of the given rank * exists with a confidence of at least 99%. * @param fraction the given normalized rank as a fraction * @return the lower bound of the value interval in which the true quantile of the given rank * exists with a confidence of at least 99%. Returns NaN if the sketch is empty. */ public double getQuantileLowerBound(final double fraction) { return getQuantile(max(0, fraction - Util.getNormalizedRankError(k_, false))); }