@Test public void testAggregate() { final DoublesUnionAggregator unionAggregator = new DoublesUnionAggregator(); DoublesUnion currentState = union1; assertEquals(3L, currentState.getResult().getN()); assertEquals(2.0D, currentState.getResult().getQuantile(0.5D), DELTA); currentState = unionAggregator.apply(currentState, union2); assertEquals(7L, currentState.getResult().getN()); assertEquals(4.0D, currentState.getResult().getQuantile(0.5D), DELTA); }
@Test public void wrapEmptyUpdateSketch() { final UpdateDoublesSketch s1 = DoublesSketch.builder().build(); final WritableMemory mem = WritableMemory.wrap(ByteBuffer.wrap(s1.toByteArray()).order(ByteOrder.nativeOrder())); final UpdateDoublesSketch s2 = DirectUpdateDoublesSketch.wrapInstance(mem); assertTrue(s2.isEmpty()); assertEquals(s2.getN(), 0); assertTrue(Double.isNaN(s2.getMinValue())); assertTrue(Double.isNaN(s2.getMaxValue())); s2.reset(); // empty: so should be a no-op assertEquals(s2.getN(), 0); }
@Test public void itemsToDoubles() { final ItemsSketch<Double> sketch1 = ItemsSketch.getInstance(Comparator.naturalOrder()); for (int i = 1; i <= 500; i++) { sketch1.update((double) i); } final byte[] bytes = sketch1.toByteArray(serDe); final UpdateDoublesSketch sketch2; sketch2 = UpdateDoublesSketch.heapify(Memory.wrap(bytes)); for (int i = 501; i <= 1000; i++) { sketch2.update(i); } Assert.assertEquals(sketch2.getN(), 1000); Assert.assertTrue(sketch2.getRetainedItems() < 1000); Assert.assertEquals(sketch2.getMinValue(), 1.0); Assert.assertEquals(sketch2.getMaxValue(), 1000.0); // based on ~1.7% normalized rank error for this particular case Assert.assertEquals(sketch2.getQuantile(0.5), 500.0, 17); }
@Test public void checkPuts() { long n1 = 1001; UpdateDoublesSketch qsk = buildAndLoadQS(32, (int)n1); long n2 = qsk.getN(); assertEquals(n2, n1); int bbCnt1 = qsk.getBaseBufferCount(); long pat1 = qsk.getBitPattern(); qsk.putBitPattern(pat1 + 1); //corrupt the pattern long pat2 = qsk.getBitPattern(); assertEquals(pat1 + 1, pat2); qsk.putBaseBufferCount(bbCnt1 + 1); //corrupt the bbCount int bbCnt2 = qsk.getBaseBufferCount(); assertEquals(bbCnt1 + 1, bbCnt2); qsk.putN(n1 + 1); //corrupt N long n3 = qsk.getN(); assertEquals(n1 + 1, n3); assertNull(qsk.getMemory()); }
@SuppressWarnings("unused") @Test public void checkToStringDetail() { int k = PreambleUtil.DEFAULT_K; int n = 1000000; UpdateDoublesSketch qs = buildAndLoadQS(k, 0); String s = qs.toString(); s = qs.toString(false, true); //println(s); qs = buildAndLoadQS(k, n); s = qs.toString(); //println(s); s = qs.toString(false, true); //println(qs.toString(false, true)); int n2 = (int)qs.getN(); assertEquals(n2, n); qs.update(Double.NaN); //ignore qs.reset(); assertEquals(qs.getN(), 0); }
@Test public void checkUnion5Direct() { //Union is direct, valid and with larger K than valid input final int k2 = 4; final int n2 = 2 * k2; //8 final int bytes = DoublesSketch.getUpdatableStorageBytes(256, 50);//big enough final WritableMemory skMem = WritableMemory.wrap(new byte[bytes]); DoublesSketch.builder().setK(256).build(skMem); final DoublesUnion union = DoublesUnionImpl.heapifyInstance(skMem); assertEquals(union.getResult().getN(), 0); assertEquals(union.getMaxK(), 256); assertEquals(union.getEffectiveK(), 256); final DoublesSketch result = union.getResult(); assertEquals(result.getK(), 256); final DoublesSketch sketchIn2 = buildAndLoadDQS(k2, n2, 17); union.update(sketchIn2); println("\nFinal" + union.getResult().toString(true, true)); assertEquals(union.getResult().getN(), n2); }
@Test public void checkUnion5() { //Union is direct, valid and with larger K than valid input final int k2 = 4; final int n2 = 2 * k2; //8 final int bytes = DoublesSketch.getUpdatableStorageBytes(256, 50);//big enough final WritableMemory skMem = WritableMemory.wrap(new byte[bytes]); DoublesSketch.builder().setK(256).build(skMem); final DoublesUnion union = DoublesUnionImpl.heapifyInstance(skMem); assertEquals(union.getResult().getN(), 0); assertEquals(union.getMaxK(), 256); assertEquals(union.getEffectiveK(), 256); final DoublesSketch result = union.getResult(); assertEquals(result.getK(), 256); final DoublesSketch sketchIn2 = buildAndLoadQS(k2, n2, 17); union.update(sketchIn2); println("\nFinal" + union.getResult().toString(true, true)); assertEquals(union.getResult().getN(), n2); }
@Test public void checkMisc() { int k = PreambleUtil.DEFAULT_K; int n = 48; int cap = 32 + ((2 * k) << 3); WritableMemory mem = WritableMemory.wrap(new byte[cap]); UpdateDoublesSketch qs = DoublesSketch.builder().setK(k).build(mem); mem = qs.getMemory(); assertEquals(mem.getCapacity(), cap); double[] combBuf = qs.getCombinedBuffer(); assertEquals(combBuf.length, 2 * k); qs = buildAndLoadDQS(k, n); qs.update(Double.NaN); int n2 = (int)qs.getN(); assertEquals(n2, n); combBuf = qs.getCombinedBuffer(); assertEquals(combBuf.length, ceilingPowerOf2(n)); // since n < k println(qs.toString(true, true)); qs.reset(); assertEquals(qs.getN(), 0); qs.putBaseBufferCount(0); }
final long n = sketch.getN();
@Test public void checkGrowFromWrappedEmptySketch() { final int k = 16; final int n = 0; final int initBytes = DoublesSketch.getUpdatableStorageBytes(k, n); //8 bytes final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(); final Memory origSketchMem = Memory.wrap(usk1.toByteArray()); try (WritableDirectHandle memHandle = WritableMemory.allocateDirect(initBytes)) { WritableMemory mem = memHandle.get(); origSketchMem.copyTo(0, mem, 0, initBytes); UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(mem); assertTrue(mem.isSameResource(usk2.getMemory())); assertEquals(mem.getCapacity(), initBytes); assertTrue(mem.isDirect()); assertTrue(usk2.isEmpty()); //update the sketch forcing it to grow on-heap for (int i = 1; i <= 5; i++) { usk2.update(i); } assertEquals(usk2.getN(), 5); WritableMemory mem2 = usk2.getMemory(); assertFalse(mem.isSameResource(mem2)); assertFalse(mem2.isDirect()); //should now be on-heap final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3); assertEquals(mem2.getCapacity(), expectedSize); } }
@Test public void checkUnion4() { //Union is direct, valid and with larger K than valid input final int k1 = 8; final int n1 = 2 * k1; //16 final int k2 = 4; final int n2 = 2 * k2; //8 final int bytes = DoublesSketch.getUpdatableStorageBytes(256, 50);//just for size final WritableMemory skMem = WritableMemory.wrap(new byte[bytes]); final UpdateDoublesSketch sketchIn1 = DoublesSketch.builder().setK(k1).build(skMem); for (int i = 0; i < n1; i++) { sketchIn1.update(i + 1); } final WritableMemory uMem = WritableMemory.wrap(new byte[bytes]); final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(uMem); //virgin 256 //DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(); //virgin 256 union.update(sketchIn1); assertEquals(union.getResult().getN(), n1); assertEquals(union.getMaxK(), 256); assertEquals(union.getEffectiveK(), k1); DoublesSketch result = union.getResult(); assertEquals(result.getN(), 16); assertEquals(result.getMaxValue(), n1, 0.0); assertEquals(result.getMinValue(), 1.0, 0.0); assertEquals(result.getK(), k1); final DoublesSketch sketchIn2 = buildAndLoadQS(k2, n2, 17); union.reset(); union.update(sketchIn2); result = union.getResult(); assertEquals(result.getMaxValue(), n2 + 17, 0.0); assertEquals(result.getMinValue(), 1.0 + 17, 0.0); println("\nFinal" + union.getResult().toString(true, true)); }
@Test public void checkUnion4Direct() { //Union is direct, valid and with larger K than valid input final int k1 = 8; final int n1 = 2 * k1; //16 final int k2 = 4; final int n2 = 2 * k2; //8 final int bytes = DoublesSketch.getUpdatableStorageBytes(256, 50);//just for size final WritableMemory skMem = WritableMemory.wrap(new byte[bytes]); final UpdateDoublesSketch sketchIn1 = DoublesSketch.builder().setK(k1).build(skMem); for (int i = 0; i < n1; i++) { sketchIn1.update(i + 1); } final WritableMemory uMem = WritableMemory.wrap(new byte[bytes]); final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(uMem); //virgin 256 union.update(sketchIn1); assertEquals(union.getResult().getN(), n1); assertEquals(union.getMaxK(), 256); assertEquals(union.getEffectiveK(), k1); DoublesSketch result = union.getResult(); assertEquals(result.getN(), 16); assertEquals(result.getMaxValue(), n1, 0.0); assertEquals(result.getMinValue(), 1.0, 0.0); assertEquals(result.getK(), k1); final DoublesSketch sketchIn2 = buildAndLoadDQS(k2, n2, 17); union.reset(); union.update(sketchIn2); result = union.getResult(); assertEquals(result.getMaxValue(), n2 + 17, 0.0); assertEquals(result.getMinValue(), 1.0 + 17, 0.0); println("\nFinal" + union.getResult().toString(true, true)); }
@Test public void checkUnion4DirectCompact() { final int k1 = 8; final int n1 = 2 * k1; //16 final int k2 = 4; final int n2 = 5 * k2; //8 final int bytes = DoublesSketch.getUpdatableStorageBytes(256, 50);//just for size final WritableMemory skMem = WritableMemory.wrap(new byte[bytes]); final UpdateDoublesSketch sketchIn0 = DoublesSketch.builder().setK(k1).build(skMem); for (int i = 0; i < n1; i++) { sketchIn0.update(i + 1); } final CompactDoublesSketch sketchIn1 = sketchIn0.compact(); final WritableMemory uMem = WritableMemory.wrap(new byte[bytes]); final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(uMem); //virgin 256 union.update(sketchIn1); assertEquals(union.getResult().getN(), n1); assertEquals(union.getMaxK(), 256); assertEquals(union.getEffectiveK(), k1); DoublesSketch result = union.getResult(); assertEquals(result.getN(), 16); assertEquals(result.getMaxValue(), n1, 0.0); assertEquals(result.getMinValue(), 1.0, 0.0); assertEquals(result.getK(), k1); final CompactDoublesSketch sketchIn2 = buildAndLoadDQS(k2, n2, 17).compact(); union.reset(); union.update(sketchIn2); result = union.getResult(); assertEquals(result.getMaxValue(), n2 + 17, 0.0); assertEquals(result.getMinValue(), 1.0 + 17, 0.0); println("\nFinal" + union.getResult().toString(true, true)); }
final long memCap = dstMem.getCapacity(); final int k = sketch.getK(); final long n = sketch.getN(); checkDirectMemCapacity(k, n, memCap);
final long n = sketch.getN();
final long memCap = dstMem.getCapacity(); final int k = sketch.getK(); final long n = sketch.getN(); checkDirectMemCapacity(k, n, memCap);