@Override public Object combine(final Object lhs, final Object rhs) { final DoublesUnion union = DoublesUnion.builder().setMaxK(k).build(); union.update((DoublesSketch) lhs); union.update((DoublesSketch) rhs); return union.getResultAndReset(); }
@Override public void reset(final ColumnValueSelector selector) { union.reset(); fold(selector); }
@Test public void checkUnion3() { //Union is direct, empty and with larger K than valid input final int k1 = 128; final int n1 = 2 * k1; final int k2 = 256; final int n2 = 2000; final DoublesSketch sketchIn1 = buildAndLoadQS(k1, n1); final int bytes = DoublesSketch.getUpdatableStorageBytes(k2, n2);//just for size final WritableMemory mem = WritableMemory.wrap(new byte[bytes]); final DoublesUnion union = DoublesUnion.builder().setMaxK(k2).build(mem); //virgin 256 union.update(sketchIn1); assertEquals(union.getMaxK(), k2); assertEquals(union.getEffectiveK(), k1); final DoublesSketch result = union.getResult(); assertEquals(result.getMaxValue(), n1, 0.0); assertEquals(result.getMinValue(), 1.0, 0.0); assertEquals(result.getK(), k1); }
@Test public void checkUpdateMemoryDirect() { final DoublesSketch qs1 = buildAndLoadDQS(256, 1000); final int bytes = qs1.getCompactStorageBytes(); final WritableMemory dstMem = WritableMemory.wrap(new byte[bytes]); qs1.putMemory(dstMem); final Memory srcMem = dstMem; final DoublesUnion union = DoublesUnion.builder().build(); //virgin union.update(srcMem); for (int i = 1000; i < 2000; i++) { union.update(i); } final DoublesSketch qs2 = union.getResult(); assertEquals(qs2.getMaxValue(), 1999, 0.0); final String s = union.toString(); println(s); //enable printing to see union.reset(); //sets to null }
@Test public void checkReverseMerge() { int k = PreambleUtil.DEFAULT_K; DoublesSketch qs1 = buildAndLoadQS(k, 1000, 0); DoublesSketch qs2 = buildAndLoadQS(2*k,1000, 1000); DoublesUnion union = DoublesUnion.heapify(qs2); union.update(qs1); //attempt merge into larger k DoublesSketch result = union.getResult(); assertEquals(result.getK(), k); }
@Test public void differentEmptySmallerK() { final int k128 = 128; final int k64 = 64; final DoublesUnion union = DoublesUnion.builder().setMaxK(k128).build(); assertTrue(union.isEmpty()); //gadget is null Assert.assertFalse(union.isDirect()); // byte[] unionByteArr = union.toByteArray(); // Assert.assertEquals(unionByteArr.length, 32 + 32); //empty final UpdateDoublesSketch sketch1 = buildAndLoadQS(k64, 0); //build smaller empty sketch union.update(sketch1); assertTrue(union.isEmpty()); //gadget is valid Assert.assertFalse(union.isDirect()); // unionByteArr = union.toByteArray(); // int udBytes = DoublesSketch.getUpdatableStorageBytes(k64, 0); // Assert.assertEquals(unionByteArr.length, udBytes); //empty Assert.assertEquals(union.getResult().getK(), 128); sketch1.update(1.0); union.update(sketch1); Assert.assertEquals(union.getResult().getK(), 128); }
@Test public void checkUnion2Direct() { final DoublesSketch qs1 = buildAndLoadDQS(256, 1000); final DoublesSketch qs2 = buildAndLoadDQS(128, 1000); final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(); //virgin 256 assertEquals(union.getEffectiveK(), 256); union.update(qs1); final DoublesSketch res1 = union.getResult(); //println(res1.toString()); assertEquals(res1.getN(), 1000); assertEquals(res1.getK(), 256); union.update(qs2); final DoublesSketch res2 = union.getResult(); assertEquals(res2.getN(), 2000); assertEquals(res2.getK(), 128); assertEquals(union.getEffectiveK(), 128); println(union.toString()); }
s1.update(2); Memory mem = Memory.wrap(s1.toByteArray(false)); DoublesUnion u = DoublesUnion.wrap(mem); DoublesSketch s2 = u.getResult(); Assert.assertEquals(s2.getMinValue(), 1.0); Assert.assertEquals(s2.getMaxValue(), 2.0); u.update(3); fail(); } catch (SketchesReadOnlyException e) { u.update(s2); fail(); } catch (SketchesReadOnlyException e) { u.update(mem); fail(); } catch (SketchesReadOnlyException e) { u.reset(); fail(); } catch (SketchesReadOnlyException e) { u.getResultAndReset(); fail(); } catch (SketchesReadOnlyException e) {
@Override public synchronized Object get() { return union.getResult(); }
@Override public synchronized void aggregate() { final DoublesSketch sketch = selector.getObject(); if (sketch == null) { return; } union.update(sketch); }
@Test public void checkUnion7() { final DoublesUnion union = DoublesUnionImpl.heapInstance(16); final DoublesSketch skEst = buildAndLoadQS(32, 64); //other is bigger, est union.update(skEst); println(skEst.toString(true, true)); println(union.toString(true, true)); final DoublesSketch result = union.getResult(); assertEquals(result.getMaxValue(), 64, 0.0); assertEquals(result.getMinValue(), 1.0, 0.0); }
@Test public void checkUnionNulls() { final DoublesUnion union = DoublesUnionImpl.heapInstance(128); final DoublesSketch sk1 = union.getResultAndReset(); final DoublesSketch sk2 = union.getResultAndReset(); assertNull(sk1); assertNull(sk2); union.update(sk2); final DoublesSketch sk3 = union.getResultAndReset(); assertNull(sk3); }
public DoublesSketchMergeAggregator(final ColumnValueSelector<DoublesSketch> selector, final int k) { this.selector = selector; union = DoublesUnion.builder().setMaxK(k).build(); }
@Test public void testIt() { java.nio.ByteBuffer bb = java.nio.ByteBuffer.allocate(1<<20).order(ByteOrder.nativeOrder()); WritableMemory mem = WritableMemory.wrap(bb); int k = 1024; DoublesSketch qsk = new DoublesSketchBuilder().setK(k).build(); DoublesUnion u1 = DoublesUnion.heapify(qsk); u1.getResult().putMemory(mem); DoublesUnion u2 = DoublesUnion.heapify(mem); DoublesSketch qsk2 = u2.getResult(); assertTrue(qsk2.isEmpty()); }
@Test public void checkWrapInstance() { final int k = 128; final int n = 1000; final UpdateDoublesSketch sketch = DoublesSketch.builder().setK(k).build(); for (int i = 1; i <= n; i++) { sketch.update(i); } final double skMedian = sketch.getQuantile(.5); Assert.assertEquals(skMedian, 500, 10); final byte[] byteArr = sketch.toByteArray(false); final WritableMemory mem = WritableMemory.wrap(byteArr); final DoublesUnion union = DoublesUnion.wrap(mem); Assert.assertFalse(union.isEmpty()); assertTrue(union.isDirect()); final DoublesSketch sketch2 = union.getResult(); final double uMedian = sketch2.getQuantile(0.5); Assert.assertEquals(skMedian, uMedian, 0.0); // check serializing again final byte[] bytesOut = union.toByteArray(); assertEquals(bytesOut.length, byteArr.length); assertEquals(bytesOut, byteArr); // wrapped, so should be exact }
@Test public void isSameResourceHeap() { DoublesUnion union = DoublesUnion.builder().build(); Assert.assertFalse(union.isSameResource(null)); }