@Override public String toString(final boolean sketchSummary, final boolean dataDetail) { final StringBuilder sb = new StringBuilder(); final String thisSimpleName = this.getClass().getSimpleName(); final int maxK = getMaxK(); final String kStr = String.format("%,d", maxK); sb.append(Util.LS).append("### Quantiles ").append(thisSimpleName).append(LS); sb.append(" maxK : ").append(kStr); if (gadget_ == null) { sb.append(HeapUpdateDoublesSketch.newInstance(maxK_).toString()); return sb.toString(); } sb.append(gadget_.toString(sketchSummary, dataDetail)); return sb.toString(); }
@SuppressWarnings("unused") @Test public void checkToStringDetail() { int k = PreambleUtil.DEFAULT_K; int n = 1000000; UpdateDoublesSketch qs = buildAndLoadQS(k, 0); String s = qs.toString(); s = qs.toString(false, true); //println(s); qs = buildAndLoadQS(k, n); s = qs.toString(); //println(s); s = qs.toString(false, true); //println(qs.toString(false, true)); int n2 = (int)qs.getN(); assertEquals(n2, n); qs.update(Double.NaN); //ignore qs.reset(); assertEquals(qs.getN(), 0); }
@Test public void checkUnion6() { final int k1 = 8; final int n1 = 2 * k1; //16 final int k2 = 16; final int n2 = 2 * k2; //32 final DoublesSketch sk1 = buildAndLoadQS(k1, n1, 0); final DoublesSketch sk2 = buildAndLoadQS(k2, n2, n1); final DoublesUnion union = DoublesUnionImpl.heapifyInstance(sk1); union.update(sk2); final DoublesSketch result = union.getResult(); assertEquals(result.getMaxValue(), n1 + n2, 0.0); assertEquals(result.getMinValue(), 1.0, 0.0); println("\nFinal" + union.getResult().toString(true, true)); }
@Test public void checkUnion6Direct() { final int k1 = 8; final int n1 = 2 * k1; //16 final int k2 = 16; final int n2 = 2 * k2; //32 final DoublesSketch sk1 = buildAndLoadDQS(k1, n1, 0); final DoublesSketch sk2 = buildAndLoadDQS(k2, n2, n1); final DoublesUnion union = DoublesUnionImpl.heapifyInstance(sk1); union.update(sk2); final DoublesSketch result = union.getResult(); assertEquals(result.getMaxValue(), n1 + n2, 0.0); assertEquals(result.getMinValue(), 1.0, 0.0); println("\nFinal" + union.getResult().toString(true, true)); }
@Test public void checkUnion5Direct() { //Union is direct, valid and with larger K than valid input final int k2 = 4; final int n2 = 2 * k2; //8 final int bytes = DoublesSketch.getUpdatableStorageBytes(256, 50);//big enough final WritableMemory skMem = WritableMemory.wrap(new byte[bytes]); DoublesSketch.builder().setK(256).build(skMem); final DoublesUnion union = DoublesUnionImpl.heapifyInstance(skMem); assertEquals(union.getResult().getN(), 0); assertEquals(union.getMaxK(), 256); assertEquals(union.getEffectiveK(), 256); final DoublesSketch result = union.getResult(); assertEquals(result.getK(), 256); final DoublesSketch sketchIn2 = buildAndLoadDQS(k2, n2, 17); union.update(sketchIn2); println("\nFinal" + union.getResult().toString(true, true)); assertEquals(union.getResult().getN(), n2); }
@Test public void checkMisc() { int k = PreambleUtil.DEFAULT_K; int n = 48; int cap = 32 + ((2 * k) << 3); WritableMemory mem = WritableMemory.wrap(new byte[cap]); UpdateDoublesSketch qs = DoublesSketch.builder().setK(k).build(mem); mem = qs.getMemory(); assertEquals(mem.getCapacity(), cap); double[] combBuf = qs.getCombinedBuffer(); assertEquals(combBuf.length, 2 * k); qs = buildAndLoadDQS(k, n); qs.update(Double.NaN); int n2 = (int)qs.getN(); assertEquals(n2, n); combBuf = qs.getCombinedBuffer(); assertEquals(combBuf.length, ceilingPowerOf2(n)); // since n < k println(qs.toString(true, true)); qs.reset(); assertEquals(qs.getN(), 0); qs.putBaseBufferCount(0); }
@Test public void checkUnion5() { //Union is direct, valid and with larger K than valid input final int k2 = 4; final int n2 = 2 * k2; //8 final int bytes = DoublesSketch.getUpdatableStorageBytes(256, 50);//big enough final WritableMemory skMem = WritableMemory.wrap(new byte[bytes]); DoublesSketch.builder().setK(256).build(skMem); final DoublesUnion union = DoublesUnionImpl.heapifyInstance(skMem); assertEquals(union.getResult().getN(), 0); assertEquals(union.getMaxK(), 256); assertEquals(union.getEffectiveK(), 256); final DoublesSketch result = union.getResult(); assertEquals(result.getK(), 256); final DoublesSketch sketchIn2 = buildAndLoadQS(k2, n2, 17); union.update(sketchIn2); println("\nFinal" + union.getResult().toString(true, true)); assertEquals(union.getResult().getN(), n2); }
@Test public void baseTest() { int n = 1 << 20; int k = 1 << 5; double[] seqArr = new double[n]; //build sequential array for (int i = 1; i <= n; i++) { seqArr[i - 1] = i; } double[] randArr = seqArr.clone(); shuffle(randArr); UpdateDoublesSketch sketch = DoublesSketch.builder().setK(k).build(); for (int i = 0; i < n; i++) { sketch.update(randArr[i]); } double[] ranks = sketch.getCDF(seqArr); double maxDelta = 0; for (int i = 0; i < n; i++) { double actRank = (double)i/n; double estRank = ranks[i]; double delta = actRank - estRank; maxDelta = Math.max(maxDelta, delta); //println("Act: " + + " \tEst: " + ranks[i]); } println("Max delta: " + maxDelta); println(sketch.toString()); }
@Test public void checkUnion4() { //Union is direct, valid and with larger K than valid input final int k1 = 8; final int n1 = 2 * k1; //16 final int k2 = 4; final int n2 = 2 * k2; //8 final int bytes = DoublesSketch.getUpdatableStorageBytes(256, 50);//just for size final WritableMemory skMem = WritableMemory.wrap(new byte[bytes]); final UpdateDoublesSketch sketchIn1 = DoublesSketch.builder().setK(k1).build(skMem); for (int i = 0; i < n1; i++) { sketchIn1.update(i + 1); } final WritableMemory uMem = WritableMemory.wrap(new byte[bytes]); final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(uMem); //virgin 256 //DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(); //virgin 256 union.update(sketchIn1); assertEquals(union.getResult().getN(), n1); assertEquals(union.getMaxK(), 256); assertEquals(union.getEffectiveK(), k1); DoublesSketch result = union.getResult(); assertEquals(result.getN(), 16); assertEquals(result.getMaxValue(), n1, 0.0); assertEquals(result.getMinValue(), 1.0, 0.0); assertEquals(result.getK(), k1); final DoublesSketch sketchIn2 = buildAndLoadQS(k2, n2, 17); union.reset(); union.update(sketchIn2); result = union.getResult(); assertEquals(result.getMaxValue(), n2 + 17, 0.0); assertEquals(result.getMinValue(), 1.0 + 17, 0.0); println("\nFinal" + union.getResult().toString(true, true)); }
@Test public void checkUnion4Direct() { //Union is direct, valid and with larger K than valid input final int k1 = 8; final int n1 = 2 * k1; //16 final int k2 = 4; final int n2 = 2 * k2; //8 final int bytes = DoublesSketch.getUpdatableStorageBytes(256, 50);//just for size final WritableMemory skMem = WritableMemory.wrap(new byte[bytes]); final UpdateDoublesSketch sketchIn1 = DoublesSketch.builder().setK(k1).build(skMem); for (int i = 0; i < n1; i++) { sketchIn1.update(i + 1); } final WritableMemory uMem = WritableMemory.wrap(new byte[bytes]); final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(uMem); //virgin 256 union.update(sketchIn1); assertEquals(union.getResult().getN(), n1); assertEquals(union.getMaxK(), 256); assertEquals(union.getEffectiveK(), k1); DoublesSketch result = union.getResult(); assertEquals(result.getN(), 16); assertEquals(result.getMaxValue(), n1, 0.0); assertEquals(result.getMinValue(), 1.0, 0.0); assertEquals(result.getK(), k1); final DoublesSketch sketchIn2 = buildAndLoadDQS(k2, n2, 17); union.reset(); union.update(sketchIn2); result = union.getResult(); assertEquals(result.getMaxValue(), n2 + 17, 0.0); assertEquals(result.getMinValue(), 1.0 + 17, 0.0); println("\nFinal" + union.getResult().toString(true, true)); }
@Test public void checkUnion4DirectCompact() { final int k1 = 8; final int n1 = 2 * k1; //16 final int k2 = 4; final int n2 = 5 * k2; //8 final int bytes = DoublesSketch.getUpdatableStorageBytes(256, 50);//just for size final WritableMemory skMem = WritableMemory.wrap(new byte[bytes]); final UpdateDoublesSketch sketchIn0 = DoublesSketch.builder().setK(k1).build(skMem); for (int i = 0; i < n1; i++) { sketchIn0.update(i + 1); } final CompactDoublesSketch sketchIn1 = sketchIn0.compact(); final WritableMemory uMem = WritableMemory.wrap(new byte[bytes]); final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(uMem); //virgin 256 union.update(sketchIn1); assertEquals(union.getResult().getN(), n1); assertEquals(union.getMaxK(), 256); assertEquals(union.getEffectiveK(), k1); DoublesSketch result = union.getResult(); assertEquals(result.getN(), 16); assertEquals(result.getMaxValue(), n1, 0.0); assertEquals(result.getMinValue(), 1.0, 0.0); assertEquals(result.getK(), k1); final CompactDoublesSketch sketchIn2 = buildAndLoadDQS(k2, n2, 17).compact(); union.reset(); union.update(sketchIn2); result = union.getResult(); assertEquals(result.getMaxValue(), n2 + 17, 0.0); assertEquals(result.getMinValue(), 1.0 + 17, 0.0); println("\nFinal" + union.getResult().toString(true, true)); }
@Override public String toString(final boolean sketchSummary, final boolean dataDetail) { final StringBuilder sb = new StringBuilder(); final String thisSimpleName = this.getClass().getSimpleName(); final int maxK = getMaxK(); final String kStr = String.format("%,d", maxK); sb.append(Util.LS).append("### Quantiles ").append(thisSimpleName).append(LS); sb.append(" maxK : ").append(kStr); if (gadget_ == null) { sb.append(HeapUpdateDoublesSketch.newInstance(maxK_).toString()); return sb.toString(); } sb.append(gadget_.toString(sketchSummary, dataDetail)); return sb.toString(); }