/** * Heapify takes the sketch image in Memory and instantiates an on-heap Sketch. * The resulting sketch will not retain any link to the source Memory. * @param srcMem a Memory image of a Sketch. * <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a> * @return a heap-based Sketch based on the given Memory */ public static DoublesSketch heapify(final Memory srcMem) { if (checkIsCompactMemory(srcMem)) { return CompactDoublesSketch.heapify(srcMem); } return UpdateDoublesSketch.heapify(srcMem); }
@Test public void wrapEmptyCompactSketch() { final CompactDoublesSketch s1 = DoublesSketch.builder().build().compact(); final Memory mem = Memory.wrap(ByteBuffer.wrap(s1.toByteArray()).order(ByteOrder.nativeOrder())); final DoublesSketch s2 = DoublesSketch.wrap(mem); assertTrue(s2.isEmpty()); assertEquals(s2.getN(), 0); assertTrue(Double.isNaN(s2.getMinValue())); assertTrue(Double.isNaN(s2.getMaxValue())); }
private void getAndCheck(String ver, int n, double quantile) { DoublesSketch.rand.setSeed(131); //make deterministic //create fileName int k = 128; double nf = 0.5; String fileName = String.format("Qk%d_n%d_v%s.bin", k, n, ver); println("fullName: "+ fileName); println("Old Median: " + quantile); //create & Read File File file = new File(getClass().getClassLoader().getResource(fileName).getFile()); byte[] byteArr2 = readFile(file); Memory srcMem = Memory.wrap(byteArr2); // heapify as update sketch DoublesSketch qs2 = UpdateDoublesSketch.heapify(srcMem); //Test the quantile double q2 = qs2.getQuantile(nf); println("New Median: " + q2); Assert.assertEquals(q2, quantile, 0.0); // same thing with compact sketch qs2 = CompactDoublesSketch.heapify(srcMem); //Test the quantile q2 = qs2.getQuantile(nf); println("New Median: " + q2); Assert.assertEquals(q2, quantile, 0.0); }
@Test public void checkEmpty() { final int k = PreambleUtil.DEFAULT_K; final UpdateDoublesSketch qs1 = buildAndLoadQS(k, 0); final byte[] byteArr = qs1.compact().toByteArray(); final byte[] byteArr2 = qs1.toByteArray(true); final Memory mem = Memory.wrap(byteArr); final HeapCompactDoublesSketch qs2 = HeapCompactDoublesSketch.heapifyInstance(mem); assertTrue(qs2.isEmpty()); assertEquals(byteArr.length, qs1.getStorageBytes()); assertEquals(byteArr, byteArr2); assertTrue(Double.isNaN(qs2.getQuantile(0.0))); assertTrue(Double.isNaN(qs2.getQuantile(1.0))); assertTrue(Double.isNaN(qs2.getQuantile(0.5))); final double[] quantiles = qs2.getQuantiles(new double[] {0.0, 0.5, 1.0}); assertNull(quantiles); //println(qs1.toString(true, true)); }
/** * Heapify takes the sketch image in Memory and instantiates an on-heap Sketch. * The resulting sketch will not retain any link to the source Memory. * @param srcMem a Memory image of a Sketch. * <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a> * @return a heap-based Sketch based on the given Memory */ public static DoublesSketch heapify(final Memory srcMem) { if (checkIsCompactMemory(srcMem)) { return CompactDoublesSketch.heapify(srcMem); } else { return UpdateDoublesSketch.heapify(srcMem); } }
@Test public void heapifyEmptyCompactSketch() { final CompactDoublesSketch s1 = DoublesSketch.builder().build().compact(); final Memory mem = Memory.wrap(s1.toByteArray()); DoublesSketch s2 = DoublesSketch.heapify(mem); Assert.assertTrue(s2.isEmpty()); }
@Test public void wrapEmptyCompactSketch() { final UpdateDoublesSketch s1 = DoublesSketch.builder().build(); final Memory mem = Memory.wrap(s1.compact().toByteArray()); DoublesSketch s2 = DoublesSketch.wrap(mem); // compact, so this is ok Assert.assertTrue(s2.isEmpty()); }
@Test public void heapifyFromCompactSketch() { final int k = 8; final int n = 177; final UpdateDoublesSketch qs = buildAndLoadQS(k, n); // assuming reverse ordered inserts final byte[] qsBytes = qs.compact().toByteArray(); final Memory qsMem = Memory.wrap(qsBytes); final HeapCompactDoublesSketch compactQs = HeapCompactDoublesSketch.heapifyInstance(qsMem); DoublesSketchTest.testSketchEquality(qs, compactQs); }
@Test public void wrapCompactSketch() { UpdateDoublesSketch s1 = DoublesSketch.builder().build(); s1.update(1); s1.update(2); //Memory mem = Memory.wrap(ByteBuffer.wrap(s1.compact().toByteArray()) // .asReadOnlyBuffer().order(ByteOrder.nativeOrder()));); final Memory mem = Memory.wrap(s1.compact().toByteArray()); final DoublesSketch s2 = DoublesSketch.wrap(mem); // compact, so this is ok Assert.assertEquals(s2.getMinValue(), 1.0); Assert.assertEquals(s2.getMaxValue(), 2.0); Assert.assertEquals(s2.getN(), 2); }
@Test public void doublesToItems() { final UpdateDoublesSketch sketch1 = DoublesSketch.builder().build(); //SerVer = 3 for (int i = 1; i <= 500; i++) { sketch1.update(i); } final CompactDoublesSketch cs = sketch1.compact(); DoublesSketchTest.testSketchEquality(sketch1, cs); //final byte[] bytes = sketch1.compact().toByteArray(); // must be compact final byte[] bytes = cs.toByteArray(); // must be compact //reconstruct with ItemsSketch final ItemsSketch<Double> sketch2 = ItemsSketch.getInstance(Memory.wrap(bytes), Comparator.naturalOrder(), serDe); for (int i = 501; i <= 1000; i++) { sketch2.update((double) i); } Assert.assertEquals(sketch2.getN(), 1000); Assert.assertTrue(sketch2.getRetainedItems() < 1000); Assert.assertEquals(sketch2.getMinValue(), 1.0); Assert.assertEquals(sketch2.getMaxValue(), 1000.0); // based on ~1.7% normalized rank error for this particular case Assert.assertEquals(sketch2.getQuantile(0.5), 500.0, 17); }
@Test public void checkCopyToHeap() { final int k = 128; final int n = 400; // HeapUpdateDoublesSketch final HeapUpdateDoublesSketch huds = (HeapUpdateDoublesSketch) buildAndLoadQS(k, n); final HeapUpdateDoublesSketch target1 = DoublesUtil.copyToHeap(huds); DoublesSketchTest.testSketchEquality(huds, target1); // DirectUpdateDoublesSketch final WritableMemory mem1 = WritableMemory.wrap(huds.toByteArray()); final DirectUpdateDoublesSketch duds = (DirectUpdateDoublesSketch) UpdateDoublesSketch.wrap(mem1); final HeapUpdateDoublesSketch target2 = DoublesUtil.copyToHeap(duds); DoublesSketchTest.testSketchEquality(huds, duds); DoublesSketchTest.testSketchEquality(duds, target2); // HeapCompactDoublesSketch final CompactDoublesSketch hcds = huds.compact(); final HeapUpdateDoublesSketch target3 = DoublesUtil.copyToHeap(hcds); DoublesSketchTest.testSketchEquality(huds, hcds); DoublesSketchTest.testSketchEquality(hcds, target3); // DirectCompactDoublesSketch final Memory mem2 = Memory.wrap(hcds.toByteArray()); final DirectCompactDoublesSketch dcds = (DirectCompactDoublesSketch) DoublesSketch.wrap(mem2); final HeapUpdateDoublesSketch target4 = DoublesUtil.copyToHeap(dcds); DoublesSketchTest.testSketchEquality(huds, dcds); DoublesSketchTest.testSketchEquality(dcds, target4); }