@Override public byte[] serialise(final DoublesSketch sketch) throws SerialisationException { return sketch.toByteArray(); }
@Override public byte[] toBytes(final DoublesSketch sketch) { if (sketch == null || sketch.isEmpty()) { return ByteArrays.EMPTY_ARRAY; } return sketch.toByteArray(true); }
@Override public void serialize(final DoublesSketch sketch, final JsonGenerator generator, final SerializerProvider provider) throws IOException { generator.writeBinary(sketch.toByteArray(true)); }
/** * Serialize this sketch to a byte array. An UpdateDoublesSketch will be serialized in * an unordered, non-compact form; a CompactDoublesSketch will be serialized in ordered, * compact form. A DirectUpdateDoublesSketch can only wrap a non-compact array, and a * DirectCompactDoublesSketch can only wrap a compact array. * * @return byte array of this sketch */ public byte[] toByteArray() { if (isCompact()) { return toByteArray(true); } return toByteArray(false); }
@Test public void checkZeroPatternReturn() { int k = PreambleUtil.DEFAULT_K; DoublesSketch qs1 = buildAndLoadQS(k, 64); byte[] byteArr = qs1.toByteArray(); Memory mem = Memory.wrap(byteArr); HeapUpdateDoublesSketch.heapifyInstance(mem); }
@Test public void checkPrintMemData2() { final int k = PreambleUtil.DEFAULT_K; final int n = 0; final DoublesSketch qs = buildAndLoadQS(k,n); final byte[] byteArr = qs.toByteArray(); final Memory mem = Memory.wrap(byteArr); println(DoublesUtil.memToString(true, true, mem)); }
@Test public void checkEmpty() { int k = PreambleUtil.DEFAULT_K; DoublesSketch qs1 = buildAndLoadQS(k, 0); byte[] byteArr = qs1.toByteArray(); Memory mem = Memory.wrap(byteArr); DoublesSketch qs2 = DoublesSketch.heapify(mem); assertTrue(qs2.isEmpty()); final int expectedSizeBytes = 8; //COMBINED_BUFFER + ((2 * MIN_K) << 3); assertEquals(byteArr.length, expectedSizeBytes); assertTrue(Double.isNaN(qs2.getQuantile(0.0))); assertTrue(Double.isNaN(qs2.getQuantile(1.0))); assertTrue(Double.isNaN(qs2.getQuantile(0.5))); double[] quantiles = qs2.getQuantiles(new double[] {0.0, 0.5, 1.0}); assertNull(quantiles); assertTrue(Double.isNaN(qs2.getRank(0))); //println(qs1.toString(true, true)); }
@Test public void checkPrintMemData() { final int k = 16; final int n = 1000; final DoublesSketch qs = buildAndLoadQS(k,n); byte[] byteArr = qs.toByteArray(false); Memory mem = Memory.wrap(byteArr); println(DoublesUtil.memToString(true, true, mem)); byteArr = qs.toByteArray(true); mem = Memory.wrap(byteArr); println(DoublesUtil.memToString(true, true, mem)); }
@Test public void checkToStringEmpty() { int k = PreambleUtil.DEFAULT_K; DoublesSketch qs = DoublesSketch.builder().setK(k).build(); byte[] byteArr = qs.toByteArray(); println(PreambleUtil.toString(byteArr, true)); }
@Override public byte[] serialise(final DoublesSketch sketch) throws SerialisationException { return sketch.toByteArray(); }
/** * Puts the current sketch into the given Memory if there is sufficient space, otherwise, * throws an error. * * @param dstMem the given memory. * @param compact if true, compacts and sorts the base buffer, which optimizes merge * performance at the cost of slightly increased serialization time. */ public void putMemory(final WritableMemory dstMem, final boolean compact) { if (isDirect() && (isCompact() == compact)) { final Memory srcMem = getMemory(); srcMem.copyTo(0, dstMem, 0, getStorageBytes()); } else { final byte[] byteArr = toByteArray(compact); final int arrLen = byteArr.length; final long memCap = dstMem.getCapacity(); if (memCap < arrLen) { throw new SketchesArgumentException( "Destination Memory not large enough: " + memCap + " < " + arrLen); } dstMem.putByteArray(0, byteArr, 0, arrLen); } }
@SuppressWarnings({ "deprecation", "unused" }) @Test public void checkDeprecatedRankError() { DoublesSketch ds = buildAndLoadQS(64, 64); double err = ds.getNormalizedRankError(); err = DoublesSketch.getNormalizedRankError(64); DoublesUnion du1 = DoublesUnionBuilder.heapify(ds); Memory mem = Memory.wrap(ds.toByteArray()); DoublesUnion du2 = DoublesUnionBuilder.heapify(mem); DoublesUnion du3 = DoublesUnionBuilder.wrap(mem); WritableMemory wmem = WritableMemory.wrap(ds.toByteArray()); DoublesUnion du4 = DoublesUnionBuilder.wrap(wmem); ItemsSketch<String> is = ItemsSketch.getInstance(64, Comparator.naturalOrder()); err = is.getNormalizedRankError(); err = ItemsSketch.getNormalizedRankError(64); }
private static void checkToFromByteArray2(int k, int n) { DoublesSketch qs = buildAndLoadQS(k, n); byte[] byteArr; Memory mem; DoublesSketch qs2; // from compact byteArr = qs.toByteArray(true); mem = Memory.wrap(byteArr); qs2 = UpdateDoublesSketch.heapify(mem); for (double f = 0.1; f < 0.95; f += 0.1) { assertEquals(qs.getQuantile(f), qs2.getQuantile(f), 0.0); } // ordered, non-compact byteArr = qs.toByteArray(false); mem = Memory.wrap(byteArr); qs2 = DoublesSketch.heapify(mem); final DoublesSketchAccessor dsa = DoublesSketchAccessor.wrap(qs2); dsa.sort(); for (double f = 0.1; f < 0.95; f += 0.1) { assertEquals(qs.getQuantile(f), qs2.getQuantile(f), 0.0); } // not ordered, not compact byteArr = qs.toByteArray(false); mem = Memory.wrap(byteArr); qs2 = DoublesSketch.heapify(mem); for (double f = 0.1; f < 0.95; f += 0.1) { assertEquals(qs.getQuantile(f), qs2.getQuantile(f), 0.0); } }
/** * Serialize this sketch to a byte array. An UpdateDoublesSketch will be serialized in * an unordered, non-compact form; a CompactDoublesSketch will be serialized in ordered, * compact form. A DirectUpdateDoublesSketch can only wrap a non-compact array, and a * DirectCompactDoublesSketch can only wrap a compact array. * * @return byte array of this sketch */ public byte[] toByteArray() { if (isCompact()) { return toByteArray(true); } else { return toByteArray(false); } }
public void quantilesCheckViaMemory() { int k = 256; int n = 1000000; DoublesSketch qs = buildAndLoadQS(k, n); double[] ranks = {0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0}; String s = getRanksTable(qs, ranks); println(s); println(""); Memory srcMem = Memory.wrap(qs.toByteArray()); HeapUpdateDoublesSketch qs2 = HeapUpdateDoublesSketch.heapifyInstance(srcMem); println(getRanksTable(qs2, ranks)); }
public void summaryCheckViaMemory() { DoublesSketch qs = buildAndLoadQS(256, 1000000); String s = qs.toString(); println(s); println(""); Memory srcMem = WritableMemory.wrap(qs.toByteArray()); HeapUpdateDoublesSketch qs2 = HeapUpdateDoublesSketch.heapifyInstance(srcMem); s = qs2.toString(); println(s); }
@Override public byte[] toBytes(final DoublesSketch sketch) { if (sketch == null || sketch.isEmpty()) { return ByteArrays.EMPTY_ARRAY; } return sketch.toByteArray(true); }
@Override public byte[] serialize() { merge(); return result.toByteArray(); }
@Override public void serialize(final DoublesSketch sketch, final JsonGenerator generator, final SerializerProvider provider) throws IOException { generator.writeBinary(sketch.toByteArray(true)); }
/** * Puts the current sketch into the given Memory if there is sufficient space, otherwise, * throws an error. * * @param dstMem the given memory. * @param compact if true, compacts and sorts the base buffer, which optimizes merge * performance at the cost of slightly increased serialization time. */ public void putMemory(final WritableMemory dstMem, final boolean compact) { if (isDirect() && (isCompact() == compact)) { final Memory srcMem = getMemory(); srcMem.copyTo(0, dstMem, 0, getStorageBytes()); } else { final byte[] byteArr = toByteArray(compact); final int arrLen = byteArr.length; final long memCap = dstMem.getCapacity(); if (memCap < arrLen) { throw new SketchesArgumentException( "Destination Memory not large enough: " + memCap + " < " + arrLen); } dstMem.putByteArray(0, byteArr, 0, arrLen); } }