@Test public void checkCheckDirectMemCapacity() { final int k = 128; DirectCompactDoublesSketch.checkDirectMemCapacity(k, (2 * k) - 1, (4 + (2 * k)) * 8); DirectCompactDoublesSketch.checkDirectMemCapacity(k, (2 * k) + 1, (4 + (3 * k)) * 8); DirectCompactDoublesSketch.checkDirectMemCapacity(k, 0, 8); try { DirectCompactDoublesSketch.checkDirectMemCapacity(k, 10000, 64); fail(); } catch (final SketchesArgumentException e) { // expected } }
/** * Returns a compact version of this sketch. If passing in a Memory object, the compact sketch * will use that direct memory; otherwise, an on-heap sketch will be returned. * @param dstMem An optional target memory to hold the sketch. * @return A compact version of this sketch */ public CompactDoublesSketch compact(final WritableMemory dstMem) { if (dstMem == null) { return HeapCompactDoublesSketch.createFromUpdateSketch(this); } return DirectCompactDoublesSketch.createFromUpdateSketch(this, dstMem); }
@Test public void wrapFromCompactSketch() { final int k = 8; final int n = 177; final DirectCompactDoublesSketch qs = buildAndLoadDCQS(k, n); // assuming ordered inserts final byte[] qsBytes = qs.toByteArray(); final Memory qsMem = Memory.wrap(qsBytes); final DirectCompactDoublesSketch compactQs = DirectCompactDoublesSketch.wrapInstance(qsMem); DoublesSketchTest.testSketchEquality(qs, compactQs); assertEquals(qsBytes.length, compactQs.getStorageBytes()); final double[] combinedBuffer = compactQs.getCombinedBuffer(); assertEquals(combinedBuffer.length, compactQs.getCombinedBufferItemCapacity()); }
@Override double[] getCombinedBuffer() { final int k = getK(); if (isEmpty()) { return new double[k << 1]; } //2K final long n = getN(); final int itemCap = computeRetainedItems(k, n); final double[] combinedBuffer = new double[itemCap]; mem_.getDoubleArray(COMBINED_BUFFER, combinedBuffer, 0, itemCap); return combinedBuffer; }
@Test public void checkEmpty() { final int k = PreambleUtil.DEFAULT_K; final DirectCompactDoublesSketch qs1 = buildAndLoadDCQS(k, 0); assertTrue(Double.isNaN(qs1.getQuantile(0.0))); assertTrue(Double.isNaN(qs1.getQuantile(1.0))); assertTrue(Double.isNaN(qs1.getQuantile(0.5))); final double[] quantiles = qs1.getQuantiles(new double[] {0.0, 0.5, 1.0}); assertNull(quantiles); final double[] combinedBuffer = qs1.getCombinedBuffer(); assertEquals(combinedBuffer.length, 2 * k); assertNotEquals(combinedBuffer.length, qs1.getCombinedBufferItemCapacity()); }
/** * Wrap this sketch around the given compact Memory image of a DoublesSketch. * * @param srcMem the given compact Memory image of a DoublesSketch that may have data, * @return a sketch that wraps the given srcMem */ static DirectCompactDoublesSketch wrapInstance(final Memory srcMem) { final long memCap = srcMem.getCapacity(); final int preLongs = extractPreLongs(srcMem); final int serVer = extractSerVer(srcMem); final int familyID = extractFamilyID(srcMem); final int flags = extractFlags(srcMem); final int k = extractK(srcMem); final boolean empty = (flags & EMPTY_FLAG_MASK) > 0; final long n = empty ? 0 : extractN(srcMem); //VALIDITY CHECKS DirectUpdateDoublesSketchR.checkPreLongs(preLongs); Util.checkFamilyID(familyID); DoublesUtil.checkDoublesSerVer(serVer, MIN_DIRECT_DOUBLES_SER_VER); checkCompact(serVer, flags); Util.checkK(k); checkDirectMemCapacity(k, n, memCap); DirectUpdateDoublesSketchR.checkEmptyAndN(empty, n); final DirectCompactDoublesSketch dds = new DirectCompactDoublesSketch(k); dds.mem_ = (WritableMemory) srcMem; return dds; }
@Test public void createFromUnsortedUpdateSketch() { final int k = 4; final int n = 13; final UpdateDoublesSketch qs = DoublesSketch.builder().setK(k).build(); for (int i = n; i > 0; --i) { qs.update(i); } final WritableMemory dstMem = WritableMemory.wrap(new byte[qs.getCompactStorageBytes()]); final DirectCompactDoublesSketch compactQs = DirectCompactDoublesSketch.createFromUpdateSketch(qs, dstMem); // don't expect equal but new base buffer should be sorted final double[] combinedBuffer = compactQs.getCombinedBuffer(); final int bbCount = compactQs.getBaseBufferCount(); for (int i = 1; i < bbCount; ++i) { assert combinedBuffer[i - 1] < combinedBuffer[i]; } }
final int k = sketch.getK(); final long n = sketch.getN(); checkDirectMemCapacity(k, n, memCap); final DirectCompactDoublesSketch dcds = new DirectCompactDoublesSketch(k); dcds.mem_ = dstMem;
/** * Wrap this sketch around the given Memory image of a DoublesSketch, compact or non-compact. * * @param srcMem the given Memory image of a DoublesSketch that may have data, * @return a sketch that wraps the given srcMem */ public static DoublesSketch wrap(final Memory srcMem) { if (checkIsCompactMemory(srcMem)) { return DirectCompactDoublesSketch.wrapInstance(srcMem); } return DirectUpdateDoublesSketchR.wrapInstance(srcMem); }
/** * Checks the validity of the direct memory capacity assuming n, k. * @param k the given value of k * @param n the given value of n * @param memCapBytes the current memory capacity in bytes */ static void checkDirectMemCapacity(final int k, final long n, final long memCapBytes) { final int reqBufBytes = getCompactStorageBytes(k, n); if (memCapBytes < reqBufBytes) { throw new SketchesArgumentException("Possible corruption: Memory capacity too small: " + memCapBytes + " < " + reqBufBytes); } }
/** * Wrap this sketch around the given compact Memory image of a DoublesSketch. * * @param srcMem the given compact Memory image of a DoublesSketch that may have data, * @return a sketch that wraps the given srcMem */ static DirectCompactDoublesSketch wrapInstance(final Memory srcMem) { final long memCap = srcMem.getCapacity(); final int preLongs = extractPreLongs(srcMem); final int serVer = extractSerVer(srcMem); final int familyID = extractFamilyID(srcMem); final int flags = extractFlags(srcMem); final int k = extractK(srcMem); final boolean empty = (flags & EMPTY_FLAG_MASK) > 0; final long n = empty ? 0 : extractN(srcMem); //VALIDITY CHECKS DirectUpdateDoublesSketchR.checkPreLongs(preLongs); Util.checkFamilyID(familyID); DoublesUtil.checkDoublesSerVer(serVer, MIN_DIRECT_DOUBLES_SER_VER); checkCompact(serVer, flags); Util.checkK(k); checkDirectMemCapacity(k, n, memCap); DirectUpdateDoublesSketchR.checkEmptyAndN(empty, n); final DirectCompactDoublesSketch dds = new DirectCompactDoublesSketch(k); dds.mem_ = (WritableMemory) srcMem; return dds; }
@Override double[] getCombinedBuffer() { final int k = getK(); if (isEmpty()) { return new double[k << 1]; } //2K final long n = getN(); final int itemCap = computeRetainedItems(k, n); final double[] combinedBuffer = new double[itemCap]; mem_.getDoubleArray(COMBINED_BUFFER, combinedBuffer, 0, itemCap); return combinedBuffer; }
final int k = sketch.getK(); final long n = sketch.getN(); checkDirectMemCapacity(k, n, memCap); final DirectCompactDoublesSketch dcds = new DirectCompactDoublesSketch(k); dcds.mem_ = dstMem;
@Test(expectedExceptions = SketchesArgumentException.class) public void wrapFromUpdateSketch() { final int k = 4; final int n = 27; final UpdateDoublesSketch qs = HeapUpdateDoublesSketchTest.buildAndLoadQS(k, n); final byte[] qsBytes = qs.toByteArray(); final Memory qsMem = Memory.wrap(qsBytes); DirectCompactDoublesSketch.wrapInstance(qsMem); fail(); }
/** * Checks the validity of the direct memory capacity assuming n, k. * @param k the given value of k * @param n the given value of n * @param memCapBytes the current memory capacity in bytes */ static void checkDirectMemCapacity(final int k, final long n, final long memCapBytes) { final int reqBufBytes = getCompactStorageBytes(k, n); if (memCapBytes < reqBufBytes) { throw new SketchesArgumentException("Possible corruption: Memory capacity too small: " + memCapBytes + " < " + reqBufBytes); } }
/** * Wrap this sketch around the given Memory image of a DoublesSketch, compact or non-compact. * * @param srcMem the given Memory image of a DoublesSketch that may have data, * @return a sketch that wraps the given srcMem */ public static DoublesSketch wrap(final Memory srcMem) { if (checkIsCompactMemory(srcMem)) { return DirectCompactDoublesSketch.wrapInstance(srcMem); } else { return DirectUpdateDoublesSketchR.wrapInstance(srcMem); } }
/** * Returns a compact version of this sketch. If passing in a Memory object, the compact sketch * will use that direct memory; otherwise, an on-heap sketch will be returned. * @param dstMem An optional target memory to hold the sketch. * @return A compact version of this sketch */ public CompactDoublesSketch compact(final WritableMemory dstMem) { if (dstMem == null) { return HeapCompactDoublesSketch.createFromUpdateSketch(this); } else { return DirectCompactDoublesSketch.createFromUpdateSketch(this, dstMem); } }