/** * Wrap this sketch around the given non-compact Memory image of a DoublesSketch. * * @param srcMem the given Memory image of a DoublesSketch that may have data, * @return a sketch that wraps the given srcMem */ public static UpdateDoublesSketch wrap(final WritableMemory srcMem) { return DirectUpdateDoublesSketch.wrapInstance(srcMem); }
/** * Wrap this sketch around the given non-compact Memory image of a DoublesSketch. * * @param srcMem the given non-compact Memory image of a DoublesSketch that may have data * @return a sketch that wraps the given srcMem */ static DirectUpdateDoublesSketch wrapInstance(final WritableMemory srcMem) { final long memCap = srcMem.getCapacity(); final int preLongs = extractPreLongs(srcMem); final int serVer = extractSerVer(srcMem); final int familyID = extractFamilyID(srcMem); final int flags = extractFlags(srcMem); final int k = extractK(srcMem); final boolean empty = (flags & EMPTY_FLAG_MASK) > 0; //Preamble flags empty state final long n = empty ? 0 : extractN(srcMem); //VALIDITY CHECKS checkPreLongs(preLongs); Util.checkFamilyID(familyID); DoublesUtil.checkDoublesSerVer(serVer, MIN_DIRECT_DOUBLES_SER_VER); checkDirectFlags(flags); //Cannot be compact Util.checkK(k); checkCompact(serVer, flags); checkDirectMemCapacity(k, n, memCap); checkEmptyAndN(empty, n); final DirectUpdateDoublesSketch dds = new DirectUpdateDoublesSketch(k); dds.mem_ = srcMem; return dds; }
final int curBBCount = getBaseBufferCount(); final int newBBCount = curBBCount + 1; //derived, not stored final int combBufItemCap = getCombinedBufferItemCapacity(); if (newBBCount > combBufItemCap) { mem_ = growCombinedMemBuffer(2 * getK()); final long curN = getN(); final long newN = curN + 1; putMaxValue(dataItem); putMinValue(dataItem); } else { if (dataItem > getMaxValue()) { putMaxValue(dataItem); } if (dataItem < getMinValue()) { putMinValue(dataItem); } final int curMemItemCap = getCombinedBufferItemCapacity(); final int itemSpaceNeeded = DoublesUpdateImpl.getRequiredItemCapacity(k_, newN); mem_ = growCombinedMemBuffer(itemSpaceNeeded); k_, DoublesSketchAccessor.wrap(this, true), getBitPattern() ); putN(newN);
/** * Returns an updatable Union object that wraps off-heap data structure of the given memory * image of a non-compact DoublesSketch. The data structures of the Union remain off-heap. * * @param mem A memory image of a non-compact DoublesSketch to be used as the data * structure for the union and will be modified. * @return a Union object */ static DoublesUnionImpl wrapInstance(final WritableMemory mem) { final DirectUpdateDoublesSketch sketch = DirectUpdateDoublesSketch.wrapInstance(mem); final DoublesUnionImpl union = new DoublesUnionImpl(sketch.getK()); union.gadget_ = sketch; return union; }
@Override public UpdateDoublesSketch getResult(final WritableMemory dstMem) { final long memCapBytes = dstMem.getCapacity(); if (gadget_ == null) { if (memCapBytes < DoublesSketch.getUpdatableStorageBytes(0, 0)) { throw new SketchesArgumentException("Insufficient capacity for result: " + memCapBytes); } return DirectUpdateDoublesSketch.newInstance(maxK_, dstMem); } gadget_.putMemory(dstMem, false); return DirectUpdateDoublesSketch.wrapInstance(dstMem); }
/** * Obtains a new Direct instance of a DoublesSketch, which may be off-heap. * * @param k Parameter that controls space usage of sketch and accuracy of estimates. * Must be greater than 1 and less than 65536 and a power of 2. * @param dstMem the destination Memory that will be initialized to hold the data for this sketch. * It must initially be at least (16 * MIN_K + 32) bytes, where MIN_K defaults to 2. As it grows * it will request more memory using the MemoryRequest callback. * @return a DirectUpdateDoublesSketch */ static DirectUpdateDoublesSketch newInstance(final int k, final WritableMemory dstMem) { // must be able to hold at least an empty sketch final long memCap = dstMem.getCapacity(); checkDirectMemCapacity(k, 0, memCap); //initialize dstMem dstMem.putLong(0, 0L); //clear pre0 insertPreLongs(dstMem, 2); insertSerVer(dstMem, DoublesSketch.DOUBLES_SER_VER); insertFamilyID(dstMem, Family.QUANTILES.getID()); insertFlags(dstMem, EMPTY_FLAG_MASK); insertK(dstMem, k); if (memCap >= COMBINED_BUFFER) { insertN(dstMem, 0L); insertMinDouble(dstMem, Double.NaN); insertMaxDouble(dstMem, Double.NaN); } final DirectUpdateDoublesSketch dds = new DirectUpdateDoublesSketch(k); dds.mem_ = dstMem; return dds; }
@Test public void checkIsSameResource() { int k = 16; WritableMemory mem = WritableMemory.wrap(new byte[(k*16) +24]); WritableMemory cmem = WritableMemory.wrap(new byte[8]); DirectUpdateDoublesSketch duds = (DirectUpdateDoublesSketch) DoublesSketch.builder().setK(k).build(mem); assertTrue(duds.isSameResource(mem)); DirectCompactDoublesSketch dcds = (DirectCompactDoublesSketch) duds.compact(cmem); assertTrue(dcds.isSameResource(cmem)); UpdateDoublesSketch uds = DoublesSketch.builder().setK(k).build(); assertFalse(uds.isSameResource(mem)); }
@Override public UpdateDoublesSketch getResult(final WritableMemory dstMem) { final long memCapBytes = dstMem.getCapacity(); if (gadget_ == null) { if (memCapBytes < DoublesSketch.getUpdatableStorageBytes(0, 0)) { throw new SketchesArgumentException("Insufficient capacity for result: " + memCapBytes); } return DirectUpdateDoublesSketch.newInstance(maxK_, dstMem); } gadget_.putMemory(dstMem, false); return DirectUpdateDoublesSketch.wrapInstance(dstMem); }
/** * Returns an updatable Union object that wraps off-heap data structure of the given memory * image of a non-compact DoublesSketch. The data structures of the Union remain off-heap. * * @param mem A memory image of a non-compact DoublesSketch to be used as the data * structure for the union and will be modified. * @return a Union object */ static DoublesUnionImpl wrapInstance(final WritableMemory mem) { final DirectUpdateDoublesSketch sketch = DirectUpdateDoublesSketch.wrapInstance(mem); final DoublesUnionImpl union = new DoublesUnionImpl(sketch.getK()); union.gadget_ = sketch; return union; }
/** * Obtains a new Direct instance of a DoublesSketch, which may be off-heap. * * @param k Parameter that controls space usage of sketch and accuracy of estimates. * Must be greater than 1 and less than 65536 and a power of 2. * @param dstMem the destination Memory that will be initialized to hold the data for this sketch. * It must initially be at least (16 * MIN_K + 32) bytes, where MIN_K defaults to 2. As it grows * it will request more memory using the MemoryRequest callback. * @return a DirectUpdateDoublesSketch */ static DirectUpdateDoublesSketch newInstance(final int k, final WritableMemory dstMem) { // must be able to hold at least an empty sketch final long memCap = dstMem.getCapacity(); checkDirectMemCapacity(k, 0, memCap); //initialize dstMem dstMem.putLong(0, 0L); //clear pre0 insertPreLongs(dstMem, 2); insertSerVer(dstMem, DoublesSketch.DOUBLES_SER_VER); insertFamilyID(dstMem, Family.QUANTILES.getID()); insertFlags(dstMem, EMPTY_FLAG_MASK); insertK(dstMem, k); if (memCap >= COMBINED_BUFFER) { insertN(dstMem, 0L); insertMinDouble(dstMem, Double.NaN); insertMaxDouble(dstMem, Double.NaN); } final DirectUpdateDoublesSketch dds = new DirectUpdateDoublesSketch(k); dds.mem_ = dstMem; return dds; }
final int curBBCount = getBaseBufferCount(); final int newBBCount = curBBCount + 1; //derived, not stored final int combBufItemCap = getCombinedBufferItemCapacity(); if (newBBCount > combBufItemCap) { mem_ = growCombinedMemBuffer(2 * getK()); final long curN = getN(); final long newN = curN + 1; putMaxValue(dataItem); putMinValue(dataItem); } else { if (dataItem > getMaxValue()) { putMaxValue(dataItem); } if (dataItem < getMinValue()) { putMinValue(dataItem); } final int curMemItemCap = getCombinedBufferItemCapacity(); final int itemSpaceNeeded = DoublesUpdateImpl.getRequiredItemCapacity(k_, newN); mem_ = growCombinedMemBuffer(itemSpaceNeeded); k_, DoublesSketchAccessor.wrap(this, true), getBitPattern() ); putN(newN);
/** * Wrap this sketch around the given non-compact Memory image of a DoublesSketch. * * @param srcMem the given non-compact Memory image of a DoublesSketch that may have data * @return a sketch that wraps the given srcMem */ static DirectUpdateDoublesSketch wrapInstance(final WritableMemory srcMem) { final long memCap = srcMem.getCapacity(); final int preLongs = extractPreLongs(srcMem); final int serVer = extractSerVer(srcMem); final int familyID = extractFamilyID(srcMem); final int flags = extractFlags(srcMem); final int k = extractK(srcMem); final boolean empty = (flags & EMPTY_FLAG_MASK) > 0; //Preamble flags empty state final long n = empty ? 0 : extractN(srcMem); //VALIDITY CHECKS checkPreLongs(preLongs); Util.checkFamilyID(familyID); DoublesUtil.checkDoublesSerVer(serVer, MIN_DIRECT_DOUBLES_SER_VER); checkDirectFlags(flags); //Cannot be compact Util.checkK(k); checkCompact(serVer, flags); checkDirectMemCapacity(k, n, memCap); checkEmptyAndN(empty, n); final DirectUpdateDoublesSketch dds = new DirectUpdateDoublesSketch(k); dds.mem_ = srcMem; return dds; }
@Test public void wrapEmptyUpdateSketch() { final UpdateDoublesSketch s1 = DoublesSketch.builder().build(); final WritableMemory mem = WritableMemory.wrap(ByteBuffer.wrap(s1.toByteArray()).order(ByteOrder.nativeOrder())); final UpdateDoublesSketch s2 = DirectUpdateDoublesSketch.wrapInstance(mem); assertTrue(s2.isEmpty()); assertEquals(s2.getN(), 0); assertTrue(Double.isNaN(s2.getMinValue())); assertTrue(Double.isNaN(s2.getMaxValue())); s2.reset(); // empty: so should be a no-op assertEquals(s2.getN(), 0); }
/** * Wrap this sketch around the given non-compact Memory image of a DoublesSketch. * * @param srcMem the given Memory image of a DoublesSketch that may have data, * @return a sketch that wraps the given srcMem */ public static UpdateDoublesSketch wrap(final WritableMemory srcMem) { return DirectUpdateDoublesSketch.wrapInstance(srcMem); }
@Test public void checkGrowFromWrappedEmptySketch() { final int k = 16; final int n = 0; final int initBytes = DoublesSketch.getUpdatableStorageBytes(k, n); //8 bytes final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(); final Memory origSketchMem = Memory.wrap(usk1.toByteArray()); try (WritableDirectHandle memHandle = WritableMemory.allocateDirect(initBytes)) { WritableMemory mem = memHandle.get(); origSketchMem.copyTo(0, mem, 0, initBytes); UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(mem); assertTrue(mem.isSameResource(usk2.getMemory())); assertEquals(mem.getCapacity(), initBytes); assertTrue(mem.isDirect()); assertTrue(usk2.isEmpty()); //update the sketch forcing it to grow on-heap for (int i = 1; i <= 5; i++) { usk2.update(i); } assertEquals(usk2.getN(), 5); WritableMemory mem2 = usk2.getMemory(); assertFalse(mem.isSameResource(mem2)); assertFalse(mem2.isDirect()); //should now be on-heap final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3); assertEquals(mem2.getCapacity(), expectedSize); } }
final WritableMemory mem = myQS.getMemory(); //myQS is empty, ok to reconfigure other.putMemory(mem, false); // not compact, but BB ordered ret = DirectUpdateDoublesSketch.wrapInstance(mem); } else { //myQS is empty and on heap ret = DoublesUtil.copyToHeap(other);
final WritableMemory mem = myQS.getMemory(); //myQS is empty, ok to reconfigure other.putMemory(mem, false); // not compact, but BB ordered ret = DirectUpdateDoublesSketch.wrapInstance(mem); } else { //myQS is empty and on heap ret = DoublesUtil.copyToHeap(other);