public static ArrayOfDoublesSketch deserializeFromByteArray(final byte[] data) { final Memory mem = Memory.wrap(data); return ArrayOfDoublesSketches.wrapSketch(mem); }
/** * This method uses locks because it can be used during indexing, * and Druid can call aggregate() and get() concurrently * https://github.com/apache/incubator-druid/pull/3956 * The returned sketch is a separate instance of ArrayOfDoublesCompactSketch * representing the current state of the aggregation, and is not affected by consequent * aggregate() calls */ @Override public Object get(final ByteBuffer buf, final int position) { final WritableMemory mem = WritableMemory.wrap(buf, ByteOrder.LITTLE_ENDIAN); final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); final Lock lock = stripedLock.getAt(ArrayOfDoublesSketchBuildBufferAggregator.lockIndex(position)).readLock(); lock.lock(); try { final ArrayOfDoublesUnion union = ArrayOfDoublesSketches.wrapUnion(region); return union.getResult(); } finally { lock.unlock(); } }
/** * This method uses locks because it can be used during indexing, * and Druid can call aggregate() and get() concurrently * https://github.com/apache/incubator-druid/pull/3956 */ @Override public void aggregate(final ByteBuffer buf, final int position) { for (int i = 0; i < valueSelectors.length; i++) { values[i] = valueSelectors[i].getDouble(); } final IndexedInts keys = keySelector.getRow(); // Wrapping memory and ArrayOfDoublesSketch is inexpensive compared to sketch operations. // Maintaining a cache of wrapped objects per buffer position like in Theta sketch aggregator // might might be considered, but it would increase complexity including relocate() support. final WritableMemory mem = WritableMemory.wrap(buf, ByteOrder.LITTLE_ENDIAN); final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); final Lock lock = stripedLock.getAt(lockIndex(position)).writeLock(); lock.lock(); try { final ArrayOfDoublesUpdatableSketch sketch = ArrayOfDoublesSketches.wrapUpdatableSketch(region); for (int i = 0, keysSize = keys.size(); i < keysSize; i++) { final String key = keySelector.lookupName(keys.get(i)); sketch.update(key, values); } } finally { lock.unlock(); } }
@Test public void serializeDeserializeEstimation() { ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < 8192; i++) { us.update(i, new double[] {1.0}); } WritableMemory wmem = WritableMemory.wrap(us.toByteArray()); ArrayOfDoublesUpdatableSketch wrappedUS = ArrayOfDoublesSketches.wrapUpdatableSketch(wmem); Assert.assertFalse(wrappedUS.isEmpty()); Assert.assertTrue(wrappedUS.isEstimationMode()); Assert.assertEquals(wrappedUS.getEstimate(), us.getEstimate()); Assert.assertEquals(wrappedUS.getThetaLong(), us.getThetaLong()); ArrayOfDoublesUpdatableSketch heapUS = ArrayOfDoublesSketches.heapifyUpdatableSketch(wmem); Assert.assertFalse(heapUS.isEmpty()); Assert.assertTrue(heapUS.isEstimationMode()); Assert.assertEquals(heapUS.getEstimate(), us.getEstimate()); Assert.assertEquals(heapUS.getThetaLong(), us.getThetaLong()); ArrayOfDoublesCompactSketch sketch1 = us.compact(); ArrayOfDoublesSketch sketch2 = ArrayOfDoublesSketches.heapifySketch(Memory.wrap(sketch1.toByteArray())); Assert.assertFalse(sketch2.isEmpty()); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), sketch1.getEstimate()); Assert.assertEquals(sketch2.getThetaLong(), sketch1.getThetaLong()); }
/** * Heapify the given Memory as an ArrayOfDoublesSketch * @param mem the given Memory * @return an ArrayOfDoublesSketch */ public static ArrayOfDoublesSketch heapifySketch(final Memory mem) { return heapifySketch(mem, DEFAULT_UPDATE_SEED); }
/** * Heapify the given Memory as an ArrayOfDoublesUnion * @param mem the given Memory * @return an ArrayOfDoublesUnion */ public static ArrayOfDoublesUnion heapifyUnion(final Memory mem) { return heapifyUnion(mem, DEFAULT_UPDATE_SEED); }
/** * Heapify the given Memory as an ArrayOfDoublesUpdatableSketch * @param mem the given Memory * @return an ArrayOfDoublesUpdatableSketch */ public static ArrayOfDoublesUpdatableSketch heapifyUpdatableSketch(final Memory mem) { return heapifyUpdatableSketch(mem, DEFAULT_UPDATE_SEED); }
@Test public void heapifyAndUpdateSketch() { ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build(); sketch1.update(1, new double[] {1}); // downcasting is not recommended, for testing only ArrayOfDoublesUpdatableSketch sketch2 = (ArrayOfDoublesUpdatableSketch) ArrayOfDoublesSketches.heapifySketch(Memory.wrap(sketch1.toByteArray())); sketch2.update(2, new double[] {1}); Assert.assertEquals(sketch2.getEstimate(), 2.0); }
/** * Heapify the given Memory as an ArrayOfDoublesUnion * @param mem the given Memory * @return an ArrayOfDoublesUnion */ public static ArrayOfDoublesUnion heapifyUnion(final Memory mem) { return heapifyUnion(mem, DEFAULT_UPDATE_SEED); }
/** * Heapify the given Memory as an ArrayOfDoublesUpdatableSketch * @param mem the given Memory * @return an ArrayOfDoublesUpdatableSketch */ public static ArrayOfDoublesUpdatableSketch heapifyUpdatableSketch(final Memory mem) { return heapifyUpdatableSketch(mem, DEFAULT_UPDATE_SEED); }
@Override public ArrayOfDoublesSketch fromByteBuffer(final ByteBuffer buffer, final int numBytes) { return ArrayOfDoublesSketches.wrapSketch(Memory.wrap(buffer, ByteOrder.LITTLE_ENDIAN).region(buffer.position(), numBytes)); }
/** * This method uses locks because it can be used during indexing, * and Druid can call aggregate() and get() concurrently * https://github.com/apache/incubator-druid/pull/3956 */ @Override public void aggregate(final ByteBuffer buf, final int position) { final ArrayOfDoublesSketch update = selector.getObject(); if (update == null) { return; } // Wrapping memory and ArrayOfDoublesUnion is inexpensive compared to union operations. // Maintaining a cache of wrapped objects per buffer position like in Theta sketch aggregator // might might be considered, but it would increase complexity including relocate() support. final WritableMemory mem = WritableMemory.wrap(buf, ByteOrder.LITTLE_ENDIAN); final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); final Lock lock = stripedLock.getAt(ArrayOfDoublesSketchBuildBufferAggregator.lockIndex(position)).writeLock(); lock.lock(); try { final ArrayOfDoublesUnion union = ArrayOfDoublesSketches.wrapUnion(region); union.update(update); } finally { lock.unlock(); } }
/** * Wrap the given WritableMemory as an ArrayOfDoublesUpdatableSketch * @param mem the given Memory * @return an ArrayOfDoublesUpdatableSketch */ public static ArrayOfDoublesUpdatableSketch wrapUpdatableSketch(final WritableMemory mem) { return wrapUpdatableSketch(mem, DEFAULT_UPDATE_SEED); }
/** * Heapify the given Memory as an ArrayOfDoublesSketch * @param mem the given Memory * @return an ArrayOfDoublesSketch */ public static ArrayOfDoublesSketch heapifySketch(final Memory mem) { return heapifySketch(mem, DEFAULT_UPDATE_SEED); }
@Test public void heapifyAndUpdateUnion() { int numUniques = 10000; int key = 0; ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < numUniques; i++) { sketch1.update(key++, new double[] {1}); } ArrayOfDoublesUnion union1 = new ArrayOfDoublesSetOperationBuilder().buildUnion(); union1.update(sketch1); ArrayOfDoublesUnion union2 = ArrayOfDoublesSketches.heapifyUnion(Memory.wrap(union1.toByteArray())); ArrayOfDoublesSketch resultSketch = union2.getResult(); Assert.assertTrue(resultSketch.isEstimationMode()); Assert.assertEquals(resultSketch.getEstimate(), numUniques, numUniques * 0.04); // make sure union update actually needs to modify the union ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < numUniques; i++) { sketch2.update(key++, new double[] {1}); } union2.update(sketch2); }
/** * This method uses locks because it can be used during indexing, * and Druid can call aggregate() and get() concurrently * https://github.com/apache/incubator-druid/pull/3956 * The returned sketch is a separate instance of ArrayOfDoublesCompactSketch * representing the current state of the aggregation, and is not affected by consequent * aggregate() calls */ @Override public Object get(final ByteBuffer buf, final int position) { final WritableMemory mem = WritableMemory.wrap(buf, ByteOrder.LITTLE_ENDIAN); final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); final Lock lock = stripedLock.getAt(lockIndex(position)).readLock(); lock.lock(); try { final ArrayOfDoublesUpdatableSketch sketch = (ArrayOfDoublesUpdatableSketch) ArrayOfDoublesSketches .wrapSketch(region); return sketch.compact(); } finally { lock.unlock(); } }
/** * Wrap the given Memory as an ArrayOfDoublesUnion * @param mem the given Memory * @return an ArrayOfDoublesUnion */ public static ArrayOfDoublesUnion wrapUnion(final Memory mem) { return wrapUnion(mem, DEFAULT_UPDATE_SEED); }
/** * Wrap the given WritableMemory as an ArrayOfDoublesUpdatableSketch * @param mem the given Memory * @return an ArrayOfDoublesUpdatableSketch */ public static ArrayOfDoublesUpdatableSketch wrapUpdatableSketch(final WritableMemory mem) { return wrapUpdatableSketch(mem, DEFAULT_UPDATE_SEED); }
@Test(expectedExceptions = SketchesArgumentException.class) public void deserializeWithWrongSeed() { ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < 8192; i++) { us.update(i, new double[] {1.0}); } ArrayOfDoublesCompactSketch sketch1 = us.compact(); ArrayOfDoublesSketches.heapifySketch(Memory.wrap(sketch1.toByteArray()), 123); } }
/** * Wrap the given Memory as an ArrayOfDoublesSketch * @param mem the given Memory * @return an ArrayOfDoublesSketch */ public static ArrayOfDoublesSketch wrapSketch(final Memory mem) { return wrapSketch(mem, DEFAULT_UPDATE_SEED); }