/** * This method uses locks because it can be used during indexing, * and Druid can call aggregate() and get() concurrently * https://github.com/apache/incubator-druid/pull/3956 */ @Override public void aggregate(final ByteBuffer buf, final int position) { for (int i = 0; i < valueSelectors.length; i++) { values[i] = valueSelectors[i].getDouble(); } final IndexedInts keys = keySelector.getRow(); // Wrapping memory and ArrayOfDoublesSketch is inexpensive compared to sketch operations. // Maintaining a cache of wrapped objects per buffer position like in Theta sketch aggregator // might might be considered, but it would increase complexity including relocate() support. final WritableMemory mem = WritableMemory.wrap(buf, ByteOrder.LITTLE_ENDIAN); final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); final Lock lock = stripedLock.getAt(lockIndex(position)).writeLock(); lock.lock(); try { final ArrayOfDoublesUpdatableSketch sketch = ArrayOfDoublesSketches.wrapUpdatableSketch(region); for (int i = 0, keysSize = keys.size(); i < keysSize; i++) { final String key = keySelector.lookupName(keys.get(i)); sketch.update(key, values); } } finally { lock.unlock(); } }
/** * Wrap the given WritableMemory as an ArrayOfDoublesUpdatableSketch * @param mem the given Memory * @return an ArrayOfDoublesUpdatableSketch */ public static ArrayOfDoublesUpdatableSketch wrapUpdatableSketch(final WritableMemory mem) { return wrapUpdatableSketch(mem, DEFAULT_UPDATE_SEED); }
/** * Wrap the given WritableMemory as an ArrayOfDoublesUpdatableSketch * @param mem the given Memory * @return an ArrayOfDoublesUpdatableSketch */ public static ArrayOfDoublesUpdatableSketch wrapUpdatableSketch(final WritableMemory mem) { return wrapUpdatableSketch(mem, DEFAULT_UPDATE_SEED); }
@Test public void serializeDeserializeEstimation() { ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < 8192; i++) { us.update(i, new double[] {1.0}); } WritableMemory wmem = WritableMemory.wrap(us.toByteArray()); ArrayOfDoublesUpdatableSketch wrappedUS = ArrayOfDoublesSketches.wrapUpdatableSketch(wmem); Assert.assertFalse(wrappedUS.isEmpty()); Assert.assertTrue(wrappedUS.isEstimationMode()); Assert.assertEquals(wrappedUS.getEstimate(), us.getEstimate()); Assert.assertEquals(wrappedUS.getThetaLong(), us.getThetaLong()); ArrayOfDoublesUpdatableSketch heapUS = ArrayOfDoublesSketches.heapifyUpdatableSketch(wmem); Assert.assertFalse(heapUS.isEmpty()); Assert.assertTrue(heapUS.isEstimationMode()); Assert.assertEquals(heapUS.getEstimate(), us.getEstimate()); Assert.assertEquals(heapUS.getThetaLong(), us.getThetaLong()); ArrayOfDoublesCompactSketch sketch1 = us.compact(); ArrayOfDoublesSketch sketch2 = ArrayOfDoublesSketches.heapifySketch(Memory.wrap(sketch1.toByteArray())); Assert.assertFalse(sketch2.isEmpty()); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), sketch1.getEstimate()); Assert.assertEquals(sketch2.getThetaLong(), sketch1.getThetaLong()); }
/** * This method uses locks because it can be used during indexing, * and Druid can call aggregate() and get() concurrently * https://github.com/apache/incubator-druid/pull/3956 */ @Override public void aggregate(final ByteBuffer buf, final int position) { for (int i = 0; i < valueSelectors.length; i++) { values[i] = valueSelectors[i].getDouble(); } final IndexedInts keys = keySelector.getRow(); // Wrapping memory and ArrayOfDoublesSketch is inexpensive compared to sketch operations. // Maintaining a cache of wrapped objects per buffer position like in Theta sketch aggregator // might might be considered, but it would increase complexity including relocate() support. final WritableMemory mem = WritableMemory.wrap(buf); final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); final Lock lock = stripedLock.getAt(lockIndex(position)).writeLock(); lock.lock(); try { final ArrayOfDoublesUpdatableSketch sketch = ArrayOfDoublesSketches.wrapUpdatableSketch(region); for (int i = 0, keysSize = keys.size(); i < keysSize; i++) { final String key = keySelector.lookupName(keys.get(i)); sketch.update(key, values); } } finally { lock.unlock(); } }