/** * This method uses locks because it can be used during indexing, * and Druid can call aggregate() and get() concurrently * https://github.com/apache/incubator-druid/pull/3956 * The returned sketch is a separate instance of ArrayOfDoublesCompactSketch * representing the current state of the aggregation, and is not affected by consequent * aggregate() calls */ @Override public Object get(final ByteBuffer buf, final int position) { final WritableMemory mem = WritableMemory.wrap(buf, ByteOrder.LITTLE_ENDIAN); final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); final Lock lock = stripedLock.getAt(ArrayOfDoublesSketchBuildBufferAggregator.lockIndex(position)).readLock(); lock.lock(); try { final ArrayOfDoublesUnion union = ArrayOfDoublesSketches.wrapUnion(region); return union.getResult(); } finally { lock.unlock(); } }
/** * This method uses locks because it can be used during indexing, * and Druid can call aggregate() and get() concurrently * https://github.com/apache/incubator-druid/pull/3956 */ @Override public void aggregate(final ByteBuffer buf, final int position) { final ArrayOfDoublesSketch update = selector.getObject(); if (update == null) { return; } // Wrapping memory and ArrayOfDoublesUnion is inexpensive compared to union operations. // Maintaining a cache of wrapped objects per buffer position like in Theta sketch aggregator // might might be considered, but it would increase complexity including relocate() support. final WritableMemory mem = WritableMemory.wrap(buf, ByteOrder.LITTLE_ENDIAN); final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); final Lock lock = stripedLock.getAt(ArrayOfDoublesSketchBuildBufferAggregator.lockIndex(position)).writeLock(); lock.lock(); try { final ArrayOfDoublesUnion union = ArrayOfDoublesSketches.wrapUnion(region); union.update(update); } finally { lock.unlock(); } }
/** * Wrap the given Memory as an ArrayOfDoublesUnion * @param mem the given Memory * @return an ArrayOfDoublesUnion */ public static ArrayOfDoublesUnion wrapUnion(final Memory mem) { return wrapUnion(mem, DEFAULT_UPDATE_SEED); }
/** * Wrap the given Memory as an ArrayOfDoublesUnion * @param mem the given Memory * @return an ArrayOfDoublesUnion */ public static ArrayOfDoublesUnion wrapUnion(final WritableMemory mem) { return wrapUnion(mem, DEFAULT_UPDATE_SEED); }
/** * Wrap the given Memory as an ArrayOfDoublesUnion * @param mem the given Memory * @return an ArrayOfDoublesUnion */ public static ArrayOfDoublesUnion wrapUnion(final WritableMemory mem) { return wrapUnion(mem, DEFAULT_UPDATE_SEED); }
/** * Wrap the given Memory as an ArrayOfDoublesUnion * @param mem the given Memory * @return an ArrayOfDoublesUnion */ public static ArrayOfDoublesUnion wrapUnion(final Memory mem) { return wrapUnion(mem, DEFAULT_UPDATE_SEED); }
ArrayOfDoublesUnion wrappedUnion = ArrayOfDoublesSketches.wrapUnion(wmem); result = wrappedUnion.getResult(); Assert.assertEquals(result.getEstimate(), 3.0);
@Test public void wrapAndTryUpdatingUnionEstimationMode() { int numUniques = 10000; int key = 0; ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < numUniques; i++) { sketch1.update(key++, new double[] {1}); } ArrayOfDoublesUnion union1 = new ArrayOfDoublesSetOperationBuilder().buildUnion(); union1.update(sketch1); ArrayOfDoublesUnion union2 = ArrayOfDoublesSketches.wrapUnion(Memory.wrap(union1.toByteArray())); ArrayOfDoublesSketch resultSketch = union2.getResult(); Assert.assertTrue(resultSketch.isEstimationMode()); Assert.assertEquals(resultSketch.getEstimate(), numUniques, numUniques * 0.04); // make sure union update actually needs to modify the union ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < numUniques; i++) { sketch2.update(key++, new double[] {1}); } boolean thrown = false; try { union2.update(sketch2); } catch (SketchesReadOnlyException e) { thrown = true; } Assert.assertTrue(thrown); }
/** * This method uses locks because it can be used during indexing, * and Druid can call aggregate() and get() concurrently * https://github.com/apache/incubator-druid/pull/3956 * The returned sketch is a separate instance of ArrayOfDoublesCompactSketch * representing the current state of the aggregation, and is not affected by consequent * aggregate() calls */ @Override public Object get(final ByteBuffer buf, final int position) { final WritableMemory mem = WritableMemory.wrap(buf); final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); final Lock lock = stripedLock.getAt(ArrayOfDoublesSketchBuildBufferAggregator.lockIndex(position)).readLock(); lock.lock(); try { final ArrayOfDoublesUnion union = ArrayOfDoublesSketches.wrapUnion(region); return union.getResult(); } finally { lock.unlock(); } }
/** * This method uses locks because it can be used during indexing, * and Druid can call aggregate() and get() concurrently * https://github.com/apache/incubator-druid/pull/3956 */ @Override public void aggregate(final ByteBuffer buf, final int position) { final ArrayOfDoublesSketch update = selector.getObject(); if (update == null) { return; } // Wrapping memory and ArrayOfDoublesUnion is inexpensive compared to union operations. // Maintaining a cache of wrapped objects per buffer position like in Theta sketch aggregator // might might be considered, but it would increase complexity including relocate() support. final WritableMemory mem = WritableMemory.wrap(buf); final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); final Lock lock = stripedLock.getAt(ArrayOfDoublesSketchBuildBufferAggregator.lockIndex(position)).writeLock(); lock.lock(); try { final ArrayOfDoublesUnion union = ArrayOfDoublesSketches.wrapUnion(region); union.update(update); } finally { lock.unlock(); } }