/** * This method uses synchronization because it can be used during indexing, * and Druid can call aggregate() and get() concurrently * https://github.com/apache/incubator-druid/pull/3956 * The returned sketch is a separate instance of ArrayOfDoublesCompactSketch * representing the current state of the aggregation, and is not affected by consequent * aggregate() calls */ @Override public synchronized Object get() { return sketch.compact(); }
public ArrayOfDoublesSketchNoOpAggregator(final int numberOfValues) { emptySketch = new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(numberOfValues).build().compact(); }
public ArrayOfDoublesSketchNoOpBufferAggregator(final int numberOfValues) { emptySketch = new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(numberOfValues).build().compact(); }
/** * This method uses locks because it can be used during indexing, * and Druid can call aggregate() and get() concurrently * https://github.com/apache/incubator-druid/pull/3956 * The returned sketch is a separate instance of ArrayOfDoublesCompactSketch * representing the current state of the aggregation, and is not affected by consequent * aggregate() calls */ @Override public Object get(final ByteBuffer buf, final int position) { final WritableMemory mem = WritableMemory.wrap(buf, ByteOrder.LITTLE_ENDIAN); final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); final Lock lock = stripedLock.getAt(lockIndex(position)).readLock(); lock.lock(); try { final ArrayOfDoublesUpdatableSketch sketch = (ArrayOfDoublesUpdatableSketch) ArrayOfDoublesSketches .wrapSketch(region); return sketch.compact(); } finally { lock.unlock(); } }
private static void generateSketches() throws Exception { Path path = FileSystems.getDefault().getPath("array_of_doubles_sketch_data.tsv"); try (BufferedWriter out = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) { Random rand = ThreadLocalRandom.current(); int key = 0; for (int i = 0; i < 20; i++) { ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder().setNominalEntries(1024) .build(); sketch.update(key++, new double[] {1}); sketch.update(key++, new double[] {1}); out.write("2015010101"); out.write('\t'); out.write("product_" + (rand.nextInt(10) + 1)); out.write('\t'); out.write(StringUtils.encodeBase64String(sketch.compact().toByteArray())); out.newLine(); } } }
/** * Gets an on-heap compact representation of the sketch * @return compact sketch */ public ArrayOfDoublesCompactSketch compact() { return compact(null); }
@Test public void emptyFromQuickSelectSketch() { ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(); ArrayOfDoublesCompactSketch sketch = us.compact(); Assert.assertTrue(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 0.0); Assert.assertEquals(sketch.getLowerBound(1), 0.0); Assert.assertEquals(sketch.getUpperBound(1), 0.0); Assert.assertEquals(sketch.getRetainedEntries(), 0); Assert.assertEquals(sketch.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch.getTheta(), 1.0); Assert.assertNotNull(sketch.getValues()); Assert.assertEquals(sketch.getValues().length, 0); ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { Assert.fail("empty sketch expected"); } }
@Test(expectedExceptions = SketchesArgumentException.class) public void fromQuickSelectSketchNotEnoughMemory() { ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.wrap(new byte[1000000])); us.update(1, new double[] {1.0}); us.compact(WritableMemory.wrap(new byte[39])); }
@Test public void emptyFromQuickSelectSketch() { ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.wrap(new byte[1000000])); ArrayOfDoublesCompactSketch sketch = us.compact(WritableMemory.wrap(new byte[1000000])); Assert.assertTrue(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 0.0); Assert.assertEquals(sketch.getLowerBound(1), 0.0); Assert.assertEquals(sketch.getUpperBound(1), 0.0); Assert.assertEquals(sketch.getRetainedEntries(), 0); Assert.assertEquals(sketch.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch.getTheta(), 1.0); Assert.assertNotNull(sketch.getValues()); Assert.assertEquals(sketch.getValues().length, 0); ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { Assert.fail("empty sketch expected"); } }
@Test public void serializeDeserializeEstimation() { ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.wrap(new byte[1000000])); for (int i = 0; i < 8192; i++) { us.update(i, new double[] {1.0}); } ArrayOfDoublesCompactSketch sketch1 = us.compact(WritableMemory.wrap(new byte[1000000])); ArrayOfDoublesSketch sketch2 = ArrayOfDoublesSketches.wrapSketch(WritableMemory.wrap(sketch1.toByteArray())); Assert.assertFalse(sketch2.isEmpty()); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), sketch1.getEstimate()); Assert.assertEquals(sketch2.getThetaLong(), sketch1.getThetaLong()); }
@Test(expectedExceptions = SketchesArgumentException.class) public void deserializeWithWrongSeed() { ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < 8192; i++) { us.update(i, new double[] {1.0}); } ArrayOfDoublesCompactSketch sketch1 = us.compact(); ArrayOfDoublesSketches.heapifySketch(Memory.wrap(sketch1.toByteArray()), 123); } }
@Test public void serializeDeserializeSmallExact() { ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(); us.update("a", new double[] {1.0}); us.update("b", new double[] {1.0}); us.update("c", new double[] {1.0}); ArrayOfDoublesCompactSketch sketch1 = us.compact(); ArrayOfDoublesSketch sketch2 = ArrayOfDoublesSketches.heapifySketch(Memory.wrap(sketch1.toByteArray())); Assert.assertFalse(sketch2.isEmpty()); Assert.assertFalse(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), 3.0); Assert.assertEquals(sketch2.getLowerBound(1), 3.0); Assert.assertEquals(sketch2.getUpperBound(1), 3.0); Assert.assertEquals(sketch2.getRetainedEntries(), 3); Assert.assertEquals(sketch2.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch2.getTheta(), 1.0); double[][] values = sketch2.getValues(); Assert.assertEquals(values.length, 3); for (double[] array: values) { Assert.assertEquals(array[0], 1.0); } }
@Test public void exactModeFromQuickSelectSketch() { ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(); us.update(1, new double[] {1.0}); us.update(2, new double[] {1.0}); us.update(3, new double[] {1.0}); us.update(1, new double[] {1.0}); us.update(2, new double[] {1.0}); us.update(3, new double[] {1.0}); ArrayOfDoublesCompactSketch sketch = us.compact(); Assert.assertFalse(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 3.0); Assert.assertEquals(sketch.getLowerBound(1), 3.0); Assert.assertEquals(sketch.getUpperBound(1), 3.0); Assert.assertEquals(sketch.getRetainedEntries(), 3); Assert.assertEquals(sketch.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch.getTheta(), 1.0); double[][] values = sketch.getValues(); Assert.assertEquals(values.length, 3); for (double[] array: values) { Assert.assertEquals(array[0], 2.0); } }
@Test public void serializeDeserializeSmallExact() { ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.wrap(new byte[1000000])); us.update("a", new double[] {1.0}); us.update("b", new double[] {1.0}); us.update("c", new double[] {1.0}); ArrayOfDoublesCompactSketch sketch1 = us.compact(WritableMemory.wrap(new byte[1000000])); ArrayOfDoublesSketch sketch2 = ArrayOfDoublesSketches.wrapSketch(WritableMemory.wrap(sketch1.toByteArray())); Assert.assertFalse(sketch2.isEmpty()); Assert.assertFalse(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), 3.0); Assert.assertEquals(sketch2.getLowerBound(1), 3.0); Assert.assertEquals(sketch2.getUpperBound(1), 3.0); Assert.assertEquals(sketch2.getRetainedEntries(), 3); Assert.assertEquals(sketch2.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch2.getTheta(), 1.0); double[][] values = sketch2.getValues(); Assert.assertEquals(values.length, 3); for (double[] array: values) { Assert.assertEquals(array[0], 1.0); } }
@Test(expectedExceptions = SketchesArgumentException.class) public void deserializeWithWrongSeed() { ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.wrap(new byte[1000000])); for (int i = 0; i < 8192; i++) { us.update(i, new double[] {1.0}); } ArrayOfDoublesCompactSketch sketch1 = us.compact(WritableMemory.wrap(new byte[1000000])); ArrayOfDoublesSketches.wrapSketch(WritableMemory.wrap(sketch1.toByteArray()), 123); }
@Test public void exactModeFromQuickSelectSketch() { ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.wrap(new byte[1000000])); us.update(1, new double[] {1.0}); us.update(2, new double[] {1.0}); us.update(3, new double[] {1.0}); us.update(1, new double[] {1.0}); us.update(2, new double[] {1.0}); us.update(3, new double[] {1.0}); ArrayOfDoublesCompactSketch sketch = us.compact(WritableMemory.wrap(new byte[1000000])); Assert.assertFalse(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 3.0); Assert.assertEquals(sketch.getLowerBound(1), 3.0); Assert.assertEquals(sketch.getUpperBound(1), 3.0); Assert.assertEquals(sketch.getRetainedEntries(), 3); Assert.assertEquals(sketch.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch.getTheta(), 1.0); Assert.assertEquals(sketch.getSeedHash(), Util.computeSeedHash(DEFAULT_UPDATE_SEED)); double[][] values = sketch.getValues(); Assert.assertEquals(values.length, 3); for (double[] array: values) { Assert.assertEquals(array[0], 2.0); } }
@Test public void heapToDirectExactTwoDoubles() { ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(2).build(); sketch1.update("a", new double[] {1, 2}); sketch1.update("b", new double[] {1, 2}); sketch1.update("c", new double[] {1, 2}); sketch1.update("d", new double[] {1, 2}); sketch1.update("a", new double[] {1, 2}); sketch1.update("b", new double[] {1, 2}); sketch1.update("c", new double[] {1, 2}); sketch1.update("d", new double[] {1, 2}); ArrayOfDoublesSketch sketch2 = new DirectArrayOfDoublesCompactSketch(Memory.wrap(sketch1.compact().toByteArray())); Assert.assertFalse(sketch2.isEmpty()); Assert.assertFalse(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), 4.0); Assert.assertEquals(sketch2.getUpperBound(1), 4.0); Assert.assertEquals(sketch2.getLowerBound(1), 4.0); Assert.assertEquals(sketch2.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch2.getTheta(), 1.0); double[][] values = sketch2.getValues(); Assert.assertEquals(values.length, 4); for (double[] array: values) { Assert.assertEquals(array.length, 2); Assert.assertEquals(array[0], 2.0); Assert.assertEquals(array[1], 4.0); } }
@Test public void serializeDeserializeEstimation() { ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < 8192; i++) { us.update(i, new double[] {1.0}); } WritableMemory wmem = WritableMemory.wrap(us.toByteArray()); ArrayOfDoublesUpdatableSketch wrappedUS = ArrayOfDoublesSketches.wrapUpdatableSketch(wmem); Assert.assertFalse(wrappedUS.isEmpty()); Assert.assertTrue(wrappedUS.isEstimationMode()); Assert.assertEquals(wrappedUS.getEstimate(), us.getEstimate()); Assert.assertEquals(wrappedUS.getThetaLong(), us.getThetaLong()); ArrayOfDoublesUpdatableSketch heapUS = ArrayOfDoublesSketches.heapifyUpdatableSketch(wmem); Assert.assertFalse(heapUS.isEmpty()); Assert.assertTrue(heapUS.isEstimationMode()); Assert.assertEquals(heapUS.getEstimate(), us.getEstimate()); Assert.assertEquals(heapUS.getThetaLong(), us.getThetaLong()); ArrayOfDoublesCompactSketch sketch1 = us.compact(); ArrayOfDoublesSketch sketch2 = ArrayOfDoublesSketches.heapifySketch(Memory.wrap(sketch1.toByteArray())); Assert.assertFalse(sketch2.isEmpty()); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), sketch1.getEstimate()); Assert.assertEquals(sketch2.getThetaLong(), sketch1.getThetaLong()); }
@Test public void directToHeapExactTwoDoubles() { ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(2).build(WritableMemory.wrap(new byte[1000000])); sketch1.update("a", new double[] {1, 2}); sketch1.update("b", new double[] {1, 2}); sketch1.update("c", new double[] {1, 2}); sketch1.update("d", new double[] {1, 2}); sketch1.update("a", new double[] {1, 2}); sketch1.update("b", new double[] {1, 2}); sketch1.update("c", new double[] {1, 2}); sketch1.update("d", new double[] {1, 2}); ArrayOfDoublesSketch sketch2 = new HeapArrayOfDoublesCompactSketch(Memory.wrap(sketch1.compact(WritableMemory.wrap(new byte[1000000])).toByteArray())); Assert.assertFalse(sketch2.isEmpty()); Assert.assertFalse(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), 4.0); Assert.assertEquals(sketch2.getUpperBound(1), 4.0); Assert.assertEquals(sketch2.getLowerBound(1), 4.0); Assert.assertEquals(sketch2.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch2.getTheta(), 1.0); double[][] values = sketch2.getValues(); Assert.assertEquals(values.length, 4); for (double[] array: values) { Assert.assertEquals(array.length, 2); Assert.assertEquals(array[0], 2.0); Assert.assertEquals(array[1], 4.0); } } }