public ArrayOfDoublesSketchMergeAggregator( final BaseObjectColumnValueSelector<ArrayOfDoublesSketch> selector, final int nominalEntries, final int numberOfValues ) { this.selector = selector; union = new ArrayOfDoublesSetOperationBuilder().setNominalEntries(nominalEntries).setNumberOfValues(numberOfValues) .buildUnion(); }
@Override public ArrayOfDoublesSketch apply(final int nominalEntries, final int numberOfValues, final ArrayOfDoublesSketch[] sketches) { if (sketches.length < 1) { throw new IAE("A-Not-B requires at least 1 sketch"); } if (sketches.length == 1) { return sketches[0]; } ArrayOfDoublesSketch result = sketches[0]; for (int i = 1; i < sketches.length; i++) { final ArrayOfDoublesAnotB aNotB = new ArrayOfDoublesSetOperationBuilder().setNumberOfValues(numberOfValues) .buildAnotB(); aNotB.update(result, sketches[i]); result = aNotB.getResult(); } return result; } };
@Override public ArrayOfDoublesSketch apply(final int nominalEntries, final int numberOfValues, final ArrayOfDoublesSketch[] sketches) { final ArrayOfDoublesIntersection intersection = new ArrayOfDoublesSetOperationBuilder() .setNominalEntries(nominalEntries).setNumberOfValues(numberOfValues).buildIntersection(); for (final ArrayOfDoublesSketch sketch : sketches) { intersection.update(sketch, COMBINER); } return intersection.getResult(); } },
@Test(expectedExceptions = SketchesArgumentException.class) public void incompatibleInputSketchMoreValues() { ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(2).build(); ArrayOfDoublesUnion union = new ArrayOfDoublesSetOperationBuilder().buildUnion(); union.update(sketch); }
@Test public void nullInput() { ArrayOfDoublesIntersection intersection = new ArrayOfDoublesSetOperationBuilder().buildIntersection(); intersection.update(null, null); ArrayOfDoublesCompactSketch result = intersection.getResult(); Assert.assertTrue(result.isEmpty()); Assert.assertEquals(result.getRetainedEntries(), 0); Assert.assertEquals(result.getEstimate(), 0.0); Assert.assertEquals(result.getLowerBound(1), 0.0); Assert.assertEquals(result.getUpperBound(1), 0.0); Assert.assertEquals(result.getValues().length, 0); }
@Test(expectedExceptions = SketchesArgumentException.class) public void incompatibleSeedA() { ArrayOfDoublesSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(1).build(); ArrayOfDoublesAnotB aNotB = new ArrayOfDoublesSetOperationBuilder().buildAnotB(); aNotB.update(sketch, null); }
@Test(expectedExceptions = SketchesArgumentException.class) public void incompatibleSeeds() { ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(1).build(); ArrayOfDoublesUnion union = new ArrayOfDoublesSetOperationBuilder().setSeed(2).buildUnion(); union.update(sketch); }
@Test(expectedExceptions = SketchesArgumentException.class) public void incompatibleInputSketchFewerValues() { ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder().build(); ArrayOfDoublesUnion union = new ArrayOfDoublesSetOperationBuilder().setNumberOfValues(2).buildUnion(); union.update(sketch); }
@Test(expectedExceptions = SketchesArgumentException.class) public void incompatibleSeeds() { ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(1).build(); ArrayOfDoublesIntersection intersection = new ArrayOfDoublesSetOperationBuilder().setSeed(2).buildIntersection(); intersection.update(sketch, combiner); } }
@Test(expectedExceptions = SketchesArgumentException.class) public void incompatibleSeeds() { ArrayOfDoublesSketch sketchA = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(1).build(); ArrayOfDoublesSketch sketchB = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(2).build(); ArrayOfDoublesAnotB aNotB = new ArrayOfDoublesSetOperationBuilder().setSeed(3).buildAnotB(); aNotB.update(sketchA, sketchB); }
@Test public void heapMixedMode() { int key = 0; ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < 1000; i++) { sketch1.update(key++, new double[] {1.0}); } key -= 500; // overlap half of the entries ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().setSamplingProbability(0.2f).build(); for (int i = 0; i < 20000; i++) { sketch2.update(key++, new double[] {1.0}); } ArrayOfDoublesUnion union = new ArrayOfDoublesSetOperationBuilder().buildUnion(); union.update(sketch1); union.update(sketch2); ArrayOfDoublesCompactSketch result = union.getResult(); Assert.assertFalse(result.isEmpty()); Assert.assertTrue(result.isEstimationMode()); Assert.assertEquals(result.getEstimate(), 20500.0, 20500 * 0.01); }
@Test public void empty() { ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build(); ArrayOfDoublesIntersection intersection = new ArrayOfDoublesSetOperationBuilder().buildIntersection(); intersection.update(sketch1, null); ArrayOfDoublesCompactSketch result = intersection.getResult(); Assert.assertTrue(result.isEmpty()); Assert.assertEquals(result.getRetainedEntries(), 0); Assert.assertEquals(result.getEstimate(), 0.0); Assert.assertEquals(result.getLowerBound(1), 0.0); Assert.assertEquals(result.getUpperBound(1), 0.0); Assert.assertEquals(result.getValues().length, 0); }
@Test(expectedExceptions = SketchesArgumentException.class) public void incompatibleSeedB() { ArrayOfDoublesSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(1).build(); ArrayOfDoublesAnotB aNotB = new ArrayOfDoublesSetOperationBuilder().buildAnotB(); aNotB.update(null, sketch); }
@Test public void heapSerializeDeserializeWithSeed() { long seed = 1; int key = 0; ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(seed).build(); for (int i = 0; i < 8192; i++) { sketch1.update(key++, new double[] {1.0}); } key -= 4096; // overlap half of the entries ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(seed).build(); for (int i = 0; i < 8192; i++) { sketch2.update(key++, new double[] {1.0}); } ArrayOfDoublesUnion union1 = new ArrayOfDoublesSetOperationBuilder().setSeed(seed).buildUnion(); union1.update(sketch1); union1.update(sketch2); ArrayOfDoublesUnion union2 = ArrayOfDoublesUnion.heapify(Memory.wrap(union1.toByteArray()), seed); ArrayOfDoublesCompactSketch result = union2.getResult(); Assert.assertEquals(result.getEstimate(), 12288.0, 12288 * 0.01); }
@Test public void heapExactModeCustomSeed() { long seed = 1234567890; ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(seed).build(); sketch1.update(1, new double[] {1}); sketch1.update(1, new double[] {1}); sketch1.update(2, new double[] {1}); sketch1.update(2, new double[] {1}); ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(seed).build(); sketch2.update(2, new double[] {1}); sketch2.update(2, new double[] {1}); sketch2.update(3, new double[] {1}); sketch2.update(3, new double[] {1}); ArrayOfDoublesIntersection intersection = new ArrayOfDoublesSetOperationBuilder().setSeed(seed).buildIntersection(); intersection.update(sketch1, combiner); intersection.update(sketch2, combiner); ArrayOfDoublesCompactSketch result = intersection.getResult(); Assert.assertFalse(result.isEmpty()); Assert.assertEquals(result.getRetainedEntries(), 1); Assert.assertEquals(result.getEstimate(), 1.0); Assert.assertEquals(result.getLowerBound(1), 1.0); Assert.assertEquals(result.getUpperBound(1), 1.0); double[][] values = result.getValues(); for (int i = 0; i < values.length; i++) Assert.assertEquals(values[i][0], 4.0); }
@Test public void exactModeCustomSeed() { long seed = 1234567890; ArrayOfDoublesUpdatableSketch sketchA = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(seed).build(); sketchA.update(1, new double[] {1}); sketchA.update(2, new double[] {1}); sketchA.update(3, new double[] {1}); sketchA.update(4, new double[] {1}); sketchA.update(5, new double[] {1}); ArrayOfDoublesUpdatableSketch sketchB = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(seed).build(); sketchB.update(3, new double[] {1}); sketchB.update(4, new double[] {1}); sketchB.update(5, new double[] {1}); sketchB.update(6, new double[] {1}); sketchB.update(7, new double[] {1}); ArrayOfDoublesAnotB aNotB = new ArrayOfDoublesSetOperationBuilder().setSeed(seed).buildAnotB(); aNotB.update(sketchA, sketchB); ArrayOfDoublesCompactSketch result = aNotB.getResult(); Assert.assertFalse(result.isEmpty()); Assert.assertEquals(result.getRetainedEntries(), 2); Assert.assertEquals(result.getEstimate(), 2.0); Assert.assertEquals(result.getLowerBound(1), 2.0); Assert.assertEquals(result.getUpperBound(1), 2.0); ArrayOfDoublesSketchIterator it = result.iterator(); while (it.next()) { Assert.assertEquals(it.getValues(), new double[] {1}); } }
private final ArrayOfDoublesUnion union = new ArrayOfDoublesSetOperationBuilder().setNominalEntries(nominalEntries) .setNumberOfValues(numberOfValues).buildUnion();
@Test public void heapEstimationMode() { int key = 0; ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < 8192; i++) { sketch1.update(key++, new double[] {1.0}); } key -= 4096; // overlap half of the entries ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < 8192; i++) { sketch2.update(key++, new double[] {1.0}); } ArrayOfDoublesUnion union = new ArrayOfDoublesSetOperationBuilder().buildUnion(); union.update(sketch1); union.update(sketch2); ArrayOfDoublesCompactSketch result = union.getResult(); Assert.assertFalse(result.isEmpty()); Assert.assertTrue(result.isEstimationMode()); Assert.assertEquals(result.getEstimate(), 12288.0, 12288 * 0.01); union.reset(); result = union.getResult(); Assert.assertTrue(result.isEmpty()); Assert.assertFalse(result.isEstimationMode()); Assert.assertEquals(result.getEstimate(), 0.0); Assert.assertEquals(result.getUpperBound(1), 0.0); Assert.assertEquals(result.getLowerBound(1), 0.0); Assert.assertEquals(result.getTheta(), 1.0); }
@Test public void notEmptyNoEntries() { ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().setSamplingProbability(0.01f).build(); sketch1.update("a", new double[] {1}); // this happens to get rejected because of sampling with low probability ArrayOfDoublesIntersection intersection = new ArrayOfDoublesSetOperationBuilder().buildIntersection(); intersection.update(sketch1, null); ArrayOfDoublesCompactSketch result = intersection.getResult(); Assert.assertTrue(result.isEmpty()); Assert.assertEquals(result.getRetainedEntries(), 0); Assert.assertEquals(result.getEstimate(), 0.0); Assert.assertEquals(result.getLowerBound(1), 0.0); Assert.assertEquals(result.getUpperBound(1), 0.0); Assert.assertEquals(result.getValues().length, 0); }
@Override public ArrayOfDoublesSketch apply(final int nominalEntries, final int numberOfValues, final ArrayOfDoublesSketch[] sketches) { final ArrayOfDoublesIntersection intersection = new ArrayOfDoublesSetOperationBuilder() .setNominalEntries(nominalEntries).setNumberOfValues(numberOfValues).buildIntersection(); for (final ArrayOfDoublesSketch sketch : sketches) { intersection.update(sketch, COMBINER); } return intersection.getResult(); } },