@Test public void sampling() { float samplingProbability = 0.001f; UpdatableSketch<Double, DoubleSummary> sketch = new UpdatableSketchBuilder<>( new DoubleSummaryFactory()).setSamplingProbability(samplingProbability).build(); sketch.update("a", 1.0); Assert.assertFalse(sketch.isEmpty()); Assert.assertTrue(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 0.0); Assert.assertTrue(sketch.getUpperBound(1) > 0.0); Assert.assertEquals(sketch.getLowerBound(1), 0.0, 0.0000001); Assert.assertEquals(sketch.getThetaLong() / (double) Long.MAX_VALUE, (double) samplingProbability); Assert.assertEquals(sketch.getTheta(), (double) samplingProbability); }
/** * Instantiate UpdatableSketch from a given Memory * @param <U> Type of update value * @param <S> Type of Summary * @param mem Memory object representing a Sketch * @param deserializer instance of SummaryDeserializer * @param summaryFactory instance of SummaryFactory * @return Sketch created from its Memory representation */ public static <U, S extends UpdatableSummary<U>> UpdatableSketch<U, S> heapifyUpdatableSketch(final Memory mem, final SummaryDeserializer<S> deserializer, final SummaryFactory<S> summaryFactory) { return new UpdatableSketch<U, S>(mem, deserializer, summaryFactory); }
private void insertOrIgnore(final long key, final U value) { setNotEmpty(); if (key >= getThetaLong()) { return; } int index = findOrInsert(key); if (index < 0) { index = ~index; insertSummary(index, getSummaryFactory().newSummary()); } summaries_[index].update(value); rebuildIfNeeded(); }
@Override public String toString() { final StringBuilder sb = new StringBuilder(); sb.append("### ").append(this.getClass().getSimpleName()).append(" SUMMARY: ").append(LS); sb.append(" Estimate : ").append(getEstimate()).append(LS); sb.append(" Upper Bound, 95% conf : ").append(getUpperBound(2)).append(LS); sb.append(" Lower Bound, 95% conf : ").append(getLowerBound(2)).append(LS); sb.append(" Theta (double) : ").append(this.getTheta()).append(LS); sb.append(" Theta (long) : ").append(this.getThetaLong()).append(LS); sb.append(" EstMode? : ").append(isEstimationMode()).append(LS); sb.append(" Empty? : ").append(isEmpty()).append(LS); sb.append(" Retained Entries : ").append(this.getRetainedEntries()).append(LS); if (this instanceof UpdatableSketch) { @SuppressWarnings("rawtypes") final UpdatableSketch updatable = (UpdatableSketch) this; sb.append(" Nominal Entries (k) : ").append(updatable.getNominalEntries()).append(LS); sb.append(" Current Capacity : ").append(updatable.getCurrentCapacity()).append(LS); sb.append(" Resize Factor : ").append(updatable.getResizeFactor().getValue()).append(LS); sb.append(" Sampling Probability (p): ").append(updatable.getSamplingProbability()).append(LS); } sb.append("### END SKETCH SUMMARY").append(LS); return sb.toString(); }
@Test public void updatesOfAllKeyTypes() { UpdatableSketch<Double, DoubleSummary> sketch = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); sketch.update(1L, 1.0); sketch.update(2.0, 1.0); byte[] bytes = { 3 }; sketch.update(bytes, 1.0); int[] ints = { 4 }; sketch.update(ints, 1.0); long[] longs = { 5L }; sketch.update(longs, 1.0); sketch.update("a", 1.0); Assert.assertEquals(sketch.getEstimate(), 6.0); }
@Test public void isEmpty() { UpdatableSketch<Double, DoubleSummary> sketch = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); Assert.assertTrue(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 0.0); Assert.assertEquals(sketch.getUpperBound(1), 0.0); Assert.assertEquals(sketch.getLowerBound(1), 0.0); Assert.assertEquals(sketch.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch.getTheta(), 1.0); Assert.assertNotNull(sketch.toString()); SketchIterator<DoubleSummary> it = sketch.iterator(); Assert.assertNotNull(it); Assert.assertFalse(it.next()); }
/** * Updates this sketch with a long key and U value. * The value is passed to update() method of the Summary object associated with the key * * @param key The given long key * @param value The given U value */ public void update(final long key, final U value) { update(new long[] {key}, value); }
@Test public void nonEmptySketchWithNoEntries() { UpdatableSketch<Double, DoubleSummary> sketch = new UpdatableSketchBuilder<>( new DoubleSummaryFactory()).setSamplingProbability(0.0001f).build(); sketch.update(0, 0.0); Assert.assertFalse(sketch.isEmpty()); Assert.assertEquals(sketch.getRetainedEntries(), 0); Filter<DoubleSummary> filter = new Filter<>(o -> true); Sketch<DoubleSummary> filteredSketch = filter.filter(sketch); Assert.assertFalse(filteredSketch.isEmpty()); Assert.assertEquals(filteredSketch.getEstimate(), sketch.getEstimate()); Assert.assertEquals(filteredSketch.getThetaLong(), sketch.getThetaLong()); Assert.assertEquals(filteredSketch.getLowerBound(1), sketch.getLowerBound(1)); Assert.assertEquals(filteredSketch.getUpperBound(1), sketch.getUpperBound(1)); }
@Test public void exactModeFromQuickSelectSketch() { UpdatableSketch<Double, DoubleSummary> us = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); us.update(1, 1.0); us.update(2, 1.0); us.update(3, 1.0); us.update(1, 1.0); us.update(2, 1.0); us.update(3, 1.0); CompactSketch<DoubleSummary> sketch = us.compact(); Assert.assertFalse(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 3.0); Assert.assertEquals(sketch.getLowerBound(1), 3.0); Assert.assertEquals(sketch.getUpperBound(1), 3.0); Assert.assertEquals(sketch.getRetainedEntries(), 3); Assert.assertEquals(sketch.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch.getTheta(), 1.0); SketchIterator<DoubleSummary> it = sketch.iterator(); int count = 0; while (it.next()) { Assert.assertEquals(it.getSummary().getValue(), 2.0); count++; } Assert.assertEquals(count, 3); }
@Test public void estimationModeWithSamplingNoResizing() { UpdatableSketch<Double, DoubleSummary> sketch = new UpdatableSketchBuilder<>( new DoubleSummaryFactory()) .setSamplingProbability(0.5f) .setResizeFactor(ResizeFactor.X1).build(); for (int i = 0; i < 16384; i++) { sketch.update(i, 1.0); } Assert.assertTrue(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 16384, 16384 * 0.01); Assert.assertTrue(sketch.getEstimate() >= sketch.getLowerBound(1)); Assert.assertTrue(sketch.getEstimate() < sketch.getUpperBound(1)); }
@Test public void doubleSummaryMaxMode() { UpdatableSketch<Double, DoubleSummary> sketch = new UpdatableSketchBuilder<>( new DoubleSummaryFactory(DoubleSummary.Mode.Max)).build(); { sketch.update(1, 1.0); Assert.assertEquals(sketch.getRetainedEntries(), 1); SketchIterator<DoubleSummary> it = sketch.iterator(); Assert.assertTrue(it.next()); Assert.assertEquals(it.getSummary().getValue(), 1.0); Assert.assertFalse(it.next()); } { sketch.update(1, 0.7); Assert.assertEquals(sketch.getRetainedEntries(), 1); SketchIterator<DoubleSummary> it = sketch.iterator(); Assert.assertTrue(it.next()); Assert.assertEquals(it.getSummary().getValue(), 1.0); Assert.assertFalse(it.next()); } { sketch.update(1, 2.0); Assert.assertEquals(sketch.getRetainedEntries(), 1); SketchIterator<DoubleSummary> it = sketch.iterator(); Assert.assertTrue(it.next()); Assert.assertEquals(it.getSummary().getValue(), 2.0); Assert.assertFalse(it.next()); } }
@Test public void serializeDeserializeExact() throws Exception { UpdatableSketch<Double, DoubleSummary> sketch1 = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); sketch1.update(1, 1.0); UpdatableSketch<Double, DoubleSummary> sketch2 = Sketches.heapifyUpdatableSketch( Memory.wrap(sketch1.toByteArray()), new DoubleSummaryDeserializer(), new DoubleSummaryFactory()); Assert.assertEquals(sketch2.getEstimate(), 1.0); SketchIterator<DoubleSummary> it = sketch2.iterator(); Assert.assertTrue(it.next()); Assert.assertEquals(it.getSummary().getValue(), 1.0); Assert.assertFalse(it.next()); // the same key, so still one unique sketch2.update(1, 1.0); Assert.assertEquals(sketch2.getEstimate(), 1.0); sketch2.update(2, 1.0); Assert.assertEquals(sketch2.getEstimate(), 2.0); }
@Test public void serializeDeserializeEstimationNoResizing() throws Exception { UpdatableSketch<Double, DoubleSummary> sketch1 = new UpdatableSketchBuilder<>( new DoubleSummaryFactory()).setResizeFactor(ResizeFactor.X1).build(); for (int j = 0; j < 10; j++) { for (int i = 0; i < 8192; i++) { sketch1.update(i, 1.0); } } sketch1.trim(); byte[] bytes = sketch1.toByteArray(); //for visual testing //TestUtil.writeBytesToFile(bytes, "UpdatableSketchWithDoubleSummary4K.bin"); Sketch<DoubleSummary> sketch2 = Sketches.heapifySketch(Memory.wrap(bytes), new DoubleSummaryDeserializer()); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), 8192, 8192 * 0.99); Assert.assertEquals(sketch1.getTheta(), sketch2.getTheta()); SketchIterator<DoubleSummary> it = sketch2.iterator(); int count = 0; while (it.next()) { Assert.assertEquals(it.getSummary().getValue(), 10.0); count++; } Assert.assertEquals(count, 4096); }
@Test public void serializeDeserializeEstimation() throws Exception { UpdatableSketch<Double, DoubleSummary> us = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); for (int i = 0; i < 8192; i++) us.update(i, 1.0); us.trim(); CompactSketch<DoubleSummary> sketch1 = us.compact(); byte[] bytes = sketch1.toByteArray(); // for visual testing //TestUtil.writeBytesToFile(bytes, "CompactSketchWithDoubleSummary4K.bin"); Sketch<DoubleSummary> sketch2 = Sketches.heapifySketch(Memory.wrap(bytes), new DoubleSummaryDeserializer()); Assert.assertFalse(sketch2.isEmpty()); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), sketch1.getEstimate()); Assert.assertEquals(sketch2.getThetaLong(), sketch1.getThetaLong()); SketchIterator<DoubleSummary> it = sketch2.iterator(); int count = 0; while (it.next()) { Assert.assertEquals(it.getSummary().getValue(), 1.0); count++; } Assert.assertEquals(count, 4096); }
@Test public void filledSketchShouldBehaveTheSame() { UpdatableSketch<Double, DoubleSummary> sketch = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); fillSketch(sketch, numberOfElements, 0.0); Filter<DoubleSummary> filter = new Filter<>(o -> true); Sketch<DoubleSummary> filteredSketch = filter.filter(sketch); Assert.assertEquals(filteredSketch.getEstimate(), sketch.getEstimate()); Assert.assertEquals(filteredSketch.getThetaLong(), sketch.getThetaLong()); Assert.assertFalse(filteredSketch.isEmpty()); Assert.assertEquals(filteredSketch.getLowerBound(1), sketch.getLowerBound(1)); Assert.assertEquals(filteredSketch.getUpperBound(1), sketch.getUpperBound(1)); }
@Test public void serializeDeserializeSampling() throws Exception { int sketchSize = 16384; int numberOfUniques = sketchSize; UpdatableSketch<Double, DoubleSummary> sketch1 = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()) .setNominalEntries(sketchSize).setSamplingProbability(0.5f).build(); for (int i = 0; i < numberOfUniques; i++) { sketch1.update(i, 1.0); } Sketch<DoubleSummary> sketch2 = Sketches.heapifySketch( Memory.wrap(sketch1.toByteArray()), new DoubleSummaryDeserializer()); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate() / numberOfUniques, 1.0, 0.01); Assert.assertEquals(sketch2.getRetainedEntries() / (double) numberOfUniques, 0.5, 0.01); Assert.assertEquals(sketch1.getTheta(), sketch2.getTheta()); }
@Test public void serialVersion1Compatibility() throws Exception { byte[] bytes = TestUtil.readBytesFromFile(getClass().getClassLoader() .getResource("UpdatableSketchWithDoubleSummary4K_serialVersion1.bin").getFile()); UpdatableSketch<Double, DoubleSummary> sketch = Sketches.heapifyUpdatableSketch( Memory.wrap(bytes), new DoubleSummaryDeserializer(), new DoubleSummaryFactory()); Assert.assertTrue(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 8192, 8192 * 0.99); Assert.assertEquals(sketch.getRetainedEntries(), 4096); int count = 0; SketchIterator<DoubleSummary> it = sketch.iterator(); while (it.next()) { Assert.assertEquals(it.getSummary().getValue(), 10.0); count++; } Assert.assertEquals(count, 4096); }
@Test public void filledSketchShouldFilterOutElements() { UpdatableSketch<Double, DoubleSummary> sketch = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); fillSketch(sketch, numberOfElements, 0.0); fillSketch(sketch, 2 * numberOfElements, 1.0); Filter<DoubleSummary> filter = new Filter<>(o -> o.getValue() < 0.5); Sketch<DoubleSummary> filteredSketch = filter.filter(sketch); Assert.assertEquals(filteredSketch.getEstimate(), (double) numberOfElements); Assert.assertEquals(filteredSketch.getThetaLong(), sketch.getThetaLong()); Assert.assertFalse(filteredSketch.isEmpty()); Assert.assertTrue(filteredSketch.getLowerBound(1) <= filteredSketch.getEstimate()); Assert.assertTrue(filteredSketch.getUpperBound(1) >= filteredSketch.getEstimate()); }
@Test public void emptyFromQuickSelectSketch() { UpdatableSketch<Double, DoubleSummary> us = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); CompactSketch<DoubleSummary> sketch = us.compact(); Assert.assertTrue(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 0.0); Assert.assertEquals(sketch.getLowerBound(1), 0.0); Assert.assertEquals(sketch.getUpperBound(1), 0.0); Assert.assertEquals(sketch.getRetainedEntries(), 0); Assert.assertEquals(sketch.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch.getTheta(), 1.0); SketchIterator<DoubleSummary> it = sketch.iterator(); Assert.assertNotNull(it); Assert.assertFalse(it.next()); }