@Test public void emptyFromNonPublicConstructorNullArray() { CompactSketch<DoubleSummary> sketch = new CompactSketch<DoubleSummary>(null, null, Long.MAX_VALUE, true); Assert.assertTrue(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 0.0); Assert.assertEquals(sketch.getLowerBound(1), 0.0); Assert.assertEquals(sketch.getUpperBound(1), 0.0); Assert.assertEquals(sketch.getRetainedEntries(), 0); Assert.assertEquals(sketch.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch.getTheta(), 1.0); SketchIterator<DoubleSummary> it = sketch.iterator(); Assert.assertNotNull(it); Assert.assertFalse(it.next()); }
@Test public void aNotBEmpty() { AnotB<DoubleSummary> aNotB = new AnotB<>(); // calling getResult() before calling update() should yield an empty set CompactSketch<DoubleSummary> result = aNotB.getResult(); Assert.assertEquals(result.getRetainedEntries(), 0); Assert.assertTrue(result.isEmpty()); Assert.assertEquals(result.getEstimate(), 0.0); Assert.assertEquals(result.getLowerBound(1), 0.0); Assert.assertEquals(result.getUpperBound(1), 0.0); aNotB.update(null, null); result = aNotB.getResult(); Assert.assertEquals(result.getRetainedEntries(), 0); Assert.assertTrue(result.isEmpty()); Assert.assertEquals(result.getEstimate(), 0.0); Assert.assertEquals(result.getLowerBound(1), 0.0); Assert.assertEquals(result.getUpperBound(1), 0.0); UpdatableSketch<Double, DoubleSummary> sketch = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); aNotB.update(sketch, sketch); result = aNotB.getResult(); Assert.assertEquals(result.getRetainedEntries(), 0); Assert.assertTrue(result.isEmpty()); Assert.assertEquals(result.getEstimate(), 0.0); Assert.assertEquals(result.getLowerBound(1), 0.0); Assert.assertEquals(result.getUpperBound(1), 0.0); }
/** * @param <S> Type of Summary * @return an empty instance of Sketch */ public static <S extends Summary> Sketch<S> createEmptySketch() { return new CompactSketch<S>(null, null, Long.MAX_VALUE, true); }
@Test public void unionEstimationMode() { int key = 0; UpdatableSketch<Double, DoubleSummary> sketch1 = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); for (int i = 0; i < 8192; i++) { sketch1.update(key++, 1.0); } key -= 4096; // overlap half of the entries UpdatableSketch<Double, DoubleSummary> sketch2 = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); for (int i = 0; i < 8192; i++) { sketch2.update(key++, 1.0); } Union<DoubleSummary> union = new Union<>(4096, new DoubleSummarySetOperations()); union.update(sketch1); union.update(sketch2); CompactSketch<DoubleSummary> result = union.getResult(); Assert.assertEquals(result.getEstimate(), 12288.0, 12288 * 0.01); Assert.assertTrue(result.getLowerBound(1) <= result.getEstimate()); Assert.assertTrue(result.getUpperBound(1) > result.getEstimate()); }
@Test public void serializeDeserializeEstimation() throws Exception { UpdatableSketch<Double, DoubleSummary> us = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); for (int i = 0; i < 8192; i++) us.update(i, 1.0); us.trim(); CompactSketch<DoubleSummary> sketch1 = us.compact(); byte[] bytes = sketch1.toByteArray(); // for visual testing //TestUtil.writeBytesToFile(bytes, "CompactSketchWithDoubleSummary4K.bin"); Sketch<DoubleSummary> sketch2 = Sketches.heapifySketch(Memory.wrap(bytes), new DoubleSummaryDeserializer()); Assert.assertFalse(sketch2.isEmpty()); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), sketch1.getEstimate()); Assert.assertEquals(sketch2.getThetaLong(), sketch1.getThetaLong()); SketchIterator<DoubleSummary> it = sketch2.iterator(); int count = 0; while (it.next()) { Assert.assertEquals(it.getSummary().getValue(), 1.0); count++; } Assert.assertEquals(count, 4096); }
int summariesBytesLength = 0; byte[][] summariesBytes = null; final int count = getRetainedEntries(); if (count > 0) { summariesBytes = new byte[count][]; ByteArrayUtil.putInt(bytes, offset, getRetainedEntries()); offset += Integer.BYTES; for (int i = 0; i < count; i++) {
@Test public void intersectionEmpty() { UpdatableSketch<Double, DoubleSummary> sketch = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); Intersection<DoubleSummary> intersection = new Intersection<>(new DoubleSummarySetOperations()); intersection.update(sketch); CompactSketch<DoubleSummary> result = intersection.getResult(); Assert.assertEquals(result.getRetainedEntries(), 0); Assert.assertTrue(result.isEmpty()); Assert.assertEquals(result.getEstimate(), 0.0); Assert.assertEquals(result.getLowerBound(1), 0.0); Assert.assertEquals(result.getUpperBound(1), 0.0); }
/** * Gets the result of this operation * @return the result of this operation as a CompactSketch */ public CompactSketch<S> getResult() { if (count_ == 0) { return new CompactSketch<S>(null, null, theta_, isEmpty_); } final CompactSketch<S> result = new CompactSketch<S>(Arrays.copyOfRange(keys_, 0, count_), Arrays.copyOfRange(summaries_, 0, count_), theta_, isEmpty_); reset(); return result; }
@Test public void unionMixedMode() { int key = 0; UpdatableSketch<Double, DoubleSummary> sketch1 = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); for (int i = 0; i < 1000; i++) { sketch1.update(key++, 1.0); //System.out.println("theta1=" + sketch1.getTheta() + " " + sketch1.getThetaLong()); } key -= 500; // overlap half of the entries UpdatableSketch<Double, DoubleSummary> sketch2 = new UpdatableSketchBuilder<> (new DoubleSummaryFactory()).setSamplingProbability(0.2f).build(); for (int i = 0; i < 20000; i++) { sketch2.update(key++, 1.0); //System.out.println("theta2=" + sketch2.getTheta() + " " + sketch2.getThetaLong()); } Union<DoubleSummary> union = new Union<>(4096, new DoubleSummarySetOperations()); union.update(sketch1); union.update(sketch2); CompactSketch<DoubleSummary> result = union.getResult(); Assert.assertEquals(result.getEstimate(), 20500.0, 20500 * 0.01); Assert.assertTrue(result.getLowerBound(1) <= result.getEstimate()); Assert.assertTrue(result.getUpperBound(1) > result.getEstimate()); }
int summariesBytesLength = 0; byte[][] summariesBytes = null; final int count = getRetainedEntries(); if (count > 0) { summariesBytes = new byte[count][]; ByteArrayUtil.putInt(bytes, offset, getRetainedEntries()); offset += Integer.BYTES; for (int i = 0; i < count; i++) {
@Test public void emptyFromNonPublicConstructor() { long[] keys = new long[0]; DoubleSummary[] summaries = (DoubleSummary[]) java.lang.reflect.Array.newInstance(DoubleSummary.class, 0); CompactSketch<DoubleSummary> sketch = new CompactSketch<DoubleSummary>(keys, summaries, Long.MAX_VALUE, true); Assert.assertTrue(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 0.0); Assert.assertEquals(sketch.getLowerBound(1), 0.0); Assert.assertEquals(sketch.getUpperBound(1), 0.0); Assert.assertEquals(sketch.getRetainedEntries(), 0); Assert.assertEquals(sketch.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch.getTheta(), 1.0); SketchIterator<DoubleSummary> it = sketch.iterator(); Assert.assertNotNull(it); Assert.assertFalse(it.next()); }
@Test public void intersectionNotEmptyNoEntries() { UpdatableSketch<Double, DoubleSummary> sketch1 = new UpdatableSketchBuilder<> (new DoubleSummaryFactory()).setSamplingProbability(0.01f).build(); sketch1.update("a", 1.0); // this happens to get rejected because of sampling with low probability Intersection<DoubleSummary> intersection = new Intersection<>(new DoubleSummarySetOperations()); intersection.update(sketch1); CompactSketch<DoubleSummary> result = intersection.getResult(); Assert.assertEquals(result.getRetainedEntries(), 0); Assert.assertFalse(result.isEmpty()); Assert.assertEquals(result.getEstimate(), 0.0); Assert.assertEquals(result.getLowerBound(1), 0.0, 0.0001); Assert.assertTrue(result.getUpperBound(1) > 0); }
/** * Gets the internal set as a CompactSketch * @return result of the intersections so far */ public CompactSketch<S> getResult() { if (isFirstCall_) { throw new SketchesStateException( "getResult() with no intervening intersections is not a legal result."); } if (sketch_ == null) { return new CompactSketch<>(null, null, theta_, isEmpty_); } return sketch_.compact(); }
@Test public void emptyFromQuickSelectSketch() { UpdatableSketch<Double, DoubleSummary> us = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); CompactSketch<DoubleSummary> sketch = us.compact(); Assert.assertTrue(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 0.0); Assert.assertEquals(sketch.getLowerBound(1), 0.0); Assert.assertEquals(sketch.getUpperBound(1), 0.0); Assert.assertEquals(sketch.getRetainedEntries(), 0); Assert.assertEquals(sketch.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch.getTheta(), 1.0); SketchIterator<DoubleSummary> it = sketch.iterator(); Assert.assertNotNull(it); Assert.assertFalse(it.next()); }
@Test public void aNotBEmptyB() { UpdatableSketch<Double, DoubleSummary> sketchA = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); sketchA.update(1, 1.0); sketchA.update(2, 1.0); UpdatableSketch<Double, DoubleSummary> sketchB = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); AnotB<DoubleSummary> aNotB = new AnotB<>(); aNotB.update(sketchA, sketchB); CompactSketch<DoubleSummary> result = aNotB.getResult(); Assert.assertEquals(result.getRetainedEntries(), 2); Assert.assertFalse(result.isEmpty()); Assert.assertEquals(result.getEstimate(), 2.0); Assert.assertEquals(result.getLowerBound(1), 2.0); Assert.assertEquals(result.getUpperBound(1), 2.0); // same thing, but compact sketches aNotB.update(sketchA.compact(), sketchB.compact()); result = aNotB.getResult(); Assert.assertEquals(result.getRetainedEntries(), 2); Assert.assertFalse(result.isEmpty()); Assert.assertEquals(result.getEstimate(), 2.0); Assert.assertEquals(result.getLowerBound(1), 2.0); Assert.assertEquals(result.getUpperBound(1), 2.0); }
/** * Instantiate Sketch from a given Memory * @param <S> Type of Summary * @param mem Memory object representing a Sketch * @param deserializer instance of SummaryDeserializer * @return Sketch created from its Memory representation */ public static <S extends Summary> Sketch<S> heapifySketch(final Memory mem, final SummaryDeserializer<S> deserializer) { final SerializerDeserializer.SketchType sketchType = SerializerDeserializer.getSketchType(mem); if (sketchType == SerializerDeserializer.SketchType.QuickSelectSketch) { return new QuickSelectSketch<S>(mem, deserializer, null); } return new CompactSketch<S>(mem, deserializer); }
@Test public void exactModeFromQuickSelectSketch() { UpdatableSketch<Double, DoubleSummary> us = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); us.update(1, 1.0); us.update(2, 1.0); us.update(3, 1.0); us.update(1, 1.0); us.update(2, 1.0); us.update(3, 1.0); CompactSketch<DoubleSummary> sketch = us.compact(); Assert.assertFalse(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 3.0); Assert.assertEquals(sketch.getLowerBound(1), 3.0); Assert.assertEquals(sketch.getUpperBound(1), 3.0); Assert.assertEquals(sketch.getRetainedEntries(), 3); Assert.assertEquals(sketch.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch.getTheta(), 1.0); SketchIterator<DoubleSummary> it = sketch.iterator(); int count = 0; while (it.next()) { Assert.assertEquals(it.getSummary().getValue(), 2.0); count++; } Assert.assertEquals(count, 3); }
@Test public void aNotBEmptyA() { UpdatableSketch<Double, DoubleSummary> sketchA = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); UpdatableSketch<Double, DoubleSummary> sketchB = new UpdatableSketchBuilder<>(new DoubleSummaryFactory()).build(); sketchB.update(1, 1.0); sketchB.update(2, 1.0); AnotB<DoubleSummary> aNotB = new AnotB<>(); aNotB.update(sketchA, sketchB); CompactSketch<DoubleSummary> result = aNotB.getResult(); Assert.assertEquals(result.getRetainedEntries(), 0); Assert.assertTrue(result.isEmpty()); Assert.assertEquals(result.getEstimate(), 0.0); Assert.assertEquals(result.getLowerBound(1), 0.0); Assert.assertEquals(result.getUpperBound(1), 0.0); }
/** * Converts the current state of the sketch into a compact sketch * @return compact sketch */ public CompactSketch<S> compact() { if (getRetainedEntries() == 0) { return new CompactSketch<>(null, null, theta_, isEmpty_); } final long[] keys = new long[getRetainedEntries()]; @SuppressWarnings("unchecked") final S[] summaries = (S[]) Array.newInstance(summaries_.getClass().getComponentType(), getRetainedEntries()); int i = 0; for (int j = 0; j < keys_.length; j++) { if (summaries_[j] != null) { keys[i] = keys_[j]; summaries[i] = summaries_[j].copy(); i++; } } return new CompactSketch<>(keys, summaries, theta_, isEmpty_); }
intersection.update(sketch2); CompactSketch<DoubleSummary> result = intersection.getResult(); Assert.assertEquals(result.getRetainedEntries(), 1); Assert.assertFalse(result.isEmpty()); Assert.assertEquals(result.getEstimate(), 1.0); Assert.assertEquals(result.getLowerBound(1), 1.0); Assert.assertEquals(result.getUpperBound(1), 1.0); SketchIterator<DoubleSummary> it = result.iterator(); Assert.assertTrue(it.next()); Assert.assertEquals(it.getSummary().getValue(), 4.0); intersection.update(null); result = intersection.getResult(); Assert.assertTrue(result.isEmpty()); Assert.assertFalse(result.isEstimationMode()); Assert.assertEquals(result.getEstimate(), 0.0); Assert.assertEquals(result.getUpperBound(1), 0.0); Assert.assertEquals(result.getLowerBound(1), 0.0); Assert.assertEquals(result.getTheta(), 1.0);