private static SummaryStatistics[] getStats(final ArrayOfDoublesSketch sketch) { final SummaryStatistics[] stats = new SummaryStatistics[sketch.getNumValues()]; Arrays.setAll(stats, i -> new SummaryStatistics()); final ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { final double[] values = it.getValues(); for (int i = 0; i < values.length; i++) { stats[i].addValue(values[i]); } } return stats; }
@Override public DoublesSketch compute(final Map<String, Object> combinedAggregators) { final ArrayOfDoublesSketch sketch = (ArrayOfDoublesSketch) getField().compute(combinedAggregators); final UpdateDoublesSketch qs = DoublesSketch.builder().setK(k).build(); final ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { qs.update(it.getValues()[column - 1]); // convert 1-based column number to zero-based index } return qs; }
@Override public double[] compute(final Map<String, Object> combinedAggregators) { final ArrayOfDoublesSketch sketch = (ArrayOfDoublesSketch) getField().compute(combinedAggregators); final SummaryStatistics[] stats = new SummaryStatistics[sketch.getNumValues()]; Arrays.setAll(stats, i -> new SummaryStatistics()); final ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { final double[] values = it.getValues(); for (int i = 0; i < values.length; i++) { stats[i].addValue(values[i]); } } final double[] means = new double[sketch.getNumValues()]; Arrays.setAll(means, i -> stats[i].getMean()); return means; }
@Override public double[] compute(final Map<String, Object> combinedAggregators) { final ArrayOfDoublesSketch sketch = (ArrayOfDoublesSketch) getField().compute(combinedAggregators); final SummaryStatistics[] stats = new SummaryStatistics[sketch.getNumValues()]; Arrays.setAll(stats, i -> new SummaryStatistics()); final ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { final double[] values = it.getValues(); for (int i = 0; i < values.length; i++) { stats[i].addValue(values[i]); } } final double[] variances = new double[sketch.getNumValues()]; Arrays.setAll(variances, i -> stats[i].getVariance()); return variances; }
private void getNoMatchSetFromSketch(final ArrayOfDoublesSketch sketch) { count_ = sketch.getRetainedEntries(); keys_ = new long[count_]; values_ = new double[count_ * numValues_]; final ArrayOfDoublesSketchIterator it = sketch.iterator(); int i = 0; while (it.next()) { keys_[i] = it.getKey(); System.arraycopy(it.getValues(), 0, values_, i * numValues_, numValues_); i++; } }
if (key < theta_) { keys_[i] = key; System.arraycopy(it.getValues(), 0, values_, i * numValues_, numValues_); i++;
if (index == -1) { keys_[count_] = it.getKey(); System.arraycopy(it.getValues(), 0, values_, count_ * numValues_, numValues_); count_++;
@Test public void heapEstimationModeFullOverlapTwoValuesAndDownsizing() { int key = 0; ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(2).build(); for (int i = 0; i < 8192; i++) { sketch1.update(key++, new double[] {1.0, 2.0}); } key = 0; // full overlap ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(2).build(); for (int i = 0; i < 8192; i++) { sketch2.update(key++, new double[] {1.0, 2.0}); } ArrayOfDoublesUnion union = new ArrayOfDoublesSetOperationBuilder().setNumberOfValues(2).setNominalEntries(1024).buildUnion(); union.update(sketch1); union.update(sketch2); ArrayOfDoublesCompactSketch result = union.getResult(); Assert.assertFalse(result.isEmpty()); Assert.assertTrue(result.isEstimationMode()); Assert.assertEquals(result.getEstimate(), 8192.0, 8192 * 0.01); Assert.assertEquals(result.getRetainedEntries(), 1024); // union was downsampled ArrayOfDoublesSketchIterator it = result.iterator(); double[] expected = {2, 4}; while (it.next()) { Assert.assertEquals(it.getValues(), expected, Arrays.toString(it.getValues()) + " != " + Arrays.toString(expected)); } }
/** * Updates the union by adding a set of entries from a given sketch * @param sketchIn sketch to add to the union */ public void update(final ArrayOfDoublesSketch sketchIn) { if (sketchIn == null) { return; } Util.checkSeedHashes(seedHash_, sketchIn.getSeedHash()); if (sketch_.getNumValues() != sketchIn.getNumValues()) { throw new SketchesArgumentException("Incompatible sketches: number of values mismatch " + sketch_.getNumValues() + " and " + sketchIn.getNumValues()); } if (sketchIn.isEmpty()) { return; } if (sketchIn.getThetaLong() < theta_) { theta_ = sketchIn.getThetaLong(); } final ArrayOfDoublesSketchIterator it = sketchIn.iterator(); while (it.next()) { sketch_.merge(it.getKey(), it.getValues()); } }
@Test public void exactMode() { ArrayOfDoublesUpdatableSketch sketchA = new ArrayOfDoublesUpdatableSketchBuilder().build(); sketchA.update(1, new double[] {1}); sketchA.update(2, new double[] {1}); sketchA.update(3, new double[] {1}); sketchA.update(4, new double[] {1}); sketchA.update(5, new double[] {1}); ArrayOfDoublesUpdatableSketch sketchB = new ArrayOfDoublesUpdatableSketchBuilder().build(); sketchB.update(3, new double[] {1}); sketchB.update(4, new double[] {1}); sketchB.update(5, new double[] {1}); sketchB.update(6, new double[] {1}); sketchB.update(7, new double[] {1}); ArrayOfDoublesAnotB aNotB = new ArrayOfDoublesSetOperationBuilder().buildAnotB(); aNotB.update(sketchA, sketchB); ArrayOfDoublesCompactSketch result = aNotB.getResult(); Assert.assertFalse(result.isEmpty()); Assert.assertEquals(result.getRetainedEntries(), 2); Assert.assertEquals(result.getEstimate(), 2.0); Assert.assertEquals(result.getLowerBound(1), 2.0); Assert.assertEquals(result.getUpperBound(1), 2.0); ArrayOfDoublesSketchIterator it = result.iterator(); while (it.next()) { Assert.assertEquals(it.getValues(), new double[] {1}); } }
@Test public void exactModeCustomSeed() { long seed = 1234567890; ArrayOfDoublesUpdatableSketch sketchA = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(seed).build(); sketchA.update(1, new double[] {1}); sketchA.update(2, new double[] {1}); sketchA.update(3, new double[] {1}); sketchA.update(4, new double[] {1}); sketchA.update(5, new double[] {1}); ArrayOfDoublesUpdatableSketch sketchB = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(seed).build(); sketchB.update(3, new double[] {1}); sketchB.update(4, new double[] {1}); sketchB.update(5, new double[] {1}); sketchB.update(6, new double[] {1}); sketchB.update(7, new double[] {1}); ArrayOfDoublesAnotB aNotB = new ArrayOfDoublesSetOperationBuilder().setSeed(seed).buildAnotB(); aNotB.update(sketchA, sketchB); ArrayOfDoublesCompactSketch result = aNotB.getResult(); Assert.assertFalse(result.isEmpty()); Assert.assertEquals(result.getRetainedEntries(), 2); Assert.assertEquals(result.getEstimate(), 2.0); Assert.assertEquals(result.getLowerBound(1), 2.0); Assert.assertEquals(result.getUpperBound(1), 2.0); ArrayOfDoublesSketchIterator it = result.iterator(); while (it.next()) { Assert.assertEquals(it.getValues(), new double[] {1}); } }
if (it.getKey() < theta_) { dstMem.putLong(keyOffset, it.getKey()); dstMem.putDoubleArray(valuesOffset, it.getValues(), 0, numValues_); keyOffset += SIZE_OF_KEY_BYTES; valuesOffset += SIZE_OF_VALUE_BYTES * numValues_;
private void getNoMatchSetFromSketch(final ArrayOfDoublesSketch sketch) { count_ = sketch.getRetainedEntries(); keys_ = new long[count_]; values_ = new double[count_ * numValues_]; final ArrayOfDoublesSketchIterator it = sketch.iterator(); int i = 0; while (it.next()) { keys_[i] = it.getKey(); System.arraycopy(it.getValues(), 0, values_, i * numValues_, numValues_); i++; } }
private static SummaryStatistics[] getStats(final ArrayOfDoublesSketch sketch) { final SummaryStatistics[] stats = new SummaryStatistics[sketch.getNumValues()]; Arrays.setAll(stats, i -> new SummaryStatistics()); final ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { final double[] values = it.getValues(); for (int i = 0; i < values.length; i++) { stats[i].addValue(values[i]); } } return stats; }
@Override public DoublesSketch compute(final Map<String, Object> combinedAggregators) { final ArrayOfDoublesSketch sketch = (ArrayOfDoublesSketch) getField().compute(combinedAggregators); final UpdateDoublesSketch qs = UpdateDoublesSketch.builder().setK(k).build(); final ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { qs.update(it.getValues()[column - 1]); // convert 1-based column number to zero-based index } return qs; }
@Override public double[] compute(final Map<String, Object> combinedAggregators) { final ArrayOfDoublesSketch sketch = (ArrayOfDoublesSketch) getField().compute(combinedAggregators); final SummaryStatistics[] stats = new SummaryStatistics[sketch.getNumValues()]; Arrays.setAll(stats, i -> new SummaryStatistics()); final ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { final double[] values = it.getValues(); for (int i = 0; i < values.length; i++) { stats[i].addValue(values[i]); } } final double[] variances = new double[sketch.getNumValues()]; Arrays.setAll(variances, i -> stats[i].getVariance()); return variances; }
@Override public double[] compute(final Map<String, Object> combinedAggregators) { final ArrayOfDoublesSketch sketch = (ArrayOfDoublesSketch) getField().compute(combinedAggregators); final SummaryStatistics[] stats = new SummaryStatistics[sketch.getNumValues()]; Arrays.setAll(stats, i -> new SummaryStatistics()); final ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { final double[] values = it.getValues(); for (int i = 0; i < values.length; i++) { stats[i].addValue(values[i]); } } final double[] means = new double[sketch.getNumValues()]; Arrays.setAll(means, i -> stats[i].getMean()); return means; }