private void getNoMatchSetFromSketch(final ArrayOfDoublesSketch sketch) { count_ = sketch.getRetainedEntries(); keys_ = new long[count_]; values_ = new double[count_ * numValues_]; final ArrayOfDoublesSketchIterator it = sketch.iterator(); int i = 0; while (it.next()) { keys_[i] = it.getKey(); System.arraycopy(it.getValues(), 0, values_, i * numValues_, numValues_); i++; } }
private static SummaryStatistics[] getStats(final ArrayOfDoublesSketch sketch) { final SummaryStatistics[] stats = new SummaryStatistics[sketch.getNumValues()]; Arrays.setAll(stats, i -> new SummaryStatistics()); final ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { final double[] values = it.getValues(); for (int i = 0; i < values.length; i++) { stats[i].addValue(values[i]); } } return stats; }
@Test public void isEmpty() { ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder(). build(WritableMemory.wrap(new byte[1000000])); Assert.assertTrue(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 0.0); Assert.assertEquals(sketch.getUpperBound(1), 0.0); Assert.assertEquals(sketch.getLowerBound(1), 0.0); Assert.assertEquals(sketch.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch.getTheta(), 1.0); ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { Assert.fail("empty sketch expected"); } }
private static long[] convertToHashTable(final ArrayOfDoublesSketch sketch) { final int size = Math.max( ceilingPowerOf2((int) Math.ceil(sketch.getRetainedEntries() / REBUILD_THRESHOLD)), 1 << MIN_LG_NOM_LONGS ); final long[] hashTable = new long[size]; final ArrayOfDoublesSketchIterator it = sketch.iterator(); final int lgSize = Integer.numberOfTrailingZeros(size); while (it.next()) { HashOperations.hashInsertOnly(hashTable, lgSize, it.getKey()); } return hashTable; }
@Override public DoublesSketch compute(final Map<String, Object> combinedAggregators) { final ArrayOfDoublesSketch sketch = (ArrayOfDoublesSketch) getField().compute(combinedAggregators); final UpdateDoublesSketch qs = DoublesSketch.builder().setK(k).build(); final ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { qs.update(it.getValues()[column - 1]); // convert 1-based column number to zero-based index } return qs; }
@Test public void isEmpty() { ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder().build(); Assert.assertTrue(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 0.0); Assert.assertEquals(sketch.getUpperBound(1), 0.0); Assert.assertEquals(sketch.getLowerBound(1), 0.0); Assert.assertEquals(sketch.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch.getTheta(), 1.0); ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { Assert.fail("empty sketch expected"); } Assert.assertNotNull(sketch.toString()); }
private static long[] convertToHashTable(final ArrayOfDoublesSketch sketch) { final int size = Math.max( ceilingPowerOf2((int) Math.ceil(sketch.getRetainedEntries() / REBUILD_THRESHOLD)), 1 << MIN_LG_NOM_LONGS ); final long[] hashTable = new long[size]; final ArrayOfDoublesSketchIterator it = sketch.iterator(); final int lgSize = Integer.numberOfTrailingZeros(size); while (it.next()) { HashOperations.hashInsertOnly(hashTable, lgSize, it.getKey()); } return hashTable; }
@Override public double[] compute(final Map<String, Object> combinedAggregators) { final ArrayOfDoublesSketch sketch = (ArrayOfDoublesSketch) getField().compute(combinedAggregators); final SummaryStatistics[] stats = new SummaryStatistics[sketch.getNumValues()]; Arrays.setAll(stats, i -> new SummaryStatistics()); final ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { final double[] values = it.getValues(); for (int i = 0; i < values.length; i++) { stats[i].addValue(values[i]); } } final double[] means = new double[sketch.getNumValues()]; Arrays.setAll(means, i -> stats[i].getMean()); return means; }
@Test public void emptyFromQuickSelectSketch() { ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(); ArrayOfDoublesCompactSketch sketch = us.compact(); Assert.assertTrue(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 0.0); Assert.assertEquals(sketch.getLowerBound(1), 0.0); Assert.assertEquals(sketch.getUpperBound(1), 0.0); Assert.assertEquals(sketch.getRetainedEntries(), 0); Assert.assertEquals(sketch.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch.getTheta(), 1.0); Assert.assertNotNull(sketch.getValues()); Assert.assertEquals(sketch.getValues().length, 0); ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { Assert.fail("empty sketch expected"); } }
values_ = new double[noMatchSize * numValues_]; final ArrayOfDoublesSketchIterator it = a.iterator(); while (it.next()) { final int index = HashOperations.hashSearch(hashTable, lgHashTableSize, it.getKey()); if (index == -1) { keys_[count_] = it.getKey(); System.arraycopy(it.getValues(), 0, values_, count_ * numValues_, numValues_); count_++;
@Override public double[] compute(final Map<String, Object> combinedAggregators) { final ArrayOfDoublesSketch sketch = (ArrayOfDoublesSketch) getField().compute(combinedAggregators); final SummaryStatistics[] stats = new SummaryStatistics[sketch.getNumValues()]; Arrays.setAll(stats, i -> new SummaryStatistics()); final ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { final double[] values = it.getValues(); for (int i = 0; i < values.length; i++) { stats[i].addValue(values[i]); } } final double[] variances = new double[sketch.getNumValues()]; Arrays.setAll(variances, i -> stats[i].getVariance()); return variances; }
@Test public void emptyFromQuickSelectSketch() { ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.wrap(new byte[1000000])); ArrayOfDoublesCompactSketch sketch = us.compact(WritableMemory.wrap(new byte[1000000])); Assert.assertTrue(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 0.0); Assert.assertEquals(sketch.getLowerBound(1), 0.0); Assert.assertEquals(sketch.getUpperBound(1), 0.0); Assert.assertEquals(sketch.getRetainedEntries(), 0); Assert.assertEquals(sketch.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch.getTheta(), 1.0); Assert.assertNotNull(sketch.getValues()); Assert.assertEquals(sketch.getValues().length, 0); ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { Assert.fail("empty sketch expected"); } }
sketch_ = createSketch(sketchIn.getRetainedEntries(), numValues_, seed_); final ArrayOfDoublesSketchIterator it = sketchIn.iterator(); while (it.next()) { sketch_.insert(it.getKey(), it.getValues()); int matchCount = 0; final ArrayOfDoublesSketchIterator it = sketchIn.iterator(); while (it.next()) { final double[] values = sketch_.find(it.getKey()); if (values != null) { matchKeys[matchCount] = it.getKey(); matchValues[matchCount] = combiner.combine(values, it.getValues()); matchCount++;
@Test public void heapEstimationModeFullOverlapTwoValuesAndDownsizing() { int key = 0; ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(2).build(); for (int i = 0; i < 8192; i++) { sketch1.update(key++, new double[] {1.0, 2.0}); } key = 0; // full overlap ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(2).build(); for (int i = 0; i < 8192; i++) { sketch2.update(key++, new double[] {1.0, 2.0}); } ArrayOfDoublesUnion union = new ArrayOfDoublesSetOperationBuilder().setNumberOfValues(2).setNominalEntries(1024).buildUnion(); union.update(sketch1); union.update(sketch2); ArrayOfDoublesCompactSketch result = union.getResult(); Assert.assertFalse(result.isEmpty()); Assert.assertTrue(result.isEstimationMode()); Assert.assertEquals(result.getEstimate(), 8192.0, 8192 * 0.01); Assert.assertEquals(result.getRetainedEntries(), 1024); // union was downsampled ArrayOfDoublesSketchIterator it = result.iterator(); double[] expected = {2, 4}; while (it.next()) { Assert.assertEquals(it.getValues(), expected, Arrays.toString(it.getValues()) + " != " + Arrays.toString(expected)); } }
Assert.assertEquals(sketch.getTheta(), 1.0); ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { Assert.fail("empty sketch expected");
/** * Updates the union by adding a set of entries from a given sketch * @param sketchIn sketch to add to the union */ public void update(final ArrayOfDoublesSketch sketchIn) { if (sketchIn == null) { return; } Util.checkSeedHashes(seedHash_, sketchIn.getSeedHash()); if (sketch_.getNumValues() != sketchIn.getNumValues()) { throw new SketchesArgumentException("Incompatible sketches: number of values mismatch " + sketch_.getNumValues() + " and " + sketchIn.getNumValues()); } if (sketchIn.isEmpty()) { return; } if (sketchIn.getThetaLong() < theta_) { theta_ = sketchIn.getThetaLong(); } final ArrayOfDoublesSketchIterator it = sketchIn.iterator(); while (it.next()) { sketch_.merge(it.getKey(), it.getValues()); } }
Assert.assertEquals(result.getUpperBound(1), 5.0); ArrayOfDoublesSketchIterator it = result.iterator(); while (it.next()) { Assert.assertEquals(it.getValues(), new double[] {1}); Assert.assertEquals(result.getUpperBound(1), 5.0); it = result.iterator(); while (it.next()) { Assert.assertEquals(it.getValues(), new double[] {1});
Assert.assertEquals(sketch.getTheta(), 1.0); ArrayOfDoublesSketchIterator it = sketch.iterator(); while (it.next()) { Assert.fail("empty sketch expected");
final ArrayOfDoublesSketchIterator it = sketch.iterator(); int actualCount = 0; while (it.next()) { if (it.getKey() < theta_) { dstMem.putLong(keyOffset, it.getKey()); dstMem.putDoubleArray(valuesOffset, it.getValues(), 0, numValues_); keyOffset += SIZE_OF_KEY_BYTES; valuesOffset += SIZE_OF_VALUE_BYTES * numValues_;