@Override protected DoublesSketch _apply(final DoublesSketch a, final DoublesSketch b) { final DoublesUnion union = DoublesUnion.builder().setMaxK(a.getK()).build(); union.update(a); union.update(b); return union.getResult(); } }
/** * Get the rank error normalized as a fraction between zero and one. * The error of this sketch is specified as a fraction of the normalized rank of the hypothetical * sorted stream of items presented to the sketch. * * <p>Suppose the sketch is presented with N values. The raw rank (0 to N-1) of an item * would be its index position in the sorted version of the input stream. If we divide the * raw rank by N, it becomes the normalized rank, which is between 0 and 1.0. * * <p>For example, choosing a K of 256 yields a normalized rank error of less than 1%. * The upper bound on the median value obtained by getQuantile(0.5) would be the value in the * hypothetical ordered stream of values at the normalized rank of 0.51. * The lower bound would be the value in the hypothetical ordered stream of values at the * normalized rank of 0.49. * * <p>The error of this sketch cannot be translated into an error (relative or absolute) of the * returned quantile values. * * @return the rank error normalized as a fraction between zero and one. * @deprecated replaced by {@link #getNormalizedRankError(boolean)} */ @Deprecated public double getNormalizedRankError() { return Util.getNormalizedRankError(getK(), true); }
/** * Computes the number of retained items (samples) in the sketch * @return the number of retained items (samples) in the sketch */ public int getRetainedItems() { return Util.computeRetainedItems(getK(), getN()); }
/** * Returns the number of bytes this sketch would require to store in compact form, which is not * updatable. * @return the number of bytes this sketch would require to store in compact form. */ public int getCompactStorageBytes() { return getCompactStorageBytes(getK(), getN()); }
/** * Returns the number of bytes this sketch would require to store in updatable form. * This uses roughly 2X the storage of the compact form. * @return the number of bytes this sketch would require to store in updatable form. */ public int getUpdatableStorageBytes() { return getUpdatableStorageBytes(getK(), getN()); }
/** * Returns a Heap DoublesUnion object that has been initialized with the data from the given * sketch. * * @param sketch A DoublesSketch to be used as a source of data only and will not be modified. * @return a DoublesUnion object */ static DoublesUnionImpl heapifyInstance(final DoublesSketch sketch) { final int k = sketch.getK(); final DoublesUnionImpl union = new DoublesUnionImpl(k); union.maxK_ = k; union.gadget_ = copyToHeap(sketch); return union; }
@Test public void checkResultAndResetDirect() { final DoublesSketch qs1 = buildAndLoadDQS(256, 0); final DoublesUnion union = DoublesUnion.heapify(qs1); final DoublesSketch qs2 = union.getResultAndReset(); assertEquals(qs2.getK(), 256); }
@Test public void checkResultAndReset() { final DoublesSketch qs1 = buildAndLoadQS(256, 0); final DoublesUnion union = DoublesUnion.heapify(qs1); final DoublesSketch qs2 = union.getResultAndReset(); assertEquals(qs2.getK(), 256); }
static byte[] toByteArray(final DoublesSketch sketch, final boolean ordered, final boolean compact) { final boolean empty = sketch.isEmpty(); //create the flags byte final int flags = (empty ? EMPTY_FLAG_MASK : 0) | (ordered ? ORDERED_FLAG_MASK : 0) | (compact ? (COMPACT_FLAG_MASK | READ_ONLY_FLAG_MASK) : 0); if (empty && !sketch.isDirect()) { //empty & on-heap final byte[] outByteArr = new byte[Long.BYTES]; final WritableMemory memOut = WritableMemory.wrap(outByteArr); final int preLongs = 1; insertPre0(memOut, preLongs, flags, sketch.getK()); return outByteArr; } //not empty || direct; flags passed for convenience return convertToByteArray(sketch, flags, ordered, compact); }
@Test public void checkUnionUpdateLogicDirectDownsampled() { final DirectUpdateDoublesSketch qs1 = (DirectUpdateDoublesSketch) buildAndLoadDQS(256, 1000); final DirectUpdateDoublesSketch qs2 = (DirectUpdateDoublesSketch) buildAndLoadDQS(128, 2000); final DoublesSketch result = DoublesUnionImpl.updateLogic(128, qs1, qs2); assertEquals(result.getMaxValue(), 2000.0, 0.0); assertEquals(result.getMinValue(), 1.0, 0.0); assertEquals(result.getN(), 3000); assertEquals(result.getK(), 128); }
@Test public void checkReverseMerge() { int k = PreambleUtil.DEFAULT_K; DoublesSketch qs1 = buildAndLoadQS(k, 1000, 0); DoublesSketch qs2 = buildAndLoadQS(2*k,1000, 1000); DoublesUnion union = DoublesUnion.heapify(qs2); union.update(qs1); //attempt merge into larger k DoublesSketch result = union.getResult(); assertEquals(result.getK(), k); }
@Test public void checkUnion2Direct() { final DoublesSketch qs1 = buildAndLoadDQS(256, 1000); final DoublesSketch qs2 = buildAndLoadDQS(128, 1000); final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(); //virgin 256 assertEquals(union.getEffectiveK(), 256); union.update(qs1); final DoublesSketch res1 = union.getResult(); //println(res1.toString()); assertEquals(res1.getN(), 1000); assertEquals(res1.getK(), 256); union.update(qs2); final DoublesSketch res2 = union.getResult(); assertEquals(res2.getN(), 2000); assertEquals(res2.getK(), 128); assertEquals(union.getEffectiveK(), 128); println(union.toString()); }
@Test public void checkUnion2() { final DoublesSketch qs1 = buildAndLoadQS(256, 1000).compact(); final DoublesSketch qs2 = buildAndLoadQS(128, 1000); final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(); //virgin 256 assertEquals(union.getEffectiveK(), 256); union.update(qs1); final DoublesSketch res1 = union.getResult(); //println(res1.toString()); assertEquals(res1.getN(), 1000); assertEquals(res1.getK(), 256); union.update(qs2); final DoublesSketch res2 = union.getResult(); assertEquals(res2.getN(), 2000); assertEquals(res2.getK(), 128); assertEquals(union.getEffectiveK(), 128); println(union.toString()); }
/** * Constructs the Auxiliary structure from the DoublesSketch * @param qs a DoublesSketch */ DoublesAuxiliary(final DoublesSketch qs ) { final int k = qs.getK(); final long n = qs.getN(); final long bitPattern = qs.getBitPattern(); final int numSamples = qs.getRetainedItems(); final DoublesSketchAccessor sketchAccessor = DoublesSketchAccessor.wrap(qs); final double[] itemsArr = new double[numSamples]; final long[] cumWtsArr = new long[numSamples + 1]; // the extra slot is very important // Populate from DoublesSketch: // copy over the "levels" and then the base buffer, all with appropriate weights populateFromDoublesSketch(k, n, bitPattern, sketchAccessor, itemsArr, cumWtsArr); // Sort the first "numSamples" slots of the two arrays in tandem, // taking advantage of the already sorted blocks of length k blockyTandemMergeSort(itemsArr, cumWtsArr, numSamples, k); final long total = QuantilesHelper.convertToPrecedingCummulative(cumWtsArr); assert total == n; auxN_ = n; auxSamplesArr_ = itemsArr; auxCumWtsArr_ = cumWtsArr; }
@Test public void checkUnion3() { //Union is direct, empty and with larger K than valid input final int k1 = 128; final int n1 = 2 * k1; final int k2 = 256; final int n2 = 2000; final DoublesSketch sketchIn1 = buildAndLoadQS(k1, n1); final int bytes = DoublesSketch.getUpdatableStorageBytes(k2, n2);//just for size final WritableMemory mem = WritableMemory.wrap(new byte[bytes]); final DoublesUnion union = DoublesUnion.builder().setMaxK(k2).build(mem); //virgin 256 union.update(sketchIn1); assertEquals(union.getMaxK(), k2); assertEquals(union.getEffectiveK(), k1); final DoublesSketch result = union.getResult(); assertEquals(result.getMaxValue(), n1, 0.0); assertEquals(result.getMinValue(), 1.0, 0.0); assertEquals(result.getK(), k1); }
@Test public void checkUnion3Direct() { //Union is direct, empty and with larger K than valid input final int k1 = 128; final int n1 = 2 * k1; final int k2 = 256; final int n2 = 2000; final DoublesSketch sketchIn1 = buildAndLoadDQS(k1, n1); final int bytes = DoublesSketch.getUpdatableStorageBytes(k2, n2);//just for size final WritableMemory mem = WritableMemory.wrap(new byte[bytes]); final DoublesUnion union = DoublesUnion.builder().setMaxK(k2).build(mem); //virgin 256 union.update(sketchIn1); assertEquals(union.getMaxK(), k2); assertEquals(union.getEffectiveK(), k1); final DoublesSketch result = union.getResult(); assertEquals(result.getMaxValue(), n1, 0.0); assertEquals(result.getMinValue(), 1.0, 0.0); assertEquals(result.getK(), k1); }
private static boolean sameStructurePredicate(final DoublesSketch mq1, final DoublesSketch mq2) { final boolean b1 = ( (mq1.getK() == mq2.getK()) && (mq1.getN() == mq2.getN()) && (mq1.getCombinedBufferItemCapacity() >= Util.computeCombinedBufferItemCapacity(mq1.getK(), mq1.getN())) && (mq2.getCombinedBufferItemCapacity() >= Util.computeCombinedBufferItemCapacity(mq2.getK(), mq2.getN())) && (mq1.getBaseBufferCount() == mq2.getBaseBufferCount()) && (mq1.getBitPattern() == mq2.getBitPattern()) ); final boolean b2; if (mq1.isEmpty()) { b2 = (Double.isNaN(mq1.getMinValue())) && (Double.isNaN(mq2.getMinValue()) && Double.isNaN(mq1.getMaxValue())) && (Double.isNaN(mq2.getMaxValue())); } else { b2 = (mq1.getMinValue() == mq2.getMinValue()) && (mq1.getMaxValue() == mq2.getMaxValue()); } return b1 && b2; }
@Test public void checkUnion5() { //Union is direct, valid and with larger K than valid input final int k2 = 4; final int n2 = 2 * k2; //8 final int bytes = DoublesSketch.getUpdatableStorageBytes(256, 50);//big enough final WritableMemory skMem = WritableMemory.wrap(new byte[bytes]); DoublesSketch.builder().setK(256).build(skMem); final DoublesUnion union = DoublesUnionImpl.heapifyInstance(skMem); assertEquals(union.getResult().getN(), 0); assertEquals(union.getMaxK(), 256); assertEquals(union.getEffectiveK(), 256); final DoublesSketch result = union.getResult(); assertEquals(result.getK(), 256); final DoublesSketch sketchIn2 = buildAndLoadQS(k2, n2, 17); union.update(sketchIn2); println("\nFinal" + union.getResult().toString(true, true)); assertEquals(union.getResult().getN(), n2); }
@Test public void checkUnion5Direct() { //Union is direct, valid and with larger K than valid input final int k2 = 4; final int n2 = 2 * k2; //8 final int bytes = DoublesSketch.getUpdatableStorageBytes(256, 50);//big enough final WritableMemory skMem = WritableMemory.wrap(new byte[bytes]); DoublesSketch.builder().setK(256).build(skMem); final DoublesUnion union = DoublesUnionImpl.heapifyInstance(skMem); assertEquals(union.getResult().getN(), 0); assertEquals(union.getMaxK(), 256); assertEquals(union.getEffectiveK(), 256); final DoublesSketch result = union.getResult(); assertEquals(result.getK(), 256); final DoublesSketch sketchIn2 = buildAndLoadDQS(k2, n2, 17); union.update(sketchIn2); println("\nFinal" + union.getResult().toString(true, true)); assertEquals(union.getResult().getN(), n2); }
@Test public void checkUnionQuantiles() { final int k = 128; final int n1 = k * 13; final int n2 = (k * 8) + (k / 2); final int n = n1 + n2; final double errorTolerance = 0.0175 * n; // assuming k = 128 final UpdateDoublesSketch sketch1 = buildAndLoadQS(k, n1); final CompactDoublesSketch sketch2 = buildAndLoadQS(k, n2, n1).compact(); final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build(); //virgin 256 union.update(sketch2); union.update(sketch1); final Memory mem = Memory.wrap(union.getResult().toByteArray(true)); final DoublesSketch result = DoublesSketch.wrap(mem); assertEquals(result.getN(), n1 + n2); assertEquals(result.getK(), k); for (double fraction = 0.05; fraction < 1.0; fraction += 0.05) { assertEquals(result.getQuantile(fraction), fraction * n, errorTolerance); } }