/** * Wrap the given WritableMemory and seed as a ArrayOfDoublesUpdatableSketch * @param mem the given Memory * @param seed the given seed * @return an ArrayOfDoublesUpdatableSketch */ public static ArrayOfDoublesUpdatableSketch wrap(final WritableMemory mem, final long seed) { return new DirectArrayOfDoublesQuickSelectSketch(mem, seed); }
@Override protected void rebuild(final int newCapacity) { final int numValues = getNumValues(); checkIfEnoughMemory(mem_, newCapacity, numValues); final int currCapacity = getCurrentCapacity(); final long[] keys = new long[currCapacity]; final double[] values = new double[currCapacity * numValues]; mem_.getLongArray(keysOffset_, keys, 0, currCapacity); mem_.getDoubleArray(valuesOffset_, values, 0, currCapacity * numValues); mem_.clear(keysOffset_, ((long) SIZE_OF_KEY_BYTES * newCapacity) + ((long) SIZE_OF_VALUE_BYTES * newCapacity * numValues)); mem_.putInt(RETAINED_ENTRIES_INT, 0); mem_.putByte(LG_CUR_CAPACITY_BYTE, (byte)Integer.numberOfTrailingZeros(newCapacity)); valuesOffset_ = keysOffset_ + (SIZE_OF_KEY_BYTES * newCapacity); lgCurrentCapacity_ = Integer.numberOfTrailingZeros(newCapacity); for (int i = 0; i < keys.length; i++) { if ((keys[i] != 0) && (keys[i] < theta_)) { insert(keys[i], Arrays.copyOfRange(values, i * numValues, (i + 1) * numValues)); } } setRebuildThreshold(); }
@Override public double[][] getValues() { final int count = getRetainedEntries(); final double[][] values = new double[count][]; if (count > 0) { long keyOffset = keysOffset_; long valuesOffset = valuesOffset_; int i = 0; for (int j = 0; j < getCurrentCapacity(); j++) { if (mem_.getLong(keyOffset) != 0) { final double[] array = new double[numValues_]; mem_.getDoubleArray(valuesOffset, array, 0, numValues_); values[i++] = array; } keyOffset += SIZE_OF_KEY_BYTES; valuesOffset += (long)SIZE_OF_VALUE_BYTES * numValues_; } } return values; }
@Override public byte[] toByteArray() { final int sizeBytes = getSerializedSizeBytes(); final byte[] byteArray = new byte[sizeBytes]; final WritableMemory mem = WritableMemory.wrap(byteArray); serializeInto(mem); return byteArray; }
@Override public void reset() { if (!isEmpty_) { isEmpty_ = true; mem_.setBits(FLAGS_BYTE, (byte) (1 << Flags.IS_EMPTY.ordinal())); } final int lgResizeFactor = mem_.getByte(LG_RESIZE_FACTOR_BYTE); final float samplingProbability = mem_.getFloat(SAMPLING_P_FLOAT); final int startingCapacity = Util.getStartingCapacity(getNominalEntries(), lgResizeFactor); theta_ = (long) (Long.MAX_VALUE * (double) samplingProbability); mem_.putLong(THETA_LONG, theta_); mem_.putByte(LG_CUR_CAPACITY_BYTE, (byte) Integer.numberOfTrailingZeros(startingCapacity)); mem_.putInt(RETAINED_ENTRIES_INT, 0); keysOffset_ = ENTRIES_START; valuesOffset_ = keysOffset_ + (SIZE_OF_KEY_BYTES * startingCapacity); mem_.clear(keysOffset_, (long) SIZE_OF_KEY_BYTES * startingCapacity); // clear keys only lgCurrentCapacity_ = Integer.numberOfTrailingZeros(startingCapacity); setRebuildThreshold(); }
valuesOffset_ = keysOffset_ + (SIZE_OF_KEY_BYTES * getCurrentCapacity()); lgCurrentCapacity_ = Integer.numberOfTrailingZeros(getCurrentCapacity()); theta_ = mem_.getLong(THETA_LONG); isEmpty_ = (mem_.getByte(FLAGS_BYTE) & (1 << Flags.IS_EMPTY.ordinal())) != 0; setRebuildThreshold();
mem_ = dstMem; final int startingCapacity = Util.getStartingCapacity(nomEntries, lgResizeFactor); checkIfEnoughMemory(dstMem, startingCapacity, numValues); mem_.putByte(PREAMBLE_LONGS_BYTE, (byte) 1); mem_.putByte(SERIAL_VERSION_BYTE, serialVersionUID); mem_.clear(keysOffset_, (long) SIZE_OF_KEY_BYTES * startingCapacity); // clear keys only lgCurrentCapacity_ = Integer.numberOfTrailingZeros(startingCapacity); setRebuildThreshold();
@Override int getSerializedSizeBytes() { return valuesOffset_ + (SIZE_OF_VALUE_BYTES * numValues_ * getCurrentCapacity()); }
@Test public void isEmptyWithSampling() { float samplingProbability = 0.1f; ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder(). setSamplingProbability(samplingProbability). build(WritableMemory.wrap(new byte[1000000])); Assert.assertTrue(sketch.isEmpty()); Assert.assertTrue(((DirectArrayOfDoublesQuickSelectSketch)sketch).isInSamplingMode()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 0.0); Assert.assertEquals(sketch.getUpperBound(1), 0.0); Assert.assertEquals(sketch.getLowerBound(1), 0.0); Assert.assertEquals( sketch.getThetaLong() / (double) Long.MAX_VALUE, (double) samplingProbability); Assert.assertEquals(sketch.getTheta(), (double) samplingProbability); }
@Override public byte[] toByteArray() { final int sizeBytes = getSerializedSizeBytes(); final byte[] byteArray = new byte[sizeBytes]; final WritableMemory mem = WritableMemory.wrap(byteArray); serializeInto(mem); return byteArray; }
@Override public void reset() { if (!isEmpty_) { isEmpty_ = true; mem_.setBits(FLAGS_BYTE, (byte) (1 << Flags.IS_EMPTY.ordinal())); } final int lgResizeFactor = mem_.getByte(LG_RESIZE_FACTOR_BYTE); final float samplingProbability = mem_.getFloat(SAMPLING_P_FLOAT); final int startingCapacity = Util.getStartingCapacity(getNominalEntries(), lgResizeFactor); theta_ = (long) (Long.MAX_VALUE * (double) samplingProbability); mem_.putLong(THETA_LONG, theta_); mem_.putByte(LG_CUR_CAPACITY_BYTE, (byte) Integer.numberOfTrailingZeros(startingCapacity)); mem_.putInt(RETAINED_ENTRIES_INT, 0); keysOffset_ = ENTRIES_START; valuesOffset_ = keysOffset_ + (SIZE_OF_KEY_BYTES * startingCapacity); mem_.clear(keysOffset_, SIZE_OF_KEY_BYTES * startingCapacity); // clear keys only lgCurrentCapacity_ = Integer.numberOfTrailingZeros(startingCapacity); setRebuildThreshold(); }
valuesOffset_ = keysOffset_ + (SIZE_OF_KEY_BYTES * getCurrentCapacity()); lgCurrentCapacity_ = Integer.numberOfTrailingZeros(getCurrentCapacity()); theta_ = mem_.getLong(THETA_LONG); isEmpty_ = (mem_.getByte(FLAGS_BYTE) & (1 << Flags.IS_EMPTY.ordinal())) != 0; setRebuildThreshold();
mem_ = dstMem; final int startingCapacity = Util.getStartingCapacity(nomEntries, lgResizeFactor); checkIfEnoughMemory(dstMem, startingCapacity, numValues); mem_.putByte(PREAMBLE_LONGS_BYTE, (byte) 1); mem_.putByte(SERIAL_VERSION_BYTE, serialVersionUID); setRebuildThreshold();
@Override public ArrayOfDoublesSketchIterator iterator() { return new DirectArrayOfDoublesSketchIterator(mem_, keysOffset_, getCurrentCapacity(), numValues_); }
@Override protected void rebuild(final int newCapacity) { final int numValues = getNumValues(); checkIfEnoughMemory(mem_, newCapacity, numValues); final int currCapacity = getCurrentCapacity(); final long[] keys = new long[currCapacity]; final double[] values = new double[currCapacity * numValues]; mem_.getLongArray(keysOffset_, keys, 0, currCapacity); mem_.getDoubleArray(valuesOffset_, values, 0, currCapacity * numValues); mem_.clear(keysOffset_, (SIZE_OF_KEY_BYTES * newCapacity) + (SIZE_OF_VALUE_BYTES * newCapacity * numValues)); mem_.putInt(RETAINED_ENTRIES_INT, 0); mem_.putByte(LG_CUR_CAPACITY_BYTE, (byte)Integer.numberOfTrailingZeros(newCapacity)); valuesOffset_ = keysOffset_ + (SIZE_OF_KEY_BYTES * newCapacity); lgCurrentCapacity_ = Integer.numberOfTrailingZeros(newCapacity); for (int i = 0; i < keys.length; i++) { if ((keys[i] != 0) && (keys[i] < theta_)) { insert(keys[i], Arrays.copyOfRange(values, i * numValues, (i + 1) * numValues)); } } setRebuildThreshold(); }
@Override protected ArrayOfDoublesQuickSelectSketch createSketch(final int size, final int numValues, final long seed) { return new DirectArrayOfDoublesQuickSelectSketch(size, 0, 1f, numValues, seed, mem_); }
@Override public double[][] getValues() { final int count = getRetainedEntries(); final double[][] values = new double[count][]; if (count > 0) { long keyOffset = keysOffset_; long valuesOffset = valuesOffset_; int i = 0; for (int j = 0; j < getCurrentCapacity(); j++) { if (mem_.getLong(keyOffset) != 0) { final double[] array = new double[numValues_]; mem_.getDoubleArray(valuesOffset, array, 0, numValues_); values[i++] = array; } keyOffset += SIZE_OF_KEY_BYTES; valuesOffset += (long)SIZE_OF_VALUE_BYTES * numValues_; } } return values; }
@Override int getSerializedSizeBytes() { return valuesOffset_ + (SIZE_OF_VALUE_BYTES * numValues_ * getCurrentCapacity()); }
/** * Creates an instance of DirectArrayOfDoublesUnion * @param nomEntries Nominal number of entries. Forced to the nearest power of 2 greater than * given value. * @param numValues Number of double values to keep for each key. * @param seed <a href="{@docRoot}/resources/dictionary.html#seed">See seed</a> * @param dstMem <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a> */ DirectArrayOfDoublesUnion(final int nomEntries, final int numValues, final long seed, final WritableMemory dstMem) { super(new DirectArrayOfDoublesQuickSelectSketch(nomEntries, 3, 1f, numValues, seed, dstMem)); mem_ = dstMem; }
@Override public ArrayOfDoublesSketchIterator iterator() { return new DirectArrayOfDoublesSketchIterator(mem_, keysOffset_, getCurrentCapacity(), numValues_); }