@Override public int getMaxIntermediateSize() { return delegate.getMaxIntermediateSizeWithNulls(); }
/** * Returns the minimum buffer capacity required for this grouper. This grouper keeps track read/write indexes * and they cannot point the same array slot at the same time. Since the read/write indexes move circularly, one * extra slot is needed in addition to the read/write slots. Finally, the required minimum buffer capacity is * 3 * record size. * * @return required minimum buffer capacity */ public static <KeyType> int requiredBufferCapacity( KeySerde<KeyType> keySerde, AggregatorFactory[] aggregatorFactories ) { int recordSize = keySerde.keySize(); for (AggregatorFactory aggregatorFactory : aggregatorFactories) { recordSize += aggregatorFactory.getMaxIntermediateSizeWithNulls(); } return recordSize * 3; }
public BufferArrayGrouper( // the buffer returned from the below supplier can have dirty bits and should be cleared during initialization final Supplier<ByteBuffer> bufferSupplier, final ColumnSelectorFactory columnSelectorFactory, final AggregatorFactory[] aggregatorFactories, final int cardinality ) { Preconditions.checkNotNull(aggregatorFactories, "aggregatorFactories"); Preconditions.checkArgument(cardinality > 0, "Cardinality must a non-zero positive number"); this.bufferSupplier = Preconditions.checkNotNull(bufferSupplier, "bufferSupplier"); this.aggregators = new BufferAggregator[aggregatorFactories.length]; this.aggregatorOffsets = new int[aggregatorFactories.length]; this.cardinalityWithMissingValue = cardinality + 1; int offset = 0; for (int i = 0; i < aggregatorFactories.length; i++) { aggregators[i] = aggregatorFactories[i].factorizeBuffered(columnSelectorFactory); aggregatorOffsets[i] = offset; offset += aggregatorFactories[i].getMaxIntermediateSizeWithNulls(); } recordSize = offset; }
@Override protected BufferAggregator[] initAggs( final AggregatorFactory[] metrics, final Supplier<InputRow> rowSupplier, final boolean deserializeComplexMetrics, final boolean concurrentEventAdd ) { selectors = new HashMap<>(); aggOffsetInBuffer = new int[metrics.length]; for (int i = 0; i < metrics.length; i++) { AggregatorFactory agg = metrics[i]; ColumnSelectorFactory columnSelectorFactory = makeColumnSelectorFactory( agg, rowSupplier, deserializeComplexMetrics ); selectors.put( agg.getName(), new OnheapIncrementalIndex.CachingColumnSelectorFactory(columnSelectorFactory, concurrentEventAdd) ); if (i == 0) { aggOffsetInBuffer[i] = 0; } else { aggOffsetInBuffer[i] = aggOffsetInBuffer[i - 1] + metrics[i - 1].getMaxIntermediateSizeWithNulls(); } } aggsTotalSize = aggOffsetInBuffer[metrics.length - 1] + metrics[metrics.length - 1].getMaxIntermediateSizeWithNulls(); return new BufferAggregator[metrics.length]; }
/** * Gives estimated max size per aggregator. It is assumed that every aggregator will have enough overhead for its own * object header and for a pointer to a selector. We are adding a overhead-factor for each object as additional 16 * bytes. * These 16 bytes or 128 bits is the object metadata for 64-bit JVM process and consists of: * <ul> * <li>Class pointer which describes the object type: 64 bits * <li>Flags which describe state of the object including hashcode: 64 bits * <ul/> * total size estimation consists of: * <ul> * <li> metrics length : Integer.BYTES * len * <li> maxAggregatorIntermediateSize : getMaxIntermediateSize per aggregator + overhead-factor(16 bytes) * </ul> * * @param incrementalIndexSchema * * @return long max aggregator size in bytes */ private static long getMaxBytesPerRowForAggregators(IncrementalIndexSchema incrementalIndexSchema) { long maxAggregatorIntermediateSize = Integer.BYTES * incrementalIndexSchema.getMetrics().length; maxAggregatorIntermediateSize += Arrays.stream(incrementalIndexSchema.getMetrics()) .mapToLong(aggregator -> aggregator.getMaxIntermediateSizeWithNulls() + Long.BYTES * 2) .sum(); return maxAggregatorIntermediateSize; }
StreamingMergeSortedGrouper( final Supplier<ByteBuffer> bufferSupplier, final KeySerde<KeyType> keySerde, final ColumnSelectorFactory columnSelectorFactory, final AggregatorFactory[] aggregatorFactories, final long queryTimeoutAtMs ) { this.bufferSupplier = bufferSupplier; this.keySerde = keySerde; this.aggregators = new BufferAggregator[aggregatorFactories.length]; this.aggregatorOffsets = new int[aggregatorFactories.length]; this.keySize = keySerde.keySize(); int offset = keySize; for (int i = 0; i < aggregatorFactories.length; i++) { aggregators[i] = aggregatorFactories[i].factorizeBuffered(columnSelectorFactory); aggregatorOffsets[i] = offset; offset += aggregatorFactories[i].getMaxIntermediateSizeWithNulls(); } this.recordSize = offset; // queryTimeoutAtMs comes from System.currentTimeMillis(), but we should use System.nanoTime() to check timeout in // this class. See increaseWriteIndex() and increaseReadIndex(). this.hasQueryTimeout = queryTimeoutAtMs != QueryContexts.NO_TIMEOUT; final long timeoutNs = hasQueryTimeout ? TimeUnit.MILLISECONDS.toNanos(queryTimeoutAtMs - System.currentTimeMillis()) : QueryContexts.NO_TIMEOUT; this.queryTimeoutAtNs = System.nanoTime() + timeoutNs; }
public BufferHashGrouper( final Supplier<ByteBuffer> bufferSupplier, final KeySerde<KeyType> keySerde, final ColumnSelectorFactory columnSelectorFactory, final AggregatorFactory[] aggregatorFactories, final int bufferGrouperMaxSize, final float maxLoadFactor, final int initialBuckets, final boolean useDefaultSorting ) { super(bufferSupplier, keySerde, aggregatorFactories, bufferGrouperMaxSize); this.aggregatorFactories = aggregatorFactories; this.maxLoadFactor = maxLoadFactor > 0 ? maxLoadFactor : DEFAULT_MAX_LOAD_FACTOR; this.initialBuckets = initialBuckets > 0 ? Math.max(MIN_INITIAL_BUCKETS, initialBuckets) : DEFAULT_INITIAL_BUCKETS; if (this.maxLoadFactor >= 1.0f) { throw new IAE("Invalid maxLoadFactor[%f], must be < 1.0", maxLoadFactor); } int offset = HASH_SIZE + keySize; for (int i = 0; i < aggregatorFactories.length; i++) { aggregators[i] = aggregatorFactories[i].factorizeBuffered(columnSelectorFactory); aggregatorOffsets[i] = offset; offset += aggregatorFactories[i].getMaxIntermediateSizeWithNulls(); } this.bucketSize = offset; this.useDefaultSorting = useDefaultSorting; }
aggregators[i] = aggregatorFactories[i].factorizeBuffered(columnSelectorFactory); aggregatorOffsets[i] = offset; offset += aggregatorFactories[i].getMaxIntermediateSizeWithNulls();
aggregators[i] = aggregatorSpec.factorizeBuffered(cursor.getColumnSelectorFactory()); metricNames[i] = aggregatorSpec.getName(); sizesRequired[i] = aggregatorSpec.getMaxIntermediateSizeWithNulls();
public <T> T[] runRelocateVerificationTest( AggregatorFactory factory, ColumnSelectorFactory selector, Class<T> clazz ) { T[] results = (T[]) Array.newInstance(clazz, 2); BufferAggregator agg = factory.factorizeBuffered(selector); ByteBuffer myBuf = ByteBuffer.allocate(10040902); agg.init(myBuf, 0); agg.aggregate(myBuf, 0); results[0] = (T) agg.get(myBuf, 0); byte[] theBytes = new byte[factory.getMaxIntermediateSizeWithNulls()]; myBuf.get(theBytes); ByteBuffer newBuf = ByteBuffer.allocate(941209); newBuf.position(7574); newBuf.put(theBytes); newBuf.position(0); agg.relocate(0, 7574, myBuf, newBuf); results[1] = (T) agg.get(newBuf, 7574); return results; }
aggregatorSizes[i] = query.getAggregatorSpecs().get(i).getMaxIntermediateSizeWithNulls(); numBytesPerRecord += aggregatorSizes[i];
numBytesPerRecord += aggregatorFactory.getMaxIntermediateSizeWithNulls();
@Override public int getMaxIntermediateSize() { return delegate.getMaxIntermediateSizeWithNulls(); }
/** * Returns the minimum buffer capacity required for this grouper. This grouper keeps track read/write indexes * and they cannot point the same array slot at the same time. Since the read/write indexes move circularly, one * extra slot is needed in addition to the read/write slots. Finally, the required minimum buffer capacity is * 3 * record size. * * @return required minimum buffer capacity */ public static <KeyType> int requiredBufferCapacity( KeySerde<KeyType> keySerde, AggregatorFactory[] aggregatorFactories ) { int recordSize = keySerde.keySize(); for (AggregatorFactory aggregatorFactory : aggregatorFactories) { recordSize += aggregatorFactory.getMaxIntermediateSizeWithNulls(); } return recordSize * 3; }
public BufferArrayGrouper( // the buffer returned from the below supplier can have dirty bits and should be cleared during initialization final Supplier<ByteBuffer> bufferSupplier, final ColumnSelectorFactory columnSelectorFactory, final AggregatorFactory[] aggregatorFactories, final int cardinality ) { Preconditions.checkNotNull(aggregatorFactories, "aggregatorFactories"); Preconditions.checkArgument(cardinality > 0, "Cardinality must a non-zero positive number"); this.bufferSupplier = Preconditions.checkNotNull(bufferSupplier, "bufferSupplier"); this.aggregators = new BufferAggregator[aggregatorFactories.length]; this.aggregatorOffsets = new int[aggregatorFactories.length]; this.cardinalityWithMissingValue = cardinality + 1; int offset = 0; for (int i = 0; i < aggregatorFactories.length; i++) { aggregators[i] = aggregatorFactories[i].factorizeBuffered(columnSelectorFactory); aggregatorOffsets[i] = offset; offset += aggregatorFactories[i].getMaxIntermediateSizeWithNulls(); } recordSize = offset; }
/** * Gives estimated max size per aggregator. It is assumed that every aggregator will have enough overhead for its own * object header and for a pointer to a selector. We are adding a overhead-factor for each object as additional 16 * bytes. * These 16 bytes or 128 bits is the object metadata for 64-bit JVM process and consists of: * <ul> * <li>Class pointer which describes the object type: 64 bits * <li>Flags which describe state of the object including hashcode: 64 bits * <ul/> * total size estimation consists of: * <ul> * <li> metrics length : Integer.BYTES * len * <li> maxAggregatorIntermediateSize : getMaxIntermediateSize per aggregator + overhead-factor(16 bytes) * </ul> * * @param incrementalIndexSchema * * @return long max aggregator size in bytes */ private static long getMaxBytesPerRowForAggregators(IncrementalIndexSchema incrementalIndexSchema) { long maxAggregatorIntermediateSize = Integer.BYTES * incrementalIndexSchema.getMetrics().length; maxAggregatorIntermediateSize += Arrays.stream(incrementalIndexSchema.getMetrics()) .mapToLong(aggregator -> aggregator.getMaxIntermediateSizeWithNulls() + Long.BYTES * 2) .sum(); return maxAggregatorIntermediateSize; }
@Override protected BufferAggregator[] initAggs( final AggregatorFactory[] metrics, final Supplier<InputRow> rowSupplier, final boolean deserializeComplexMetrics, final boolean concurrentEventAdd ) { selectors = Maps.newHashMap(); aggOffsetInBuffer = new int[metrics.length]; for (int i = 0; i < metrics.length; i++) { AggregatorFactory agg = metrics[i]; ColumnSelectorFactory columnSelectorFactory = makeColumnSelectorFactory( agg, rowSupplier, deserializeComplexMetrics ); selectors.put( agg.getName(), new OnheapIncrementalIndex.CachingColumnSelectorFactory(columnSelectorFactory, concurrentEventAdd) ); if (i == 0) { aggOffsetInBuffer[i] = 0; } else { aggOffsetInBuffer[i] = aggOffsetInBuffer[i - 1] + metrics[i - 1].getMaxIntermediateSizeWithNulls(); } } aggsTotalSize = aggOffsetInBuffer[metrics.length - 1] + metrics[metrics.length - 1].getMaxIntermediateSizeWithNulls(); return new BufferAggregator[metrics.length]; }
StreamingMergeSortedGrouper( final Supplier<ByteBuffer> bufferSupplier, final KeySerde<KeyType> keySerde, final ColumnSelectorFactory columnSelectorFactory, final AggregatorFactory[] aggregatorFactories, final long queryTimeoutAtMs ) { this.bufferSupplier = bufferSupplier; this.keySerde = keySerde; this.aggregators = new BufferAggregator[aggregatorFactories.length]; this.aggregatorOffsets = new int[aggregatorFactories.length]; this.keySize = keySerde.keySize(); int offset = keySize; for (int i = 0; i < aggregatorFactories.length; i++) { aggregators[i] = aggregatorFactories[i].factorizeBuffered(columnSelectorFactory); aggregatorOffsets[i] = offset; offset += aggregatorFactories[i].getMaxIntermediateSizeWithNulls(); } this.recordSize = offset; // queryTimeoutAtMs comes from System.currentTimeMillis(), but we should use System.nanoTime() to check timeout in // this class. See increaseWriteIndex() and increaseReadIndex(). this.hasQueryTimeout = queryTimeoutAtMs != QueryContexts.NO_TIMEOUT; final long timeoutNs = hasQueryTimeout ? TimeUnit.MILLISECONDS.toNanos(queryTimeoutAtMs - System.currentTimeMillis()) : QueryContexts.NO_TIMEOUT; this.queryTimeoutAtNs = System.nanoTime() + timeoutNs; }
public BufferHashGrouper( final Supplier<ByteBuffer> bufferSupplier, final KeySerde<KeyType> keySerde, final ColumnSelectorFactory columnSelectorFactory, final AggregatorFactory[] aggregatorFactories, final int bufferGrouperMaxSize, final float maxLoadFactor, final int initialBuckets, final boolean useDefaultSorting ) { super(bufferSupplier, keySerde, aggregatorFactories, bufferGrouperMaxSize); this.aggregatorFactories = aggregatorFactories; this.maxLoadFactor = maxLoadFactor > 0 ? maxLoadFactor : DEFAULT_MAX_LOAD_FACTOR; this.initialBuckets = initialBuckets > 0 ? Math.max(MIN_INITIAL_BUCKETS, initialBuckets) : DEFAULT_INITIAL_BUCKETS; if (this.maxLoadFactor >= 1.0f) { throw new IAE("Invalid maxLoadFactor[%f], must be < 1.0", maxLoadFactor); } int offset = HASH_SIZE + keySize; for (int i = 0; i < aggregatorFactories.length; i++) { aggregators[i] = aggregatorFactories[i].factorizeBuffered(columnSelectorFactory); aggregatorOffsets[i] = offset; offset += aggregatorFactories[i].getMaxIntermediateSizeWithNulls(); } this.bucketSize = offset; this.useDefaultSorting = useDefaultSorting; }
aggregators[i] = aggregatorFactories[i].factorizeBuffered(columnSelectorFactory); aggregatorOffsets[i] = offset; offset += aggregatorFactories[i].getMaxIntermediateSizeWithNulls();