@Override public void ensureCapacity(long size) { headBlockIndex.ensureCapacity(size); headPosition.ensureCapacity(size); tailBlockIndex.ensureCapacity(size); tailPosition.ensureCapacity(size); groupEntryCount.ensureCapacity(size); }
@Override public void ensureCapacity(long size) { headBlockIndex.ensureCapacity(size); headPosition.ensureCapacity(size); tailBlockIndex.ensureCapacity(size); tailPosition.ensureCapacity(size); groupEntryCount.ensureCapacity(size); }
private void resizeNodeArrays(int newBucketCount) { // every per-bucket array needs to be updated counts.ensureCapacity(newBucketCount); valuePositions.ensureCapacity(newBucketCount); nextPointers.ensureCapacity(newBucketCount); valueAndGroupHashes.ensureCapacity(newBucketCount); groupIds.ensureCapacity(newBucketCount); }
private void resizeNodeArrays(int newBucketCount) { // every per-bucket array needs to be updated counts.ensureCapacity(newBucketCount); valuePositions.ensureCapacity(newBucketCount); nextPointers.ensureCapacity(newBucketCount); valueAndGroupHashes.ensureCapacity(newBucketCount); groupIds.ensureCapacity(newBucketCount); }
private SingleTypedHistogram(Type type, int expectedSize, int hashCapacity, BlockBuilder values) { this.type = type; this.expectedSize = expectedSize; this.hashCapacity = hashCapacity; this.values = values; checkArgument(expectedSize > 0, "expectedSize must be greater than zero"); maxFill = calculateMaxFill(hashCapacity); mask = hashCapacity - 1; hashPositions = new IntBigArray(-1); hashPositions.ensureCapacity(hashCapacity); counts = new LongBigArray(); counts.ensureCapacity(hashCapacity); }
@Override public void ensureCapacity(long size) { partitionCounts.ensureCapacity(size); counts.ensureCapacity(size); envelopes.ensureCapacity(size); samples.ensureCapacity(size); } }
private SingleTypedHistogram(Type type, int expectedSize, int hashCapacity, BlockBuilder values) { this.type = type; this.expectedSize = expectedSize; this.hashCapacity = hashCapacity; this.values = values; checkArgument(expectedSize > 0, "expectedSize must be greater than zero"); maxFill = calculateMaxFill(hashCapacity); mask = hashCapacity - 1; hashPositions = new IntBigArray(-1); hashPositions.ensureCapacity(hashCapacity); counts = new LongBigArray(); counts.ensureCapacity(hashCapacity); }
@VisibleForTesting public ValueStore(int expectedSize, BlockBuilder values) { bucketCount = computeBucketCount(expectedSize, MAX_FILL_RATIO); mask = bucketCount - 1; maxFill = calculateMaxFill(bucketCount, MAX_FILL_RATIO); this.values = values; buckets = new IntBigArray(-1); buckets.ensureCapacity(bucketCount); valueHashes = new LongBigArray(-1); valueHashes.ensureCapacity(bucketCount); }
@VisibleForTesting public ValueStore(int expectedSize, BlockBuilder values) { bucketCount = computeBucketCount(expectedSize, MAX_FILL_RATIO); mask = bucketCount - 1; maxFill = calculateMaxFill(bucketCount, MAX_FILL_RATIO); this.values = values; buckets = new IntBigArray(-1); buckets.ensureCapacity(bucketCount); valueHashes = new LongBigArray(-1); valueHashes.ensureCapacity(bucketCount); }
public DictionaryBuilder(int expectedSize) { checkArgument(expectedSize >= 0, "expectedSize must not be negative"); // todo we can do better int expectedEntries = min(expectedSize, DEFAULT_MAX_PAGE_SIZE_IN_BYTES / EXPECTED_BYTES_PER_ENTRY); // it is guaranteed expectedEntries * EXPECTED_BYTES_PER_ENTRY will not overflow this.elementBlock = new VariableWidthBlockBuilder( null, expectedEntries, expectedEntries * EXPECTED_BYTES_PER_ENTRY); // first position is always null this.elementBlock.appendNull(); int hashSize = arraySize(expectedSize, FILL_RATIO); this.maxFill = calculateMaxFill(hashSize); this.hashMask = hashSize - 1; blockPositionByHash.ensureCapacity(hashSize); blockPositionByHash.fill(EMPTY_SLOT); this.containsNullElement = false; }
public DictionaryBuilder(int expectedSize) { checkArgument(expectedSize >= 0, "expectedSize must not be negative"); // todo we can do better int expectedEntries = min(expectedSize, DEFAULT_MAX_PAGE_SIZE_IN_BYTES / EXPECTED_BYTES_PER_ENTRY); // it is guaranteed expectedEntries * EXPECTED_BYTES_PER_ENTRY will not overflow this.elementBlock = new VariableWidthBlockBuilder( null, expectedEntries, expectedEntries * EXPECTED_BYTES_PER_ENTRY); // first position is always null this.elementBlock.appendNull(); int hashSize = arraySize(expectedSize, FILL_RATIO); this.maxFill = calculateMaxFill(hashSize); this.hashMask = hashSize - 1; blockPositionByHash.ensureCapacity(hashSize); blockPositionByHash.fill(EMPTY_SLOT); this.containsNullElement = false; }
public BigintGroupByHash(int hashChannel, boolean outputRawHash, int expectedSize, UpdateMemory updateMemory) { checkArgument(hashChannel >= 0, "hashChannel must be at least zero"); checkArgument(expectedSize > 0, "expectedSize must be greater than zero"); this.hashChannel = hashChannel; this.outputRawHash = outputRawHash; hashCapacity = arraySize(expectedSize, FILL_RATIO); maxFill = calculateMaxFill(hashCapacity); mask = hashCapacity - 1; values = new LongBigArray(); values.ensureCapacity(hashCapacity); groupIds = new IntBigArray(-1); groupIds.ensureCapacity(hashCapacity); valuesByGroupId = new LongBigArray(); valuesByGroupId.ensureCapacity(hashCapacity); // This interface is used for actively reserving memory (push model) for rehash. // The caller can also query memory usage on this object (pull model) this.updateMemory = requireNonNull(updateMemory, "updateMemory is null"); }
public BigintGroupByHash(int hashChannel, boolean outputRawHash, int expectedSize, UpdateMemory updateMemory) { checkArgument(hashChannel >= 0, "hashChannel must be at least zero"); checkArgument(expectedSize > 0, "expectedSize must be greater than zero"); this.hashChannel = hashChannel; this.outputRawHash = outputRawHash; hashCapacity = arraySize(expectedSize, FILL_RATIO); maxFill = calculateMaxFill(hashCapacity); mask = hashCapacity - 1; values = new LongBigArray(); values.ensureCapacity(hashCapacity); groupIds = new IntBigArray(-1); groupIds.ensureCapacity(hashCapacity); valuesByGroupId = new LongBigArray(); valuesByGroupId.ensureCapacity(hashCapacity); // This interface is used for actively reserving memory (push model) for rehash. // The caller can also query memory usage on this object (pull model) this.updateMemory = requireNonNull(updateMemory, "updateMemory is null"); }
protected AbstractGroupCollectionAggregationState(PageBuilder pageBuilder) { this.headBlockIndex = new ShortBigArray(NULL); this.headPosition = new IntBigArray(NULL); this.nextBlockIndex = new ShortBigArray(NULL); this.nextPosition = new IntBigArray(NULL); this.tailBlockIndex = new ShortBigArray(NULL); this.tailPosition = new IntBigArray(NULL); this.currentPageBuilder = pageBuilder; this.values = new ArrayList<>(); this.sumPositions = new LongArrayList(); this.groupEntryCount = new IntBigArray(); values.add(currentPageBuilder); sumPositions.add(0L); valueBlocksRetainedSizeInBytes = 0; totalPositions = 0; capacity = 1024; nextBlockIndex.ensureCapacity(capacity); nextPosition.ensureCapacity(capacity); groupEntryCount.ensureCapacity(capacity); }
protected AbstractGroupCollectionAggregationState(PageBuilder pageBuilder) { this.headBlockIndex = new ShortBigArray(NULL); this.headPosition = new IntBigArray(NULL); this.nextBlockIndex = new ShortBigArray(NULL); this.nextPosition = new IntBigArray(NULL); this.tailBlockIndex = new ShortBigArray(NULL); this.tailPosition = new IntBigArray(NULL); this.currentPageBuilder = pageBuilder; this.values = new ArrayList<>(); this.sumPositions = new LongArrayList(); this.groupEntryCount = new IntBigArray(); values.add(currentPageBuilder); sumPositions.add(0L); valueBlocksRetainedSizeInBytes = 0; totalPositions = 0; capacity = 1024; nextBlockIndex.ensureCapacity(capacity); nextPosition.ensureCapacity(capacity); groupEntryCount.ensureCapacity(capacity); }
private void rehash(int size) { int newHashSize = arraySize(size + 1, FILL_RATIO); hashMask = newHashSize - 1; maxFill = calculateMaxFill(newHashSize); blockPositionByHash.ensureCapacity(newHashSize); blockPositionByHash.fill(EMPTY_SLOT); // the first element of elementBlock is always null for (int blockPosition = 1; blockPosition < elementBlock.getPositionCount(); blockPosition++) { blockPositionByHash.set(getHashPositionOfElement(elementBlock, blockPosition), blockPosition); } }
private void rehash(int size) { int newHashSize = arraySize(size + 1, FILL_RATIO); hashMask = newHashSize - 1; maxFill = calculateMaxFill(newHashSize); blockPositionByHash.ensureCapacity(newHashSize); blockPositionByHash.fill(EMPTY_SLOT); // the first element of elementBlock is always null for (int blockPosition = 1; blockPosition < elementBlock.getPositionCount(); blockPosition++) { blockPositionByHash.set(getHashPositionOfElement(elementBlock, blockPosition), blockPosition); } }
@Override public void writeBlock(Block block) { checkState(!closed); checkArgument(block.getPositionCount() > 0, "Block is empty"); if (directEncoded) { directColumnWriter.writeBlock(block); return; } // record values values.ensureCapacity(rowGroupValueCount + block.getPositionCount()); for (int position = 0; position < block.getPositionCount(); position++) { int index = dictionary.putIfAbsent(block, position); values.set(rowGroupValueCount, index); rowGroupValueCount++; totalValueCount++; if (!block.isNull(position)) { // todo min/max statistics only need to be updated if value was not already in the dictionary, but non-null count does statisticsBuilder.addValue(type.getSlice(block, position)); rawBytes += block.getSliceLength(position); totalNonNullValueCount++; } } }
private IntIterator hashSortedGroupIds() { IntBigArray groupIds = new IntBigArray(); groupIds.ensureCapacity(groupByHash.getGroupCount()); for (int i = 0; i < groupByHash.getGroupCount(); i++) { groupIds.set(i, i); } groupIds.sort(0, groupByHash.getGroupCount(), (leftGroupId, rightGroupId) -> Long.compare(groupByHash.getRawHash(leftGroupId), groupByHash.getRawHash(rightGroupId))); return new AbstractIntIterator() { private final int totalPositions = groupByHash.getGroupCount(); private int position; @Override public boolean hasNext() { return position < totalPositions; } @Override public int nextInt() { return groupIds.get(position++); } }; }
public GroupedTypedHistogram(Type type, int expectedCount) { checkArgument(expectedCount > 0, "expectedSize must be greater than zero"); this.type = type; this.bucketId = expectedCount; this.bucketCount = computeBucketCount(expectedCount, MAX_FILL_RATIO); this.mask = bucketCount - 1; this.maxFill = calculateMaxFill(bucketCount, MAX_FILL_RATIO); this.values = type.createBlockBuilder(null, computeBucketCount(expectedCount, GroupedTypedHistogram.MAX_FILL_RATIO)); // buckets and node-arrays (bucket "points" to a node, so 1:1 relationship) buckets = new IntBigArray(-1); buckets.ensureCapacity(bucketCount); counts = new LongBigArray(); valuePositions = new IntBigArray(); valueAndGroupHashes = new LongBigArray(); nextPointers = new IntBigArray(NULL); groupIds = new LongBigArray(-1); // here, one bucket is one node in the hash structure (vs a bucket may be a chain of nodes in closed-hashing with linked list hashing) // ie, this is open-address hashing resizeNodeArrays(bucketCount); // end bucket/node based arrays // per-group arrays: size will be set by external call, same as groups since the number will be the same headPointers = new LongBigArray(NULL); // index into counts/valuePositions nextNodePointer = 0; bucketNodeFactory = this.new BucketNodeFactory(); valueStore = new ValueStore(expectedCount, values); }