private void clearPartitions() { for (InMemoryPartition<T> p : this.partitions) { p.clearAllMemory(this.availableMemory); } this.partitions.clear(); this.compactionMemory.clearAllMemory(availableMemory); }
@Override public String toString() { return String.format("Partition %d - %d records, %d partition blocks, %d bucket overflow blocks", getPartitionNumber(), getRecordCount(), getBlockCount(), this.numOverflowSegments); }
if (this.closed || partitionNumber >= this.partitions.size() || this.partitions.get(partitionNumber).isCompacted()) { return; this.compactionMemory.clearAllMemory(availableMemory); this.compactionMemory.allocateSegments(1); this.compactionMemory.pushDownPages(); T tempHolder = this.buildSideSerializer.createInstance(); final int numPartitions = this.partitions.size(); while (numInSegment < countInSegment) { pointer = segment.getLong(pointerOffset); tempHolder = partition.readRecordAt(pointer, tempHolder); pointer = this.compactionMemory.appendRecord(tempHolder); segment.putLong(pointerOffset, pointer); pointerOffset += POINTER_LEN; this.compactionMemory.setPartitionNumber(partitionNumber); this.partitions.add(partitionNumber, compactionMemory); this.partitions.get(partitionNumber).overflowSegments = partition.overflowSegments; this.partitions.get(partitionNumber).numOverflowSegments = partition.numOverflowSegments; this.partitions.get(partitionNumber).nextOverflowBucket = partition.nextOverflowBucket; this.partitions.get(partitionNumber).setIsCompacted(true); this.compactionMemory.resetRecordCounter(); this.compactionMemory.setPartitionNumber(-1); this.compactionMemory.overflowSegments = null; this.compactionMemory.numOverflowSegments = 0; this.compactionMemory.nextOverflowBucket = 0;
private long insertRecordIntoPartition(T record, InMemoryPartition<T> partition, boolean fragments) throws IOException { try { long pointer = partition.appendRecord(record); if (fragments) { partition.setIsCompacted(false); if ((pointer >> this.pageSizeInBits) > this.compactionMemory.getBlockCount()) { this.compactionMemory.allocateSegments((int)(pointer >> this.pageSizeInBits)); final int partitionNumber = partition.getPartitionNumber(); compactPartition(partitionNumber); long newPointer = partition.appendRecord(record); if ((newPointer >> this.pageSizeInBits) > this.compactionMemory.getBlockCount()) { this.compactionMemory.allocateSegments((int)(newPointer >> this.pageSizeInBits));
/** * @return number of memory segments in the smallest partition */ private int getMinPartition() { int minPartition = Integer.MAX_VALUE; for(InMemoryPartition<T> p1 : this.partitions) { if(p1.getBlockCount() < minPartition) { minPartition = p1.getBlockCount(); } } return minPartition; }
if (posHashCode == bucket && oldBucketCount < NUM_ENTRIES_PER_BUCKET) { bucketOffset = (bucket % bucketsPerSegment) * HASH_BUCKET_SIZE; insertBucketEntryFromStart(segment, bucketOffset, hash, pointer, partition.getPartitionNumber()); oldBucketCount++; insertBucketEntryFromStart(newSegment, bucketOffset, hash, pointer, partition.getPartitionNumber()); newBucketCount++; this.availableMemory.addAll(partition.resetOverflowBuckets()); bucketInSegmentPos = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS; bucket = this.buckets[bucketArrayPos]; insertBucketEntryFromStart(bucket, bucketInSegmentPos, hash, pointer, partition.getPartitionNumber());
private void createPartitions(int numPartitions) { this.partitions.clear(); ListMemorySegmentSource memSource = new ListMemorySegmentSource(this.availableMemory); for (int i = 0; i < numPartitions; i++) { this.partitions.add(new InMemoryPartition<T>(this.buildSideSerializer, i, memSource, this.segmentSize, pageSizeInBits)); } this.compactionMemory = new InMemoryPartition<T>(this.buildSideSerializer, -1, memSource, this.segmentSize, pageSizeInBits); }
T valueAtPosition = partition.readRecordAt(pointer); if (this.buildSideComparator.equalToReference(valueAtPosition)) { long newPointer = insertRecordIntoPartition(record, partition, true);
if (this.closed || partitionNumber >= this.partitions.size() || this.partitions.get(partitionNumber).isCompacted()) { return; this.compactionMemory.clearAllMemory(availableMemory); this.compactionMemory.allocateSegments(1); this.compactionMemory.pushDownPages(); T tempHolder = this.buildSideSerializer.createInstance(); final int numPartitions = this.partitions.size(); while (numInSegment < countInSegment) { pointer = segment.getLong(pointerOffset); tempHolder = partition.readRecordAt(pointer, tempHolder); pointer = this.compactionMemory.appendRecord(tempHolder); segment.putLong(pointerOffset, pointer); pointerOffset += POINTER_LEN; this.compactionMemory.setPartitionNumber(partitionNumber); this.partitions.add(partitionNumber, compactionMemory); this.partitions.get(partitionNumber).overflowSegments = partition.overflowSegments; this.partitions.get(partitionNumber).numOverflowSegments = partition.numOverflowSegments; this.partitions.get(partitionNumber).nextOverflowBucket = partition.nextOverflowBucket; this.partitions.get(partitionNumber).setIsCompacted(true); this.compactionMemory.resetRecordCounter(); this.compactionMemory.setPartitionNumber(-1); this.compactionMemory.overflowSegments = null; this.compactionMemory.numOverflowSegments = 0; this.compactionMemory.nextOverflowBucket = 0;
private long insertRecordIntoPartition(T record, InMemoryPartition<T> partition, boolean fragments) throws IOException { try { long pointer = partition.appendRecord(record); if (fragments) { partition.setIsCompacted(false); if ((pointer >> this.pageSizeInBits) > this.compactionMemory.getBlockCount()) { this.compactionMemory.allocateSegments((int)(pointer >> this.pageSizeInBits)); final int partitionNumber = partition.getPartitionNumber(); compactPartition(partitionNumber); long newPointer = partition.appendRecord(record); if ((newPointer >> this.pageSizeInBits) > this.compactionMemory.getBlockCount()) { this.compactionMemory.allocateSegments((int)(newPointer >> this.pageSizeInBits));
/** * Size of all memory segments owned by the partitions of this hash table excluding the compaction partition * * @return size in bytes */ private long getPartitionSize() { long numSegments = 0; for(InMemoryPartition<T> p : this.partitions) { numSegments += p.getBlockCount(); } return numSegments*this.segmentSize; }
if (posHashCode == bucket && oldBucketCount < NUM_ENTRIES_PER_BUCKET) { bucketOffset = (bucket % bucketsPerSegment) * HASH_BUCKET_SIZE; insertBucketEntryFromStart(segment, bucketOffset, hash, pointer, partition.getPartitionNumber()); oldBucketCount++; insertBucketEntryFromStart(newSegment, bucketOffset, hash, pointer, partition.getPartitionNumber()); newBucketCount++; this.availableMemory.addAll(partition.resetOverflowBuckets()); bucketInSegmentPos = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS; bucket = this.buckets[bucketArrayPos]; insertBucketEntryFromStart(bucket, bucketInSegmentPos, hash, pointer, partition.getPartitionNumber());
private void createPartitions(int numPartitions) { this.partitions.clear(); ListMemorySegmentSource memSource = new ListMemorySegmentSource(this.availableMemory); for (int i = 0; i < numPartitions; i++) { this.partitions.add(new InMemoryPartition<T>(this.buildSideSerializer, i, memSource, this.segmentSize, pageSizeInBits)); } this.compactionMemory = new InMemoryPartition<T>(this.buildSideSerializer, -1, memSource, this.segmentSize, pageSizeInBits); }
T valueAtPosition = partition.readRecordAt(pointer); if (this.buildSideComparator.equalToReference(valueAtPosition)) { long newPointer = insertRecordIntoPartition(record, partition, true);
if (this.closed || partitionNumber >= this.partitions.size() || this.partitions.get(partitionNumber).isCompacted()) { return; this.compactionMemory.clearAllMemory(availableMemory); this.compactionMemory.allocateSegments(1); this.compactionMemory.pushDownPages(); T tempHolder = this.buildSideSerializer.createInstance(); final int numPartitions = this.partitions.size(); while (numInSegment < countInSegment) { pointer = segment.getLong(pointerOffset); tempHolder = partition.readRecordAt(pointer, tempHolder); pointer = this.compactionMemory.appendRecord(tempHolder); segment.putLong(pointerOffset, pointer); pointerOffset += POINTER_LEN; this.compactionMemory.setPartitionNumber(partitionNumber); this.partitions.add(partitionNumber, compactionMemory); this.partitions.get(partitionNumber).overflowSegments = partition.overflowSegments; this.partitions.get(partitionNumber).numOverflowSegments = partition.numOverflowSegments; this.partitions.get(partitionNumber).nextOverflowBucket = partition.nextOverflowBucket; this.partitions.get(partitionNumber).setIsCompacted(true); this.compactionMemory.resetRecordCounter(); this.compactionMemory.setPartitionNumber(-1); this.compactionMemory.overflowSegments = null; this.compactionMemory.numOverflowSegments = 0; this.compactionMemory.nextOverflowBucket = 0;
private long insertRecordIntoPartition(T record, InMemoryPartition<T> partition, boolean fragments) throws IOException { try { long pointer = partition.appendRecord(record); if (fragments) { partition.setIsCompacted(false); if ((pointer >> this.pageSizeInBits) > this.compactionMemory.getBlockCount()) { this.compactionMemory.allocateSegments((int)(pointer >> this.pageSizeInBits)); final int partitionNumber = partition.getPartitionNumber(); compactPartition(partitionNumber); long newPointer = partition.appendRecord(record); if ((newPointer >> this.pageSizeInBits) > this.compactionMemory.getBlockCount()) { this.compactionMemory.allocateSegments((int)(newPointer >> this.pageSizeInBits));
@Override public String toString() { return String.format("Partition %d - %d records, %d partition blocks, %d bucket overflow blocks", getPartitionNumber(), getRecordCount(), getBlockCount(), this.numOverflowSegments); }
/** * @return number of memory segments in the smallest partition */ private int getMinPartition() { int minPartition = Integer.MAX_VALUE; for(InMemoryPartition<T> p1 : this.partitions) { if(p1.getBlockCount() < minPartition) { minPartition = p1.getBlockCount(); } } return minPartition; }
if (posHashCode == bucket && oldBucketCount < NUM_ENTRIES_PER_BUCKET) { bucketOffset = (bucket % bucketsPerSegment) * HASH_BUCKET_SIZE; insertBucketEntryFromStart(segment, bucketOffset, hash, pointer, partition.getPartitionNumber()); oldBucketCount++; insertBucketEntryFromStart(newSegment, bucketOffset, hash, pointer, partition.getPartitionNumber()); newBucketCount++; this.availableMemory.addAll(partition.resetOverflowBuckets()); bucketInSegmentPos = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS; bucket = this.buckets[bucketArrayPos]; insertBucketEntryFromStart(bucket, bucketInSegmentPos, hash, pointer, partition.getPartitionNumber());
private void createPartitions(int numPartitions) { this.partitions.clear(); ListMemorySegmentSource memSource = new ListMemorySegmentSource(this.availableMemory); for (int i = 0; i < numPartitions; i++) { this.partitions.add(new InMemoryPartition<T>(this.buildSideSerializer, i, memSource, this.segmentSize, pageSizeInBits)); } this.compactionMemory = new InMemoryPartition<T>(this.buildSideSerializer, -1, memSource, this.segmentSize, pageSizeInBits); }