private void initTable(int numBuckets, byte numPartitions) { final int bucketsPerSegment = this.bucketsPerSegmentMask + 1; final int numSegs = (numBuckets >>> this.bucketsPerSegmentBits) + ( (numBuckets & this.bucketsPerSegmentMask) == 0 ? 0 : 1); final MemorySegment[] table = new MemorySegment[numSegs]; // go over all segments that are part of the table for (int i = 0, bucket = 0; i < numSegs && bucket < numBuckets; i++) { final MemorySegment seg = getNextBuffer(); // go over all buckets in the segment for (int k = 0; k < bucketsPerSegment && bucket < numBuckets; k++, bucket++) { final int bucketOffset = k * HASH_BUCKET_SIZE; // compute the partition that the bucket corresponds to final byte partition = assignPartition(bucket, numPartitions); // initialize the header fields seg.put(bucketOffset + HEADER_PARTITION_OFFSET, partition); seg.putInt(bucketOffset + HEADER_COUNT_OFFSET, 0); seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET); } table[i] = seg; } this.buckets = table; this.numBuckets = numBuckets; }
private void readInitialSolutionSet(CompactingHashTable<X> solutionSet, MutableObjectIterator<X> solutionSetInput) throws IOException { solutionSet.open(); solutionSet.buildTableWithUniqueKey(solutionSetInput); }
/** * Closes the hash table. This effectively releases all internal structures and closes all * open files and removes them. The call to this method is valid both as a cleanup after the * complete inputs were properly processed, and as an cancellation call, which cleans up * all resources that are currently held by the hash join. If another process still access the hash * table after close has been called no operations will be performed. */ @Override public void close() { // make sure that we close only once synchronized (this.stateLock) { if (this.closed) { return; } this.closed = true; } LOG.debug("Closing hash table and releasing resources."); // release the table structure releaseTable(); // clear the memory in the partitions clearPartitions(); }
/** * Initialize the hash table */ @Override public void open() { synchronized (stateLock) { if (!closed) { throw new IllegalStateException("currently not closed."); } closed = false; } // create the partitions final int partitionFanOut = getPartitioningFanOutNoEstimates(this.availableMemory.size()); createPartitions(partitionFanOut); // set up the table structure. the write behind buffers are taken away, as are one buffer per partition final int numBuckets = getInitialTableSize(this.availableMemory.size(), this.segmentSize, partitionFanOut, this.avgRecordLen); initTable(numBuckets, (byte) partitionFanOut); }
if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT1>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
compactPartition(i); if(this.availableMemory.size() >= additionalSegments) { break; bucketOffset = k * HASH_BUCKET_SIZE; seg.put(bucketOffset + HEADER_PARTITION_OFFSET, assignPartition(bucket, (byte)numPartitions)); seg.putInt(bucketOffset + HEADER_COUNT_OFFSET, 0); seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET); seg = getNextBuffer(); seg.put(bucketOffset + HEADER_PARTITION_OFFSET, assignPartition(bucket, (byte)numPartitions)); seg.putInt(bucketOffset + HEADER_COUNT_OFFSET, 0); seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET); if (posHashCode == bucket && oldBucketCount < NUM_ENTRIES_PER_BUCKET) { bucketOffset = (bucket % bucketsPerSegment) * HASH_BUCKET_SIZE; insertBucketEntryFromStart(segment, bucketOffset, hash, pointer, partition.getPartitionNumber()); oldBucketCount++; insertBucketEntryFromStart(newSegment, bucketOffset, hash, pointer, partition.getPartitionNumber()); newBucketCount++; bucketInSegmentPos = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS; bucket = this.buckets[bucketArrayPos]; insertBucketEntryFromStart(bucket, bucketInSegmentPos, hash, pointer, partition.getPartitionNumber());
int numPages = getMemoryManager().computeNumberOfPages(hashjoinMemorySize); memSegments = getMemoryManager().allocatePages(getContainingTask(), numPages); hashTable = new CompactingHashTable<BT>(solutionTypeSerializer, solutionTypeComparator, memSegments); success = true; return hashTable; if (hashTable != null) { try { hashTable.close(); } catch (Throwable t) { log.error("Error closing the solution set hash table after unsuccessful creation.", t);
compactPartition(partitionNumber); getMemoryConsumptionString() + " Message: " + ex.getMessage());
private void streamSolutionSetToFinalOutput(CompactingHashTable<X> hashTable) throws IOException { final MutableObjectIterator<X> results = hashTable.getEntryIterator(); final Collector<X> output = this.finalOutputCollector; X record = solutionTypeSerializer.getSerializer().createInstance(); while ((record = results.next(record)) != null) { output.collect(record); } }
solutionSet.close();
/** * Initialize the hash table */ @Override public void open() { synchronized (stateLock) { if (!closed) { throw new IllegalStateException("currently not closed."); } closed = false; } // create the partitions final int partitionFanOut = getPartitioningFanOutNoEstimates(this.availableMemory.size()); createPartitions(partitionFanOut); // set up the table structure. the write behind buffers are taken away, as are one buffer per partition final int numBuckets = getInitialTableSize(this.availableMemory.size(), this.segmentSize, partitionFanOut, this.avgRecordLen); initTable(numBuckets, (byte) partitionFanOut); }
if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT1>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
compactPartition(i); if(this.availableMemory.size() >= additionalSegments) { break; bucketOffset = k * HASH_BUCKET_SIZE; seg.put(bucketOffset + HEADER_PARTITION_OFFSET, assignPartition(bucket, (byte)numPartitions)); seg.putInt(bucketOffset + HEADER_COUNT_OFFSET, 0); seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET); seg = getNextBuffer(); seg.put(bucketOffset + HEADER_PARTITION_OFFSET, assignPartition(bucket, (byte)numPartitions)); seg.putInt(bucketOffset + HEADER_COUNT_OFFSET, 0); seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET); if (posHashCode == bucket && oldBucketCount < NUM_ENTRIES_PER_BUCKET) { bucketOffset = (bucket % bucketsPerSegment) * HASH_BUCKET_SIZE; insertBucketEntryFromStart(segment, bucketOffset, hash, pointer, partition.getPartitionNumber()); oldBucketCount++; insertBucketEntryFromStart(newSegment, bucketOffset, hash, pointer, partition.getPartitionNumber()); newBucketCount++; bucketInSegmentPos = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS; bucket = this.buckets[bucketArrayPos]; insertBucketEntryFromStart(bucket, bucketInSegmentPos, hash, pointer, partition.getPartitionNumber());
int numPages = getMemoryManager().computeNumberOfPages(hashjoinMemorySize); memSegments = getMemoryManager().allocatePages(getContainingTask(), numPages); hashTable = new CompactingHashTable<BT>(solutionTypeSerializer, solutionTypeComparator, memSegments); success = true; return hashTable; if (hashTable != null) { try { hashTable.close(); } catch (Throwable t) { log.error("Error closing the solution set hash table after unsuccessful creation.", t);
compactPartition(partitionNumber); getMemoryConsumptionString() + " Message: " + ex.getMessage());
private void streamSolutionSetToFinalOutput(CompactingHashTable<X> hashTable) throws IOException { final MutableObjectIterator<X> results = hashTable.getEntryIterator(); final Collector<X> output = this.finalOutputCollector; X record = solutionTypeSerializer.getSerializer().createInstance(); while ((record = results.next(record)) != null) { output.collect(record); } }
solutionSet.close();
/** * Initialize the hash table */ @Override public void open() { synchronized (stateLock) { if (!closed) { throw new IllegalStateException("currently not closed."); } closed = false; } // create the partitions final int partitionFanOut = getPartitioningFanOutNoEstimates(this.availableMemory.size()); createPartitions(partitionFanOut); // set up the table structure. the write behind buffers are taken away, as are one buffer per partition final int numBuckets = getInitialTableSize(this.availableMemory.size(), this.segmentSize, partitionFanOut, this.avgRecordLen); initTable(numBuckets, (byte) partitionFanOut); }
if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT1>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
compactPartition(i); if(this.availableMemory.size() >= additionalSegments) { break; bucketOffset = k * HASH_BUCKET_SIZE; seg.put(bucketOffset + HEADER_PARTITION_OFFSET, assignPartition(bucket, (byte)numPartitions)); seg.putInt(bucketOffset + HEADER_COUNT_OFFSET, 0); seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET); seg = getNextBuffer(); seg.put(bucketOffset + HEADER_PARTITION_OFFSET, assignPartition(bucket, (byte)numPartitions)); seg.putInt(bucketOffset + HEADER_COUNT_OFFSET, 0); seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET); if (posHashCode == bucket && oldBucketCount < NUM_ENTRIES_PER_BUCKET) { bucketOffset = (bucket % bucketsPerSegment) * HASH_BUCKET_SIZE; insertBucketEntryFromStart(segment, bucketOffset, hash, pointer, partition.getPartitionNumber()); oldBucketCount++; insertBucketEntryFromStart(newSegment, bucketOffset, hash, pointer, partition.getPartitionNumber()); newBucketCount++; bucketInSegmentPos = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS; bucket = this.buckets[bucketArrayPos]; insertBucketEntryFromStart(bucket, bucketInSegmentPos, hash, pointer, partition.getPartitionNumber());