/** * Returns a new inMemoryPartition object. * This is required as a plug for ReOpenableMutableHashTable. */ protected HashPartition<BT, PT> getNewInMemoryPartition(int number, int recursionLevel) { return new HashPartition<BT, PT>(this.buildSideSerializer, this.probeSideSerializer, number, recursionLevel, this.availableMemory.remove(this.availableMemory.size() - 1), this, this.segmentSize); }
/** * This method clears all partitions currently residing (partially) in memory. It releases all memory * and deletes all spilled partitions. * <p> * This method is intended for a hard cleanup in the case that the join is aborted. */ protected void clearPartitions() { for (int i = this.partitionsBeingBuilt.size() - 1; i >= 0; --i) { final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(i); try { p.clearAllMemory(this.availableMemory); } catch (Exception e) { LOG.error("Error during partition cleanup.", e); } } this.partitionsBeingBuilt.clear(); }
if (p.isInMemory() && p.getNumOccupiedMemorySegments() > largestNumBlocks) { largestNumBlocks = p.getNumOccupiedMemorySegments(); largestPartNum = i; int numBuffersFreed = p.spillPartition(this.availableMemory, this.ioManager, this.currentEnumerator.next(), this.writeBehindBuffers); this.writeBehindBuffersAvailable += numBuffersFreed;
for (int i = 0; i < this.partitionsBeingBuilt.size(); i++) { final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(i); p.setFurtherPatitioning(this.furtherPartitioning); buffersAvailable += p.finalizeProbePhase(this.availableMemory, this.partitionsPending, this.buildSideOuterJoin); this.currentSpilledBuildSide = this.ioManager.createBlockChannelReader(p.getBuildSideChannel().getChannelID()); final ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(currentSpilledBuildSide, memory, p.getBuildSideBlockCount(), p.getLastSegmentLimit(), false); final ChannelReaderInputViewIterator<BT> inIter = new ChannelReaderInputViewIterator<BT>(inView, this.availableMemory, this.buildSideSerializer); this.currentSpilledProbeSide = this.ioManager.createBlockChannelReader(p.getProbeSideChannel().getChannelID(), returnQueue); returnQueue, memory, this.availableMemory, this.probeSideSerializer, p.getProbeSideBlockCount()); this.probeIterator.set(probeReader); this.currentRecursionDepth = p.getRecursionLevel() + 1;
final int nextRecursionLevel = p.getRecursionLevel() + 1; if (nextRecursionLevel > MAX_RECURSION_DEPTH) { throw new RuntimeException("Hash join exceeded maximum number of recursions, without reducing " long numBuckets = p.getBuildSideRecordCount() / NUM_ENTRIES_PER_BUCKET + 1; final long totalBuffersNeeded = 2 * (numBuckets / (this.bucketsPerSegmentMask + 1)) + p.getBuildSideBlockCount() + 2; ensureNumBuffersReturned(p.getBuildSideBlockCount()); final BulkBlockChannelReader reader = this.ioManager.createBulkBlockChannelReader(p.getBuildSideChannel().getChannelID(), this.availableMemory, p.getBuildSideBlockCount()); final HashPartition<BT, PT> newPart = new HashPartition<BT, PT>(this.buildSideSerializer, this.probeSideSerializer, 0, nextRecursionLevel, partitionBuffers, p.getBuildSideRecordCount(), this.segmentSize, p.getLastSegmentLimit()); final HashPartition<BT, PT>.PartitionIterator pIter = newPart.getPartitionIterator(this.buildSideComparator); BT record = this.buildSideSerializer.createInstance(); final int avgRecordLenPartition = (int) (((long) p.getBuildSideBlockCount()) * this.segmentSize / p.getBuildSideRecordCount()); final BlockChannelReader<MemorySegment> inReader = this.ioManager.createBlockChannelReader(p.getBuildSideChannel().getChannelID()); final ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(inReader, segments, p.getBuildSideBlockCount(), p.getLastSegmentLimit(), false); final ChannelReaderInputViewIterator<BT> inIter = new ChannelReaderInputViewIterator<BT>(inView, this.availableMemory, this.buildSideSerializer); part.finalizeBuildPhase(this.ioManager, this.currentEnumerator, this.writeBehindBuffers);
if (!isInMemory()) { throw new RuntimeException("Bug in Hybrid Hash Join: " + "Request to spill a partition that has already been spilled."); if (getNumOccupiedMemorySegments() < 2) { throw new RuntimeException("Bug in Hybrid Hash Join: " + "Request to spill a partition with less than two buffers.");
p.finalizeBuildPhase(this.ioManager, this.currentEnumerator, this.writeBehindBuffers);
return super.finalizeProbePhase(freeMemory, spilledPartitions, keepUnprobedSpilledPartitions);
for (int i = 0; i < this.partitionsBeingBuilt.size(); i++) { final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(i); p.setFurtherPatitioning(this.furtherPartitioning); buffersAvailable += p.finalizeProbePhase(this.availableMemory, this.partitionsPending, this.buildSideOuterJoin); this.currentSpilledBuildSide = this.ioManager.createBlockChannelReader(p.getBuildSideChannel().getChannelID()); final ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(currentSpilledBuildSide, memory, p.getBuildSideBlockCount(), p.getLastSegmentLimit(), false); final ChannelReaderInputViewIterator<BT> inIter = new ChannelReaderInputViewIterator<BT>(inView, this.availableMemory, this.buildSideSerializer); this.currentSpilledProbeSide = this.ioManager.createBlockChannelReader(p.getProbeSideChannel().getChannelID(), returnQueue); returnQueue, memory, this.availableMemory, this.probeSideSerializer, p.getProbeSideBlockCount()); this.probeIterator.set(probeReader); this.currentRecursionDepth = p.getRecursionLevel() + 1;
final int nextRecursionLevel = p.getRecursionLevel() + 1; if (nextRecursionLevel > MAX_RECURSION_DEPTH) { throw new RuntimeException("Hash join exceeded maximum number of recursions, without reducing " long numBuckets = p.getBuildSideRecordCount() / NUM_ENTRIES_PER_BUCKET + 1; final long totalBuffersNeeded = 2 * (numBuckets / (this.bucketsPerSegmentMask + 1)) + p.getBuildSideBlockCount() + 2; ensureNumBuffersReturned(p.getBuildSideBlockCount()); final BulkBlockChannelReader reader = this.ioManager.createBulkBlockChannelReader(p.getBuildSideChannel().getChannelID(), this.availableMemory, p.getBuildSideBlockCount()); final HashPartition<BT, PT> newPart = new HashPartition<BT, PT>(this.buildSideSerializer, this.probeSideSerializer, 0, nextRecursionLevel, partitionBuffers, p.getBuildSideRecordCount(), this.segmentSize, p.getLastSegmentLimit()); final HashPartition<BT, PT>.PartitionIterator pIter = newPart.getPartitionIterator(this.buildSideComparator); BT record = this.buildSideSerializer.createInstance(); final int avgRecordLenPartition = (int) (((long) p.getBuildSideBlockCount()) * this.segmentSize / p.getBuildSideRecordCount()); final BlockChannelReader<MemorySegment> inReader = this.ioManager.createBlockChannelReader(p.getBuildSideChannel().getChannelID()); final ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(inReader, segments, p.getBuildSideBlockCount(), p.getLastSegmentLimit(), false); final ChannelReaderInputViewIterator<BT> inIter = new ChannelReaderInputViewIterator<BT>(inView, this.availableMemory, this.buildSideSerializer); part.finalizeBuildPhase(this.ioManager, this.currentEnumerator, this.writeBehindBuffers);
if (!isInMemory()) { throw new RuntimeException("Bug in Hybrid Hash Join: " + "Request to spill a partition that has already been spilled."); if (getNumOccupiedMemorySegments() < 2) { throw new RuntimeException("Bug in Hybrid Hash Join: " + "Request to spill a partition with less than two buffers.");
p.finalizeBuildPhase(this.ioManager, this.currentEnumerator, this.writeBehindBuffers);
return super.finalizeProbePhase(freeMemory, spilledPartitions, keepUnprobedSpilledPartitions);
for (int i = 0; i < this.partitionsBeingBuilt.size(); i++) { final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(i); p.setFurtherPatitioning(this.furtherPartitioning); buffersAvailable += p.finalizeProbePhase(this.availableMemory, this.partitionsPending, this.buildSideOuterJoin); this.currentSpilledBuildSide = this.ioManager.createBlockChannelReader(p.getBuildSideChannel().getChannelID()); final ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(currentSpilledBuildSide, memory, p.getBuildSideBlockCount(), p.getLastSegmentLimit(), false); final ChannelReaderInputViewIterator<BT> inIter = new ChannelReaderInputViewIterator<BT>(inView, this.availableMemory, this.buildSideSerializer); this.currentSpilledProbeSide = this.ioManager.createBlockChannelReader(p.getProbeSideChannel().getChannelID(), returnQueue); returnQueue, memory, this.availableMemory, this.probeSideSerializer, p.getProbeSideBlockCount()); this.probeIterator.set(probeReader); this.currentRecursionDepth = p.getRecursionLevel() + 1;
final int nextRecursionLevel = p.getRecursionLevel() + 1; if (nextRecursionLevel > MAX_RECURSION_DEPTH) { throw new RuntimeException("Hash join exceeded maximum number of recursions, without reducing " long numBuckets = p.getBuildSideRecordCount() / NUM_ENTRIES_PER_BUCKET + 1; final long totalBuffersNeeded = 2 * (numBuckets / (this.bucketsPerSegmentMask + 1)) + p.getBuildSideBlockCount() + 2; ensureNumBuffersReturned(p.getBuildSideBlockCount()); final BulkBlockChannelReader reader = this.ioManager.createBulkBlockChannelReader(p.getBuildSideChannel().getChannelID(), this.availableMemory, p.getBuildSideBlockCount()); final HashPartition<BT, PT> newPart = new HashPartition<BT, PT>(this.buildSideSerializer, this.probeSideSerializer, 0, nextRecursionLevel, partitionBuffers, p.getBuildSideRecordCount(), this.segmentSize, p.getLastSegmentLimit()); final HashPartition<BT, PT>.PartitionIterator pIter = newPart.getPartitionIterator(this.buildSideComparator); BT record = this.buildSideSerializer.createInstance(); final int avgRecordLenPartition = (int) (((long) p.getBuildSideBlockCount()) * this.segmentSize / p.getBuildSideRecordCount()); final BlockChannelReader<MemorySegment> inReader = this.ioManager.createBlockChannelReader(p.getBuildSideChannel().getChannelID()); final ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(inReader, segments, p.getBuildSideBlockCount(), p.getLastSegmentLimit(), false); final ChannelReaderInputViewIterator<BT> inIter = new ChannelReaderInputViewIterator<BT>(inView, this.availableMemory, this.buildSideSerializer); part.finalizeBuildPhase(this.ioManager, this.currentEnumerator, this.writeBehindBuffers);
if (p.isInMemory() && p.getNumOccupiedMemorySegments() > largestNumBlocks) { largestNumBlocks = p.getNumOccupiedMemorySegments(); largestPartNum = i; int numBuffersFreed = p.spillPartition(this.availableMemory, this.ioManager, this.currentEnumerator.next(), this.writeBehindBuffers); this.writeBehindBuffersAvailable += numBuffersFreed;
if (!isInMemory()) { throw new RuntimeException("Bug in Hybrid Hash Join: " + "Request to spill a partition that has already been spilled."); if (getNumOccupiedMemorySegments() < 2) { throw new RuntimeException("Bug in Hybrid Hash Join: " + "Request to spill a partition with less than two buffers.");
@Override public void clearAllMemory(List<MemorySegment> target) { if (initialBuildSideChannel != null) { try { this.initialBuildSideWriter.closeAndDelete(); } catch (IOException ioex) { throw new RuntimeException("Error deleting the partition files. Some temporary files might not be removed."); } } super.clearAllMemory(target); }
/** * Returns a new inMemoryPartition object. * This is required as a plug for ReOpenableMutableHashTable. */ protected HashPartition<BT, PT> getNewInMemoryPartition(int number, int recursionLevel) { return new HashPartition<BT, PT>(this.buildSideSerializer, this.probeSideSerializer, number, recursionLevel, this.availableMemory.remove(this.availableMemory.size() - 1), this, this.segmentSize); }
p.finalizeBuildPhase(this.ioManager, this.currentEnumerator, this.writeBehindBuffers);