/** * Inserts the given object into the current buffer. This method returns a pointer that * can be used to address the written record in this partition, if it is in-memory. The returned * pointers have no expressiveness in the case where the partition is spilled. * * @param record The object to be written to the partition. * @return A pointer to the object in the partition, or <code>-1</code>, if the partition is spilled. * @throws IOException Thrown, when this is a spilled partition and the write failed. */ public final long insertIntoBuildBuffer(BT record) throws IOException { this.buildSideRecordCounter++; if (isInMemory()) { final long pointer = this.buildSideWriteBuffer.getCurrentPointer(); this.buildSideSerializer.serialize(record, this.buildSideWriteBuffer); return isInMemory() ? pointer : -1; } else { this.buildSideSerializer.serialize(record, this.buildSideWriteBuffer); return -1; } }
/** * Inserts the given object into the current buffer. This method returns a pointer that * can be used to address the written record in this partition, if it is in-memory. The returned * pointers have no expressiveness in the case where the partition is spilled. * * @param record The object to be written to the partition. * @return A pointer to the object in the partition, or <code>-1</code>, if the partition is spilled. * @throws IOException Thrown, when this is a spilled partition and the write failed. */ public final long insertIntoBuildBuffer(BT record) throws IOException { this.buildSideRecordCounter++; if (isInMemory()) { final long pointer = this.buildSideWriteBuffer.getCurrentPointer(); this.buildSideSerializer.serialize(record, this.buildSideWriteBuffer); return isInMemory() ? pointer : -1; } else { this.buildSideSerializer.serialize(record, this.buildSideWriteBuffer); return -1; } }
/** * Inserts the given object into the current buffer. This method returns a pointer that * can be used to address the written record in this partition, if it is in-memory. The returned * pointers have no expressiveness in the case where the partition is spilled. * * @param record The object to be written to the partition. * @return A pointer to the object in the partition, or <code>-1</code>, if the partition is spilled. * @throws IOException Thrown, when this is a spilled partition and the write failed. */ public final long insertIntoBuildBuffer(BT record) throws IOException { this.buildSideRecordCounter++; if (isInMemory()) { final long pointer = this.buildSideWriteBuffer.getCurrentPointer(); this.buildSideSerializer.serialize(record, this.buildSideWriteBuffer); return isInMemory() ? pointer : -1; } else { this.buildSideSerializer.serialize(record, this.buildSideWriteBuffer); return -1; } }
/** * Inserts the given object into the current buffer. This method returns a pointer that * can be used to address the written record in this partition, if it is in-memory. The returned * pointers have no expressiveness in the case where the partition is spilled. * * @param record The object to be written to the partition. * @return A pointer to the object in the partition, or <code>-1</code>, if the partition is spilled. * @throws IOException Thrown, when this is a spilled partition and the write failed. */ public final long insertIntoBuildBuffer(BT record) throws IOException { this.buildSideRecordCounter++; if (isInMemory()) { final long pointer = this.buildSideWriteBuffer.getCurrentPointer(); this.buildSideSerializer.serialize(record, this.buildSideWriteBuffer); return isInMemory() ? pointer : -1; } else { this.buildSideSerializer.serialize(record, this.buildSideWriteBuffer); return -1; } }
@Override protected boolean prepareNextPartition() throws IOException { // check if there will be further partition processing. this.furtherPartitioning = false; for (int i = 0; i < this.partitionsBeingBuilt.size(); i++) { final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(i); if (!p.isInMemory() && p.getProbeSideRecordCount() != 0) { furtherPartitioning = true; break; } } if (furtherPartitioning) { ((ReOpenableMutableHashTable<BT, PT>) this).storeInitialHashTable(); } return super.prepareNextPartition(); }
@Override protected boolean prepareNextPartition() throws IOException { // check if there will be further partition processing. this.furtherPartitioning = false; for (int i = 0; i < this.partitionsBeingBuilt.size(); i++) { final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(i); if (!p.isInMemory() && p.getProbeSideRecordCount() != 0) { furtherPartitioning = true; break; } } if (furtherPartitioning) { ((ReOpenableMutableHashTable<BT, PT>) this).storeInitialHashTable(); } return super.prepareNextPartition(); }
@Override protected boolean prepareNextPartition() throws IOException { // check if there will be further partition processing. this.furtherPartitioning = false; for (int i = 0; i < this.partitionsBeingBuilt.size(); i++) { final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(i); if (!p.isInMemory() && p.getProbeSideRecordCount() != 0) { furtherPartitioning = true; break; } } if (furtherPartitioning) { ((ReOpenableMutableHashTable<BT, PT>) this).storeInitialHashTable(); } return super.prepareNextPartition(); }
/** * Move to next bucket, return true while move to a on heap bucket, return false while move to a spilled bucket * or there is no more bucket. */ private boolean moveToNextBucket() { scanCount++; if (scanCount > totalBucketNumber - 1) { return false; } // move to next bucket, update all the current bucket status with new bucket information. final int bucketArrayPos = scanCount >> this.bucketsPerSegmentBits; final int currentBucketInSegmentOffset = (scanCount & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS; MemorySegment currentBucket = this.buckets[bucketArrayPos]; final int partitionNumber = currentBucket.get(currentBucketInSegmentOffset + HEADER_PARTITION_OFFSET); final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(partitionNumber); if (p.isInMemory()) { setBucket(currentBucket, p.overflowSegments, p, currentBucketInSegmentOffset); return true; } else { return false; } }
/** * Move to next bucket, return true while move to a on heap bucket, return false while move to a spilled bucket * or there is no more bucket. */ private boolean moveToNextBucket() { scanCount++; if (scanCount > totalBucketNumber - 1) { return false; } // move to next bucket, update all the current bucket status with new bucket information. final int bucketArrayPos = scanCount >> this.bucketsPerSegmentBits; final int currentBucketInSegmentOffset = (scanCount & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS; MemorySegment currentBucket = this.buckets[bucketArrayPos]; final int partitionNumber = currentBucket.get(currentBucketInSegmentOffset + HEADER_PARTITION_OFFSET); final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(partitionNumber); if (p.isInMemory()) { setBucket(currentBucket, p.overflowSegments, p, currentBucketInSegmentOffset); return true; } else { return false; } }
@Override protected boolean prepareNextPartition() throws IOException { // check if there will be further partition processing. this.furtherPartitioning = false; for (int i = 0; i < this.partitionsBeingBuilt.size(); i++) { final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(i); if (!p.isInMemory() && p.getProbeSideRecordCount() != 0) { furtherPartitioning = true; break; } } if (furtherPartitioning) { ((ReOpenableMutableHashTable<BT, PT>) this).storeInitialHashTable(); } return super.prepareNextPartition(); }
public void prepareProbePhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { if (isInMemory()) { return; } // ATTENTION: The following lines are duplicated code from finalizeBuildPhase this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); }
public void prepareProbePhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { if (isInMemory()) { return; } // ATTENTION: The following lines are duplicated code from finalizeBuildPhase this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); }
public void prepareProbePhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { if (isInMemory()) { return; } // ATTENTION: The following lines are duplicated code from finalizeBuildPhase this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); }
public void prepareProbePhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { if (isInMemory()) { return; } // ATTENTION: The following lines are duplicated code from finalizeBuildPhase this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); }
public HashBucketIterator<BT, PT> getMatchesFor(PT record) throws IOException { final TypeComparator<PT> probeAccessors = this.probeSideComparator; final int hash = hash(probeAccessors.hash(record), this.currentRecursionDepth); final int posHashCode = hash % this.numBuckets; // get the bucket for the given hash code final int bucketArrayPos = posHashCode >> this.bucketsPerSegmentBits; final int bucketInSegmentOffset = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS; final MemorySegment bucket = this.buckets[bucketArrayPos]; // get the basic characteristics of the bucket final int partitionNumber = bucket.get(bucketInSegmentOffset + HEADER_PARTITION_OFFSET); final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(partitionNumber); // for an in-memory partition, process set the return iterators, else spill the probe records if (p.isInMemory()) { this.recordComparator.setReference(record); this.bucketIterator.set(bucket, p.overflowSegments, p, hash, bucketInSegmentOffset); return this.bucketIterator; } else { throw new IllegalStateException("Method is not applicable to partially spilled hash tables."); } }
public HashBucketIterator<BT, PT> getMatchesFor(PT record) throws IOException { final TypeComparator<PT> probeAccessors = this.probeSideComparator; final int hash = hash(probeAccessors.hash(record), this.currentRecursionDepth); final int posHashCode = hash % this.numBuckets; // get the bucket for the given hash code final int bucketArrayPos = posHashCode >> this.bucketsPerSegmentBits; final int bucketInSegmentOffset = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS; final MemorySegment bucket = this.buckets[bucketArrayPos]; // get the basic characteristics of the bucket final int partitionNumber = bucket.get(bucketInSegmentOffset + HEADER_PARTITION_OFFSET); final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(partitionNumber); // for an in-memory partition, process set the return iterators, else spill the probe records if (p.isInMemory()) { this.recordComparator.setReference(record); this.bucketIterator.set(bucket, p.overflowSegments, p, hash, bucketInSegmentOffset); return this.bucketIterator; } else { throw new IllegalStateException("Method is not applicable to partially spilled hash tables."); } }
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment(); this.partitionBuffers = this.buildSideWriteBuffer.close(); if (!isInMemory()) { // close the channel. note that in the spilled case, the build-side-buffer will have sent off // the last segment and it will be returned to the write-behind-buffer queue. this.buildSideChannel.close(); // create the channel for the probe side and claim one buffer for it this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); // creating the ChannelWriterOutputView without memory will cause it to draw one segment from the // write behind queue, which is the spare segment we had above. this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); } }
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment(); this.partitionBuffers = this.buildSideWriteBuffer.close(); if (!isInMemory()) { // close the channel. note that in the spilled case, the build-side-buffer will have sent off // the last segment and it will be returned to the write-behind-buffer queue. this.buildSideChannel.close(); // create the channel for the probe side and claim one buffer for it this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); // creating the ChannelWriterOutputView without memory will cause it to draw one segment from the // write behind queue, which is the spare segment we had above. this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); } }
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment(); this.partitionBuffers = this.buildSideWriteBuffer.close(); if (!isInMemory()) { // close the channel. note that in the spilled case, the build-side-buffer will have sent off // the last segment and it will be returned to the write-behind-buffer queue. this.buildSideChannel.close(); // create the channel for the probe side and claim one buffer for it this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); // creating the ChannelWriterOutputView without memory will cause it to draw one segment from the // write behind queue, which is the spare segment we had above. this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); } }
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment(); this.partitionBuffers = this.buildSideWriteBuffer.close(); if (!isInMemory()) { // close the channel. note that in the spilled case, the build-side-buffer will have sent off // the last segment and it will be returned to the write-behind-buffer queue. this.buildSideChannel.close(); // create the channel for the probe side and claim one buffer for it this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); // creating the ChannelWriterOutputView without memory will cause it to draw one segment from the // write behind queue, which is the spare segment we had above. this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); } }