public HashTableProber(TypeComparator<PT> probeTypeComparator, TypePairComparator<PT, T> pairComparator) { super(probeTypeComparator, pairComparator); }
/** * @return String containing a summary of the memory consumption for error messages */ private String getMemoryConsumptionString() { return "InPlaceMutableHashTable memory stats:\n" + "Total memory: " + numAllMemorySegments * segmentSize + "\n" + "Free memory: " + freeMemorySegments.size() * segmentSize + "\n" + "Bucket area: " + numBuckets * 8 + "\n" + "Record area: " + recordArea.getTotalSize() + "\n" + "Staging area: " + stagingSegments.size() * segmentSize + "\n" + "Num of elements: " + numElements + "\n" + "Holes total size: " + holes; }
@Override public void close() { // make sure that we close only once synchronized (stateLock) { if (closed) { // We have to do this here, because the ctor already allocates a segment to the record area and // the staging area, even before we are opened. So we might have segments to free, even if we // are closed. recordArea.giveBackSegments(); freeMemorySegments.addAll(stagingSegments); stagingSegments.clear(); return; } closed = true; } LOG.debug("Closing InPlaceMutableHashTable and releasing resources."); releaseBucketSegments(); recordArea.giveBackSegments(); freeMemorySegments.addAll(stagingSegments); stagingSegments.clear(); numElements = 0; holes = 0; }
pointerToAppended = recordArea.appendPointerAndRecord(END_OF_LIST ,record); } catch (EOFException ex) { compactOrThrow();
pointerToAppended = recordArea.appendPointerAndRecord(END_OF_LIST ,record); } catch (EOFException ex) { compactOrThrow();
pointerToAppended = recordArea.appendPointerAndRecord(END_OF_LIST ,record); } catch (EOFException ex) { compactOrThrow();
pointerToAppended = recordArea.appendPointerAndRecord(END_OF_LIST ,record); } catch (EOFException ex) { compactOrThrow();
/** * Inserts the given record into the hash table. * Note: this method doesn't care about whether a record with the same key is already present. * @param record The record to insert. * @throws IOException (EOFException specifically, if memory ran out) */ @Override public void insert(T record) throws IOException { if (closed) { return; } final int hashCode = MathUtils.jenkinsHash(buildSideComparator.hash(record)); final int bucket = hashCode & numBucketsMask; final int bucketSegmentIndex = bucket >>> numBucketsPerSegmentBits; // which segment contains the bucket final MemorySegment bucketSegment = bucketSegments[bucketSegmentIndex]; final int bucketOffset = (bucket & numBucketsPerSegmentMask) << bucketSizeBits; // offset of the bucket in the segment final long firstPointer = bucketSegment.getLong(bucketOffset); try { final long newFirstPointer = recordArea.appendPointerAndRecord(firstPointer, record); bucketSegment.putLong(bucketOffset, newFirstPointer); } catch (EOFException ex) { compactOrThrow(); insert(record); return; } numElements++; resizeTableIfNecessary(); }
/** Same as above, but the number of bucket segments of the new table can be specified. */ private void rebuild(long newNumBucketSegments) throws IOException { // Get new bucket segments releaseBucketSegments(); allocateBucketSegments((int)newNumBucketSegments); T record = buildSideSerializer.createInstance(); try { EntryIterator iter = getEntryIterator(); recordArea.resetAppendPosition(); recordArea.setWritePosition(0); while ((record = iter.next(record)) != null && !closed) { final int hashCode = MathUtils.jenkinsHash(buildSideComparator.hash(record)); final int bucket = hashCode & numBucketsMask; final int bucketSegmentIndex = bucket >>> numBucketsPerSegmentBits; // which segment contains the bucket final MemorySegment bucketSegment = bucketSegments[bucketSegmentIndex]; final int bucketOffset = (bucket & numBucketsPerSegmentMask) << bucketSizeBits; // offset of the bucket in the segment final long firstPointer = bucketSegment.getLong(bucketOffset); long ptrToAppended = recordArea.noSeekAppendPointerAndRecord(firstPointer, record); bucketSegment.putLong(bucketOffset, ptrToAppended); } recordArea.freeSegmentsAfterAppendPosition(); holes = 0; } catch (EOFException ex) { throw new RuntimeException("Bug in InPlaceMutableHashTable: we shouldn't get out of memory during a rebuild, " + "because we aren't allocating any new memory."); } }
/** * Inserts the given record into the hash table. * Note: this method doesn't care about whether a record with the same key is already present. * @param record The record to insert. * @throws IOException (EOFException specifically, if memory ran out) */ @Override public void insert(T record) throws IOException { if (closed) { return; } final int hashCode = MathUtils.jenkinsHash(buildSideComparator.hash(record)); final int bucket = hashCode & numBucketsMask; final int bucketSegmentIndex = bucket >>> numBucketsPerSegmentBits; // which segment contains the bucket final MemorySegment bucketSegment = bucketSegments[bucketSegmentIndex]; final int bucketOffset = (bucket & numBucketsPerSegmentMask) << bucketSizeBits; // offset of the bucket in the segment final long firstPointer = bucketSegment.getLong(bucketOffset); try { final long newFirstPointer = recordArea.appendPointerAndRecord(firstPointer, record); bucketSegment.putLong(bucketOffset, newFirstPointer); } catch (EOFException ex) { compactOrThrow(); insert(record); return; } numElements++; resizeTableIfNecessary(); }
/** * Inserts the given record into the hash table. * Note: this method doesn't care about whether a record with the same key is already present. * @param record The record to insert. * @throws IOException (EOFException specifically, if memory ran out) */ @Override public void insert(T record) throws IOException { if (closed) { return; } final int hashCode = MathUtils.jenkinsHash(buildSideComparator.hash(record)); final int bucket = hashCode & numBucketsMask; final int bucketSegmentIndex = bucket >>> numBucketsPerSegmentBits; // which segment contains the bucket final MemorySegment bucketSegment = bucketSegments[bucketSegmentIndex]; final int bucketOffset = (bucket & numBucketsPerSegmentMask) << bucketSizeBits; // offset of the bucket in the segment final long firstPointer = bucketSegment.getLong(bucketOffset); try { final long newFirstPointer = recordArea.appendPointerAndRecord(firstPointer, record); bucketSegment.putLong(bucketOffset, newFirstPointer); } catch (EOFException ex) { compactOrThrow(); insert(record); return; } numElements++; resizeTableIfNecessary(); }
/** * Inserts the given record into the hash table. * Note: this method doesn't care about whether a record with the same key is already present. * @param record The record to insert. * @throws IOException (EOFException specifically, if memory ran out) */ @Override public void insert(T record) throws IOException { if (closed) { return; } final int hashCode = MathUtils.jenkinsHash(buildSideComparator.hash(record)); final int bucket = hashCode & numBucketsMask; final int bucketSegmentIndex = bucket >>> numBucketsPerSegmentBits; // which segment contains the bucket final MemorySegment bucketSegment = bucketSegments[bucketSegmentIndex]; final int bucketOffset = (bucket & numBucketsPerSegmentMask) << bucketSizeBits; // offset of the bucket in the segment final long firstPointer = bucketSegment.getLong(bucketOffset); try { final long newFirstPointer = recordArea.appendPointerAndRecord(firstPointer, record); bucketSegment.putLong(bucketOffset, newFirstPointer); } catch (EOFException ex) { compactOrThrow(); insert(record); return; } numElements++; resizeTableIfNecessary(); }
/** Same as above, but the number of bucket segments of the new table can be specified. */ private void rebuild(long newNumBucketSegments) throws IOException { // Get new bucket segments releaseBucketSegments(); allocateBucketSegments((int)newNumBucketSegments); T record = buildSideSerializer.createInstance(); try { EntryIterator iter = getEntryIterator(); recordArea.resetAppendPosition(); recordArea.setWritePosition(0); while ((record = iter.next(record)) != null && !closed) { final int hashCode = MathUtils.jenkinsHash(buildSideComparator.hash(record)); final int bucket = hashCode & numBucketsMask; final int bucketSegmentIndex = bucket >>> numBucketsPerSegmentBits; // which segment contains the bucket final MemorySegment bucketSegment = bucketSegments[bucketSegmentIndex]; final int bucketOffset = (bucket & numBucketsPerSegmentMask) << bucketSizeBits; // offset of the bucket in the segment final long firstPointer = bucketSegment.getLong(bucketOffset); long ptrToAppended = recordArea.noSeekAppendPointerAndRecord(firstPointer, record); bucketSegment.putLong(bucketOffset, ptrToAppended); } recordArea.freeSegmentsAfterAppendPosition(); holes = 0; } catch (EOFException ex) { throw new RuntimeException("Bug in InPlaceMutableHashTable: we shouldn't get out of memory during a rebuild, " + "because we aren't allocating any new memory."); } }
/** Same as above, but the number of bucket segments of the new table can be specified. */ private void rebuild(long newNumBucketSegments) throws IOException { // Get new bucket segments releaseBucketSegments(); allocateBucketSegments((int)newNumBucketSegments); T record = buildSideSerializer.createInstance(); try { EntryIterator iter = getEntryIterator(); recordArea.resetAppendPosition(); recordArea.setWritePosition(0); while ((record = iter.next(record)) != null && !closed) { final int hashCode = MathUtils.jenkinsHash(buildSideComparator.hash(record)); final int bucket = hashCode & numBucketsMask; final int bucketSegmentIndex = bucket >>> numBucketsPerSegmentBits; // which segment contains the bucket final MemorySegment bucketSegment = bucketSegments[bucketSegmentIndex]; final int bucketOffset = (bucket & numBucketsPerSegmentMask) << bucketSizeBits; // offset of the bucket in the segment final long firstPointer = bucketSegment.getLong(bucketOffset); long ptrToAppended = recordArea.noSeekAppendPointerAndRecord(firstPointer, record); bucketSegment.putLong(bucketOffset, ptrToAppended); } recordArea.freeSegmentsAfterAppendPosition(); holes = 0; } catch (EOFException ex) { throw new RuntimeException("Bug in InPlaceMutableHashTable: we shouldn't get out of memory during a rebuild, " + "because we aren't allocating any new memory."); } }
/** Same as above, but the number of bucket segments of the new table can be specified. */ private void rebuild(long newNumBucketSegments) throws IOException { // Get new bucket segments releaseBucketSegments(); allocateBucketSegments((int)newNumBucketSegments); T record = buildSideSerializer.createInstance(); try { EntryIterator iter = getEntryIterator(); recordArea.resetAppendPosition(); recordArea.setWritePosition(0); while ((record = iter.next(record)) != null && !closed) { final int hashCode = MathUtils.jenkinsHash(buildSideComparator.hash(record)); final int bucket = hashCode & numBucketsMask; final int bucketSegmentIndex = bucket >>> numBucketsPerSegmentBits; // which segment contains the bucket final MemorySegment bucketSegment = bucketSegments[bucketSegmentIndex]; final int bucketOffset = (bucket & numBucketsPerSegmentMask) << bucketSizeBits; // offset of the bucket in the segment final long firstPointer = bucketSegment.getLong(bucketOffset); long ptrToAppended = recordArea.noSeekAppendPointerAndRecord(firstPointer, record); bucketSegment.putLong(bucketOffset, ptrToAppended); } recordArea.freeSegmentsAfterAppendPosition(); holes = 0; } catch (EOFException ex) { throw new RuntimeException("Bug in InPlaceMutableHashTable: we shouldn't get out of memory during a rebuild, " + "because we aren't allocating any new memory."); } }
@Override public void close() { // make sure that we close only once synchronized (stateLock) { if (closed) { // We have to do this here, because the ctor already allocates a segment to the record area and // the staging area, even before we are opened. So we might have segments to free, even if we // are closed. recordArea.giveBackSegments(); freeMemorySegments.addAll(stagingSegments); stagingSegments.clear(); return; } closed = true; } LOG.debug("Closing InPlaceMutableHashTable and releasing resources."); releaseBucketSegments(); recordArea.giveBackSegments(); freeMemorySegments.addAll(stagingSegments); stagingSegments.clear(); numElements = 0; holes = 0; }
@Override public void close() { // make sure that we close only once synchronized (stateLock) { if (closed) { // We have to do this here, because the ctor already allocates a segment to the record area and // the staging area, even before we are opened. So we might have segments to free, even if we // are closed. recordArea.giveBackSegments(); freeMemorySegments.addAll(stagingSegments); stagingSegments.clear(); return; } closed = true; } LOG.debug("Closing InPlaceMutableHashTable and releasing resources."); releaseBucketSegments(); recordArea.giveBackSegments(); freeMemorySegments.addAll(stagingSegments); stagingSegments.clear(); numElements = 0; holes = 0; }
@Override public void close() { // make sure that we close only once synchronized (stateLock) { if (closed) { // We have to do this here, because the ctor already allocates a segment to the record area and // the staging area, even before we are opened. So we might have segments to free, even if we // are closed. recordArea.giveBackSegments(); freeMemorySegments.addAll(stagingSegments); stagingSegments.clear(); return; } closed = true; } LOG.debug("Closing InPlaceMutableHashTable and releasing resources."); releaseBucketSegments(); recordArea.giveBackSegments(); freeMemorySegments.addAll(stagingSegments); stagingSegments.clear(); numElements = 0; holes = 0; }
public HashTableProber(TypeComparator<PT> probeTypeComparator, TypePairComparator<PT, T> pairComparator) { super(probeTypeComparator, pairComparator); }
public HashTableProber(TypeComparator<PT> probeTypeComparator, TypePairComparator<PT, T> pairComparator) { super(probeTypeComparator, pairComparator); }