/** Called when this index is closed. */ synchronized void clear() { maps = new Maps(); tombstones.clear(); ramBytesUsedCurrent.set(0); // NOTE: we can't zero this here, because a refresh thread could be calling InternalEngine.pruneDeletedTombstones at the same time, // and this will lead to an assert trip. Presumably it's fine if our ramBytesUsedTombstones is non-zero after clear since the index // is being closed: //ramBytesUsedTombstones.set(0); if (mgr != null) { mgr.removeListener(this); mgr = null; } }
/** * Adds this uid/version to the pending adds map iff the map needs safe access. */ void maybePutIndexUnderLock(BytesRef uid, IndexVersionValue version) { assert assertKeyedLockHeldByCurrentThread(uid); Maps maps = this.maps; if (maps.isSafeAccessMode()) { putIndexUnderLock(uid, version); } else { // Even though we don't store a record of the indexing operation (and mark as unsafe), // we should still remove any previous delete for this uuid (avoid accidental accesses). // Not this should not hurt performance because the tombstone is small (or empty) when unsafe is relevant. removeTombstoneUnderLock(uid); maps.current.markAsUnsafe(); assert putAssertionMap(uid, version); } }
/** * Adds this uid/version to the pending adds map iff the map needs safe access. */ void maybePutIndexUnderLock(BytesRef uid, IndexVersionValue version) { assert assertKeyedLockHeldByCurrentThread(uid); Maps maps = this.maps; if (maps.isSafeAccessMode()) { putIndexUnderLock(uid, version); } else { // Even though we don't store a record of the indexing operation (and mark as unsafe), // we should still remove any previous delete for this uuid (avoid accidental accesses). // Not this should not hurt performance because the tombstone is small (or empty) when unsafe is relevant. removeTombstoneUnderLock(uid); maps.current.markAsUnsafe(); assert putAssertionMap(uid, version); } }
boolean isSafeAccessRequired() { return maps.isSafeAccessMode(); }
boolean isSafeAccessRequired() { return maps.isSafeAccessMode(); }
void remove(BytesRef uid, DeleteVersionValue deleted) { VersionValue previousValue = current.remove(uid); current.updateMinDeletedTimestamp(deleted); if (previousValue != null) { long uidRAMBytesUsed = BASE_BYTES_PER_BYTESREF + uid.bytes.length; adjustRam(-(BASE_BYTES_PER_CHM_ENTRY + previousValue.ramBytesUsed() + uidRAMBytesUsed)); } if (old != VersionLookup.EMPTY) { // we also need to remove it from the old map here to make sure we don't read this stale value while // we are in the middle of a refresh. Most of the time the old map is an empty map so we can skip it there. old.remove(uid); } }
/** Called when this index is closed. */ synchronized void clear() { maps = new Maps(); tombstones.clear(); ramBytesUsedCurrent.set(0); // NOTE: we can't zero this here, because a refresh thread could be calling InternalEngine.pruneDeletedTombstones at the same time, // and this will lead to an assert trip. Presumably it's fine if our ramBytesUsedTombstones is non-zero after clear since the index // is being closed: //ramBytesUsedTombstones.set(0); if (mgr != null) { mgr.removeListener(this); mgr = null; } }
private boolean canRemoveTombstone(long maxTimestampToPrune, long maxSeqNoToPrune, DeleteVersionValue versionValue) { // check if the value is old enough and safe to be removed final boolean isTooOld = versionValue.time < maxTimestampToPrune; final boolean isSafeToPrune = versionValue.seqNo <= maxSeqNoToPrune; // version value can't be removed it's // not yet flushed to lucene ie. it's part of this current maps object final boolean isNotTrackedByCurrentMaps = versionValue.time < maps.getMinDeleteTimestamp(); return isTooOld && isSafeToPrune && isNotTrackedByCurrentMaps; }
void put(BytesRef uid, VersionValue version) { long uidRAMBytesUsed = BASE_BYTES_PER_BYTESREF + uid.bytes.length; long ramAccounting = BASE_BYTES_PER_CHM_ENTRY + version.ramBytesUsed() + uidRAMBytesUsed; VersionValue previousValue = current.put(uid, version); ramAccounting += previousValue == null ? 0 : -(BASE_BYTES_PER_CHM_ENTRY + previousValue.ramBytesUsed() + uidRAMBytesUsed); adjustRam(ramAccounting); }
private boolean canRemoveTombstone(long maxTimestampToPrune, long maxSeqNoToPrune, DeleteVersionValue versionValue) { // check if the value is old enough and safe to be removed final boolean isTooOld = versionValue.time < maxTimestampToPrune; final boolean isSafeToPrune = versionValue.seqNo <= maxSeqNoToPrune; // version value can't be removed it's // not yet flushed to lucene ie. it's part of this current maps object final boolean isNotTrackedByCurrentMaps = versionValue.time < maps.getMinDeleteTimestamp(); return isTooOld && isSafeToPrune && isNotTrackedByCurrentMaps; }
void put(BytesRef uid, VersionValue version) { long uidRAMBytesUsed = BASE_BYTES_PER_BYTESREF + uid.bytes.length; long ramAccounting = BASE_BYTES_PER_CHM_ENTRY + version.ramBytesUsed() + uidRAMBytesUsed; VersionValue previousValue = current.put(uid, version); ramAccounting += previousValue == null ? 0 : -(BASE_BYTES_PER_CHM_ENTRY + previousValue.ramBytesUsed() + uidRAMBytesUsed); adjustRam(ramAccounting); }
/** * Called when this index is closed. */ synchronized void clear() { maps = new Maps(); tombstones.clear(); // NOTE: we can't zero this here, because a refresh thread could be calling InternalEngine.pruneDeletedTombstones at the same time, // and this will lead to an assert trip. Presumably it's fine if our ramBytesUsedTombstones is non-zero after clear since the index // is being closed: //ramBytesUsedTombstones.set(0); }
void putDeleteUnderLock(BytesRef uid, DeleteVersionValue version) { assert assertKeyedLockHeldByCurrentThread(uid); assert uid.bytes.length == uid.length : "Oversized _uid! UID length: " + uid.length + ", bytes length: " + uid.bytes.length; putTombstone(uid, version); maps.remove(uid, version); }
@Override public void afterRefresh(boolean didRefresh) throws IOException { // We can now drop old because these operations are now visible via the newly opened searcher. Even if didRefresh is false, which // means Lucene did not actually open a new reader because it detected no changes, it's possible old has some entries in it, which // is fine: it means they were actually already included in the previously opened reader, so we can still safely drop them in that // case. This is because we assign new maps (in beforeRefresh) slightly before Lucene actually flushes any segments for the // reopen, and so any concurrent indexing requests can still sneak in a few additions to that current map that are in fact reflected // in the previous reader. We don't touch tombstones here: they expire on their own index.gc_deletes timeframe: maps = maps.invalidateOldMap(); assert (unsafeKeysMap = unsafeKeysMap.invalidateOldMap()) != null; }
@Override public void beforeRefresh() throws IOException { // Start sending all updates after this point to the new // map. While reopen is running, any lookup will first // try this new map, then fallback to old, then to the // current searcher: maps = new Maps(ConcurrentCollections.<BytesRef,VersionValue>newConcurrentMapWithAggressiveConcurrency(), maps.current); // This is not 100% correct, since concurrent indexing ops can change these counters in between our execution of the previous // line and this one, but that should be minor, and the error won't accumulate over time: ramBytesUsedCurrent.set(0); }
void remove(BytesRef uid, DeleteVersionValue deleted) { VersionValue previousValue = current.remove(uid); current.updateMinDeletedTimestamp(deleted); if (previousValue != null) { long uidRAMBytesUsed = BASE_BYTES_PER_BYTESREF + uid.bytes.length; adjustRam(-(BASE_BYTES_PER_CHM_ENTRY + previousValue.ramBytesUsed() + uidRAMBytesUsed)); } if (old != VersionLookup.EMPTY) { // we also need to remove it from the old map here to make sure we don't read this stale value while // we are in the middle of a refresh. Most of the time the old map is an empty map so we can skip it there. old.remove(uid); } }
@Override public void afterRefresh(boolean didRefresh) throws IOException { // We can now drop old because these operations are now visible via the newly opened searcher. Even if didRefresh is false, which // means Lucene did not actually open a new reader because it detected no changes, it's possible old has some entries in it, which // is fine: it means they were actually already included in the previously opened reader, so we can still safely drop them in that // case. This is because we assign new maps (in beforeRefresh) slightly before Lucene actually flushes any segments for the // reopen, and so any concurrent indexing requests can still sneak in a few additions to that current map that are in fact reflected // in the previous reader. We don't touch tombstones here: they expire on their own index.gc_deletes timeframe: maps = new Maps(maps.current, ConcurrentCollections.<BytesRef,VersionValue>newConcurrentMapWithAggressiveConcurrency()); }
/** Sync'd because we replace old mgr. */ synchronized void setManager(ReferenceManager newMgr) { if (mgr != null) { mgr.removeListener(this); } mgr = newMgr; // In case InternalEngine closes & opens a new IndexWriter/SearcherManager, all deletes are made visible, so we clear old and // current here. This is safe because caller holds writeLock here (so no concurrent adds/deletes can be happeninge): maps = new Maps(); // So we are notified when reopen starts and finishes mgr.addListener(this); }
private boolean putAssertionMap(BytesRef uid, IndexVersionValue version) { assert assertKeyedLockHeldByCurrentThread(uid); assert uid.bytes.length == uid.length : "Oversized _uid! UID length: " + uid.length + ", bytes length: " + uid.bytes.length; unsafeKeysMap.put(uid, version); return true; }
void putDeleteUnderLock(BytesRef uid, DeleteVersionValue version) { assert assertKeyedLockHeldByCurrentThread(uid); assert uid.bytes.length == uid.length : "Oversized _uid! UID length: " + uid.length + ", bytes length: " + uid.bytes.length; putTombstone(uid, version); maps.remove(uid, version); }