/** * Removes this uid from the pending deletes map. */ void removeTombstoneUnderLock(BytesRef uid) { assert assertKeyedLockHeldByCurrentThread(uid); long uidRAMBytesUsed = BASE_BYTES_PER_BYTESREF + uid.bytes.length; final VersionValue prev = tombstones.remove(uid); if (prev != null) { assert prev.isDelete(); long v = ramBytesUsedTombstones.addAndGet(-(BASE_BYTES_PER_CHM_ENTRY + prev.ramBytesUsed() + uidRAMBytesUsed)); assert v >= 0 : "bytes=" + v; } }
@Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; DeleteVersionValue that = (DeleteVersionValue) o; return time == that.time; }
@Override public int hashCode() { int result = super.hashCode(); result = 31 * result + (int) (time ^ (time >>> 32)); return result; }
void put(BytesRef uid, VersionValue version) { long uidRAMBytesUsed = BASE_BYTES_PER_BYTESREF + uid.bytes.length; long ramAccounting = BASE_BYTES_PER_CHM_ENTRY + version.ramBytesUsed() + uidRAMBytesUsed; VersionValue previousValue = current.put(uid, version); ramAccounting += previousValue == null ? 0 : -(BASE_BYTES_PER_CHM_ENTRY + previousValue.ramBytesUsed() + uidRAMBytesUsed); adjustRam(ramAccounting); }
private OpVsLuceneDocStatus compareOpToLuceneDocBasedOnVersions(final Operation op) throws IOException { assert op.seqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO : "op is resolved based on versions but have a seq#"; assert op.version() >= 0 : "versions should be non-negative. got " + op.version(); final VersionValue versionValue = resolveDocVersion(op, false); if (versionValue == null) { return OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND; } else { return op.versionType().isVersionConflictForWrites(versionValue.version, op.version(), versionValue.isDelete()) ? OpVsLuceneDocStatus.OP_STALE_OR_EQUAL : OpVsLuceneDocStatus.OP_NEWER; } }
private OpVsLuceneDocStatus compareOpToLuceneDocBasedOnVersions(final Operation op) throws IOException { assert op.version() >= 0 : "versions should be non-negative. got " + op.version(); final VersionValue versionValue = resolveDocVersion(op); if (versionValue == null) { return OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND; } else { return op.versionType().isVersionConflictForWrites(versionValue.getVersion(), op.version(), versionValue.isDelete()) ? OpVsLuceneDocStatus.OP_STALE_OR_EQUAL : OpVsLuceneDocStatus.OP_NEWER; } }
/** resolves the current version of the document, returning null if not found */ private VersionValue resolveDocVersion(final Operation op) throws IOException { assert incrementVersionLookup(); // used for asserting in tests //VersionValue versionValue = versionMap.getUnderLock(op.uid()); VersionValue versionValue = null; if (versionValue == null) { assert incrementIndexVersionLookup(); // used for asserting in tests final long currentVersion = loadCurrentVersionFromIndex(op.uid()); if (currentVersion != Versions.NOT_FOUND) { versionValue = new VersionValue(currentVersion); } } else if (engineConfig.isEnableGcDeletes() && versionValue.isDelete() && (engineConfig.getThreadPool().relativeTimeInMillis() - versionValue.getTime()) > getGcDeletesInMillis()) { versionValue = null; } return versionValue; }
if (versionValue.isDelete()) { return GetResult.NOT_EXISTS; if (versionValue.getLocation() != null) { try { Translog.Operation operation = translog.readOperation(versionValue.getLocation()); if (operation != null) {
/** Removes this uid from the pending deletes map. */ void removeTombstoneUnderLock(BytesRef uid) { long uidRAMBytesUsed = BASE_BYTES_PER_BYTESREF + uid.bytes.length; final VersionValue prev = tombstones.remove(uid); if (prev != null) { assert prev.delete(); long v = ramBytesUsedTombstones.addAndGet(-(BASE_BYTES_PER_CHM_ENTRY + prev.ramBytesUsed() + uidRAMBytesUsed)); assert v >= 0: "bytes=" + v; } final VersionValue curVersion = maps.current.get(uid); if (curVersion != null && curVersion.delete()) { // We now shift accounting of the BytesRef from tombstones to current, because a refresh would clear this RAM. This should be // uncommon, because with the default refresh=1s and gc_deletes=60s, deletes should be cleared from current long before we drop // them from tombstones: ramBytesUsedCurrent.addAndGet(curVersion.ramBytesUsed() + uidRAMBytesUsed); } }
currentVersion = loadCurrentVersionFromIndex(index.uid()); } else { if (engineConfig.isEnableGcDeletes() && versionValue.delete() && (engineConfig.getThreadPool().estimatedTimeInMillis() - versionValue.time()) > engineConfig.getGcDeletesInMillis()) { currentVersion = versionValue.version(); created = versionValue.delete(); // we have a delete which is not GC'ed... } else { created = false; versionMap.putUnderLock(index.uid().bytes(), new VersionValue(updatedVersion, translogLocation)); index.setTranslogLocation(translogLocation); indexingService.postIndexUnderLock(index);
private void innerCreate(Create create) throws IOException { if (engineConfig.isOptimizeAutoGenerateId() && create.autoGeneratedId() && !create.canHaveDuplicates()) { // We don't need to lock because this ID cannot be concurrently updated: innerCreateNoLock(create, Versions.NOT_FOUND, null); } else { try (Releasable ignored = acquireLock(create.uid())) { final long currentVersion; final VersionValue versionValue; versionValue = versionMap.getUnderLock(create.uid().bytes()); if (versionValue == null) { currentVersion = loadCurrentVersionFromIndex(create.uid()); } else { if (engineConfig.isEnableGcDeletes() && versionValue.delete() && (engineConfig.getThreadPool().estimatedTimeInMillis () - versionValue.time()) > engineConfig.getGcDeletesInMillis()) { currentVersion = Versions.NOT_FOUND; // deleted, and GC } else { currentVersion = versionValue.version(); } } innerCreateNoLock(create, currentVersion, versionValue); } } }
@Override public GetResult get(Get get) throws EngineException { try (ReleasableLock lock = readLock.acquire()) { ensureOpen(); if (get.realtime()) { VersionValue versionValue = versionMap.getUnderLock(get.uid().bytes()); if (versionValue != null) { if (versionValue.delete()) { return GetResult.NOT_EXISTS; } if (get.versionType().isVersionConflictForReads(versionValue.version(), get.version())) { Uid uid = Uid.createUid(get.uid().text()); throw new VersionConflictEngineException(shardId, uid.type(), uid.id(), versionValue.version(), get.version()); } Translog.Operation op = translog.read(versionValue.translogLocation()); if (op != null) { return new GetResult(true, versionValue.version(), op.getSource()); } } } // no version, get the version from the index, we know that we refresh on flush return getFromSearcher(get); } }
if ((versionValue != null && versionValue.delete() == false) || (versionValue == null && currentVersion != Versions.NOT_FOUND)) { if (create.origin() == Operation.Origin.RECOVERY) { return; versionMap.putUnderLock(create.uid().bytes(), new VersionValue(updatedVersion, translogLocation)); create.setTranslogLocation(translogLocation); indexingService.postCreateUnderLock(create);
index(index.docs(), indexWriter); versionMap.putUnderLock(index.uid().bytes(), new VersionValue(plan.versionForIndexing)); return new IndexResult(plan.versionForIndexing, plan.currentNotFoundOrDeleted); } catch (Exception ex) {
private void putTombstone(BytesRef uid, DeleteVersionValue version) { long uidRAMBytesUsed = BASE_BYTES_PER_BYTESREF + uid.bytes.length; // Also enroll the delete into tombstones, and account for its RAM too: final VersionValue prevTombstone = tombstones.put(uid, version); long accountRam = (BASE_BYTES_PER_CHM_ENTRY + version.ramBytesUsed() + uidRAMBytesUsed); // Deduct tombstones bytes used for the version we just removed or replaced: if (prevTombstone != null) { accountRam -= (BASE_BYTES_PER_CHM_ENTRY + prevTombstone.ramBytesUsed() + uidRAMBytesUsed); } if (accountRam != 0) { long v = ramBytesUsedTombstones.addAndGet(accountRam); assert v >= 0: "bytes=" + v; } }
/** * Asserts that the doc in the index operation really doesn't exist */ private boolean assertDocDoesNotExist(final Index index, final boolean allowDeleted) throws IOException { // NOTE this uses direct access to the version map since we are in the assertion code where we maintain a secondary // map in the version map such that we don't need to refresh if we are unsafe; final VersionValue versionValue = versionMap.getVersionForAssert(index.uid().bytes()); if (versionValue != null) { if (versionValue.isDelete() == false || allowDeleted == false) { throw new AssertionError("doc [" + index.type() + "][" + index.id() + "] exists in version map (version " + versionValue + ")"); } } else { try (Searcher searcher = acquireSearcher("assert doc doesn't exist", SearcherScope.INTERNAL)) { final long docsWithId = searcher.searcher().count(new TermQuery(index.uid())); if (docsWithId > 0) { throw new AssertionError("doc [" + index.type() + "][" + index.id() + "] exists [" + docsWithId + "] times in index"); } } } return true; }
private OpVsLuceneDocStatus compareOpToLuceneDocBasedOnVersions(final Operation op) throws IOException { assert op.version() >= 0 : "versions should be non-negative. got " + op.version(); final VersionValue versionValue = resolveDocVersion(op); if (versionValue == null) { return OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND; } else { return op.versionType().isVersionConflictForWrites(versionValue.getVersion(), op.version(), versionValue.isDelete()) ? OpVsLuceneDocStatus.OP_STALE_OR_EQUAL : OpVsLuceneDocStatus.OP_NEWER; } }
/** resolves the current version of the document, returning null if not found */ private VersionValue resolveDocVersion(final Operation op) throws IOException { assert incrementVersionLookup(); // used for asserting in tests VersionValue versionValue = versionMap.getUnderLock(op.uid()); if (versionValue == null) { assert incrementIndexVersionLookup(); // used for asserting in tests final long currentVersion = loadCurrentVersionFromIndex(op.uid()); if (currentVersion != Versions.NOT_FOUND) { versionValue = new VersionValue(currentVersion); } } else if (engineConfig.isEnableGcDeletes() && versionValue.isDelete() && (engineConfig.getThreadPool().relativeTimeInMillis() - versionValue.getTime()) > getGcDeletesInMillis()) { versionValue = null; } return versionValue; }
if (versionValue.isDelete()) { return GetResult.NOT_EXISTS; if (versionValue.getLocation() != null) { try { Translog.Operation operation = translog.readOperation(versionValue.getLocation()); if (operation != null) {
if (prev.delete() == false) { prevBytes += prev.ramBytesUsed() + uidRAMBytesUsed; if (version.delete() == false) { newBytes += version.ramBytesUsed() + uidRAMBytesUsed; if (version.delete()) { ramBytesUsedTombstones.addAndGet(BASE_BYTES_PER_CHM_ENTRY + version.ramBytesUsed() + uidRAMBytesUsed); if (prevTombstone == null && prev != null && prev.delete()) { ramBytesUsedCurrent.addAndGet(-(prev.ramBytesUsed() + uidRAMBytesUsed)); long v = ramBytesUsedTombstones.addAndGet(-(BASE_BYTES_PER_CHM_ENTRY + prevTombstone.ramBytesUsed() + uidRAMBytesUsed)); assert v >= 0: "bytes=" + v;