private DeleteResult deleteInLucene(Delete delete, DeletionStrategy plan) throws IOException { assert assertMaxSeqNoOfUpdatesIsAdvanced(delete.uid(), plan.seqNoOfDeletion, false, false); try { if (softDeleteEnabled) { final ParsedDocument tombstone = engineConfig.getTombstoneDocSupplier().newDeleteTombstoneDoc(delete.type(), delete.id()); assert tombstone.docs().size() == 1 : "Tombstone doc should have single doc [" + tombstone + "]"; tombstone.updateSeqID(plan.seqNoOfDeletion, delete.primaryTerm()); tombstone.version().setLongValue(plan.versionOfDeletion); final ParseContext.Document doc = tombstone.docs().get(0); indexWriter.addDocument(doc); } else { indexWriter.softUpdateDocument(delete.uid(), doc, softDeletesField); indexWriter.deleteDocuments(delete.uid()); versionMap.putDeleteUnderLock(delete.uid().bytes(), new DeleteVersionValue(plan.versionOfDeletion, plan.seqNoOfDeletion, delete.primaryTerm(), engineConfig.getThreadPool().relativeTimeInMillis())); ex, plan.versionOfDeletion, delete.primaryTerm(), plan.seqNoOfDeletion, plan.currentlyDeleted == false); } else { throw ex;
static Engine.Delete prepareDelete(String type, String id, Term uid, long version, VersionType versionType, Engine.Operation.Origin origin) { long startTime = System.nanoTime(); return new Engine.Delete(type, id, uid, version, versionType, origin, startTime); }
protected final DeletionStrategy planDeletionAsNonPrimary(Delete delete) throws IOException { assertNonPrimaryOrigin(delete); assert delete.versionType().versionTypeForReplicationAndRecovery() == delete.versionType() : "resolving out of order delivery based on versioning but version type isn't fit for it. got [" + delete.versionType() + "]"; maxSeqNoOfNonAppendOnlyOperations.updateAndGet(curr -> Math.max(delete.seqNo(), curr)); assert maxSeqNoOfNonAppendOnlyOperations.get() >= delete.seqNo() : "max_seqno of non-append-only was not updated;" + "max_seqno non-append-only [" + maxSeqNoOfNonAppendOnlyOperations.get() + "], seqno of delete [" + delete.seqNo() + "]"; if (delete.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO && delete.seqNo() <= localCheckpointTracker.getCheckpoint()) { plan = DeletionStrategy.processButSkipLucene(false, delete.seqNo(), delete.version()); } else { final OpVsLuceneDocStatus opVsLucene; if (delete.seqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO) { plan = DeletionStrategy.processAsStaleOp(softDeleteEnabled, false, delete.seqNo(), delete.version()); } else { plan = DeletionStrategy.processNormally(opVsLucene == OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND, delete.seqNo(), delete.version());
this(delete.type(), delete.id(), delete.uid(), deleteResult.getSeqNo(), delete.primaryTerm(), deleteResult.getVersion(), delete.versionType());
logger.trace("[translog] recover [delete] op of [{}][{}]", uid.type(), uid.id()); engine.delete(new Engine.Delete(uid.type(), uid.id(), delete.uid(), delete.version(), delete.versionType().versionTypeForReplicationAndRecovery(), Engine.Operation.Origin.RECOVERY, System.nanoTime(), false)); break;
logger.trace("[translog] recover [delete] op of [{}][{}]", delete.type(), delete.id()); final Engine.Delete engineDelete = new Engine.Delete(delete.type(), delete.id(), delete.uid(), delete.version(), delete.versionType().versionTypeForReplicationAndRecovery(), origin, System.nanoTime()); delete(engine, engineDelete);
@Override public DeleteResult delete(Delete delete) throws IOException { versionMap.enforceSafeAccess(); assert Objects.equals(delete.uid().field(), uidField) : delete.uid().field(); assert assertVersionType(delete); assert assertIncomingSequenceNumber(delete.origin(), delete.seqNo()); final DeleteResult deleteResult; try (ReleasableLock ignored = readLock.acquire(); Releasable ignored2 = versionMap.acquireLock(delete.uid().bytes())) { ensureOpen(); lastWriteNanos = delete.startTime(); final DeletionStrategy plan = deletionStrategyForOperation(delete); plan.versionOfDeletion, getPrimaryTerm(), plan.seqNoOfDeletion, plan.currentlyDeleted == false); if (delete.origin().isFromTranslog() == false) { final Translog.Location location; if (deleteResult.getResultType() == Result.Type.SUCCESS) { } else if (deleteResult.getSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) { final NoOp noOp = new NoOp(deleteResult.getSeqNo(), delete.primaryTerm(), delete.origin(), delete.startTime(), deleteResult.getFailure().toString()); location = innerNoOp(noOp).getTranslogLocation(); } else { deleteResult.setTook(System.nanoTime() - delete.startTime()); deleteResult.freeze(); } catch (RuntimeException | IOException e) {
protected final DeletionStrategy planDeletionAsNonPrimary(Delete delete) throws IOException { assertNonPrimaryOrigin(delete); assert delete.versionType().versionTypeForReplicationAndRecovery() == delete.versionType() : "resolving out of order delivery based on versioning but version type isn't fit for it. got [" + delete.versionType() + "]"; maxSeqNoOfNonAppendOnlyOperations.updateAndGet(curr -> Math.max(delete.seqNo(), curr)); assert maxSeqNoOfNonAppendOnlyOperations.get() >= delete.seqNo() : "max_seqno of non-append-only was not updated;" + "max_seqno non-append-only [" + maxSeqNoOfNonAppendOnlyOperations.get() + "], seqno of delete [" + delete.seqNo() + "]"; if (delete.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO && delete.seqNo() <= localCheckpointTracker.getCheckpoint()) { plan = DeletionStrategy.processButSkipLucene(false, delete.seqNo(), delete.version()); } else { final OpVsLuceneDocStatus opVsLucene; if (delete.seqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO) { plan = DeletionStrategy.processAsStaleOp(softDeleteEnabled, false, delete.seqNo(), delete.version()); } else { plan = DeletionStrategy.processNormally(opVsLucene == OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND, delete.seqNo(), delete.version());
private void innerDelete(Delete delete) throws IOException { try (Releasable ignored = acquireLock(delete.uid())) { final long currentVersion; VersionValue versionValue = versionMap.getUnderLock(delete.uid().bytes()); if (versionValue == null) { currentVersion = loadCurrentVersionFromIndex(delete.uid()); } else { if (engineConfig.isEnableGcDeletes() && versionValue.delete() && (engineConfig.getThreadPool().estimatedTimeInMillis() - long expectedVersion = delete.version(); if (delete.versionType().isVersionConflictForWrites(currentVersion, expectedVersion)) { if (delete.origin() == Operation.Origin.RECOVERY) { return; } else { throw new VersionConflictEngineException(shardId, delete.type(), delete.id(), currentVersion, expectedVersion); updatedVersion = delete.versionType().updateVersion(currentVersion, expectedVersion); final boolean found; if (currentVersion == Versions.NOT_FOUND) { indexWriter.deleteDocuments(delete.uid()); found = true; delete.updateVersion(updatedVersion, found); Translog.Location translogLocation = translog.add(new Translog.Delete(delete)); versionMap.putUnderLock(delete.uid().bytes(), new DeleteVersionValue(updatedVersion, engineConfig.getThreadPool() .estimatedTimeInMillis(), translogLocation)); delete.setTranslogLocation(translogLocation); indexingService.postDeleteUnderLock(delete);
@Override public DeleteResult delete(Delete delete) throws IOException { assert Objects.equals(delete.uid().field(), uidField) : delete.uid().field(); assert assertVersionType(delete); final DeleteResult deleteResult; try (ReleasableLock ignored = readLock.acquire(); Releasable ignored2 = acquireLock(delete.uid())) { ensureOpen(); lastWriteNanos = delete.startTime(); final DeletionStrategy plan; if (delete.origin() == Operation.Origin.PRIMARY) { plan = planDeletionAsPrimary(delete); } else { delete.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) { Translog.Location location = translog.add(new Translog.Delete(delete, deleteResult)); deleteResult.setTranslogLocation(location); deleteResult.setTook(System.nanoTime() - delete.startTime()); deleteResult.freeze(); } catch (RuntimeException | IOException e) {
private DeleteResult deleteInLucene(Delete delete, DeletionStrategy plan) throws IOException { assert assertMaxSeqNoOfUpdatesIsAdvanced(delete.uid(), plan.seqNoOfDeletion, false, false); try { if (softDeleteEnabled) { final ParsedDocument tombstone = engineConfig.getTombstoneDocSupplier().newDeleteTombstoneDoc(delete.type(), delete.id()); assert tombstone.docs().size() == 1 : "Tombstone doc should have single doc [" + tombstone + "]"; tombstone.updateSeqID(plan.seqNoOfDeletion, delete.primaryTerm()); tombstone.version().setLongValue(plan.versionOfDeletion); final ParseContext.Document doc = tombstone.docs().get(0); indexWriter.addDocument(doc); } else { indexWriter.softUpdateDocument(delete.uid(), doc, softDeletesField); indexWriter.deleteDocuments(delete.uid()); versionMap.putDeleteUnderLock(delete.uid().bytes(), new DeleteVersionValue(plan.versionOfDeletion, plan.seqNoOfDeletion, delete.primaryTerm(), engineConfig.getThreadPool().relativeTimeInMillis()));
@Override public DeleteResult delete(Delete delete) throws IOException { assert Objects.equals(delete.uid().field(), uidField) : delete.uid().field(); assert assertVersionType(delete); final DeleteResult deleteResult; try (ReleasableLock ignored = readLock.acquire(); Releasable ignored2 = acquireLock(delete.uid())) { ensureOpen(); lastWriteNanos = delete.startTime(); final DeletionStrategy plan; if (delete.origin() == Operation.Origin.PRIMARY) { plan = planDeletionAsPrimary(delete); } else { delete.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) { deleteResult.setTranslogLocation( translog.add( delete.estimatedSizeInBytes() )); deleteResult.setTook(System.nanoTime() - delete.startTime()); deleteResult.freeze(); } catch (RuntimeException | IOException e) {
private DeletionStrategy planDeletionAsPrimary(Delete delete) throws IOException { assert delete.origin() == Operation.Origin.PRIMARY : "planing as primary but got " + delete.origin(); /* // resolve operation from external to internal final VersionValue versionValue = resolveDocVersion(delete); assert incrementVersionLookup(); final long currentVersion; final boolean currentlyDeleted; if (versionValue == null) { currentVersion = Versions.NOT_FOUND; currentlyDeleted = true; } else { currentVersion = versionValue.getVersion(); currentlyDeleted = versionValue.isDelete(); } final DeletionStrategy plan; if (delete.versionType().isVersionConflictForWrites(currentVersion, delete.version(), currentlyDeleted)) { plan = DeletionStrategy.skipDueToVersionConflict( new VersionConflictEngineException(shardId, delete, currentVersion, currentlyDeleted), currentVersion, currentlyDeleted); } else { plan = DeletionStrategy.processNormally(currentlyDeleted, delete.versionType().updateVersion(currentVersion, delete.version())); } return plan; */ return DeletionStrategy.processNormally(false, 1L); }
private DeletionStrategy planDeletionAsPrimary(Delete delete) throws IOException { assert delete.origin() == Operation.Origin.PRIMARY : "planing as primary but got " + delete.origin(); // resolve operation from external to internal final VersionValue versionValue = resolveDocVersion(delete); assert incrementVersionLookup(); final long currentVersion; final boolean currentlyDeleted; if (versionValue == null) { currentVersion = Versions.NOT_FOUND; currentlyDeleted = true; } else { currentVersion = versionValue.version; currentlyDeleted = versionValue.isDelete(); } final DeletionStrategy plan; if (delete.versionType().isVersionConflictForWrites(currentVersion, delete.version(), currentlyDeleted)) { final VersionConflictEngineException e = new VersionConflictEngineException(shardId, delete, currentVersion, currentlyDeleted); plan = DeletionStrategy.skipDueToVersionConflict(e, currentVersion, getPrimaryTerm(), currentlyDeleted); } else { plan = DeletionStrategy.processNormally( currentlyDeleted, generateSeqNoForOperation(delete), delete.versionType().updateVersion(currentVersion, delete.version())); } return plan; }
private Engine.Operation convertToEngineOp(Translog.Operation operation, Engine.Operation.Origin origin) { switch (operation.opType()) { case INDEX: final Translog.Index index = (Translog.Index) operation; final String indexName = mapperService.index().getName(); final Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), mapperService.getIndexSettings().getIndexVersionCreated(), source(indexName, index.type(), index.id(), index.source(), XContentFactory.xContentType(index.source())) .routing(index.routing()).parent(index.parent()), index.seqNo(), index.primaryTerm(), index.version(), index.versionType().versionTypeForReplicationAndRecovery(), origin, index.getAutoGeneratedIdTimestamp(), true); return engineIndex; case DELETE: final Translog.Delete delete = (Translog.Delete) operation; final Engine.Delete engineDelete = new Engine.Delete(delete.type(), delete.id(), delete.uid(), delete.seqNo(), delete.primaryTerm(), delete.version(), delete.versionType().versionTypeForReplicationAndRecovery(), origin, System.nanoTime()); return engineDelete; case NO_OP: final Translog.NoOp noOp = (Translog.NoOp) operation; final Engine.NoOp engineNoOp = new Engine.NoOp(noOp.seqNo(), noOp.primaryTerm(), origin, System.nanoTime(), noOp.reason()); return engineNoOp; default: throw new IllegalStateException("No operation defined for [" + operation + "]"); } }
protected final DeletionStrategy planDeletionAsPrimary(Delete delete) throws IOException { assert delete.origin() == Operation.Origin.PRIMARY : "planing as primary but got " + delete.origin(); assert getMaxSeqNoOfUpdatesOrDeletes() != SequenceNumbers.UNASSIGNED_SEQ_NO : "max_seq_no_of_updates is not initialized"; // resolve operation from external to internal final VersionValue versionValue = resolveDocVersion(delete); assert incrementVersionLookup(); final long currentVersion; final boolean currentlyDeleted; if (versionValue == null) { currentVersion = Versions.NOT_FOUND; currentlyDeleted = true; } else { currentVersion = versionValue.version; currentlyDeleted = versionValue.isDelete(); } final DeletionStrategy plan; if (delete.versionType().isVersionConflictForWrites(currentVersion, delete.version(), currentlyDeleted)) { final VersionConflictEngineException e = new VersionConflictEngineException(shardId, delete, currentVersion, currentlyDeleted); plan = DeletionStrategy.skipDueToVersionConflict(e, currentVersion, getPrimaryTerm(), currentlyDeleted); } else { plan = DeletionStrategy.processNormally( currentlyDeleted, generateSeqNoForOperation(delete), delete.versionType().updateVersion(currentVersion, delete.version())); advanceMaxSeqNoOfUpdatesOrDeletes(plan.seqNoOfDeletion); } return plan; }
private DeletionStrategy planDeletionAsPrimary(Delete delete) throws IOException { assert delete.origin() == Operation.Origin.PRIMARY : "planing as primary but got " + delete.origin(); // resolve operation from external to internal final VersionValue versionValue = resolveDocVersion(delete); assert incrementVersionLookup(); final long currentVersion; final boolean currentlyDeleted; if (versionValue == null) { currentVersion = Versions.NOT_FOUND; currentlyDeleted = true; } else { currentVersion = versionValue.getVersion(); currentlyDeleted = versionValue.isDelete(); } final DeletionStrategy plan; if (delete.versionType().isVersionConflictForWrites(currentVersion, delete.version(), currentlyDeleted)) { plan = DeletionStrategy.skipDueToVersionConflict( new VersionConflictEngineException(shardId, delete, currentVersion, currentlyDeleted), currentVersion, currentlyDeleted); } else { plan = DeletionStrategy.processNormally(currentlyDeleted, delete.versionType().updateVersion(currentVersion, delete.version())); } return plan; }
private DeleteResult deleteInLucene(Delete delete, DeletionStrategy plan) throws IOException { try { if (plan.currentlyDeleted == false) { // any exception that comes from this is a either an ACE or a fatal exception there // can't be any document failures coming from this indexWriter.deleteDocuments(delete.uid()); } /* versionMap.putUnderLock(delete.uid().bytes(), new DeleteVersionValue(plan.versionOfDeletion, engineConfig.getThreadPool().relativeTimeInMillis())); */ return new DeleteResult(plan.versionOfDeletion, plan.currentlyDeleted == false); } catch (Exception ex) { if (indexWriter.getTragicException() == null) { // there is no tragic event and such it must be a document level failure return new DeleteResult(ex, plan.versionOfDeletion, plan.currentlyDeleted == false); } else { throw ex; } } }
private NoOpResult innerNoOp(final NoOp noOp) throws IOException { assert readLock.isHeldByCurrentThread() || writeLock.isHeldByCurrentThread(); assert noOp.seqNo() > SequenceNumbers.NO_OPS_PERFORMED; final long seqNo = noOp.seqNo(); try { final NoOpResult noOpResult = new NoOpResult(getPrimaryTerm(), noOp.seqNo()); if (noOp.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) { final Translog.Location location = translog.add(new Translog.NoOp(noOp.seqNo(), noOp.primaryTerm(), noOp.reason())); noOpResult.setTranslogLocation(location); } noOpResult.setTook(System.nanoTime() - noOp.startTime()); noOpResult.freeze(); return noOpResult; } finally { if (seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO) { localCheckpointTracker.markSeqNoAsCompleted(seqNo); } } }