@Override public boolean shouldPeriodicallyFlush() { ensureOpen(); final long translogGenerationOfLastCommit = Long.parseLong(lastCommittedSegmentInfos.userData.get(Translog.TRANSLOG_GENERATION_KEY)); final long flushThreshold = config().getIndexSettings().getFlushThresholdSize().getBytes(); if (translog.sizeInBytesByMinGen(translogGenerationOfLastCommit) < flushThreshold) { return false; } /* * We flush to reduce the size of uncommitted translog but strictly speaking the uncommitted size won't always be * below the flush-threshold after a flush. To avoid getting into an endless loop of flushing, we only enable the * periodically flush condition if this condition is disabled after a flush. The condition will change if the new * commit points to the later generation the last commit's(eg. gen-of-last-commit < gen-of-new-commit)[1]. * * When the local checkpoint equals to max_seqno, and translog-gen of the last commit equals to translog-gen of * the new commit, we know that the last generation must contain operations because its size is above the flush * threshold and the flush-threshold is guaranteed to be higher than an empty translog by the setting validation. * This guarantees that the new commit will point to the newly rolled generation. In fact, this scenario only * happens when the generation-threshold is close to or above the flush-threshold; otherwise we have rolled * generations as the generation-threshold was reached, then the first condition (eg. [1]) is already satisfied. * * This method is to maintain translog only, thus IndexWriter#hasUncommittedChanges condition is not considered. */ final long translogGenerationOfNewCommit = translog.getMinGenerationForSeqNo(localCheckpointTracker.getCheckpoint() + 1).translogFileGeneration; return translogGenerationOfLastCommit < translogGenerationOfNewCommit || localCheckpointTracker.getCheckpoint() == localCheckpointTracker.getMaxSeqNo(); }
private boolean assertMaxSeqNoOfUpdatesIsAdvanced(Term id, long seqNo, boolean allowDeleted, boolean relaxIfGapInSeqNo) { final long maxSeqNoOfUpdates = getMaxSeqNoOfUpdatesOrDeletes(); // If the primary is on an old version which does not replicate msu, we need to relax this assertion for that. if (maxSeqNoOfUpdates == SequenceNumbers.UNASSIGNED_SEQ_NO) { assert config().getIndexSettings().getIndexVersionCreated().before(Version.V_6_5_0); return true; } // We treat a delete on the tombstones on replicas as a regular document, then use updateDocument (not addDocument). if (allowDeleted) { final VersionValue versionValue = versionMap.getVersionForAssert(id.bytes()); if (versionValue != null && versionValue.isDelete()) { return true; } } // Operations can be processed on a replica in a different order than on the primary. If the order on the primary is index-1, // delete-2, index-3, and the order on a replica is index-1, index-3, delete-2, then the msu of index-3 on the replica is 2 // even though it is an update (overwrites index-1). We should relax this assertion if there is a pending gap in the seq_no. if (relaxIfGapInSeqNo && getLocalCheckpoint() < maxSeqNoOfUpdates) { return true; } assert seqNo <= maxSeqNoOfUpdates : "id=" + id + " seq_no=" + seqNo + " msu=" + maxSeqNoOfUpdates; return true; }
assert config().getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha1) : "index is newly created but op has no sequence numbers. op: " + delete; opVsLucene = compareOpToLuceneDocBasedOnVersions(delete);
assert config().getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha1) : "index is newly created but op has no sequence numbers. op: " + index; opVsLucene = compareOpToLuceneDocBasedOnVersions(index);
/** * Forces a refresh if the versionMap is using too much RAM */ private void checkVersionMapRefresh() { if (versionMap.ramBytesUsedForRefresh() > config().getVersionMapSize().bytes() && versionMapRefreshPending.getAndSet(true) == false) { try { if (isClosed.get()) { // no point... return; } // Now refresh to clear versionMap: engineConfig.getThreadPool().executor(ThreadPool.Names.REFRESH).execute(new Runnable() { @Override public void run() { try { refresh("version_table_full"); } catch (EngineClosedException ex) { // ignore } } }); } catch (EsRejectedExecutionException ex) { // that is fine too.. we might be shutting down } } }
@Override protected void handleMergeException(final Directory dir, final Throwable exc) { logger.error("failed to merge", exc); if (config().getMergeSchedulerConfig().isNotifyOnMergeFailure()) { engineConfig.getThreadPool().generic().execute(new AbstractRunnable() { @Override public void onFailure(Throwable t) { logger.debug("merge failure action rejected", t); } @Override protected void doRun() throws Exception { MergePolicy.MergeException e = new MergePolicy.MergeException(exc, dir); failEngine("merge failed", e); } }); } } }
@Override public boolean shouldPeriodicallyFlush() { ensureOpen(); final long translogGenerationOfLastCommit = Long.parseLong(lastCommittedSegmentInfos.userData.get(Translog.TRANSLOG_GENERATION_KEY)); final long flushThreshold = config().getIndexSettings().getFlushThresholdSize().getBytes(); if (translog.sizeInBytesByMinGen(translogGenerationOfLastCommit) < flushThreshold) { return false; } /* * We flush to reduce the size of uncommitted translog but strictly speaking the uncommitted size won't always be * below the flush-threshold after a flush. To avoid getting into an endless loop of flushing, we only enable the * periodically flush condition if this condition is disabled after a flush. The condition will change if the new * commit points to the later generation the last commit's(eg. gen-of-last-commit < gen-of-new-commit)[1]. * * When the local checkpoint equals to max_seqno, and translog-gen of the last commit equals to translog-gen of * the new commit, we know that the last generation must contain operations because its size is above the flush * threshold and the flush-threshold is guaranteed to be higher than an empty translog by the setting validation. * This guarantees that the new commit will point to the newly rolled generation. In fact, this scenario only * happens when the generation-threshold is close to or above the flush-threshold; otherwise we have rolled * generations as the generation-threshold was reached, then the first condition (eg. [1]) is already satisfied. * * This method is to maintain translog only, thus IndexWriter#hasUncommittedChanges condition is not considered. */ final long translogGenerationOfNewCommit = translog.getMinGenerationForSeqNo(localCheckpointTracker.getCheckpoint() + 1).translogFileGeneration; return translogGenerationOfLastCommit < translogGenerationOfNewCommit || localCheckpointTracker.getCheckpoint() == localCheckpointTracker.getMaxSeqNo(); }
IndexWriter createWriter(boolean create) throws IOException { try { final IndexWriterConfig iwc = new IndexWriterConfig(engineConfig.getAnalyzer()); iwc.setCommitOnClose(false); // we by default don't commit on close iwc.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND); iwc.setIndexDeletionPolicy(deletionPolicy); // with tests.verbose, lucene sets this up: plumb to align with filesystem stream boolean verbose = false; try { verbose = Boolean.parseBoolean(System.getProperty("tests.verbose")); } catch (Exception ignore) { } iwc.setInfoStream(verbose ? InfoStream.getDefault() : new LoggerInfoStream(logger)); iwc.setMergeScheduler(mergeScheduler); MergePolicy mergePolicy = config().getMergePolicy(); // Give us the opportunity to upgrade old segments while performing // background merges mergePolicy = new ElasticsearchMergePolicy(mergePolicy); iwc.setMergePolicy(mergePolicy); iwc.setSimilarity(engineConfig.getSimilarity()); iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac()); iwc.setCodec(engineConfig.getCodec()); iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh return new IndexWriter(store.directory(), iwc); } catch (LockObtainFailedException ex) { logger.warn("could not lock IndexWriter", ex); throw ex; } }
private boolean assertMaxSeqNoOfUpdatesIsAdvanced(Term id, long seqNo, boolean allowDeleted, boolean relaxIfGapInSeqNo) { final long maxSeqNoOfUpdates = getMaxSeqNoOfUpdatesOrDeletes(); // If the primary is on an old version which does not replicate msu, we need to relax this assertion for that. if (maxSeqNoOfUpdates == SequenceNumbers.UNASSIGNED_SEQ_NO) { assert config().getIndexSettings().getIndexVersionCreated().before(Version.V_6_5_0); return true; } // We treat a delete on the tombstones on replicas as a regular document, then use updateDocument (not addDocument). if (allowDeleted) { final VersionValue versionValue = versionMap.getVersionForAssert(id.bytes()); if (versionValue != null && versionValue.isDelete()) { return true; } } // Operations can be processed on a replica in a different order than on the primary. If the order on the primary is index-1, // delete-2, index-3, and the order on a replica is index-1, index-3, delete-2, then the msu of index-3 on the replica is 2 // even though it is an update (overwrites index-1). We should relax this assertion if there is a pending gap in the seq_no. if (relaxIfGapInSeqNo && getLocalCheckpoint() < maxSeqNoOfUpdates) { return true; } assert seqNo <= maxSeqNoOfUpdates : "id=" + id + " seq_no=" + seqNo + " msu=" + maxSeqNoOfUpdates; return true; }
MergePolicy mergePolicy = config().getMergePolicy();
assert config().getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha1) : "index is newly created but op has no sequence numbers. op: " + delete; opVsLucene = compareOpToLuceneDocBasedOnVersions(delete);
assert config().getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha1) : "index is newly created but op has no sequence numbers. op: " + index; opVsLucene = compareOpToLuceneDocBasedOnVersions(index);