void openEditLogForWrite(int layoutVersion) throws IOException { assert editLog != null : "editLog must be initialized"; editLog.openForWrite(layoutVersion); storage.writeTransactionIdFileToStorage(editLog.getCurSegmentTxId()); }
while (lastAppliedTxId < editLog.getCurSegmentTxId() - 1) { long target = editLog.getCurSegmentTxId(); LOG.info("Loading edits into backupnode to try to catch up from txid " + lastAppliedTxId + " to " + target); if (s.getFirstTxId() != editLog.getCurSegmentTxId()) { editStreams.add(s); if (lastAppliedTxId != editLog.getCurSegmentTxId() - 1) { LOG.debug("Logs rolled while catching up to current segment"); return false; // drop lock and try again to load local logs Collection<EditLogInputStream> editStreams = getEditLog().selectInputStreams( getEditLog().getCurSegmentTxId(), getEditLog().getCurSegmentTxId()); if (s.getFirstTxId() == getEditLog().getCurSegmentTxId()) { stream = s; LOG.warn("Unable to find stream starting with " + editLog.getCurSegmentTxId() + ". This indicates that there is an error in synchronization in BackupImage"); return false;
CheckpointSignature(FSImage fsImage) { super(fsImage.getStorage()); blockpoolID = fsImage.getBlockPoolID(); mostRecentCheckpointTxId = fsImage.getStorage().getMostRecentCheckpointTxId(); curSegmentTxId = fsImage.getEditLog().getCurSegmentTxId(); }
/** * Remote namenode just has started a log segment, start log segment locally. */ public synchronized void startLogSegment(long txid, boolean abortCurrentLogSegment, int layoutVersion) throws IOException { LOG.info("Started a new log segment at txid " + txid); if (isSegmentOpen()) { if (getLastWrittenTxId() == txid - 1) { //In sync with the NN, so end and finalize the current segment` endCurrentLogSegment(false); } else { //Missed some transactions: probably lost contact with NN temporarily. final String mess = "Cannot start a new log segment at txid " + txid + " since only up to txid " + getLastWrittenTxId() + " have been written in the log segment starting at " + getCurSegmentTxId() + "."; if (abortCurrentLogSegment) { //Mark the current segment as aborted. LOG.warn(mess); abortCurrentLogSegment(); } else { throw new IOException(mess); } } } setNextTxId(txid); startLogSegment(txid, layoutVersion); }
/** * Get the correct number of transactions since last edit log roll. * This method holds a lock of FSEditLog and must not be used for metrics. */ private long getCorrectTransactionsSinceLastLogRoll() { if (isInStandbyState() || !getEditLog().isSegmentOpen()) { return 0; } else { return getEditLog().getLastWrittenTxId() - getEditLog().getCurSegmentTxId() + 1; } }
CheckpointSignature rollEditLog(int layoutVersion) throws IOException { getEditLog().rollEditLog(layoutVersion); // Record this log segment ID in all of the storage directories, so // we won't miss this log segment on a restart if the edits directories // go missing. storage.writeTransactionIdFileToStorage(getEditLog().getCurSegmentTxId()); //Update NameDirSize Metric getStorage().updateNameDirSize(); return new CheckpointSignature(this); }
@Override public Boolean get() { return editLog.getCurSegmentTxId() > startTxId; } }, 1000, 5000);
void openEditLogForWrite() throws IOException { assert editLog != null : "editLog must be initialized"; editLog.openForWrite(); storage.writeTransactionIdFileToStorage(editLog.getCurSegmentTxId()); }
void openEditLogForWrite() throws IOException { assert editLog != null : "editLog must be initialized"; editLog.openForWrite(); storage.writeTransactionIdFileToStorage(editLog.getCurSegmentTxId()); }
@Override public void run() { while (fsRunning && shouldRun) { try { FSEditLog editLog = getFSImage().getEditLog(); long numEdits = editLog.getLastWrittenTxId() - editLog.getCurSegmentTxId(); if (numEdits > rollThreshold) { FSNamesystem.LOG.info("NameNode rolling its own edit log because" + " number of edits in open segment exceeds threshold of " + rollThreshold); rollEditLog(); } } catch (Exception e) { FSNamesystem.LOG.error("Swallowing exception in " + NameNodeEditLogRoller.class.getSimpleName() + ":", e); } try { Thread.sleep(sleepIntervalMs); } catch (InterruptedException e) { FSNamesystem.LOG.info(NameNodeEditLogRoller.class.getSimpleName() + " was interrupted, exiting"); break; } } }
CheckpointSignature rollEditLog() throws IOException { getEditLog().rollEditLog(); // Record this log segment ID in all of the storage directories, so // we won't miss this log segment on a restart if the edits directories // go missing. storage.writeTransactionIdFileToStorage(getEditLog().getCurSegmentTxId()); return new CheckpointSignature(this); }
CheckpointSignature(FSImage fsImage) { super(fsImage.getStorage()); blockpoolID = fsImage.getBlockPoolID(); mostRecentCheckpointTxId = fsImage.getStorage().getMostRecentCheckpointTxId(); curSegmentTxId = fsImage.getEditLog().getCurSegmentTxId(); }
CheckpointSignature(FSImage fsImage) { super(fsImage.getStorage()); blockpoolID = fsImage.getBlockPoolID(); mostRecentCheckpointTxId = fsImage.getStorage().getMostRecentCheckpointTxId(); curSegmentTxId = fsImage.getEditLog().getCurSegmentTxId(); }
CheckpointSignature rollEditLog() throws IOException { getEditLog().rollEditLog(); // Record this log segment ID in all of the storage directories, so // we won't miss this log segment on a restart if the edits directories // go missing. storage.writeTransactionIdFileToStorage(getEditLog().getCurSegmentTxId()); return new CheckpointSignature(this); }
@Override public void run() { while (fsRunning && shouldRun) { try { FSEditLog editLog = getFSImage().getEditLog(); long numEdits = editLog.getLastWrittenTxId() - editLog.getCurSegmentTxId(); if (numEdits > rollThreshold) { FSNamesystem.LOG.info("NameNode rolling its own edit log because" + " number of edits in open segment exceeds threshold of " + rollThreshold); rollEditLog(); } } catch (Exception e) { FSNamesystem.LOG.error("Swallowing exception in " + NameNodeEditLogRoller.class.getSimpleName() + ":", e); } try { Thread.sleep(sleepIntervalMs); } catch (InterruptedException e) { FSNamesystem.LOG.info(NameNodeEditLogRoller.class.getSimpleName() + " was interrupted, exiting"); break; } } }
editLog.getLastWrittenTxId() + "in the log segment starting at " + editLog.getCurSegmentTxId() + ". Aborting this " + "log segment."); editLog.abortCurrentLogSegment();
@Test(timeout=60000) public void testEditLogAutoroll() throws Exception { // Make some edits final long startTxId = editLog.getCurSegmentTxId(); for (int i=0; i<11; i++) { fs.mkdirs(new Path("testEditLogAutoroll-" + i)); } // Wait for the NN to autoroll GenericTestUtils.waitFor(new Supplier<Boolean>() { @Override public Boolean get() { return editLog.getCurSegmentTxId() > startTxId; } }, 1000, 5000); // Transition to standby and make sure the roller stopped nn0.transitionToStandby(); GenericTestUtils.assertNoThreadsMatching( ".*" + NameNodeEditLogRoller.class.getSimpleName() + ".*"); } }
while (lastAppliedTxId < editLog.getCurSegmentTxId() - 1) { long target = editLog.getCurSegmentTxId(); LOG.info("Loading edits into backupnode to try to catch up from txid " + lastAppliedTxId + " to " + target); if (s.getFirstTxId() != editLog.getCurSegmentTxId()) { editStreams.add(s); if (lastAppliedTxId != editLog.getCurSegmentTxId() - 1) { LOG.debug("Logs rolled while catching up to current segment"); return false; // drop lock and try again to load local logs Collection<EditLogInputStream> editStreams = getEditLog().selectInputStreams( getEditLog().getCurSegmentTxId(), getEditLog().getCurSegmentTxId()); if (s.getFirstTxId() == getEditLog().getCurSegmentTxId()) { stream = s; LOG.warn("Unable to find stream starting with " + editLog.getCurSegmentTxId() + ". This indicates that there is an error in synchronization in BackupImage"); return false;
editLog.getLastWrittenTxId() + "in the log segment starting at " + editLog.getCurSegmentTxId() + ". Aborting this " + "log segment."); editLog.abortCurrentLogSegment();
NamenodeProtocols nnRpc = nn.getRpcServer(); nnRpc.rollEditLog(); assertEquals(bnImage.getEditLog().getCurSegmentTxId(), nn.getFSImage().getEditLog().getCurSegmentTxId()); nn.getFSImage().getEditLog().getCurSegmentTxId()); assertTrue("Should not have finalized " + editsLog, editsLog.isInProgress());