/** * Check that the logs are non-overlapping sequences of transactions, * in sorted order. They do not need to be contiguous. * @throws IllegalStateException if incorrect */ private void checkState() { Preconditions.checkNotNull(logs); RemoteEditLog prev = null; for (RemoteEditLog log : logs) { if (prev != null) { if (log.getStartTxId() <= prev.getEndTxId()) { throw new IllegalStateException( "Invalid log manifest (log " + log + " overlaps " + prev + ")\n" + this); } } prev = log; } }
static String getParamStringForLog(RemoteEditLog log, StorageInfo remoteStorageInfo) { return "getedit=1&" + START_TXID_PARAM + "=" + log.getStartTxId() + "&" + END_TXID_PARAM + "=" + log.getEndTxId() + "&" + STORAGEINFO_PARAM + "=" + remoteStorageInfo.toColonSeparatedString(); }
static void rollForwardByApplyingLogs( RemoteEditLogManifest manifest, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); List<EditLogInputStream> editsStreams = Lists.newArrayList(); for (RemoteEditLog log : manifest.getLogs()) { if (log.getEndTxId() > dstImage.getLastAppliedTxId()) { File f = dstStorage.findFinalizedEditsFile( log.getStartTxId(), log.getEndTxId()); editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), log.getEndTxId(), true)); } } LOG.info("Checkpointer about to load edits from " + editsStreams.size() + " stream(s)."); dstImage.loadEdits(editsStreams, dstNamesystem); } }
.getRoot()); assert log.getStartTxId() > 0 && log.getEndTxId() > 0 : "bad log: " + log; File finalEditsFile = jnStorage.getFinalizedEditsFile(log.getStartTxId(), log.getEndTxId()); log.getStartTxId(), log.getEndTxId()); try { moveSuccess = journal.moveTmpSegmentToCurrent(tmpEditsFile, finalEditsFile, log.getEndTxId()); } catch (IOException e) { LOG.info("Could not move %s to current directory.", tmpEditsFile);
logs.add(bestLog); curStartTxId = bestLog.getEndTxId() + 1;
static void downloadEditsToStorage(URL fsName, RemoteEditLog log, NNStorage dstStorage) throws IOException { assert log.getStartTxId() > 0 && log.getEndTxId() > 0 : "bad log: " + log; String fileid = ImageServlet.getParamStringForLog( log, dstStorage); String finalFileName = NNStorage.getFinalizedEditsFileName( log.getStartTxId(), log.getEndTxId()); log.getStartTxId(), log.getEndTxId(), milliTime); List<File> tmpFiles = dstStorage.getFiles(NameNodeDirType.EDITS, tmpFileName); log.getStartTxId(), log.getEndTxId(), milliTime); File finalizedFile = NNStorage.getFinalizedEditsFile(sd, log.getStartTxId(), log.getEndTxId()); if (LOG.isDebugEnabled()) { LOG.debug("Renaming " + tmpFile + " to " + finalizedFile);
URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId()); long endTxId = remoteLog.getEndTxId();
public static RemoteEditLogProto convert(RemoteEditLog log) { return RemoteEditLogProto.newBuilder() .setStartTxId(log.getStartTxId()) .setEndTxId(log.getEndTxId()) .setIsInProgress(log.isInProgress()).build(); }
/** * Check that the logs are non-overlapping sequences of transactions, * in sorted order. They do not need to be contiguous. * @throws IllegalStateException if incorrect */ private void checkState() { Preconditions.checkNotNull(logs); RemoteEditLog prev = null; for (RemoteEditLog log : logs) { if (prev != null) { if (log.getStartTxId() <= prev.getEndTxId()) { throw new IllegalStateException( "Invalid log manifest (log " + log + " overlaps " + prev + ")\n" + this); } } prev = log; } }
static String getParamStringForLog(RemoteEditLog log, StorageInfo remoteStorageInfo) { return "getedit=1&" + START_TXID_PARAM + "=" + log.getStartTxId() + "&" + END_TXID_PARAM + "=" + log.getEndTxId() + "&" + STORAGEINFO_PARAM + "=" + remoteStorageInfo.toColonSeparatedString(); }
static String getParamStringForLog(RemoteEditLog log, StorageInfo remoteStorageInfo) { return "getedit=1&" + START_TXID_PARAM + "=" + log.getStartTxId() + "&" + END_TXID_PARAM + "=" + log.getEndTxId() + "&" + STORAGEINFO_PARAM + "=" + remoteStorageInfo.toColonSeparatedString(); }
/** * Check that the logs are non-overlapping sequences of transactions, * in sorted order. They do not need to be contiguous. * @throws IllegalStateException if incorrect */ private void checkState() { Preconditions.checkNotNull(logs); RemoteEditLog prev = null; for (RemoteEditLog log : logs) { if (prev != null) { if (log.getStartTxId() <= prev.getEndTxId()) { throw new IllegalStateException( "Invalid log manifest (log " + log + " overlaps " + prev + ")\n" + this); } } prev = log; } }
static void rollForwardByApplyingLogs( RemoteEditLogManifest manifest, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); List<EditLogInputStream> editsStreams = Lists.newArrayList(); for (RemoteEditLog log : manifest.getLogs()) { if (log.getEndTxId() > dstImage.getLastAppliedTxId()) { File f = dstStorage.findFinalizedEditsFile( log.getStartTxId(), log.getEndTxId()); editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), log.getEndTxId(), true)); } } LOG.info("Checkpointer about to load edits from " + editsStreams.size() + " stream(s)."); dstImage.loadEdits(editsStreams, dstNamesystem); } }
static void rollForwardByApplyingLogs( RemoteEditLogManifest manifest, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); List<EditLogInputStream> editsStreams = Lists.newArrayList(); for (RemoteEditLog log : manifest.getLogs()) { if (log.getEndTxId() > dstImage.getLastAppliedTxId()) { File f = dstStorage.findFinalizedEditsFile( log.getStartTxId(), log.getEndTxId()); editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), log.getEndTxId(), true)); } } LOG.info("Checkpointer about to load edits from " + editsStreams.size() + " stream(s)."); dstImage.loadEdits(editsStreams, dstNamesystem); } }
@Override public void selectInputStreams(Collection<EditLogInputStream> streams, long fromTxnId, boolean inProgressOk) throws IOException { QuorumCall<AsyncLogger, RemoteEditLogManifest> q = loggers.getEditLogManifest(fromTxnId, inProgressOk); Map<AsyncLogger, RemoteEditLogManifest> resps = loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs, "selectInputStreams"); LOG.debug("selectInputStream manifests:\n" + Joiner.on("\n").withKeyValueSeparator(": ").join(resps)); final PriorityQueue<EditLogInputStream> allStreams = new PriorityQueue<EditLogInputStream>(64, JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR); for (Map.Entry<AsyncLogger, RemoteEditLogManifest> e : resps.entrySet()) { AsyncLogger logger = e.getKey(); RemoteEditLogManifest manifest = e.getValue(); for (RemoteEditLog remoteLog : manifest.getLogs()) { URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId()); EditLogInputStream elis = EditLogFileInputStream.fromUrl( connectionFactory, url, remoteLog.getStartTxId(), remoteLog.getEndTxId(), remoteLog.isInProgress()); allStreams.add(elis); } } JournalSet.chainAndMakeRedundantStreams(streams, allStreams, fromTxnId); }
private static void compare(RemoteEditLog l1, RemoteEditLog l2) { assertEquals(l1.getEndTxId(), l2.getEndTxId()); assertEquals(l1.getStartTxId(), l2.getStartTxId()); }
static void downloadEditsToStorage(URL fsName, RemoteEditLog log, NNStorage dstStorage) throws IOException { assert log.getStartTxId() > 0 && log.getEndTxId() > 0 : "bad log: " + log; String fileid = ImageServlet.getParamStringForLog( log, dstStorage); String finalFileName = NNStorage.getFinalizedEditsFileName( log.getStartTxId(), log.getEndTxId()); log.getStartTxId(), log.getEndTxId(), milliTime); List<File> tmpFiles = dstStorage.getFiles(NameNodeDirType.EDITS, tmpFileName); log.getStartTxId(), log.getEndTxId(), milliTime); File finalizedFile = NNStorage.getFinalizedEditsFile(sd, log.getStartTxId(), log.getEndTxId()); if (LOG.isDebugEnabled()) { LOG.debug("Renaming " + tmpFile + " to " + finalizedFile);
@Override public void selectInputStreams(Collection<EditLogInputStream> streams, long fromTxnId, boolean inProgressOk) throws IOException { QuorumCall<AsyncLogger, RemoteEditLogManifest> q = loggers.getEditLogManifest(fromTxnId, inProgressOk); Map<AsyncLogger, RemoteEditLogManifest> resps = loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs, "selectInputStreams"); LOG.debug("selectInputStream manifests:\n" + Joiner.on("\n").withKeyValueSeparator(": ").join(resps)); final PriorityQueue<EditLogInputStream> allStreams = new PriorityQueue<EditLogInputStream>(64, JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR); for (Map.Entry<AsyncLogger, RemoteEditLogManifest> e : resps.entrySet()) { AsyncLogger logger = e.getKey(); RemoteEditLogManifest manifest = e.getValue(); for (RemoteEditLog remoteLog : manifest.getLogs()) { URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId()); EditLogInputStream elis = EditLogFileInputStream.fromUrl( connectionFactory, url, remoteLog.getStartTxId(), remoteLog.getEndTxId(), remoteLog.isInProgress()); allStreams.add(elis); } } JournalSet.chainAndMakeRedundantStreams(streams, allStreams, fromTxnId); }
public static RemoteEditLogProto convert(RemoteEditLog log) { return RemoteEditLogProto.newBuilder() .setStartTxId(log.getStartTxId()) .setEndTxId(log.getEndTxId()) .setIsInProgress(log.isInProgress()).build(); }
public static RemoteEditLogProto convert(RemoteEditLog log) { return RemoteEditLogProto.newBuilder() .setStartTxId(log.getStartTxId()) .setEndTxId(log.getEndTxId()) .setIsInProgress(log.isInProgress()).build(); }