try { if (prevTxId != HdfsServerConstants.INVALID_TXID) { LOG.info("Fast-forwarding stream '" + streams[curIdx].getName() + "' to transaction ID " + (prevTxId + 1)); streams[curIdx].skipUntil(prevTxId + 1); case OK: try { FSEditLogOp op = streams[curIdx].readOp(); if (op == null) { state = State.EOF; if (streams[curIdx].getLastTxId() == prevTxId) { return null; } else { throw new PrematureEOFException("got premature end-of-file " + "at txid " + prevTxId + "; expected file to go up to " + streams[curIdx].getLastTxId()); long oldLast = streams[curIdx].getLastTxId(); long newLast = streams[curIdx + 1].getLastTxId(); if (newLast < oldLast) { throw new IOException("We encountered an error reading " + streams[curIdx].getName() + ". During automatic edit log " + "failover, we noticed that all of the remaining edit log " + "streams are shorter than the current one! The best " + streams[curIdx].getName() + "; failing over to edit log " + streams[curIdx + 1].getName(), prevException); curIdx++;
/** * Load an edit log, and apply the changes to the in-memory structure * This is where we apply edits that we've been writing to disk all * along. */ long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId, long maxTxnsToRead, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = createStartupProgressStep(edits); prog.beginStep(Phase.LOADING_EDITS, step); fsNamesys.writeLock(); try { long startTime = monotonicNow(); FSImage.LOG.info("Start loading edits file " + edits.getName() + " maxTxnsToRead = " + maxTxnsToRead); long numEdits = loadEditRecords(edits, false, expectedStartingTxId, maxTxnsToRead, startOpt, recovery); FSImage.LOG.info("Edits file " + edits.getName() + " of size " + edits.length() + " edits # " + numEdits + " loaded in " + (monotonicNow()-startTime)/1000 + " seconds"); return numEdits; } finally { edits.close(); fsNamesys.writeUnlock("loadFSEdits"); prog.endStep(Phase.LOADING_EDITS, step); } }
@Override public int compare(EditLogInputStream a, EditLogInputStream b) { return ComparisonChain.start() .compare(a.getFirstTxId(), b.getFirstTxId()) .compare(a.getLastTxId(), b.getLastTxId()) .result(); } });
/** * Creates a Step used for updating startup progress, populated with * information from the given edits. The step always includes the log's name. * If the log has a known length, then the length is included in the step too. * * @param edits EditLogInputStream to use for populating step * @return Step populated with information from edits * @throws IOException thrown if there is an I/O error */ private static Step createStartupProgressStep(EditLogInputStream edits) throws IOException { long length = edits.length(); String name = edits.getCurrentStreamName(); return length != -1 ? new Step(name, length) : new Step(name); } }
while (true) { long txid; lastPos = in.getPosition(); try { if ((txid = in.scanNextOp()) == HdfsServerConstants.INVALID_TXID) { break; + " while determining its valid length. Position was " + lastPos, t); in.resync(); FSImage.LOG.warn("After resync, position is " + in.getPosition()); continue;
long lastTxId = in.getLastTxId(); long numTxns = (lastTxId - expectedStartingTxId) + 1; StartupProgress prog = NameNode.getStartupProgress(); FSEditLogOp op; try { op = in.readOp(); if (op == null) { break; check203UpgradeFailure(in.getVersion(true), e); String errorMessage = formatEditLogReplayError(in, recentOpcodeOffsets, expectedTxId); "We failed to read txId " + expectedTxId, recovery, "skipping the bad section in the log"); in.resync(); continue; in.getPosition(); if (op.hasTransactionId()) { if (op.getTransactionId() > expectedTxId) { in.getVersion(true), lastInodeId); if (lastInodeId < inodeId) { lastInodeId = inodeId; } catch (MetaRecoveryContext.RequestStopException e) { MetaRecoveryContext.LOG.warn("Stopped reading edit log at " +
public void loadEdits() throws IOException { try { visitor.start(inputStream.getVersion(true)); while (true) { try { FSEditLogOp op = inputStream.readOp(); if (op == null) break; inputStream.getPosition()); visitor.close(e); throw e; inputStream.resync(); } catch (RuntimeException e) { if (!recoveryMode) { inputStream.getPosition()); visitor.close(e); throw e; inputStream.resync();
static EditLogValidation scanEditLog(EditLogInputStream in) { long lastPos = 0; long lastTxId = HdfsConstants.INVALID_TXID; long numValid = 0; FSEditLogOp op = null; while (true) { lastPos = in.getPosition(); try { if ((op = in.readOp()) == null) { // TODO break; } } catch (Throwable t) { FSImage.LOG.warn("Caught exception after reading " + numValid + " ops from " + in + " while determining its valid length." + "Position was " + lastPos, t); in.resync(); FSImage.LOG.warn("After resync, position is " + in.getPosition()); continue; } if (lastTxId == HdfsConstants.INVALID_TXID || op.getTransactionId() > lastTxId) { lastTxId = op.getTransactionId(); } numValid++; } return new EditLogValidation(lastPos, lastTxId, false); }
/** * Go through the next operation from the stream storage. * @return the txid of the next operation. */ protected long scanNextOp() throws IOException { FSEditLogOp next = readOp(); return next != null ? next.txid : HdfsServerConstants.INVALID_TXID; }
private static EditLogInputStream getJournalInputStream(JournalManager jm, long txId, boolean inProgressOk) throws IOException { final PriorityQueue<EditLogInputStream> allStreams = new PriorityQueue<EditLogInputStream>(64, JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR); jm.selectInputStreams(allStreams, txId, inProgressOk); EditLogInputStream elis = null, ret; try { while ((elis = allStreams.poll()) != null) { if (elis.getFirstTxId() > txId) { break; } if (elis.getLastTxId() < txId) { elis.close(); continue; } elis.skipUntil(txId); ret = elis; elis = null; return ret; } } finally { IOUtils.cleanup(LOG, allStreams.toArray(new EditLogInputStream[0])); IOUtils.cleanup(LOG, elis); } return null; }
/** * Load an edit log, and apply the changes to the in-memory structure * This is where we apply edits that we've been writing to disk all * along. */ int loadFSEdits(EditLogInputStream edits, long expectedStartingTxId) throws IOException { long startTime = now(); currentTxId = expectedStartingTxId; int numEdits = loadFSEdits(edits, true); FSImage.LOG.info("Edits file " + edits.getName() + " of size " + edits.length() + " edits # " + numEdits + " loaded in " + (now()-startTime)/1000 + " seconds."); return numEdits; }
/** * Returns the name of the currently active underlying stream. The default * implementation returns the same value as getName unless overridden by the * subclass. * * @return String name of the currently active underlying stream */ public String getCurrentStreamName() { return getName(); }
try { FSEditLogOp op; while ((op = in.readOp()) != null) { if (logVersion <= FSConstants.STORED_TXIDS) { long diskTxid = op.txid; in.close(); sb.append("Error replaying edit log at offset " + in.getPosition()); if (recentOpcodeOffsets[0] != -1) { Arrays.sort(recentOpcodeOffsets);
assertEquals(1, stream.getFirstTxId()); assertEquals(3, stream.getLastTxId()); assertNull(stream.readOp()); } finally { IOUtils.cleanup(LOG, streams.toArray(new Closeable[0])); assertEquals(1, streams.size()); EditLogInputStream stream = streams.get(0); assertEquals(1, stream.getFirstTxId()); assertEquals(3, stream.getLastTxId()); verifyEdits(streams, 1, 3); } finally { try { assertEquals(2, streams.size()); assertEquals(4, streams.get(1).getFirstTxId()); assertEquals(6, streams.get(1).getLastTxId());
static OfflineEditsLoader createLoader(OfflineEditsVisitor visitor, String inputFileName, boolean xmlInput, OfflineEditsViewer.Flags flags) throws IOException { if (xmlInput) { return new OfflineEditsXmlLoader(visitor, new File(inputFileName), flags); } else { File file = null; EditLogInputStream elis = null; OfflineEditsLoader loader = null; try { file = new File(inputFileName); elis = new EditLogFileInputStream(file, HdfsServerConstants.INVALID_TXID, HdfsServerConstants.INVALID_TXID, false); loader = new OfflineEditsBinaryLoader(visitor, elis, flags); } finally { if ((loader == null) && (elis != null)) { elis.close(); } } return loader; } } }
private static long readAllEdits(Collection<EditLogInputStream> streams, long startTxId) throws IOException { FSEditLogOp op; long nextTxId = startTxId; long numTx = 0; for (EditLogInputStream s : streams) { while (true) { op = s.readOp(); if (op == null) break; if (op.getTransactionId() != nextTxId) { throw new IOException("out of order transaction ID! expected " + nextTxId + " but got " + op.getTransactionId() + " when " + "reading " + s.getName()); } numTx++; nextTxId = op.getTransactionId() + 1; } } return numTx; }
@Test public void testReadURL() throws Exception { HttpURLConnection conn = mock(HttpURLConnection.class); doReturn(new ByteArrayInputStream(FAKE_LOG_DATA)).when(conn).getInputStream(); doReturn(HttpURLConnection.HTTP_OK).when(conn).getResponseCode(); doReturn(Integer.toString(FAKE_LOG_DATA.length)).when(conn).getHeaderField("Content-Length"); URLConnectionFactory factory = mock(URLConnectionFactory.class); doReturn(conn).when(factory).openConnection(Mockito.<URL> any(), anyBoolean()); URL url = new URL("http://localhost/fakeLog"); EditLogInputStream elis = EditLogFileInputStream.fromUrl(factory, url, HdfsConstants.INVALID_TXID, HdfsConstants.INVALID_TXID, false); // Read the edit log and verify that we got all of the data. EnumMap<FSEditLogOpCodes, Holder<Integer>> counts = FSImageTestUtil .countEditLogOpTypes(elis); assertThat(counts.get(FSEditLogOpCodes.OP_ADD).held, is(1)); assertThat(counts.get(FSEditLogOpCodes.OP_SET_GENSTAMP_V1).held, is(1)); assertThat(counts.get(FSEditLogOpCodes.OP_CLOSE).held, is(1)); // Check that length header was picked up. assertEquals(FAKE_LOG_DATA.length, elis.length()); elis.close(); } }
@Override public long getLastTxId() { return streams[curIdx].getLastTxId(); }
@Override public long getPosition() { return streams[curIdx].getPosition(); }
@Override public long length() throws IOException { return streams[curIdx].length(); }