static Iterable<EditLogInputStream> getEditLogStreams(NNStorage storage) throws IOException { FSImagePreTransactionalStorageInspector inspector = new FSImagePreTransactionalStorageInspector(); storage.inspectStorageDirs(inspector); List<EditLogInputStream> editStreams = new ArrayList<EditLogInputStream>(); for (File f : inspector.getLatestEditsFiles()) { editStreams.add(new EditLogFileInputStream(f)); } return editStreams; } }
dataIn = new DataInputStream(tracker); try { logVersion = readLogVersion(dataIn, verifyLayoutVersion); } catch (EOFException eofe) { throw new LogHeaderCorruptException("No header found in log");
EditLogInputStream elis = EditLogFileInputStream.fromUrl( connectionFactory, url, remoteLog.getStartTxId(), endTxId, remoteLog.isInProgress());
/** * @param file File being scanned and validated. * @param maxTxIdToScan Maximum Tx ID to try to scan. * The scan returns after reading this or a higher * ID. The file portion beyond this ID is * potentially being updated. * @return Result of the validation * @throws IOException */ static FSEditLogLoader.EditLogValidation scanEditLog(File file, long maxTxIdToScan, boolean verifyVersion) throws IOException { EditLogFileInputStream in; try { in = new EditLogFileInputStream(file); // read the header, initialize the inputstream, but do not check the // layoutversion in.getVersion(verifyVersion); } catch (LogHeaderCorruptException e) { LOG.warn("Log file " + file + " has no valid header", e); return new FSEditLogLoader.EditLogValidation(0, HdfsServerConstants.INVALID_TXID, true); } try { return FSEditLogLoader.scanEditLog(in, maxTxIdToScan); } finally { IOUtils.closeStream(in); } }
case UNINIT: try { init(true); } catch (Throwable e) { LOG.error("caught exception initializing " + this, e); return nextOpImpl(skipBrokenEdits); case OPEN: op = reader.readOp(skipBrokenEdits); if (LOG.isDebugEnabled()) { LOG.debug("skipping " + skipAmt + " bytes at the end " + "of edit log '" + getName() + "': reached txid " + txId + " out of " + lastTxId);
EditLogFileInputStream in; try { in = new EditLogFileInputStream(file); in.getVersion(false); } catch (LogHeaderCorruptException e) { LOG.warn("Log file " + file + " has no valid header", e); while (true) { long txid = HdfsConstants.INVALID_TXID; lastPos = in.getPosition(); try { if ((txid = in.scanNextOp()) == HdfsConstants.INVALID_TXID) { break; + " while determining its valid length. Position was " + lastPos, t); in.resync(); FSImage.LOG.warn("After resync, position is " + in.getPosition()); continue;
/** * Load and merge edits from two edits files * * @param sd storage directory * @return number of edits loaded * @throws IOException */ int loadFSEdits(StorageDirectory sd) throws IOException { int numEdits = 0; EditLogFileInputStream edits = new EditLogFileInputStream(getImageFile(sd, NameNodeFile.EDITS)); FSEditLogLoader loader = new FSEditLogLoader(namesystem); numEdits += loader.loadFSEdits(edits, editLog.getCurrentTxId()); edits.close(); File editsNew = getImageFile(sd, NameNodeFile.EDITS_NEW); if (editsNew.exists() && editsNew.length() > 0) { edits = new EditLogFileInputStream(editsNew); numEdits += loader.loadFSEdits(edits, loader.getCurrentTxId()); edits.close(); } editLog.setStartTransactionId(loader.getCurrentTxId()); // update the counts. getFSNamesystem().dir.updateCountForINodeWithQuota(); return numEdits; }
elfos.close(); elfos = null; elfis = new EditLogFileInputStream(TEST_LOG_NAME); FSEditLogOp op = elfis.readOp(); if (op == null) break; elfos.close(); if (elfis != null) elfis.close();
elfos.close(); elfos = null; elfis = new EditLogFileInputStream(TEST_LOG_NAME); elfis.setMaxOpSize(elts.getMaxOpSize()); try { while (true) { op = elfis.nextOp(); if (op == null) { break; try { while (true) { op = elfis.nextValidOp(); if (op == null) { break;
@Override public String toString() { return getName(); }
@Override protected FSEditLogOp nextOp() throws IOException { return nextOpImpl(false); }
/** * Find out where the edit log ends. * This will update the lastTxId of the EditLogFile or * mark it as corrupt if it is. */ public void validateLog() throws IOException { EditLogValidation val = EditLogFileInputStream.validateEditLog(file); this.lastTxId = val.getEndTxId(); this.hasCorruptHeader = val.hasCorruptHeader(); }
@Override public int getVersion(boolean verifyVersion) throws IOException { if (state == State.UNINIT) { init(verifyVersion); } return logVersion; }
@Override protected long scanNextOp() throws IOException { Preconditions.checkState(state == State.OPEN); FSEditLogOp cachedNext = getCachedOp(); return cachedNext == null ? reader.scanOp() : cachedNext.txid; }
/** * Find out where the edit log ends. * This will update the lastTxId of the EditLogFile or * mark it as corrupt if it is. * @param maxTxIdToScan Maximum Tx ID to try to scan. * The scan returns after reading this or a higher ID. * The file portion beyond this ID is potentially being * updated. * @param verifyVersion Whether the scan should verify the layout version */ public void scanLog(long maxTxIdToScan, boolean verifyVersion) throws IOException { EditLogValidation val = EditLogFileInputStream.scanEditLog(file, maxTxIdToScan, verifyVersion); this.lastTxId = val.getEndTxId(); this.hasCorruptHeader = val.hasCorruptHeader(); }
EditLogFileInputStream in; try { in = new EditLogFileInputStream(file); in.getVersion(false); } catch (LogHeaderCorruptException e) { LOG.warn("Log file " + file + " has no valid header", e); while (true) { long txid = HdfsConstants.INVALID_TXID; lastPos = in.getPosition(); try { if ((txid = in.scanNextOp()) == HdfsConstants.INVALID_TXID) { break; + " while determining its valid length. Position was " + lastPos, t); in.resync(); FSImage.LOG.warn("After resync, position is " + in.getPosition()); continue;
case UNINIT: try { init(true); } catch (Throwable e) { LOG.error("caught exception initializing " + this, e); return nextOpImpl(skipBrokenEdits); case OPEN: op = reader.readOp(skipBrokenEdits); if (LOG.isDebugEnabled()) { LOG.debug("skipping " + skipAmt + " bytes at the end " + "of edit log '" + getName() + "': reached txid " + txId + " out of " + lastTxId);
static FSEditLogLoader.EditLogValidation validateEditLog(File file) throws IOException { EditLogFileInputStream in; try { in = new EditLogFileInputStream(file); in.getVersion(true); // causes us to read the header } catch (LogHeaderCorruptException e) { // If the header is malformed or the wrong value, this indicates a corruption LOG.warn("Log file " + file + " has no valid header", e); return new FSEditLogLoader.EditLogValidation(0, HdfsConstants.INVALID_TXID, true); } try { return FSEditLogLoader.validateEditLog(in); } finally { IOUtils.closeStream(in); } }
@Override public String toString() { return getName(); }
@Override protected FSEditLogOp nextValidOp() { try { return nextOpImpl(true); } catch (Throwable e) { LOG.error("nextValidOp: got exception while reading " + this, e); return null; } }