static Iterable<EditLogInputStream> getEditLogStreams(NNStorage storage) throws IOException { FSImagePreTransactionalStorageInspector inspector = new FSImagePreTransactionalStorageInspector(); storage.inspectStorageDirs(inspector); List<EditLogInputStream> editStreams = new ArrayList<EditLogInputStream>(); for (File f : inspector.getLatestEditsFiles()) { editStreams.add(new EditLogFileInputStream(f)); } return editStreams; } }
static OfflineEditsLoader createLoader(OfflineEditsVisitor visitor, String inputFileName, boolean xmlInput, OfflineEditsViewer.Flags flags) throws IOException { if (xmlInput) { return new OfflineEditsXmlLoader(visitor, new File(inputFileName), flags); } else { File file = null; EditLogInputStream elis = null; OfflineEditsLoader loader = null; try { file = new File(inputFileName); elis = new EditLogFileInputStream(file, HdfsServerConstants.INVALID_TXID, HdfsServerConstants.INVALID_TXID, false); loader = new OfflineEditsBinaryLoader(visitor, elis, flags); } finally { if ((loader == null) && (elis != null)) { elis.close(); } } return loader; } } }
/** * Open an EditLogInputStream for the given URL. * * @param connectionFactory * the URLConnectionFactory used to create the connection. * @param url * the url hosting the log * @param startTxId * the expected starting txid * @param endTxId * the expected ending txid * @param inProgress * whether the log is in-progress * @return a stream from which edits may be read */ public static EditLogInputStream fromUrl( URLConnectionFactory connectionFactory, URL url, long startTxId, long endTxId, boolean inProgress) { return new EditLogFileInputStream(new URLLog(connectionFactory, url), startTxId, endTxId, inProgress); }
EditLogFileInputStream elfis = new EditLogFileInputStream(elf.getFile(), elf.getFirstTxId(), elf.getLastTxId(), elf.isInProgress()); LOG.debug("selecting edit log stream " + elf);
/** * @param file File being scanned and validated. * @param maxTxIdToScan Maximum Tx ID to try to scan. * The scan returns after reading this or a higher * ID. The file portion beyond this ID is * potentially being updated. * @return Result of the validation * @throws IOException */ static FSEditLogLoader.EditLogValidation scanEditLog(File file, long maxTxIdToScan, boolean verifyVersion) throws IOException { EditLogFileInputStream in; try { in = new EditLogFileInputStream(file); // read the header, initialize the inputstream, but do not check the // layoutversion in.getVersion(verifyVersion); } catch (LogHeaderCorruptException e) { LOG.warn("Log file " + file + " has no valid header", e); return new FSEditLogLoader.EditLogValidation(0, HdfsServerConstants.INVALID_TXID, true); } try { return FSEditLogLoader.scanEditLog(in, maxTxIdToScan); } finally { IOUtils.closeStream(in); } }
static void rollForwardByApplyingLogs( RemoteEditLogManifest manifest, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); List<EditLogInputStream> editsStreams = Lists.newArrayList(); for (RemoteEditLog log : manifest.getLogs()) { if (log.getEndTxId() > dstImage.getLastAppliedTxId()) { File f = dstStorage.findFinalizedEditsFile( log.getStartTxId(), log.getEndTxId()); editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), log.getEndTxId(), true)); } } LOG.info("Checkpointer about to load edits from " + editsStreams.size() + " stream(s)."); dstImage.loadEdits(editsStreams, dstNamesystem); } }
static Iterable<EditLogInputStream> getEditLogStreams(NNStorage storage) throws IOException { FSImagePreTransactionalStorageInspector inspector = new FSImagePreTransactionalStorageInspector(); storage.inspectStorageDirs(inspector); List<EditLogInputStream> editStreams = new ArrayList<EditLogInputStream>(); for (File f : inspector.getLatestEditsFiles()) { editStreams.add(new EditLogFileInputStream(f)); } return editStreams; } }
static Iterable<EditLogInputStream> getEditLogStreams(NNStorage storage) throws IOException { FSImagePreTransactionalStorageInspector inspector = new FSImagePreTransactionalStorageInspector(); storage.inspectStorageDirs(inspector); List<EditLogInputStream> editStreams = new ArrayList<EditLogInputStream>(); for (File f : inspector.getLatestEditsFiles()) { editStreams.add(new EditLogFileInputStream(f)); } return editStreams; } }
private void verifyEditLogs(FSNamesystem namesystem, FSImage fsimage) throws IOException { // Verify that we can read in all the transactions that we have written. // If there were any corruptions, it is likely that the reading in // of these transactions will throw an exception. for (Iterator<StorageDirectory> it = fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) { File editFile = FSImage.getImageFile(it.next(), NameNodeFile.EDITS); System.out.println("Verifying file: " + editFile); int numEdits = new FSEditLogLoader(namesystem).loadFSEdits( new EditLogFileInputStream(editFile)); System.out.println("Number of edits: " + numEdits); } }
static OfflineEditsLoader createLoader(OfflineEditsVisitor visitor, String inputFileName, boolean xmlInput, OfflineEditsViewer.Flags flags) throws IOException { if (xmlInput) { return new OfflineEditsXmlLoader(visitor, new File(inputFileName), flags); } else { File file = null; EditLogInputStream elis = null; OfflineEditsLoader loader = null; try { file = new File(inputFileName); elis = new EditLogFileInputStream(file, HdfsConstants.INVALID_TXID, HdfsConstants.INVALID_TXID, false); loader = new OfflineEditsBinaryLoader(visitor, elis, flags); } finally { if ((loader == null) && (elis != null)) { elis.close(); } } return loader; } } }
/** * Open an EditLogInputStream for the given URL. * * @param connectionFactory * the URLConnectionFactory used to create the connection. * @param url * the url hosting the log * @param startTxId * the expected starting txid * @param endTxId * the expected ending txid * @param inProgress * whether the log is in-progress * @return a stream from which edits may be read */ public static EditLogInputStream fromUrl( URLConnectionFactory connectionFactory, URL url, long startTxId, long endTxId, boolean inProgress) { return new EditLogFileInputStream(new URLLog(connectionFactory, url), startTxId, endTxId, inProgress); }
/** * Open an EditLogInputStream for the given URL. * * @param connectionFactory * the URLConnectionFactory used to create the connection. * @param url * the url hosting the log * @param startTxId * the expected starting txid * @param endTxId * the expected ending txid * @param inProgress * whether the log is in-progress * @return a stream from which edits may be read */ public static EditLogInputStream fromUrl( URLConnectionFactory connectionFactory, URL url, long startTxId, long endTxId, boolean inProgress) { return new EditLogFileInputStream(new URLLog(connectionFactory, url), startTxId, endTxId, inProgress); }
static FSEditLogLoader.EditLogValidation validateEditLog(File file) throws IOException { EditLogFileInputStream in; try { in = new EditLogFileInputStream(file); } catch (LogHeaderCorruptException corrupt) { // If it's missing its header, this is equivalent to no transactions FSImage.LOG.warn("Log at " + file + " has no valid header", corrupt); return new FSEditLogLoader.EditLogValidation(0, HdfsConstants.INVALID_TXID, HdfsConstants.INVALID_TXID); } try { return FSEditLogLoader.validateEditLog(in); } finally { IOUtils.closeStream(in); } }
static FSEditLogLoader.EditLogValidation validateEditLog(File file) throws IOException { EditLogFileInputStream in; try { in = new EditLogFileInputStream(file); in.getVersion(true); // causes us to read the header } catch (LogHeaderCorruptException e) { // If the header is malformed or the wrong value, this indicates a corruption LOG.warn("Log file " + file + " has no valid header", e); return new FSEditLogLoader.EditLogValidation(0, HdfsConstants.INVALID_TXID, true); } try { return FSEditLogLoader.validateEditLog(in); } finally { IOUtils.closeStream(in); } }
static void rollForwardByApplyingLogs( RemoteEditLogManifest manifest, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); List<EditLogInputStream> editsStreams = Lists.newArrayList(); for (RemoteEditLog log : manifest.getLogs()) { if (log.getEndTxId() > dstImage.getLastAppliedTxId()) { File f = dstStorage.findFinalizedEditsFile( log.getStartTxId(), log.getEndTxId()); editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), log.getEndTxId(), true)); } } LOG.info("Checkpointer about to load edits from " + editsStreams.size() + " stream(s)."); dstImage.loadEdits(editsStreams, dstNamesystem); } }
/** * @param editLog a path of an edit log file * @return the count of each type of operation in the log file * @throws Exception if there is an error reading it */ public static EnumMap<FSEditLogOpCodes,Holder<Integer>> countEditLogOpTypes( File editLog) throws Exception { EditLogInputStream elis = new EditLogFileInputStream(editLog); try { return countEditLogOpTypes(elis); } finally { IOUtils.closeStream(elis); } }
static FSEditLogLoader.EditLogValidation validateEditLog(File file) throws IOException { EditLogFileInputStream in; try { in = new EditLogFileInputStream(file); in.getVersion(true); // causes us to read the header } catch (LogHeaderCorruptException e) { // If the header is malformed or the wrong value, this indicates a corruption LOG.warn("Log file " + file + " has no valid header", e); return new FSEditLogLoader.EditLogValidation(0, HdfsConstants.INVALID_TXID, true); } try { return FSEditLogLoader.validateEditLog(in); } finally { IOUtils.closeStream(in); } }
/** * Load and merge edits from two edits files * * @param sd storage directory * @return number of edits loaded * @throws IOException */ int loadFSEdits(StorageDirectory sd) throws IOException { int numEdits = 0; EditLogFileInputStream edits = new EditLogFileInputStream(getImageFile(sd, NameNodeFile.EDITS)); FSEditLogLoader loader = new FSEditLogLoader(namesystem); numEdits += loader.loadFSEdits(edits, editLog.getCurrentTxId()); edits.close(); File editsNew = getImageFile(sd, NameNodeFile.EDITS_NEW); if (editsNew.exists() && editsNew.length() > 0) { edits = new EditLogFileInputStream(editsNew); numEdits += loader.loadFSEdits(edits, loader.getCurrentTxId()); edits.close(); } editLog.setStartTransactionId(loader.getCurrentTxId()); // update the counts. getFSNamesystem().dir.updateCountForINodeWithQuota(); return numEdits; }
static void rollForwardByApplyingLogs( RemoteEditLogManifest manifest, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); List<EditLogInputStream> editsStreams = Lists.newArrayList(); for (RemoteEditLog log : manifest.getLogs()) { if (log.getEndTxId() > dstImage.getLastAppliedTxId()) { File f = dstStorage.findFinalizedEditsFile( log.getStartTxId(), log.getEndTxId()); editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), log.getEndTxId(), true)); } } LOG.info("Checkpointer about to load edits from " + editsStreams.size() + " stream(s)."); dstImage.loadEdits(editsStreams, dstNamesystem); } }
private long verifyEditLogs(FSNamesystem namesystem, FSImage fsimage, String logFileName, long startTxId) throws IOException { long numEdits = -1; // Verify that we can read in all the transactions that we have written. // If there were any corruptions, it is likely that the reading in // of these transactions will throw an exception. for (StorageDirectory sd : fsimage.getStorage().dirIterable(NameNodeDirType.EDITS)) { File editFile = new File(sd.getCurrentDir(), logFileName); System.out.println("Verifying file: " + editFile); FSEditLogLoader loader = new FSEditLogLoader(namesystem, startTxId); long numEditsThisLog = loader.loadFSEdits( new EditLogFileInputStream(editFile), startTxId); System.out.println("Number of edits: " + numEditsThisLog); assertTrue(numEdits == -1 || numEditsThisLog == numEdits); numEdits = numEditsThisLog; } assertTrue(numEdits != -1); return numEdits; }