/** * If there is an IO Error on any log operations on storage directory, * remove any stream associated with that directory */ synchronized void processIOError(StorageDirectory sd) { // Try to remove stream only if one should exist if (!sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) return; if (editStreams == null || editStreams.size() <= 1) { FSNamesystem.LOG.fatal( "Fatal Error : All storage directories are inaccessible."); Runtime.getRuntime().exit(-1); } for (int idx = 0; idx < editStreams.size(); idx++) { File parentStorageDir = ((EditLogFileOutputStream)editStreams .get(idx)).getFile() .getParentFile().getParentFile(); if (parentStorageDir.getName().equals(sd.getRoot().getName())) editStreams.remove(idx); } }
if (sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) { imageExists = getImageFile(sd, NameNodeFile.IMAGE).exists(); imageDirs.add(sd.getRoot().getCanonicalPath()); if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) { editsExists = getImageFile(sd, NameNodeFile.EDITS).exists(); editsDirs.add(sd.getRoot().getCanonicalPath()); if (sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE) && (latestNameCheckpointTime < checkpointTime) && imageExists) { latestNameCheckpointTime = checkpointTime; latestNameSD = sd; if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS) && (latestEditsCheckpointTime < checkpointTime) && editsExists) { latestEditsCheckpointTime = checkpointTime;
if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) editLog.processIOError(sd); StorageDirectory sd = it.next(); if (!sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) { File editsFile = getImageFile(sd, NameNodeFile.EDITS); editsFile.delete(); if (!sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) { File imageFile = getImageFile(sd, NameNodeFile.IMAGE); imageFile.delete(); LOG.error("Cannot write file " + sd.getRoot(), e); if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) editLog.processIOError(sd);
if (sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) { imageExists = getImageFile(sd, NameNodeFile.IMAGE).exists(); imageDirs.add(sd.getRoot().getCanonicalPath()); if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) { editsExists = getImageFile(sd, NameNodeFile.EDITS).exists(); editsDirs.add(sd.getRoot().getCanonicalPath()); if (sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE) && (latestNameCheckpointTime < checkpointTime) && imageExists) { latestNameCheckpointTime = checkpointTime; latestNameSD = sd; if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS) && (latestEditsCheckpointTime < checkpointTime) && editsExists) { latestEditsCheckpointTime = checkpointTime;
if (!ckpt.renameTo(curFile)) { if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) editLog.processIOError(sd); it.remove(); StorageDirectory sd = it.next(); if (!sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) { File editsFile = getImageFile(sd, NameNodeFile.EDITS); editsFile.delete(); if (!sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) { File imageFile = getImageFile(sd, NameNodeFile.IMAGE); imageFile.delete(); LOG.error("Cannot write file " + sd.getRoot(), e); if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) editLog.processIOError(sd); it.remove();
if (sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) { try { long txid = Long.parseLong(imageMatch.group(1));
if (sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) { try { long txid = Long.parseLong(imageMatch.group(1));
if (sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) { imageExists = NNStorage.getStorageFile(sd, NameNodeFile.IMAGE).exists(); imageDirs.add(sd.getRoot().getCanonicalPath()); if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) { editsExists = NNStorage.getStorageFile(sd, NameNodeFile.EDITS).exists(); editsDirs.add(sd.getRoot().getCanonicalPath()); if (sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE) && (latestNameCheckpointTime < checkpointTime) && imageExists) { latestNameCheckpointTime = checkpointTime; if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS) && (latestEditsCheckpointTime < checkpointTime) && editsExists) { latestEditsCheckpointTime = checkpointTime;
if (sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) { imageExists = NNStorage.getStorageFile(sd, NameNodeFile.IMAGE).exists(); imageDirs.add(sd.getRoot().getCanonicalPath()); if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) { editsExists = NNStorage.getStorageFile(sd, NameNodeFile.EDITS).exists(); editsDirs.add(sd.getRoot().getCanonicalPath()); if (sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE) && (latestNameCheckpointTime < checkpointTime) && imageExists) { latestNameCheckpointTime = checkpointTime; if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS) && (latestEditsCheckpointTime < checkpointTime) && editsExists) { latestEditsCheckpointTime = checkpointTime;
/** * verify that edits log and fsimage are in different directories and of a correct size */ private void verifyDifferentDirs(FSImage img, long expectedImgSize, long expectedEditsSize) { StorageDirectory sd =null; for (Iterator<StorageDirectory> it = img.dirIterator(); it.hasNext();) { sd = it.next(); if(sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) { File imf = FSImage.getImageFile(sd, NameNodeFile.IMAGE); LOG.info("--image file " + imf.getAbsolutePath() + "; len = " + imf.length() + "; expected = " + expectedImgSize); assertEquals(expectedImgSize, imf.length()); } else if(sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) { File edf = FSImage.getImageFile(sd, NameNodeFile.EDITS); LOG.info("-- edits file " + edf.getAbsolutePath() + "; len = " + edf.length() + "; expected = " + expectedEditsSize); assertEquals(expectedEditsSize, edf.length()); } else { fail("Image/Edits directories are not different"); } } } /**
/** * Record new checkpoint time in order to * distinguish healthy directories from the removed ones. * If there is an error writing new checkpoint time, the corresponding * storage directory is removed from the list. */ void incrementCheckpointTime() { this.checkpointTime++; // Write new checkpoint time in all storage directories for(Iterator<StorageDirectory> it = dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); try { writeCheckpointTime(sd); } catch(IOException e) { // Close any edits stream associated with this dir and remove directory if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) editLog.processIOError(sd); it.remove(); } } }
/** * verify that edits log and fsimage are in different directories and of a correct size */ private void verifyDifferentDirs(FSImage img, long expectedImgSize, long expectedEditsSize) { StorageDirectory sd =null; for (Iterator<StorageDirectory> it = img.getStorage().dirIterator(); it.hasNext();) { sd = it.next(); if(sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) { img.getStorage(); File imf = NNStorage.getStorageFile(sd, NameNodeFile.IMAGE, 0); LOG.info("--image file " + imf.getAbsolutePath() + "; len = " + imf.length() + "; expected = " + expectedImgSize); assertEquals(expectedImgSize, imf.length()); } else if(sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) { img.getStorage(); File edf = NNStorage.getStorageFile(sd, NameNodeFile.EDITS, 0); LOG.info("-- edits file " + edf.getAbsolutePath() + "; len = " + edf.length() + "; expected = " + expectedEditsSize); assertEquals(expectedEditsSize, edf.length()); } else { fail("Image/Edits directories are not different"); } } } /**
/** * Record new checkpoint time in order to * distinguish healthy directories from the removed ones. * If there is an error writing new checkpoint time, the corresponding * storage directory is removed from the list. */ void incrementCheckpointTime() { this.checkpointTime++; // Write new checkpoint time in all storage directories for(Iterator<StorageDirectory> it = dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); try { writeCheckpointTime(sd); } catch(IOException e) { // Close any edits stream associated with this dir and remove directory LOG.warn("incrementCheckpointTime failed on " + sd.getRoot().getPath() + ";type="+sd.getStorageDirType()); if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) editLog.processIOError(sd); //add storage to the removed list removedStorageDirs.add(sd); it.remove(); } } }
/** * test */ public void printStorages(FSImage fs) { LOG.info("current storages and corresoponding sizes:"); for(Iterator<StorageDirectory> it = fs.dirIterator(); it.hasNext(); ) { StorageDirectory sd = it.next(); if(sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) { File imf = FSImage.getImageFile(sd, NameNodeFile.IMAGE); LOG.info(" image file " + imf.getAbsolutePath() + "; len = " + imf.length()); } if(sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) { File edf = FSImage.getImageFile(sd, NameNodeFile.EDITS); LOG.info(" edits file " + edf.getAbsolutePath() + "; len = " + edf.length()); } } }
/** * If there is an IO Error on any log operations on storage directory, * remove any stream associated with that directory */ synchronized void processIOError(StorageDirectory sd) { // Try to remove stream only if one should exist if (!sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) return; if (editStreams == null || editStreams.size() <= 1) { FSNamesystem.LOG.fatal( "Fatal Error : All storage directories are inaccessible."); Runtime.getRuntime().exit(-1); } for (int idx = 0; idx < editStreams.size(); idx++) { File parentStorageDir = ((EditLogFileOutputStream)editStreams .get(idx)).getFile() .getParentFile().getParentFile(); if (parentStorageDir.getName().equals(sd.getRoot().getName())) editStreams.remove(idx); } }
public boolean hasNext() { if (storageDirs.isEmpty() || nextIndex >= storageDirs.size()) return false; if (dirType != null) { while (nextIndex < storageDirs.size()) { if (getStorageDir(nextIndex).getStorageDirType().isOfType(dirType)) break; nextIndex++; } if (nextIndex >= storageDirs.size()) return false; } return true; }
public boolean hasNext() { if (storageDirs.isEmpty() || nextIndex >= storageDirs.size()) return false; if (dirType != null) { while (nextIndex < storageDirs.size()) { if (getStorageDir(nextIndex).getStorageDirType().isOfType(dirType)) break; nextIndex++; } if (nextIndex >= storageDirs.size()) return false; } return true; }
public StorageDirectory next() { StorageDirectory sd = getStorageDir(nextIndex); prevIndex = nextIndex; nextIndex++; if (dirType != null) { while (nextIndex < storageDirs.size()) { if (getStorageDir(nextIndex).getStorageDirType().isOfType(dirType)) break; nextIndex++; } } return sd; }
public StorageDirectory next() { StorageDirectory sd = getStorageDir(nextIndex); prevIndex = nextIndex; nextIndex++; if (dirType != null) { while (nextIndex < storageDirs.size()) { if (getStorageDir(nextIndex).getStorageDirType().isOfType(dirType)) break; nextIndex++; } } return sd; }
private boolean shouldReturnNextDir() { StorageDirectory sd = getStorageDir(nextIndex); return (dirType == null || sd.getStorageDirType().isOfType(dirType)) && (includeShared || !sd.isShared()); } }