/** Get a path from the local FS for reading. We search through all the * configured dirs for the file's existence and return the complete * path to the file when we find one */ public Path getLocalPathToRead(String pathStr, Configuration conf) throws IOException { Context ctx = confChanged(conf); int numDirs = ctx.localDirs.length; int numDirsSearched = 0; //remove the leading slash from the path (to make sure that the uri //resolution results in a valid path on the dir being checked) if (pathStr.startsWith("/")) { pathStr = pathStr.substring(1); } while (numDirsSearched < numDirs) { Path file = new Path(ctx.localDirs[numDirsSearched], pathStr); if (ctx.localFS.exists(file)) { return file; } numDirsSearched++; } //no path found throw new DiskErrorException ("Could not find " + pathStr +" in any of" + " the configured local directories"); }
/** * Returns {@link DiskValidator} instance corresponding to its name. * The diskValidator parameter can be "basic" for {@link BasicDiskValidator} * or "read-write" for {@link ReadWriteDiskValidator}. * @param diskValidator canonical class name, for example, "basic" * @throws DiskErrorException if the class cannot be located */ @SuppressWarnings("unchecked") public static DiskValidator getInstance(String diskValidator) throws DiskErrorException { @SuppressWarnings("rawtypes") Class clazz; if (diskValidator.equalsIgnoreCase(BasicDiskValidator.NAME)) { clazz = BasicDiskValidator.class; } else if (diskValidator.equalsIgnoreCase(ReadWriteDiskValidator.NAME)) { clazz = ReadWriteDiskValidator.class; } else { try { clazz = Class.forName(diskValidator); } catch (ClassNotFoundException cnfe) { throw new DiskErrorException(diskValidator + " DiskValidator class not found.", cnfe); } } return getInstance(clazz); } }
if (!dir.isDirectory()) { metric.diskCheckFailed(); throw new DiskErrorException(dir + " is not a directory!"); throw new DiskErrorException("Data in file has been corrupted."); throw new DiskErrorException("Disk Check failed!", e); } finally { } catch (IOException e) { metric.diskCheckFailed(); throw new DiskErrorException("File deletion failed!", e);
/** * Create the directory if it doesn't exist and check that dir is readable, * writable and executable * * @param dir * @throws DiskErrorException */ public static void checkDir(File dir) throws DiskErrorException { if (!mkdirsWithExistsCheck(dir)) { throw new DiskErrorException("Cannot create directory: " + dir.toString()); } checkAccessByFileMethods(dir); }
public static void checkDir(File dir) throws DiskErrorException { if (!mkdirsWithExistsCheck(dir)) throw new DiskErrorException("can not create directory: " + dir.toString()); if (!dir.isDirectory()) throw new DiskErrorException("not a directory: " + dir.toString()); if (!dir.canRead()) throw new DiskErrorException("directory is not readable: " + dir.toString()); if (!dir.canWrite()) throw new DiskErrorException("directory is not writable: " + dir.toString()); }
/** * Checks that the given file is a directory and that the current running * process can read, write, and execute it. * * @param dir File to check * @throws DiskErrorException if dir is not a directory, not readable, not * writable, or not executable */ private static void checkDirAccess(File dir) throws DiskErrorException { if (!dir.isDirectory()) { throw new DiskErrorException("Not a directory: " + dir.toString()); } checkAccessByFileMethods(dir); }
/** * Create the directory if it doesn't exist and check that dir is readable, * writable and executable * * @param dir * @throws DiskErrorException */ public static void checkDir(File dir) throws DiskErrorException { if (!mkdirsWithExistsCheck(dir)) { throw new DiskErrorException("Cannot create directory: " + dir.toString()); } checkDirAccess(dir); }
public static void checkDir(File dir) throws DiskErrorException { if (!mkdirsWithExistsCheck(dir)) throw new DiskErrorException("can not create directory: " + dir.toString()); if (!dir.isDirectory()) throw new DiskErrorException("not a directory: " + dir.toString()); if (!dir.canRead()) throw new DiskErrorException("directory is not readable: " + dir.toString()); if (!dir.canWrite()) throw new DiskErrorException("directory is not writable: " + dir.toString()); }
/** * Create the directory if it doesn't exist and check that dir is readable, * writable and executable * * @param dir * @throws DiskErrorException */ public static void checkDir(File dir) throws DiskErrorException { if (!mkdirsWithExistsCheck(dir)) { throw new DiskErrorException("Cannot create directory: " + dir.toString()); } checkDirAccess(dir); }
/** * Create the directory if it doesn't exist and check that dir is readable, * writable and executable * * @param dir * @throws DiskErrorException */ public static void checkDir(File dir) throws DiskErrorException { if (!mkdirsWithExistsCheck(dir)) { throw new DiskErrorException("Cannot create directory: " + dir.toString()); } checkDirAccess(dir); }
throw new DiskErrorException("Invalid value configured for " + DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY + " - " + maxVolumeFailuresTolerated + ". Value configured is >= " throw new DiskErrorException("Too many failed volumes - " + "current valid volumes: " + goodLocations.size() + ", volumes configured: " + dataDirs.size() throw new DiskErrorException("Too many failed volumes - " + "current valid volumes: " + goodLocations.size() + ", volumes configured: " + dataDirs.size() throw new DiskErrorException("All directories in " + DFS_DATANODE_DATA_DIR_KEY + " are invalid: " + failedLocations);
throw new DiskErrorException("Invalid value configured for " + "dfs.datanode.failed.volumes.tolerated - " + volFailuresTolerated + ". Value configured is either less than maxVolumeFailureLimit or greater than " throw new DiskErrorException( "Too many failed volumes - " + "current valid volumes: " + storage.getNumStorageDirs() + ", volumes configured: " throw new DiskErrorException( "Too many failed volumes - " + "current valid volumes: " + storage.getNumStorageDirs() + ", volumes configured: "
if (volFailuresTolerated < MAX_VOLUME_FAILURE_TOLERATED_LIMIT || volFailuresTolerated >= volsConfigured) { throw new DiskErrorException("Invalid value configured for " + "dfs.datanode.failed.volumes.tolerated - " + volFailuresTolerated + ". Value configured is either greater than -1 or >= "
throw new DiskErrorException("Invalid value configured for " + DFS_DATANODE_DISK_CHECK_TIMEOUT_KEY + " - " + maxAllowedTimeForCheckMs + " (should be > 0)"); throw new DiskErrorException("Invalid value configured for " + DFS_DATANODE_DISK_CHECK_MIN_GAP_KEY + " - " + minDiskCheckGapMs + " (should be >= 0)"); throw new DiskErrorException("Invalid value configured for " + DFS_DATANODE_DISK_CHECK_TIMEOUT_KEY + " - " + diskCheckTimeout + " (should be >= 0)"); throw new DiskErrorException("Invalid value configured for " + DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY + " - " + maxVolumeFailuresTolerated + " "
throw new DiskErrorException("Invalid value configured for " + DFS_DATANODE_DISK_CHECK_TIMEOUT_KEY + " - " + maxAllowedTimeForCheckMs + " (should be > 0)"); throw new DiskErrorException("Invalid value configured for " + DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY + " - " + maxVolumeFailuresTolerated + " "
@Test public void testReduceOutOfDiskSpace() throws Throwable { LOG.info("testReduceOutOfDiskSpace"); Fetcher<Text,Text> underTest = new FakeFetcher<Text,Text>(job, id, ss, mm, r, metrics, except, key, connection); String replyHash = SecureShuffleUtils.generateHash(encHash.getBytes(), key); ShuffleHeader header = new ShuffleHeader(map1ID.toString(), 10, 10, 1); ByteArrayOutputStream bout = new ByteArrayOutputStream(); header.write(new DataOutputStream(bout)); ByteArrayInputStream in = new ByteArrayInputStream(bout.toByteArray()); when(connection.getResponseCode()).thenReturn(200); when(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_NAME)) .thenReturn(ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); when(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_VERSION)) .thenReturn(ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); when(connection.getHeaderField(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH)) .thenReturn(replyHash); when(connection.getInputStream()).thenReturn(in); when(mm.reserve(any(TaskAttemptID.class), anyLong(), anyInt())) .thenThrow(new DiskErrorException("No disk space available")); underTest.copyFromHost(host); verify(ss).reportLocalError(any(IOException.class)); }
throw new DiskErrorException("No space available in any of the local directories."); throw new DiskErrorException("Could not find any valid local " + "directory for " + pathStr);
/** * Performs some disk IO by writing to a new file in the given directory * and sync'ing file contents to disk. * * This increases the likelihood of catching catastrophic disk/controller * failures sooner. * * @param dir directory to be checked. * @throws DiskErrorException if we hit an error while trying to perform * disk IO against the file. */ private static void doDiskIo(File dir) throws DiskErrorException { try { IOException ioe = null; for (int i = 0; i < DISK_IO_MAX_ITERATIONS; ++i) { final File file = getFileNameForDiskIoCheck(dir, i+1); try { diskIoCheckWithoutNativeIo(file); return; } catch (IOException e) { // Let's retry a few times before we really give up and // declare the disk as bad. ioe = e; } } throw ioe; // Just rethrow the last exception to signal failure. } catch(IOException e) { throw new DiskErrorException("Error checking directory " + dir, e); } }
/** * Checks that the current running process can read, write, and execute the * given directory by using methods of the File object. * * @param dir File to check * @throws DiskErrorException if dir is not readable, not writable, or not * executable */ private static void checkAccessByFileMethods(File dir) throws DiskErrorException { if (!dir.isDirectory()) { throw new DiskErrorException("Not a directory: " + dir.toString()); } if (!FileUtil.canRead(dir)) { throw new DiskErrorException("Directory is not readable: " + dir.toString()); } if (!FileUtil.canWrite(dir)) { throw new DiskErrorException("Directory is not writable: " + dir.toString()); } if (!FileUtil.canExecute(dir)) { throw new DiskErrorException("Directory is not executable: " + dir.toString()); } }
private static void checkDirInternal(File dir) throws DiskErrorException { if (!mkdirsWithExistsCheck(dir)) { throw new DiskErrorException("Cannot create directory: " + dir.toString()); } checkAccessByFileMethods(dir); }