/** * Creates a new object for a child entry in this directory * @param child the basename will be appended to this object's path * @return PathData for the child * @throws IOException if this object does not exist or is not a directory */ public PathData getPathDataForChild(PathData child) throws IOException { checkIfExists(FileTypeRequirement.SHOULD_BE_DIRECTORY); return new PathData(fs, getStringForChildPath(child.path)); }
/** * Expand the given argument into a list of {@link PathData} objects. * The default behavior is to expand globs. Commands may override to * perform other expansions on an argument. * @param arg string pattern to expand * @return list of {@link PathData} objects * @throws IOException if anything goes wrong... */ protected List<PathData> expandArgument(String arg) throws IOException { PathData[] items = PathData.expandAsGlob(arg, getConf()); if (items.length == 0) { // it's a glob that failed to match throw new PathNotFoundException(arg); } return Arrays.asList(items); }
@Override protected void processPath(PathData item) throws IOException { if (!item.stat.isDirectory()) { throw new PathIsNotDirectoryException(item.toString()); } }
protected PathData getTargetPath(PathData src) throws IOException { PathData target; // on the first loop, the dst may be directory or a file, so only create // a child path if dst is a dir; after recursion, it's always a dir if ((getDepth() > 0) || (dst.exists && dst.stat.isDirectory())) { target = dst.getPathDataForChild(src); } else if (dst.representsDirectory()) { // see if path looks like a dir target = dst.getPathDataForChild(src); } else { target = dst; } return target; }
/** * Creates an object to wrap the given parameters as fields. The string * used to create the path will be recorded since the Path object does not * return exactly the same string used to initialize it. * @param fs the FileSystem * @param pathString a String of the path * @param stat the FileStatus (may be null if the path doesn't exist) */ private PathData(FileSystem fs, String pathString, FileStatus stat) throws IOException { this.fs = fs; this.uri = stringToUri(pathString); this.path = fs.makeQualified(new Path(uri)); setStat(stat); if (Path.WINDOWS) { inferredSchemeFromPath = checkIfSchemeInferredFromPath(pathString); } }
@Test (timeout = 30000) public void testQualifiedUriContents() throws Exception { String dirString = fs.makeQualified(new Path("d1")).toString(); PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString(dirString+"/f1", dirString+"/f1.1", dirString+"/f2"), sortedString(items) ); }
items = new PathData[]{ new PathData(fs, pattern, null) }; } else { case HAS_SCHEME: // use as-is, but remove authority if necessary if (globUri.getAuthority() == null) { matchUri = removeAuthority(matchUri); globMatch = uriToString(matchUri, false); break; case SCHEMELESS_ABSOLUTE: // take just the uri's path case RELATIVE: // make it relative to the current working dir URI cwdUri = fs.getWorkingDirectory().toUri(); globMatch = relativize(cwdUri, matchUri, stat.isDirectory()); break; items[i++] = new PathData(fs, globMatch, stat);
@Test (timeout = 30000) public void testToFile() throws Exception { PathData item = new PathData(".", conf); assertEquals(new File(testDir.toString()), item.toFile()); item = new PathData("d1/f1", conf); assertEquals(new File(testDir + "/d1/f1"), item.toFile()); item = new PathData(testDir + "/d1/f1", conf); assertEquals(new File(testDir + "/d1/f1"), item.toFile()); }
/** * Returns a new PathData with the given extension. * @param extension for the suffix * @return PathData * @throws IOException shouldn't happen */ public PathData suffix(String extension) throws IOException { return new PathData(fs, this+extension); }
/** * Wait for all files in waitList to have length equal to newLength. */ private void waitForRecovery() throws IOException { for(PathData item : waitList) { out.println("Waiting for " + item + " ..."); out.flush(); for(;;) { item.refreshStatus(); if(item.stat.getLen() == newLength) break; try {Thread.sleep(1000);} catch(InterruptedException ignored) {} } out.println("Truncated " + item + " to length: " + newLength); out.flush(); } } }
@Override protected void processOptions(LinkedList<String> args) throws IOException { try { CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "nl", "skip-empty-file"); cf.parse(args); delimiter = cf.getOpt("nl") ? "\n" : null; skipEmptyFileDelimiter = cf.getOpt("skip-empty-file"); dst = new PathData(new URI(args.removeLast()), getConf()); if (dst.exists && dst.stat.isDirectory()) { throw new PathIsDirectoryException(dst.toString()); } srcs = new LinkedList<PathData>(); } catch (URISyntaxException e) { throw new IOException("unexpected URISyntaxException", e); } }
/** * The last arg is expected to be a remote path, if only one argument is * given then the destination will be the remote user's directory * @param args is the list of arguments * @throws PathIOException if path doesn't exist or matches too many times */ protected void getRemoteDestination(LinkedList<String> args) throws IOException { if (args.size() < 2) { dst = new PathData(Path.CUR_DIR, getConf()); } else { String pathString = args.removeLast(); // if the path is a glob, then it must match one and only one path PathData[] items = PathData.expandAsGlob(pathString, getConf()); switch (items.length) { case 0: throw new PathNotFoundException(pathString); case 1: dst = items[0]; break; default: throw new PathIOException(pathString, "Too many matches"); } } }
if (dst.exists) { if (!dst.stat.isDirectory()) { throw new PathIsNotDirectoryException(dst.toString()); PathIOException e = new PathIOException(dst.toString()); e.setOperation("mkdir"); throw e; dst.refreshStatus(); // need to update stat to know it exists now
@Override protected void processNonexistentPath(PathData item) throws IOException { if (!item.parentExists()) { throw new PathNotFoundException(item.toString()) .withFullyQualifiedPath(item.path.toUri().toString()); } touchz(item); }
@Override protected void processArguments(LinkedList<PathData> args) throws IOException { if (!dst.exists) { dst.fs.create(dst.path, false).close(); } FileInputStream is = null; try (FSDataOutputStream fos = dst.fs.append(dst.path)) { if (readStdin) { if (args.size() == 0) { IOUtils.copyBytes(System.in, fos, DEFAULT_IO_LENGTH); } else { throw new IOException( "stdin (-) must be the sole input argument when present"); } } // Read in each input file and write to the target. for (PathData source : args) { is = new FileInputStream(source.toFile()); IOUtils.copyBytes(is, fos, DEFAULT_IO_LENGTH); IOUtils.closeStream(is); is = null; } } finally { if (is != null) { IOUtils.closeStream(is); } } } }
/** * Gets the directory listing for a path and invokes * {@link #processPaths(PathData, PathData...)} * @param item {@link PathData} for directory to recurse into * @throws IOException if anything goes wrong... */ protected void recursePath(PathData item) throws IOException { try { depth++; processPaths(item, item.getDirectoryContents()); } finally { depth--; } }
private static String relativize(URI cwdUri, URI srcUri, boolean isDir) { String uriPath = srcUri.getPath(); String cwdPath = cwdUri.getPath(); if (cwdPath.equals(uriPath)) { return Path.CUR_DIR; } // find common ancestor int lastSep = findLongestDirPrefix(cwdPath, uriPath, isDir); StringBuilder relPath = new StringBuilder(); // take the remaining path fragment after the ancestor if (lastSep < uriPath.length()) { relPath.append(uriPath.substring(lastSep+1)); } // if cwd has a path fragment after the ancestor, convert them to ".." if (lastSep < cwdPath.length()) { while (lastSep != -1) { if (relPath.length() != 0) relPath.insert(0, Path.SEPARATOR); relPath.insert(0, ".."); lastSep = cwdPath.indexOf(Path.SEPARATOR, lastSep+1); } } return relPath.toString(); }
/** * Gets the directory listing for a path and invokes * {@link #processPaths(PathData, PathData...)} * @param item {@link PathData} for directory to recurse into * @throws IOException if anything goes wrong... */ protected void recursePath(PathData item) throws IOException { try { depth++; if (isSorted()) { // use the non-iterative method for listing because explicit sorting is // required. Iterators not guaranteed to return sorted elements processPaths(item, item.getDirectoryContents()); } else { processPaths(item, item.getDirectoryContentsIterator()); } } finally { depth--; } }
/** * Returns a RemoteIterator for PathData objects of the items contained in the * given directory. * @return remote iterator of PathData objects for its children * @throws IOException if anything else goes wrong... */ public RemoteIterator<PathData> getDirectoryContentsIterator() throws IOException { checkIfExists(FileTypeRequirement.SHOULD_BE_DIRECTORY); final RemoteIterator<FileStatus> stats = this.fs.listStatusIterator(path); return new RemoteIterator<PathData>() { @Override public boolean hasNext() throws IOException { return stats.hasNext(); } @Override public PathData next() throws IOException { FileStatus file = stats.next(); String child = getStringForChildPath(file.getPath()); return new PathData(fs, child, file); } }; }
@Test (timeout = 30000) public void testQualifiedUriContents() throws Exception { String dirString = fs.makeQualified(new Path("d1")).toString(); PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString(dirString+"/f1", dirString+"/f1.1", dirString+"/f2"), sortedString(items) ); }