@SuppressWarnings("deprecation") public static FileStatus validateTargetDir(Path path, Configuration conf) throws IOException { FileSystem fs = path.getFileSystem(conf); FileStatus fstatus = null; try { fstatus = fs.getFileStatus(path); } catch (FileNotFoundException fe) { // do nothing } return (fstatus != null && fstatus.isDir()) ? fstatus : null; }
@Override public int compareTo(HdfsFile other) { FileStatus fs1 = this.fs; FileStatus fs2 = other.fs; // This is probably dead code, no directories should be present. // directories before files if(fs1.isDir()) return fs2.isDir() ? 0 : -1; if(fs2.isDir()) return fs1.isDir() ? 0 : 1; // if both are of same type, sort lexicographically if(this.type == other.type) { String f1 = fs1.getPath().getName(), f2 = fs2.getPath().getName(); return f1.compareToIgnoreCase(f2); } // Move the index files to the end, this is a heuristic as they might be // available in the page cache if(this.type == FileType.INDEX) { return 1; } else { return -1; } }
public static void listStatusRecursively(FileSystem fs, FileStatus fileStatus, PathFilter filter, List<FileStatus> results) throws IOException { if (fileStatus.isDir()) { for (FileStatus stat : fs.listStatus(fileStatus.getPath(), filter)) { listStatusRecursively(fs, stat, results); } } else { results.add(fileStatus); } }
public static Collection<String> getAllFilesInPath(final Path path, final Configuration conf) throws IOException { List<String> filesInPath = new ArrayList<>(); FileSystem fs = path.getFileSystem(conf); FileStatus fileStatus = fs.getFileStatus(path); if (fileStatus.isDir()) { FileStatus[] fileStatuses = fs.listStatus(path, FileDump.HIDDEN_AND_SIDE_FILE_FILTER); for (FileStatus fileInPath : fileStatuses) { if (fileInPath.isDir()) { filesInPath.addAll(getAllFilesInPath(fileInPath.getPath(), conf)); } else { filesInPath.add(fileInPath.getPath().toString()); } } } else { filesInPath.add(path.toString()); } return filesInPath; } }
private static void moveFile(FileSystem fs, FileStatus file, Path dst) throws IOException, HiveException { Path srcFilePath = file.getPath(); String fileName = srcFilePath.getName(); Path dstFilePath = new Path(dst, fileName); if (file.isDir()) { renameOrMoveFiles(fs, srcFilePath, dstFilePath); } else { if (fs.exists(dstFilePath)) { int suffix = 0; do { suffix++; dstFilePath = new Path(dst, fileName + "_" + suffix); } while (fs.exists(dstFilePath)); } if (!fs.rename(srcFilePath, dstFilePath)) { throw new HiveException("Unable to move: " + srcFilePath + " to: " + dst); } } }
private static void handleHadoopClasspathExtras(Configuration conf, Map<String, String> env) throws IOException { if(!TempletonUtils.isset(conf.get(JobSubmissionConstants.HADOOP_CLASSPATH_EXTRAS))) { return; } LOG.debug(HADOOP_CLASSPATH_EXTRAS + "=" + conf.get(HADOOP_CLASSPATH_EXTRAS)); String[] files = conf.getStrings(HADOOP_CLASSPATH_EXTRAS); StringBuilder paths = new StringBuilder(); FileSystem fs = FileSystem.getLocal(conf);//these have been localized already for(String f : files) { Path p = new Path(f); FileStatus fileStatus = fs.getFileStatus(p); paths.append(f); if(fileStatus.isDir()) { paths.append(File.separator).append("*"); } paths.append(File.pathSeparator); } paths.setLength(paths.length() - 1); prependPathToVariable(HADOOP_CLASSPATH, env, paths.toString()); } /**
Path dirPath = inputPaths.get(0); try { FileSystem inpFs = dirPath.getFileSystem(conf); List<FileStatus> status = HiveStatsUtils.getFileStatusRecurse( dirPath, listBucketingCtx.getSkewedColNames().size(), inpFs); boolean succeed = true; for (FileStatus s : status) { if (s.isDir()) { newInputPath.add(s.getPath()); } else {
public boolean isDir(Path f) throws MetaException { FileSystem fs; try { fs = getFs(f); FileStatus fstatus = fs.getFileStatus(f); if (!fstatus.isDir()) { return false; } } catch (FileNotFoundException e) { return false; } catch (IOException e) { MetaStoreUtils.logAndThrowMetaException(e); } return true; }
/** * Is the given directory in ACID format? * @param directory the partition directory to check * @param conf the query configuration * @return true, if it is an ACID directory * @throws IOException */ public static boolean isAcid(Path directory, Configuration conf) throws IOException { FileSystem fs = directory.getFileSystem(conf); for(FileStatus file: fs.listStatus(directory)) { String filename = file.getPath().getName(); if (filename.startsWith(BASE_PREFIX) || filename.startsWith(DELTA_PREFIX) || filename.startsWith(DELETE_DELTA_PREFIX)) { if (file.isDir()) { return true; } } } return false; }
throws IOException { LOG.info("Path pattern = " + pathPattern); FileStatus srcs[] = fs.globStatus(new Path(pathPattern)); Arrays.sort(srcs); break; if (src.isDir()) { LOG.info("Got directory: " + src.getPath()); AddPathReturnStatus ret = addPath(fs, src.getPath().toString() + "/*", sizeLeft, fileLimit, retPathList); if (ret == null) { allFile &= ret.allFile; } else { LOG.info("Got file: " + src.getPath()); hasFile = true; retPathList.add(src.getPath());
private long sizeOfPath(FileSystem fs, Path path) throws IOException { long size = 0; FileStatus[] statuses = fs.listStatus(path); if(statuses != null) { for(FileStatus status: statuses) { if(status.isDir()) size += sizeOfPath(fs, status.getPath()); else size += status.getLen(); } } return size; }
index=tmpIndex.incrementAndGet(); Path tmp=new Path(tmpparent,String.valueOf(index)); if(checkzip&&!fs.getFileStatus(src).isDir())
@SuppressWarnings("deprecation") public static FileStatus validateTargetDir(Path path, Configuration conf) throws IOException { FileSystem fs = path.getFileSystem(conf); FileStatus fstatus = null; try { fstatus = fs.getFileStatus(path); } catch (FileNotFoundException fe) { // do nothing } return (fstatus != null && fstatus.isDir()) ? fstatus : null; }
public int compare(FileStatus fs1, FileStatus fs2) { // directories before files if(fs1.isDir()) return fs2.isDir() ? 0 : -1; if(fs2.isDir()) return fs1.isDir() ? 0 : 1; String f1 = fs1.getPath().getName(), f2 = fs2.getPath().getName(); // if both same, lexicographically if((f1.contains(".index") && f2.contains(".index")) || (f1.contains(".data") && f2.contains(".data"))) { return f1.compareToIgnoreCase(f2); } if(f1.contains(".index")) { return 1; } else { return -1; } } }
Path dirPath = inputPaths.get(0); try { FileSystem inpFs = dirPath.getFileSystem(conf); FileStatus[] status = HiveStatsUtils.getFileStatusRecurse(dirPath, listBucketingCtx boolean succeed = true; for (int i = 0; i < status.length; ++i) { if (status[i].isDir()) { newInputPath.add(status[i].getPath()); } else {
public boolean accept(final Path path) { try { return fs.getFileStatus(path).isDir() && !path.getName().startsWith("_"); } catch (IOException ioe) { return false; } } };
/** * Is the given directory in ACID format? * @param directory the partition directory to check * @param conf the query configuration * @return true, if it is an ACID directory * @throws IOException */ public static boolean isAcid(Path directory, Configuration conf) throws IOException { FileSystem fs = directory.getFileSystem(conf); for(FileStatus file: fs.listStatus(directory)) { String filename = file.getPath().getName(); if (filename.startsWith(BASE_PREFIX) || filename.startsWith(DELTA_PREFIX) || filename.startsWith(DELETE_DELTA_PREFIX)) { if (file.isDir()) { return true; } } } return false; }
throws IOException { LOG.info("Path pattern = " + pathPattern); FileStatus srcs[] = fs.globStatus(new Path(pathPattern)); Arrays.sort(srcs); break; if (src.isDir()) { LOG.info("Got directory: " + src.getPath()); AddPathReturnStatus ret = addPath(fs, src.getPath().toString() + "/*", sizeLeft, fileLimit, retPathList); if (ret == null) { allFile &= ret.allFile; } else { LOG.info("Got file: " + src.getPath()); hasFile = true; retPathList.add(src.getPath());
public static FileStatus[] matchFilesOrDir(FileSystem fs, Path path) throws IOException { FileStatus[] srcs = fs.globStatus(path, new PathFilter() { @Override public boolean accept(Path p) { String name = p.getName(); return name.equals(EximUtil.METADATA_NAME) ? true : !name.startsWith("_") && !name.startsWith("."); } }); if ((srcs != null) && srcs.length == 1) { if (srcs[0].isDir()) { srcs = fs.listStatus(srcs[0].getPath(), new PathFilter() { @Override public boolean accept(Path p) { String name = p.getName(); return !name.startsWith("_") && !name.startsWith("."); } }); } } return (srcs); }
@Override public UfsStatus getStatus(String path) throws IOException { Path tPath = new Path(path); FileSystem hdfs = getFs(); FileStatus fs = hdfs.getFileStatus(tPath); if (!fs.isDir()) { // Return file status. String contentHash = UnderFileSystemUtils.approximateContentHash(fs.getLen(), fs.getModificationTime()); return new UfsFileStatus(path, contentHash, fs.getLen(), fs.getModificationTime(), fs.getOwner(), fs.getGroup(), fs.getPermission().toShort()); } // Return directory status. return new UfsDirectoryStatus(path, fs.getOwner(), fs.getGroup(), fs.getPermission().toShort(), fs.getModificationTime()); }