private List<FileStatus> getFiles() throws IOException { // get all the files that are involved in the splits final List<FileStatus> files = new ArrayList<FileStatus>(); final FileSystem fs = this.filePath.getFileSystem(); final FileStatus pathFile = fs.getFileStatus(this.filePath); if (pathFile.isDir()) { // input is directory. list all contained files final FileStatus[] partials = fs.listStatus(this.filePath); for (int i = 0; i < partials.length; i++) if (!partials[i].isDir()) files.add(partials[i]); } else files.add(pathFile); return files; }
protected List<FileStatus> getFiles() throws IOException { // get all the files that are involved in the splits List<FileStatus> files = new ArrayList<FileStatus>(); final FileSystem fs = this.filePath.getFileSystem(); final FileStatus pathFile = fs.getFileStatus(this.filePath); if (pathFile.isDir()) { // input is directory. list all contained files final FileStatus[] partials = fs.listStatus(this.filePath); for (int i = 0; i < partials.length; i++) { if (!partials[i].isDir()) { files.add(partials[i]); } } } else { files.add(pathFile); } return files; }
protected List<FileStatus> getFiles() throws IOException { // get all the files that are involved in the splits List<FileStatus> files = new ArrayList<FileStatus>(); final FileSystem fs = this.filePath.getFileSystem(); final FileStatus pathFile = fs.getFileStatus(this.filePath); if (pathFile.isDir()) { // input is directory. list all contained files final FileStatus[] partials = fs.listStatus(this.filePath); for (int i = 0; i < partials.length; i++) { if (!partials[i].isDir()) { files.add(partials[i]); } } } else { files.add(pathFile); } return files; }
/** * Set minNumSplits to number of files. */ @Override public FileInputSplit[] createInputSplits(int minNumSplits) throws IOException { int numAvroFiles = 0; final Path path = this.filePath; // get all the files that are involved in the splits final FileSystem fs = path.getFileSystem(); final FileStatus pathFile = fs.getFileStatus(path); if (!acceptFile(pathFile)) { throw new IOException("The given file does not pass the file-filter"); } if (pathFile.isDir()) { // input is directory. list all contained files final FileStatus[] dir = fs.listStatus(path); for (int i = 0; i < dir.length; i++) { if (!dir[i].isDir() && acceptFile(dir[i])) { numAvroFiles++; } } } else { numAvroFiles = 1; } return super.createInputSplits(numAvroFiles); }
@Override public FSDataInputStream open(final Path f) throws IOException { final FileStatus fileStatus = getFileStatus(f); // Will throw FileNotFoundException if f does not exist // Make sure f is not a directory if (fileStatus.isDir()) { throw new IOException("Cannot open " + f.toUri() + " because it is a directory"); } final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f); if (!bop.hasBucket() || !bop.hasObject()) { throw new IOException(f.toUri() + " cannot be opened"); } return new S3DataInputStream(this.s3Client, bop.getBucket(), bop.getObject()); }
@Override public FSDataInputStream open(final Path f) throws IOException { final FileStatus fileStatus = getFileStatus(f); // Will throw FileNotFoundException if f does not exist // Make sure f is not a directory if (fileStatus.isDir()) { throw new IOException("Cannot open " + f.toUri() + " because it is a directory"); } final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f); if (!bop.hasBucket() || !bop.hasObject()) { throw new IOException(f.toUri() + " cannot be opened"); } return new S3DataInputStream(this.s3Client, bop.getBucket(), bop.getObject()); }
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException { FileSystem sFS = sourcePath.getFileSystem(); FileSystem tFS = targetPath.getFileSystem(); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) { tFS.mkdirs(targetPath); FileStatus[] contents = sFS.listStatus(sourcePath); for (FileStatus content : contents) { String distPath = content.getPath().toString(); if (content.isDir()) { if (distPath.endsWith("/")) { distPath = distPath.substring(0, distPath.length() - 1); } } String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/")); copy(content.getPath(), new Path(localPath), executable); } } else { try { FSDataOutputStream lfsOutput = tFS.create(targetPath, false); FSDataInputStream fsInput = sFS.open(sourcePath); IOUtils.copyBytes(fsInput, lfsOutput); new File(targetPath.toString()).setExecutable(executable); } catch (IOException ioe) { } } } }
/** * Returns the number of blocks this file/directory consists of * assuming the file system's standard block size. * * @param file * the file * @return the number of block's the file/directory consists of * @throws IOException */ public int getNumberOfBlocks(final FileStatus file) throws IOException { int numberOfBlocks = 0; if (file == null) { return 0; } // For a file, this is easy if (!file.isDir()) { return getNumberOfBlocks(file.getLen(), file.getBlockSize()); } // file is a directory final FileStatus[] files = this.listStatus(file.getPath()); for (int i = 0; i < files.length; i++) { if (!files[i].isDir()) { numberOfBlocks += getNumberOfBlocks(files[i].getLen(), files[i].getBlockSize()); } } return numberOfBlocks; }
/** * Returns the number of blocks this file/directory consists of * assuming the file system's standard block size. * * @param file * the file * @return the number of block's the file/directory consists of * @throws IOException */ public int getNumberOfBlocks(final FileStatus file) throws IOException { int numberOfBlocks = 0; if (file == null) { return 0; } // For a file, this is easy if (!file.isDir()) { return getNumberOfBlocks(file.getLen(), file.getBlockSize()); } // file is a directory final FileStatus[] files = this.listStatus(file.getPath()); for (int i = 0; i < files.length; i++) { if (!files[i].isDir()) { numberOfBlocks += getNumberOfBlocks(files[i].getLen(), files[i].getBlockSize()); } } return numberOfBlocks; }
FileSystem fs = nephelePath.getFileSystem(); FileStatus fileStatus = fs.getFileStatus(nephelePath); if (!fileStatus.isDir()) { return Arrays.asList(openInput(inputFormatClass, path, configuration));
FileSystem fs = nephelePath.getFileSystem(); FileStatus fileStatus = fs.getFileStatus(nephelePath); if (!fileStatus.isDir()) { return Arrays.asList(openInput(inputFormatClass, path, configuration));
if (f.isDir()) { return -1;
if (f.isDir()) { return -1;
@Override public boolean preVisit(final Operator visitable) { int degree = GenericTestPlan.this.getDegreeOfParallelism(); if (visitable instanceof GenericDataSource<?>) degree = 1; else if (degree > 1 && visitable instanceof FileDataSink) try { final Path path = new Path(((FileDataSink) visitable).getFilePath()); final FileSystem fs = path.getFileSystem(); final FileStatus f = fs.getFileStatus(path); if (!f.isDir()) { fs.delete(path, false); fs.mkdirs(path); } } catch (final IOException e) { e.printStackTrace(); } if (visitable.getDegreeOfParallelism() == -1) visitable.setDegreeOfParallelism(degree); return true; } });
return this.exists(outPath) && this.getFileStatus(outPath).isDir(); } else {
final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f); if (fileStatus.isDir()) {
final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f); if (fileStatus.isDir()) {