private List<FileStatus> getFiles() throws IOException { // get all the files that are involved in the splits final List<FileStatus> files = new ArrayList<FileStatus>(); final FileSystem fs = this.filePath.getFileSystem(); final FileStatus pathFile = fs.getFileStatus(this.filePath); if (pathFile.isDir()) { // input is directory. list all contained files final FileStatus[] partials = fs.listStatus(this.filePath); for (int i = 0; i < partials.length; i++) if (!partials[i].isDir()) files.add(partials[i]); } else files.add(pathFile); return files; }
protected List<FileStatus> getFiles() throws IOException { // get all the files that are involved in the splits List<FileStatus> files = new ArrayList<FileStatus>(); final FileSystem fs = this.filePath.getFileSystem(); final FileStatus pathFile = fs.getFileStatus(this.filePath); if (pathFile.isDir()) { // input is directory. list all contained files final FileStatus[] partials = fs.listStatus(this.filePath); for (int i = 0; i < partials.length; i++) { if (!partials[i].isDir()) { files.add(partials[i]); } } } else { files.add(pathFile); } return files; }
protected List<FileStatus> getFiles() throws IOException { // get all the files that are involved in the splits List<FileStatus> files = new ArrayList<FileStatus>(); final FileSystem fs = this.filePath.getFileSystem(); final FileStatus pathFile = fs.getFileStatus(this.filePath); if (pathFile.isDir()) { // input is directory. list all contained files final FileStatus[] partials = fs.listStatus(this.filePath); for (int i = 0; i < partials.length; i++) { if (!partials[i].isDir()) { files.add(partials[i]); } } } else { files.add(pathFile); } return files; }
FileStatus[] fss = fs.listStatus(filePath); files.ensureCapacity(fss.length);
return Arrays.asList(openInput(inputFormatClass, path, configuration)); FileStatus[] list = fs.listStatus(nephelePath); List<F> formats = new ArrayList<F>(); for (int index = 0; index < list.length; index++) {
return Arrays.asList(openInput(inputFormatClass, path, configuration)); FileStatus[] list = fs.listStatus(nephelePath); List<F> formats = new ArrayList<F>(); for (int index = 0; index < list.length; index++) {
FileStatus[] fss = fs.listStatus(filePath); files.ensureCapacity(fss.length);
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException { FileSystem sFS = sourcePath.getFileSystem(); FileSystem tFS = targetPath.getFileSystem(); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) { tFS.mkdirs(targetPath); FileStatus[] contents = sFS.listStatus(sourcePath); for (FileStatus content : contents) { String distPath = content.getPath().toString(); if (content.isDir()) { if (distPath.endsWith("/")) { distPath = distPath.substring(0, distPath.length() - 1); } } String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/")); copy(content.getPath(), new Path(localPath), executable); } } else { try { FSDataOutputStream lfsOutput = tFS.create(targetPath, false); FSDataInputStream fsInput = sFS.open(sourcePath); IOUtils.copyBytes(fsInput, lfsOutput); new File(targetPath.toString()).setExecutable(executable); } catch (IOException ioe) { } } } }
/** * Set minNumSplits to number of files. */ @Override public FileInputSplit[] createInputSplits(int minNumSplits) throws IOException { int numAvroFiles = 0; final Path path = this.filePath; // get all the files that are involved in the splits final FileSystem fs = path.getFileSystem(); final FileStatus pathFile = fs.getFileStatus(path); if (!acceptFile(pathFile)) { throw new IOException("The given file does not pass the file-filter"); } if (pathFile.isDir()) { // input is directory. list all contained files final FileStatus[] dir = fs.listStatus(path); for (int i = 0; i < dir.length; i++) { if (!dir[i].isDir() && acceptFile(dir[i])) { numAvroFiles++; } } } else { numAvroFiles = 1; } return super.createInputSplits(numAvroFiles); }
final FileStatus[] dir = fs.listStatus(path); for (int i = 0; i < dir.length; i++) { if (!dir[i].isDir() && acceptFile(dir[i])) {
final FileStatus[] dir = fs.listStatus(path); for (int i = 0; i < dir.length; i++) { if (!dir[i].isDir() && acceptFile(dir[i])) {
final FileStatus[] dir = fs.listStatus(path); for (int i = 0; i < dir.length; i++) { if (!dir[i].isDir()) {
final FileStatus[] dir = fs.listStatus(path); for (int i = 0; i < dir.length; i++) { if (!dir[i].isDir()) {
/** * Returns the number of blocks this file/directory consists of * assuming the file system's standard block size. * * @param file * the file * @return the number of block's the file/directory consists of * @throws IOException */ public int getNumberOfBlocks(final FileStatus file) throws IOException { int numberOfBlocks = 0; if (file == null) { return 0; } // For a file, this is easy if (!file.isDir()) { return getNumberOfBlocks(file.getLen(), file.getBlockSize()); } // file is a directory final FileStatus[] files = this.listStatus(file.getPath()); for (int i = 0; i < files.length; i++) { if (!files[i].isDir()) { numberOfBlocks += getNumberOfBlocks(files[i].getLen(), files[i].getBlockSize()); } } return numberOfBlocks; }
/** * Returns the number of blocks this file/directory consists of * assuming the file system's standard block size. * * @param file * the file * @return the number of block's the file/directory consists of * @throws IOException */ public int getNumberOfBlocks(final FileStatus file) throws IOException { int numberOfBlocks = 0; if (file == null) { return 0; } // For a file, this is easy if (!file.isDir()) { return getNumberOfBlocks(file.getLen(), file.getBlockSize()); } // file is a directory final FileStatus[] files = this.listStatus(file.getPath()); for (int i = 0; i < files.length; i++) { if (!files[i].isDir()) { numberOfBlocks += getNumberOfBlocks(files[i].getLen(), files[i].getBlockSize()); } } return numberOfBlocks; }