/** * Check if exists. * * @param f * source file */ public boolean exists(final Path f) throws IOException { try { return (getFileStatus(f) != null); } catch (FileNotFoundException e) { return false; } }
/** * Check if exists. * * @param f * source file */ public boolean exists(final Path f) throws IOException { try { return (getFileStatus(f) != null); } catch (FileNotFoundException e) { return false; } }
private List<FileStatus> getFiles() throws IOException { // get all the files that are involved in the splits final List<FileStatus> files = new ArrayList<FileStatus>(); final FileSystem fs = this.filePath.getFileSystem(); final FileStatus pathFile = fs.getFileStatus(this.filePath); if (pathFile.isDir()) { // input is directory. list all contained files final FileStatus[] partials = fs.listStatus(this.filePath); for (int i = 0; i < partials.length; i++) if (!partials[i].isDir()) files.add(partials[i]); } else files.add(pathFile); return files; }
protected List<FileStatus> getFiles() throws IOException { // get all the files that are involved in the splits List<FileStatus> files = new ArrayList<FileStatus>(); final FileSystem fs = this.filePath.getFileSystem(); final FileStatus pathFile = fs.getFileStatus(this.filePath); if (pathFile.isDir()) { // input is directory. list all contained files final FileStatus[] partials = fs.listStatus(this.filePath); for (int i = 0; i < partials.length; i++) { if (!partials[i].isDir()) { files.add(partials[i]); } } } else { files.add(pathFile); } return files; }
@Override public int getMaximumNumberOfSubtasks(final AbstractInvokable invokable) { int numberOfBlocks = -1; if (this.path == null) { return -1; } try { final FileSystem fs = this.path.getFileSystem(); final FileStatus f = fs.getFileStatus(this.path); numberOfBlocks = fs.getNumberOfBlocks(f); } catch (IOException e) { return -1; } return (int) Math.min(numberOfBlocks, invokable.getMaximumNumberOfSubtasks()); } }
@Override public int getMaximumNumberOfSubtasks(final AbstractInvokable invokable) { int numberOfBlocks = -1; if (this.path == null) { return -1; } try { final FileSystem fs = this.path.getFileSystem(); final FileStatus f = fs.getFileStatus(this.path); numberOfBlocks = fs.getNumberOfBlocks(f); } catch (IOException e) { return -1; } return (int) Math.min(numberOfBlocks, invokable.getMaximumNumberOfSubtasks()); } }
protected List<FileStatus> getFiles() throws IOException { // get all the files that are involved in the splits List<FileStatus> files = new ArrayList<FileStatus>(); final FileSystem fs = this.filePath.getFileSystem(); final FileStatus pathFile = fs.getFileStatus(this.filePath); if (pathFile.isDir()) { // input is directory. list all contained files final FileStatus[] partials = fs.listStatus(this.filePath); for (int i = 0; i < partials.length; i++) { if (!partials[i].isDir()) { files.add(partials[i]); } } } else { files.add(pathFile); } return files; }
final FileStatus file = fs.getFileStatus(jar); out.writeLong(file.getLen());
final FileStatus status = fs.getFileStatus(storePath);
final FileStatus status = fs.getFileStatus(storePath);
Path nephelePath = new Path(path); FileSystem fs = nephelePath.getFileSystem(); FileStatus fileStatus = fs.getFileStatus(nephelePath); if (!fileStatus.isDir()) { return Arrays.asList(openInput(inputFormatClass, path, configuration));
final FileStatus file = fs.getFileStatus(filePath); long latestModTime = file.getModificationTime();
Path nephelePath = new Path(path); FileSystem fs = nephelePath.getFileSystem(); FileStatus fileStatus = fs.getFileStatus(nephelePath); if (!fileStatus.isDir()) { return Arrays.asList(openInput(inputFormatClass, path, configuration));
final FileStatus file = fs.getFileStatus(filePath); long latestModTime = file.getModificationTime();
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException { FileSystem sFS = sourcePath.getFileSystem(); FileSystem tFS = targetPath.getFileSystem(); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) { tFS.mkdirs(targetPath); FileStatus[] contents = sFS.listStatus(sourcePath); for (FileStatus content : contents) { String distPath = content.getPath().toString(); if (content.isDir()) { if (distPath.endsWith("/")) { distPath = distPath.substring(0, distPath.length() - 1); } } String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/")); copy(content.getPath(), new Path(localPath), executable); } } else { try { FSDataOutputStream lfsOutput = tFS.create(targetPath, false); FSDataInputStream fsInput = sFS.open(sourcePath); IOUtils.copyBytes(fsInput, lfsOutput); new File(targetPath.toString()).setExecutable(executable); } catch (IOException ioe) { } } } }
/** * Set minNumSplits to number of files. */ @Override public FileInputSplit[] createInputSplits(int minNumSplits) throws IOException { int numAvroFiles = 0; final Path path = this.filePath; // get all the files that are involved in the splits final FileSystem fs = path.getFileSystem(); final FileStatus pathFile = fs.getFileStatus(path); if (!acceptFile(pathFile)) { throw new IOException("The given file does not pass the file-filter"); } if (pathFile.isDir()) { // input is directory. list all contained files final FileStatus[] dir = fs.listStatus(path); for (int i = 0; i < dir.length; i++) { if (!dir[i].isDir() && acceptFile(dir[i])) { numAvroFiles++; } } } else { numAvroFiles = 1; } return super.createInputSplits(numAvroFiles); }
@Override public boolean preVisit(final Operator visitable) { int degree = GenericTestPlan.this.getDegreeOfParallelism(); if (visitable instanceof GenericDataSource<?>) degree = 1; else if (degree > 1 && visitable instanceof FileDataSink) try { final Path path = new Path(((FileDataSink) visitable).getFilePath()); final FileSystem fs = path.getFileSystem(); final FileStatus f = fs.getFileStatus(path); if (!f.isDir()) { fs.delete(path, false); fs.mkdirs(path); } } catch (final IOException e) { e.printStackTrace(); } if (visitable.getDegreeOfParallelism() == -1) visitable.setDegreeOfParallelism(degree); return true; } });
@Override public void checkConfiguration(final AbstractInvokable invokable) throws IllegalConfigurationException { // Check if the user has specified a path if (this.path == null) { throw new IllegalConfigurationException(this.getName() + " does not specify an input path"); } // Check if the path is valid try { final FileSystem fs = this.path.getFileSystem(); final FileStatus f = fs.getFileStatus(this.path); if (f == null) { throw new IOException(this.path.toString() + " led to a null object"); } } catch (IOException e) { throw new IllegalConfigurationException("Cannot access file or directory: " + StringUtils.stringifyException(e)); } // register the path in the configuration invokable.getTaskConfiguration() .setString(AbstractFileInputTask.INPUT_PATH_CONFIG_KEY, this.path.toString()); // Finally, see if the task itself has a valid configuration super.checkConfiguration(invokable); }
@Override public void checkConfiguration(final AbstractInvokable invokable) throws IllegalConfigurationException { // Check if the user has specified a path if (this.path == null) { throw new IllegalConfigurationException(this.getName() + " does not specify an input path"); } // Check if the path is valid try { final FileSystem fs = this.path.getFileSystem(); final FileStatus f = fs.getFileStatus(this.path); if (f == null) { throw new IOException(this.path.toString() + " led to a null object"); } } catch (IOException e) { throw new IllegalConfigurationException("Cannot access file or directory: " + StringUtils.stringifyException(e)); } // register the path in the configuration invokable.getTaskConfiguration() .setString(AbstractFileInputTask.INPUT_PATH_CONFIG_KEY, this.path.toString()); // Finally, see if the task itself has a valid configuration super.checkConfiguration(invokable); }
FileStatus fileStatus = fs.getFileStatus(normalizedPath);