/** * Returns the FileSystem that owns this Path. * * @return the FileSystem that owns this Path * @throws IOException * thrown if the file system could not be retrieved */ public FileSystem getFileSystem() throws IOException { return FileSystem.get(this.toUri()); }
inputFormat.configure(configuration); final FileSystem fs = FileSystem.get(normalizedPath.toUri()); FileStatus fileStatus = fs.getFileStatus(normalizedPath);
inputFormat.configure(configuration); final FileSystem fs = FileSystem.get(normalizedPath.toUri()); FileStatus fileStatus = fs.getFileStatus(normalizedPath);
/** * Returns the FileSystem that owns this Path. * * @return the FileSystem that owns this Path * @throws IOException * thrown if the file system could not be retrieved */ public FileSystem getFileSystem() throws IOException { return FileSystem.get(this.toUri()); }
/** * Returns a reference to the {@link FileSystem} instance for accessing the * local file system. * * @return a reference to the {@link FileSystem} instance for accessing the * local file system. * @throws IOException * thrown if a reference to the file system instance could not be obtained */ public static FileSystem getLocalFileSystem() throws IOException { URI localUri; try { localUri = OperatingSystem.isWindows() ? new URI("file:/") : new URI("file:///"); } catch (URISyntaxException e) { throw new IOException("Cannot create URI for local file system"); } return get(localUri); }
/** * Returns a reference to the {@link FileSystem} instance for accessing the * local file system. * * @return a reference to the {@link FileSystem} instance for accessing the * local file system. * @throws IOException * thrown if a reference to the file system instance could not be obtained */ public static FileSystem getLocalFileSystem() throws IOException { URI localUri; try { localUri = OperatingSystem.isWindows() ? new URI("file:/") : new URI("file:///"); } catch (URISyntaxException e) { throw new IOException("Cannot create URI for local file system"); } return get(localUri); }
/** * register cache files in program level * @param filePath The files must be stored in a place that can be accessed from all workers (most commonly HDFS) * @param name user defined name of that file * @throws java.io.IOException */ public void registerCachedFile(String filePath, String name) throws IOException { if (!this.cacheFile.containsKey(name)) { try { URI u = new URI(filePath); if (!u.getPath().startsWith("/")) { u = new URI(new File(filePath).getAbsolutePath()); } FileSystem fs = FileSystem.get(u); if (fs.exists(new Path(u.getPath()))) { this.cacheFile.put(name, u.toString()); } else { throw new RuntimeException("File " + u.toString() + " doesn't exist."); } } catch (URISyntaxException ex) { throw new RuntimeException("Invalid path: " + filePath, ex); } } else { throw new RuntimeException("cache file " + name + "already exists!"); } }
/** * register cache files in program level * @param entry contains all relevant information * @param name user defined name of that file * @throws java.io.IOException */ public void registerCachedFile(String name, DistributedCacheEntry entry) throws IOException { if (!this.cacheFile.containsKey(name)) { try { URI u = new URI(entry.filePath); if (!u.getPath().startsWith("/")) { u = new File(entry.filePath).toURI(); } FileSystem fs = FileSystem.get(u); if (fs.exists(new Path(u.getPath()))) { this.cacheFile.put(name, new DistributedCacheEntry(u.toString(), entry.isExecutable)); } else { throw new IOException("File " + u.toString() + " doesn't exist."); } } catch (URISyntaxException ex) { throw new IOException("Invalid path: " + entry.filePath, ex); } } else { throw new IOException("cache file " + name + "already exists!"); } }
final FileSystem fs = FileSystem.get(path.toUri());
final FileSystem fs = FileSystem.get(path.toUri());
@Override public SequentialStatistics getStatistics(final BaseStatistics cachedStats) { final FileBaseStatistics cachedFileStats = cachedStats != null && cachedStats instanceof FileBaseStatistics ? (FileBaseStatistics) cachedStats : null; try { final Path filePath = this.filePath; // get the filesystem final FileSystem fs = FileSystem.get(filePath.toUri()); final ArrayList<FileStatus> allFiles = new ArrayList<FileStatus>(1); // let the file input format deal with the up-to-date check and the basic size final FileBaseStatistics stats = this.getFileStats(cachedFileStats, filePath, fs, allFiles); if (stats == null) return null; // check whether the file stats are still sequential stats (in that case they are still valid) if (stats instanceof SequentialStatistics) return (SequentialStatistics) stats; return this.createStatistics(allFiles, stats); } catch (final IOException ioex) { if (LOG.isWarnEnabled()) LOG.warn(String.format("Could not determine complete statistics for file '%s' due to an I/O error: %s", this.filePath, StringUtils.stringifyException(ioex))); } catch (final Throwable t) { if (LOG.isErrorEnabled()) LOG.error(String.format("Unexpected problem while getting the file statistics for file '%s' due to %s", this.filePath, StringUtils.stringifyException(t))); } // no stats available return null; }
final FileSystem fs = FileSystem.get(filePath.toUri()); final ArrayList<FileStatus> allFiles = new ArrayList<FileStatus>(1);
final FileSystem fs = FileSystem.get(filePath.toUri()); final ArrayList<FileStatus> allFiles = new ArrayList<FileStatus>(1);
@Override public void run() { try { final FileSystem fs = FileSystem.get(this.split.getPath().toUri()); this.fdis = fs.open(this.split.getPath()); // check for canceling and close the stream in that case, because no one will obtain it if (this.aborted) { final FSDataInputStream f = this.fdis; this.fdis = null; f.close(); } } catch (Throwable t) { this.error = t; } }
@Override public void run() { try { final FileSystem fs = FileSystem.get(this.split.getPath().toUri()); this.fdis = fs.open(this.split.getPath()); // check for canceling and close the stream in that case, because no one will obtain it if (this.aborted) { final FSDataInputStream f = this.fdis; this.fdis = null; f.close(); } } catch (Throwable t) { this.error = t; } }
final FileSystem fs = FileSystem.get(filePath.toUri()); final ArrayList<FileStatus> allFiles = new ArrayList<FileStatus>(1);
final FileSystem fs = FileSystem.get(filePath.toUri()); final ArrayList<FileStatus> allFiles = new ArrayList<FileStatus>(1);