/** * Returns the FileSystem that owns this Path. * * @return the FileSystem that owns this Path * @throws IOException * thrown if the file system could not be retrieved */ public FileSystem getFileSystem() throws IOException { return FileSystem.get(this.toUri()); }
/** * Returns the FileSystem that owns this Path. * * @return the FileSystem that owns this Path * @throws IOException * thrown if the file system could not be retrieved */ public FileSystem getFileSystem() throws IOException { return FileSystem.get(this.toUri()); }
/** * Fixes the path if it denotes a local (relative) file without the proper protocol prefix. */ private static Path normalizePath(Path path) { URI uri = path.toUri(); if (uri.getScheme() == null) { try { uri = new URI("file", uri.getHost(), uri.getPath(), uri.getFragment()); path = new Path(uri.toString()); } catch (URISyntaxException e) { throw new IllegalArgumentException("path is invalid", e); } } return path; } }
/** * Fixes the path if it denotes a local (relative) file without the proper protocol prefix. */ private static Path normalizePath(Path path) { URI uri = path.toUri(); if (uri.getScheme() == null) { try { uri = new URI("file", uri.getHost(), uri.getPath(), uri.getFragment()); path = new Path(uri.toString()); } catch (URISyntaxException e) { throw new IllegalArgumentException("path is invalid", e); } } return path; } }
private File pathToFile(Path path) { if (!path.isAbsolute()) { path = new Path(getWorkingDirectory(), path); } return new File(path.toUri().getPath()); }
final FileSystem fs = FileSystem.get(path.toUri());
final FileSystem fs = FileSystem.get(path.toUri());
private File pathToFile(Path path) { if (!path.isAbsolute()) { path = new Path(getWorkingDirectory(), path); } return new File(path.toUri().getPath()); }
@Override public FSDataInputStream open(final Path f) throws IOException { final FileStatus fileStatus = getFileStatus(f); // Will throw FileNotFoundException if f does not exist // Make sure f is not a directory if (fileStatus.isDir()) { throw new IOException("Cannot open " + f.toUri() + " because it is a directory"); } final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f); if (!bop.hasBucket() || !bop.hasObject()) { throw new IOException(f.toUri() + " cannot be opened"); } return new S3DataInputStream(this.s3Client, bop.getBucket(), bop.getObject()); }
@Override public SequentialStatistics getStatistics(final BaseStatistics cachedStats) { final FileBaseStatistics cachedFileStats = cachedStats != null && cachedStats instanceof FileBaseStatistics ? (FileBaseStatistics) cachedStats : null; try { final Path filePath = this.filePath; // get the filesystem final FileSystem fs = FileSystem.get(filePath.toUri()); final ArrayList<FileStatus> allFiles = new ArrayList<FileStatus>(1); // let the file input format deal with the up-to-date check and the basic size final FileBaseStatistics stats = this.getFileStats(cachedFileStats, filePath, fs, allFiles); if (stats == null) return null; // check whether the file stats are still sequential stats (in that case they are still valid) if (stats instanceof SequentialStatistics) return (SequentialStatistics) stats; return this.createStatistics(allFiles, stats); } catch (final IOException ioex) { if (LOG.isWarnEnabled()) LOG.warn(String.format("Could not determine complete statistics for file '%s' due to an I/O error: %s", this.filePath, StringUtils.stringifyException(ioex))); } catch (final Throwable t) { if (LOG.isErrorEnabled()) LOG.error(String.format("Unexpected problem while getting the file statistics for file '%s' due to %s", this.filePath, StringUtils.stringifyException(t))); } // no stats available return null; }
@Override public FSDataOutputStream create(final Path f, final boolean overwrite, final int bufferSize, final short replication, final long blockSize) throws IOException { if (!overwrite && exists(f)) { throw new IOException(f.toUri() + " already exists"); } final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f); if (!bop.hasBucket() || !bop.hasObject()) { throw new IOException(f.toUri() + " is not a valid path to create a new file"); } if (bufferSize < S3DataOutputStream.MINIMUM_MULTIPART_SIZE) { throw new IOException("Provided buffer must be at least " + S3DataOutputStream.MINIMUM_MULTIPART_SIZE + " bytes"); } final byte[] buf = new byte[bufferSize]; // TODO: Use memory manager to allocate larger pages return new S3DataOutputStream(this.s3Client, bop.getBucket(), bop.getObject(), buf, this.useRRS); }
@Override public FSDataOutputStream create(final Path f, final boolean overwrite, final int bufferSize, final short replication, final long blockSize) throws IOException { if (!overwrite && exists(f)) { throw new IOException(f.toUri() + " already exists"); } final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f); if (!bop.hasBucket() || !bop.hasObject()) { throw new IOException(f.toUri() + " is not a valid path to create a new file"); } if (bufferSize < S3DataOutputStream.MINIMUM_MULTIPART_SIZE) { throw new IOException("Provided buffer must be at least " + S3DataOutputStream.MINIMUM_MULTIPART_SIZE + " bytes"); } final byte[] buf = new byte[bufferSize]; // TODO: Use memory manager to allocate larger pages return new S3DataOutputStream(this.s3Client, bop.getBucket(), bop.getObject(), buf, this.useRRS); }
@Override public FSDataInputStream open(final Path f) throws IOException { final FileStatus fileStatus = getFileStatus(f); // Will throw FileNotFoundException if f does not exist // Make sure f is not a directory if (fileStatus.isDir()) { throw new IOException("Cannot open " + f.toUri() + " because it is a directory"); } final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f); if (!bop.hasBucket() || !bop.hasObject()) { throw new IOException(f.toUri() + " cannot be opened"); } return new S3DataInputStream(this.s3Client, bop.getBucket(), bop.getObject()); }
/** * Checks if FileDataSource is correctly connected. In case that the * contract is incorrectly connected a RuntimeException is thrown. * * @param fileSource * FileDataSource that is checked. */ private void checkFileDataSource(FileDataSourceBase<?> fileSource) { String path = fileSource.getFilePath(); if (path == null) { throw new InvalidProgramException("File path of FileDataSource is null."); } if (path.length() == 0) { throw new InvalidProgramException("File path of FileDataSource is empty string."); } try { Path p = new Path(path); String scheme = p.toUri().getScheme(); if (scheme == null) { throw new InvalidProgramException("File path \"" + path + "\" of FileDataSource has no file system scheme (like 'file:// or hdfs://')."); } } catch (Exception e) { throw new InvalidProgramException("File path \"" + path + "\" of FileDataSource is an invalid path: " + e.getMessage()); } }
/** * Checks if FileDataSource is correctly connected. In case that the * contract is incorrectly connected a RuntimeException is thrown. * * @param fileSource * FileDataSource that is checked. */ private void checkFileDataSource(FileDataSourceBase<?> fileSource) { String path = fileSource.getFilePath(); if (path == null) { throw new InvalidProgramException("File path of FileDataSource is null."); } if (path.length() == 0) { throw new InvalidProgramException("File path of FileDataSource is empty string."); } try { Path p = new Path(path); String scheme = p.toUri().getScheme(); if (scheme == null) { throw new InvalidProgramException("File path \"" + path + "\" of FileDataSource has no file system scheme (like 'file:// or hdfs://')."); } } catch (Exception e) { throw new InvalidProgramException("File path \"" + path + "\" of FileDataSource is an invalid path: " + e.getMessage()); } }
/** * Checks if FileDataSink is correctly connected. In case that the * contract is incorrectly connected a RuntimeException is thrown. * * @param fileSink * FileDataSink that is checked. */ private void checkFileDataSink(FileDataSinkBase<?> fileSink) { String path = fileSink.getFilePath(); if (path == null) { throw new InvalidProgramException("File path of FileDataSink is null."); } if (path.length() == 0) { throw new InvalidProgramException("File path of FileDataSink is empty string."); } try { Path p = new Path(path); String scheme = p.toUri().getScheme(); if (scheme == null) { throw new InvalidProgramException("File path \"" + path + "\" of FileDataSink has no file system scheme (like 'file:// or hdfs://')."); } } catch (Exception e) { throw new InvalidProgramException("File path \"" + path + "\" of FileDataSink is an invalid path: " + e.getMessage()); } checkDataSink(fileSink); }
@Override public void run() { try { final FileSystem fs = FileSystem.get(this.split.getPath().toUri()); this.fdis = fs.open(this.split.getPath()); // check for canceling and close the stream in that case, because no one will obtain it if (this.aborted) { final FSDataInputStream f = this.fdis; this.fdis = null; f.close(); } } catch (Throwable t) { this.error = t; } }
@Override public void run() { try { final FileSystem fs = FileSystem.get(this.split.getPath().toUri()); this.fdis = fs.open(this.split.getPath()); // check for canceling and close the stream in that case, because no one will obtain it if (this.aborted) { final FSDataInputStream f = this.fdis; this.fdis = null; f.close(); } } catch (Throwable t) { this.error = t; } }
inputFormat.configure(configuration); final FileSystem fs = FileSystem.get(normalizedPath.toUri()); FileStatus fileStatus = fs.getFileStatus(normalizedPath);
inputFormat.configure(configuration); final FileSystem fs = FileSystem.get(normalizedPath.toUri()); FileStatus fileStatus = fs.getFileStatus(normalizedPath);