if(!this.isDistributedFS()) { return false; if(this.exists(outPath)) { this.delete(outPath, true); } catch(IOException ioe) { if(!this.exists(outPath)) { this.mkdirs(outPath); return this.exists(outPath) && this.getFileStatus(outPath).isDir(); } else { return !this.exists(outPath);
private List<FileStatus> getFiles() throws IOException { // get all the files that are involved in the splits final List<FileStatus> files = new ArrayList<FileStatus>(); final FileSystem fs = this.filePath.getFileSystem(); final FileStatus pathFile = fs.getFileStatus(this.filePath); if (pathFile.isDir()) { // input is directory. list all contained files final FileStatus[] partials = fs.listStatus(this.filePath); for (int i = 0; i < partials.length; i++) if (!partials[i].isDir()) files.add(partials[i]); } else files.add(pathFile); return files; }
final FileSystem fs = FileSystem.getLocalFileSystem(); if (!fs.exists(this.userJars.get(i))) { throw new IOException("Cannot find jar file " + this.userJars.get(i)); final FileStatus file = fs.getFileStatus(jar); out.writeLong(file.getLen()); final FSDataInputStream inStream = fs.open(this.userJars.get(i)); final byte[] buf = new byte[BUFFERSIZE]; int read = inStream.read(buf, 0, buf.length);
private FSDataOutputStream getOutputStream() throws IOException { if (this.outputStream == null) { this.outputStream = this.fs.create(this.checkpointFile, false, this.buf.length, this.replication, this.fs.getDefaultBlockSize()); } return this.outputStream; }
@Override public int getMaximumNumberOfSubtasks(final AbstractInvokable invokable) { int numberOfBlocks = -1; if (this.path == null) { return -1; } try { final FileSystem fs = this.path.getFileSystem(); final FileStatus f = fs.getFileStatus(this.path); numberOfBlocks = fs.getNumberOfBlocks(f); } catch (IOException e) { return -1; } return (int) Math.min(numberOfBlocks, invokable.getMaximumNumberOfSubtasks()); } }
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException { FileSystem sFS = sourcePath.getFileSystem(); FileSystem tFS = targetPath.getFileSystem(); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) { tFS.mkdirs(targetPath); FileStatus[] contents = sFS.listStatus(sourcePath); for (FileStatus content : contents) { String distPath = content.getPath().toString(); if (content.isDir()) { if (distPath.endsWith("/")) { distPath = distPath.substring(0, distPath.length() - 1); } } String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/")); copy(content.getPath(), new Path(localPath), executable); } } else { try { FSDataOutputStream lfsOutput = tFS.create(targetPath, false); FSDataInputStream fsInput = sFS.open(sourcePath); IOUtils.copyBytes(fsInput, lfsOutput); new File(targetPath.toString()).setExecutable(executable); } catch (IOException ioe) { } } } }
final FileStatus pathFile = fs.getFileStatus(path); final FileStatus[] dir = fs.listStatus(path); for (int i = 0; i < dir.length; i++) { if (!dir[i].isDir() && acceptFile(dir[i])) { int splitNum = 0; for (final FileStatus file : files) { final BlockLocation[] blocks = fs.getFileBlockLocations(file, 0, file.getLen()); Set<String> hosts = new HashSet<String>(); for(BlockLocation block : blocks) { final BlockLocation[] blocks = fs.getFileBlockLocations(file, 0, len); Arrays.sort(blocks); final BlockLocation[] blocks = fs.getFileBlockLocations(file, 0, 0); String[] hosts; if (blocks.length > 0) {
inputFormat.configure(configuration); final FileSystem fs = FileSystem.get(normalizedPath.toUri()); FileStatus fileStatus = fs.getFileStatus(normalizedPath); BlockLocation[] blocks = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen()); inputFormat.open(new FileInputSplit(0, new Path(path), 0, fileStatus.getLen(), blocks[0].getHosts())); return inputFormat;
if (!fs.exists(storePath)) { throw new IOException(storePath + " does not exist!"); final FileStatus status = fs.getFileStatus(storePath); final FSDataInputStream inStream = fs.open(storePath); final byte[] buf = new byte[8192]; // 8K Buffer*/ int read = inStream.read(buf, 0, buf.length);
@Override public boolean preVisit(final Operator visitable) { int degree = GenericTestPlan.this.getDegreeOfParallelism(); if (visitable instanceof GenericDataSource<?>) degree = 1; else if (degree > 1 && visitable instanceof FileDataSink) try { final Path path = new Path(((FileDataSink) visitable).getFilePath()); final FileSystem fs = path.getFileSystem(); final FileStatus f = fs.getFileStatus(path); if (!f.isDir()) { fs.delete(path, false); fs.mkdirs(path); } } catch (final IOException e) { e.printStackTrace(); } if (visitable.getDegreeOfParallelism() == -1) visitable.setDegreeOfParallelism(degree); return true; } });
final FileStatus f = fs.getFileStatus(path); fs.mkdirs(path); return -1; } else {
@Override public FileInputSplit[] createInputSplits(int minNumSplits) throws IOException { List<FileStatus> files = this.getFiles(); final FileSystem fs = this.filePath.getFileSystem(); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? fs.getDefaultBlockSize() : this.blockSize; final List<FileInputSplit> inputSplits = new ArrayList<FileInputSplit>(minNumSplits); for (FileStatus file : files) { long splitSize = blockSize; for (long pos = 0, length = file.getLen(); pos < length; pos += splitSize) { long remainingLength = Math.min(pos + splitSize, length) - pos; // get the block locations and make sure they are in order with respect to their offset final BlockLocation[] blocks = fs.getFileBlockLocations(file, pos, remainingLength); Arrays.sort(blocks); inputSplits.add(new FileInputSplit(inputSplits.size(), file.getPath(), pos, remainingLength, blocks[0].getHosts())); } } if (inputSplits.size() < minNumSplits) { LOG.warn(String.format( "With the given block size %d, the file %s cannot be split into %d blocks. Filling up with empty splits...", blockSize, this.filePath, minNumSplits)); FileStatus last = files.get(files.size() - 1); final BlockLocation[] blocks = fs.getFileBlockLocations(last, 0, last.getLen()); for (int index = files.size(); index < minNumSplits; index++) { inputSplits.add(new FileInputSplit(index, last.getPath(), last.getLen(), 0, blocks[0].getHosts())); } } return inputSplits.toArray(new FileInputSplit[0]); }
/** * Returns the number of blocks this file/directory consists of * assuming the file system's standard block size. * * @param file * the file * @return the number of block's the file/directory consists of * @throws IOException */ public int getNumberOfBlocks(final FileStatus file) throws IOException { int numberOfBlocks = 0; if (file == null) { return 0; } // For a file, this is easy if (!file.isDir()) { return getNumberOfBlocks(file.getLen(), file.getBlockSize()); } // file is a directory final FileStatus[] files = this.listStatus(file.getPath()); for (int i = 0; i < files.length; i++) { if (!files[i].isDir()) { numberOfBlocks += getNumberOfBlocks(files[i].getLen(), files[i].getBlockSize()); } } return numberOfBlocks; }
/** * Check if exists. * * @param f * source file */ public boolean exists(final Path f) throws IOException { try { return (getFileStatus(f) != null); } catch (FileNotFoundException e) { return false; } }
/** * Reads library data from the given stream. * * @param in * the stream to read the library data from * @throws IOException * throws if an error occurs while reading from the stream */ private void readLibraryFromStreamInternal(final DataInput in) throws IOException { final String libraryFileName = StringRecord.readString(in); if (libraryFileName == null) { throw new IOException("libraryFileName is null!"); } final long length = in.readLong(); if (length > (long) Integer.MAX_VALUE) { throw new IOException("Submitted jar file " + libraryFileName + " is too large"); } final byte[] buf = new byte[(int) length]; in.readFully(buf); final Path storePath = new Path(this.libraryCachePath + "/" + libraryFileName); synchronized (this.fs) { // Check if file already exists in our library cache, if not write it to the cache directory if (!fs.exists(storePath)) { final FSDataOutputStream fos = fs.create(storePath, false); fos.write(buf, 0, buf.length); fos.close(); } } }
private FSDataInputStream getInputStream() throws IOException { if (this.inputStream == null) { this.inputStream = this.fs.open(this.checkpointFile, this.buf.length); } return this.inputStream; }
@Override public void run() { try { final FileSystem fs = FileSystem.get(this.split.getPath().toUri()); this.fdis = fs.open(this.split.getPath()); // check for canceling and close the stream in that case, because no one will obtain it if (this.aborted) { final FSDataInputStream f = this.fdis; this.fdis = null; f.close(); } } catch (Throwable t) { this.error = t; } }
if(!fs.isDistributedFS()) { if(!fs.initOutPathLocalFS(p, writeMode, false)) { if(!fs.isDistributedFS()) { if(!fs.initOutPathLocalFS(p, writeMode, true)) { this.fdos = fs.create(p, false); break; case OVERWRITE: this.fdos = fs.create(p, true); break; default:
/** * Returns the FileSystem that owns this Path. * * @return the FileSystem that owns this Path * @throws IOException * thrown if the file system could not be retrieved */ public FileSystem getFileSystem() throws IOException { return FileSystem.get(this.toUri()); }
/** * register cache files in program level * @param filePath The files must be stored in a place that can be accessed from all workers (most commonly HDFS) * @param name user defined name of that file * @throws java.io.IOException */ public void registerCachedFile(String filePath, String name) throws IOException { if (!this.cacheFile.containsKey(name)) { try { URI u = new URI(filePath); if (!u.getPath().startsWith("/")) { u = new URI(new File(filePath).getAbsolutePath()); } FileSystem fs = FileSystem.get(u); if (fs.exists(new Path(u.getPath()))) { this.cacheFile.put(name, u.toString()); } else { throw new RuntimeException("File " + u.toString() + " doesn't exist."); } } catch (URISyntaxException ex) { throw new RuntimeException("Invalid path: " + filePath, ex); } } else { throw new RuntimeException("cache file " + name + "already exists!"); } }