/** * Checks if the given library is in the local cache. * * @param cacheName * The name of the library to be checked for. * @return the path object of the library if it is cached, <code>null</code> otherwise * @throws IOException * thrown if no access to the file system could be obtained */ private Path containsInternal(final String cacheName) throws IOException { // Create a path object from the external name string final Path p = new Path(this.libraryCachePath + File.separator + cacheName); synchronized (this.fs) { if (fs.exists(p)) { return p; } } return null; }
/** * Checks if the given library is in the local cache. * * @param cacheName * The name of the library to be checked for. * @return the path object of the library if it is cached, <code>null</code> otherwise * @throws IOException * thrown if no access to the file system could be obtained */ private Path containsInternal(final String cacheName) throws IOException { // Create a path object from the external name string final Path p = new Path(this.libraryCachePath + File.separator + cacheName); synchronized (this.fs) { if (fs.exists(p)) { return p; } } return null; }
/** * register cache files in program level * @param filePath The files must be stored in a place that can be accessed from all workers (most commonly HDFS) * @param name user defined name of that file * @throws java.io.IOException */ public void registerCachedFile(String filePath, String name) throws IOException { if (!this.cacheFile.containsKey(name)) { try { URI u = new URI(filePath); if (!u.getPath().startsWith("/")) { u = new URI(new File(filePath).getAbsolutePath()); } FileSystem fs = FileSystem.get(u); if (fs.exists(new Path(u.getPath()))) { this.cacheFile.put(name, u.toString()); } else { throw new RuntimeException("File " + u.toString() + " doesn't exist."); } } catch (URISyntaxException ex) { throw new RuntimeException("Invalid path: " + filePath, ex); } } else { throw new RuntimeException("cache file " + name + "already exists!"); } }
/** * register cache files in program level * @param entry contains all relevant information * @param name user defined name of that file * @throws java.io.IOException */ public void registerCachedFile(String name, DistributedCacheEntry entry) throws IOException { if (!this.cacheFile.containsKey(name)) { try { URI u = new URI(entry.filePath); if (!u.getPath().startsWith("/")) { u = new File(entry.filePath).toURI(); } FileSystem fs = FileSystem.get(u); if (fs.exists(new Path(u.getPath()))) { this.cacheFile.put(name, new DistributedCacheEntry(u.toString(), entry.isExecutable)); } else { throw new IOException("File " + u.toString() + " doesn't exist."); } } catch (URISyntaxException ex) { throw new IOException("Invalid path: " + entry.filePath, ex); } } else { throw new IOException("cache file " + name + "already exists!"); } }
if (!fs.exists(this.userJars.get(i))) { throw new IOException("Cannot find jar file " + this.userJars.get(i));
if (!fs.exists(this.userJars.get(i))) { throw new IOException("Cannot find jar file " + this.userJars.get(i));
/** * Reads library data from the given stream. * * @param in * the stream to read the library data from * @throws IOException * throws if an error occurs while reading from the stream */ private void readLibraryFromStreamInternal(final DataInput in) throws IOException { final String libraryFileName = StringRecord.readString(in); if (libraryFileName == null) { throw new IOException("libraryFileName is null!"); } final long length = in.readLong(); if (length > (long) Integer.MAX_VALUE) { throw new IOException("Submitted jar file " + libraryFileName + " is too large"); } final byte[] buf = new byte[(int) length]; in.readFully(buf); final Path storePath = new Path(this.libraryCachePath + "/" + libraryFileName); synchronized (this.fs) { // Check if file already exists in our library cache, if not write it to the cache directory if (!fs.exists(storePath)) { final FSDataOutputStream fos = fs.create(storePath, false); fos.write(buf, 0, buf.length); fos.close(); } } }
/** * Reads library data from the given stream. * * @param in * the stream to read the library data from * @throws IOException * throws if an error occurs while reading from the stream */ private void readLibraryFromStreamInternal(final DataInput in) throws IOException { final String libraryFileName = StringRecord.readString(in); if (libraryFileName == null) { throw new IOException("libraryFileName is null!"); } final long length = in.readLong(); if (length > (long) Integer.MAX_VALUE) { throw new IOException("Submitted jar file " + libraryFileName + " is too large"); } final byte[] buf = new byte[(int) length]; in.readFully(buf); final Path storePath = new Path(this.libraryCachePath + "/" + libraryFileName); synchronized (this.fs) { // Check if file already exists in our library cache, if not write it to the cache directory if (!fs.exists(storePath)) { final FSDataOutputStream fos = fs.create(storePath, false); fos.write(buf, 0, buf.length); fos.close(); } } }
if (!this.fs.exists(storePath)) { final FSDataOutputStream fos = this.fs.create(storePath, false); fos.write(buf, 0, buf.length);
if (!this.fs.exists(storePath)) { final FSDataOutputStream fos = this.fs.create(storePath, false); fos.write(buf, 0, buf.length);
if (!fs.exists(storePath)) { throw new IOException(storePath + " does not exist!");
if (!fs.exists(storePath)) { throw new IOException(storePath + " does not exist!");
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException { FileSystem sFS = sourcePath.getFileSystem(); FileSystem tFS = targetPath.getFileSystem(); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) { tFS.mkdirs(targetPath); FileStatus[] contents = sFS.listStatus(sourcePath); for (FileStatus content : contents) { String distPath = content.getPath().toString(); if (content.isDir()) { if (distPath.endsWith("/")) { distPath = distPath.substring(0, distPath.length() - 1); } } String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/")); copy(content.getPath(), new Path(localPath), executable); } } else { try { FSDataOutputStream lfsOutput = tFS.create(targetPath, false); FSDataInputStream fsInput = sFS.open(sourcePath); IOUtils.copyBytes(fsInput, lfsOutput); new File(targetPath.toString()).setExecutable(executable); } catch (IOException ioe) { } } } }