private FSDataOutputStream getOutputStream() throws IOException { if (this.outputStream == null) { this.outputStream = this.fs.create(this.checkpointFile, false, this.buf.length, this.replication, this.fs.getDefaultBlockSize()); } return this.outputStream; }
private FSDataOutputStream getOutputStream() throws IOException { if (this.outputStream == null) { this.outputStream = this.fs.create(this.checkpointFile, false, this.buf.length, this.replication, this.fs.getDefaultBlockSize()); } return this.outputStream; }
/** * Reads library data from the given stream. * * @param in * the stream to read the library data from * @throws IOException * throws if an error occurs while reading from the stream */ private void readLibraryFromStreamInternal(final DataInput in) throws IOException { final String libraryFileName = StringRecord.readString(in); if (libraryFileName == null) { throw new IOException("libraryFileName is null!"); } final long length = in.readLong(); if (length > (long) Integer.MAX_VALUE) { throw new IOException("Submitted jar file " + libraryFileName + " is too large"); } final byte[] buf = new byte[(int) length]; in.readFully(buf); final Path storePath = new Path(this.libraryCachePath + "/" + libraryFileName); synchronized (this.fs) { // Check if file already exists in our library cache, if not write it to the cache directory if (!fs.exists(storePath)) { final FSDataOutputStream fos = fs.create(storePath, false); fos.write(buf, 0, buf.length); fos.close(); } } }
/** * Reads library data from the given stream. * * @param in * the stream to read the library data from * @throws IOException * throws if an error occurs while reading from the stream */ private void readLibraryFromStreamInternal(final DataInput in) throws IOException { final String libraryFileName = StringRecord.readString(in); if (libraryFileName == null) { throw new IOException("libraryFileName is null!"); } final long length = in.readLong(); if (length > (long) Integer.MAX_VALUE) { throw new IOException("Submitted jar file " + libraryFileName + " is too large"); } final byte[] buf = new byte[(int) length]; in.readFully(buf); final Path storePath = new Path(this.libraryCachePath + "/" + libraryFileName); synchronized (this.fs) { // Check if file already exists in our library cache, if not write it to the cache directory if (!fs.exists(storePath)) { final FSDataOutputStream fos = fs.create(storePath, false); fos.write(buf, 0, buf.length); fos.close(); } } }
final FSDataOutputStream fos = this.fs.create(storePath, false); fos.write(buf, 0, buf.length); fos.close();
final FSDataOutputStream fos = this.fs.create(storePath, false); fos.write(buf, 0, buf.length); fos.close();
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException { FileSystem sFS = sourcePath.getFileSystem(); FileSystem tFS = targetPath.getFileSystem(); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) { tFS.mkdirs(targetPath); FileStatus[] contents = sFS.listStatus(sourcePath); for (FileStatus content : contents) { String distPath = content.getPath().toString(); if (content.isDir()) { if (distPath.endsWith("/")) { distPath = distPath.substring(0, distPath.length() - 1); } } String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/")); copy(content.getPath(), new Path(localPath), executable); } } else { try { FSDataOutputStream lfsOutput = tFS.create(targetPath, false); FSDataInputStream fsInput = sFS.open(sourcePath); IOUtils.copyBytes(fsInput, lfsOutput); new File(targetPath.toString()).setExecutable(executable); } catch (IOException ioe) { } } } }