/** * Convenience method for {@link #open(TachyonURI, OpenOptions)} with default options. */ public TachyonFile open(TachyonURI path) throws IOException, InvalidPathException, TachyonException { return open(path, OpenOptions.defaults()); }
public int report(TachyonURI path) throws IOException { try { TachyonFile fd = mTfs.open(path); mTfs.reportLostFile(fd); System.out .println(path + " with file id " + fd.getFileId() + " has reported been report lost."); listLineages(); return 0; } catch (TachyonException e) { throw new IOException(e.getMessage()); } }
private List<FileInfo> listStatusSortedByIncreasingCreationTime(TachyonURI path) throws IOException { List<FileInfo> files = null; try { TachyonFile fd = mTfs.open(path); files = mTfs.listStatus(fd); } catch (TachyonException e) { throw new IOException(e.getMessage()); } Collections.sort(files, new Comparator<FileInfo>() { @Override public int compare(FileInfo fileInfo, FileInfo fileInfo2) { long t1 = fileInfo.creationTimeMs; long t2 = fileInfo2.creationTimeMs; if (t1 < t2) { return -1; } if (t1 == t2) { return 0; } return 1; } }); return files; }
private boolean read(TachyonFileSystem tachyonFileSystem, TachyonURI filePath, ReadType readType) throws IOException, TachyonException { InStreamOptions clientOptions = new InStreamOptions.Builder(ClientContext.getConf()) .setReadType(readType).build(); TachyonFile file = tachyonFileSystem.open(filePath); DataInputStream input = new DataInputStream(tachyonFileSystem.getInStream(file, clientOptions)); boolean passes = true; try { int length = input.readInt(); for (int i = 0; i < length; i ++) { passes &= (input.readInt() == i); } } finally { input.close(); } return passes; }
private boolean readFile(TachyonFileSystem tachyonClient) throws IOException, TachyonException { boolean pass = true; for (int i = 0; i < mNumFiles; i ++) { TachyonURI filePath = new TachyonURI(mFileFolder + "/part-" + i); LOG.debug("Reading data from {}", filePath); TachyonFile file = tachyonClient.open(filePath); FileInStream is = tachyonClient.getInStream(file); FileInfo info = tachyonClient.getInfo(file); ByteBuffer buf = ByteBuffer.allocate((int) info.getBlockSizeBytes()); is.read(buf.array()); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mNumFiles; k ++) { pass = pass && (buf.getInt() == k); } is.close(); } return pass; }
/** * Pins the given file or folder (recursively pinning all children if a folder). Pinned files are * never evicted from memory. * * @param path The TachyonURI path as the input of the command * @return 0 if command is successful, -1 if an error occurred */ public int pin(TachyonURI path) { try { TachyonFile fd = mTfs.open(path); SetStateOptions options = new SetStateOptions.Builder(mTachyonConf).setPinned(true).build(); mTfs.setState(fd, options); System.out.println("File '" + path + "' was successfully pinned."); return 0; } catch (Exception e) { e.printStackTrace(); System.out.println("File '" + path + "' could not be pinned."); return -1; } }
/** * Unpins the given file or folder (recursively unpinning all children if a folder). Pinned files * are never evicted from memory, so this method will allow such files to be evicted. * * @param path The TachyonURI path as the input of the command * @return 0 if command is successful, -1 if an error occurred * @throws IOException */ public int unpin(TachyonURI path) throws IOException { try { SetStateOptions options = new SetStateOptions.Builder(mTachyonConf).setPinned(false).build(); TachyonFile fd = mTfs.open(path); mTfs.setState(fd, options); System.out.println("File '" + path + "' was successfully unpinned."); return 0; } catch (Exception e) { e.printStackTrace(); System.out.println("File '" + path + "' could not be unpinned."); return -1; } }
/** * Renames a file or directory specified by argv. Will fail if the new path name already exists. * * @param argv [] Array of arguments given by the user's input from the terminal * @return 0 if command is successful, -1 if an error occurred * @throws IOException */ public int rename(String[] argv) throws IOException { TachyonURI srcPath = new TachyonURI(argv[1]); TachyonURI dstPath = new TachyonURI(argv[2]); try { TachyonFile fd = mTfs.open(srcPath); if (mTfs.rename(fd, dstPath)) { System.out.println("Renamed " + srcPath + " to " + dstPath); return 0; } else { System.out.println("mv: Failed to rename " + srcPath + " to " + dstPath); return -1; } } catch (TachyonException e) { throw new IOException(e.getMessage()); } }
/** * Removes the file specified by argv. * * @param path The TachyonURI path as the input of the command * @return 0 if command is successful, -1 if an error occurred * @throws IOException */ public int rm(TachyonURI path) throws IOException { TachyonFile fd; FileInfo fInfo; try { fd = mTfs.open(path); fInfo = mTfs.getInfo(fd); } catch (TachyonException e) { throw new IOException(e.getMessage()); } if (fInfo.isFolder) { System.out.println("rm: cannot remove a directory, please try rmr <path>"); return -1; } try { mTfs.delete(fd); System.out.println(path + " has been removed"); return 0; } catch (IOException ioe) { return -1; } catch (TachyonException e) { throw new IOException(e.getMessage()); } }
/** * Free the given file or folder from tachyon in-memory (recursively freeing all children if a * folder) * * @param path The TachyonURI path as the input of the command * @return 0 if command if successful, -1 if an error occurred */ public int free(TachyonURI path) throws IOException { try { FreeOptions options = new FreeOptions.Builder(mTachyonConf).setRecursive(true).build(); TachyonFile fd = mTfs.open(path); mTfs.free(fd, options); System.out.println(path + " was successfully freed from memory."); return 0; } catch (TachyonException e) { return -1; } }
/** * Removes the file or directory specified by argv. Will remove all files and directories in the * directory if a directory is specified. * * @param path The TachyonURI path as the input of the command * @return 0 if command is successful, -1 if an error occurred */ public int rmr(TachyonURI path) { try { DeleteOptions options = new DeleteOptions.Builder(mTachyonConf).setRecursive(true).build(); TachyonFile fd = mTfs.open(path); mTfs.delete(fd, options); System.out.println(path + " has been removed"); return 0; } catch (IOException e) { return -1; } catch (TachyonException e) { return -1; } }
/** * Displays the file's all blocks info * * @param path The TachyonURI path as the input of the command * @return 0 if command is successful, -1 if an error occurred * @throws IOException */ public int fileinfo(TachyonURI path) throws IOException { TachyonFile fd; FileInfo fInfo; try { fd = mTfs.open(path); fInfo = mTfs.getInfo(fd); } catch (TachyonException e) { throw new IOException(e.getMessage()); } if (fInfo.isFolder) { System.out.println(path + " is a directory path so does not have file blocks."); return -1; } System.out.println(path + " with file id " + fd.getFileId() + " has the following blocks: "); for (long blockId : fInfo.getBlockIds()) { System.out.println(TachyonBlockStore.get().getInfo(blockId)); } return 0; }
/** * Returns a list of files at a given <code>path</code>. * * @param tfs a TachyonFileSystem handler * @param path a path in tachyon file system * @return a list of strings representing the file names under the given path * @throws IOException if <code>path</code> does not exist or is invalid */ public static List<String> listFiles(TachyonFileSystem tfs, String path) throws IOException { try { TachyonFile file = tfs.open(new TachyonURI(path)); if (file == null) { throw new IOException(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(path)); } List<FileInfo> infos = tfs.listStatus(file); List<String> res = new ArrayList<String>(); for (FileInfo info : infos) { res.add(info.getPath()); if (info.isFolder) { res.addAll(listFiles(tfs, info.getPath())); } } return res; } catch (TachyonException e) { throw new IOException(e.getMessage()); } }
private FileOutStream getOrCreate(TachyonFileSystem tachyonFileSystem, TachyonURI filePath, boolean deleteIfExists, OutStreamOptions clientOptions) throws IOException, TachyonException { TachyonFile file; try { file = tachyonFileSystem.open(filePath); } catch (Exception e) { file = null; } if (file == null) { // file doesn't exist yet, so create it return tachyonFileSystem.getOutStream(filePath, clientOptions); } else if (deleteIfExists) { // file exists, so delete it and recreate tachyonFileSystem.delete(file); return tachyonFileSystem.getOutStream(filePath, clientOptions); } // file exists and deleteIfExists is false throw new FileAlreadyExistsException(" file exists but deleteIfExists is false"); }
private long[] countHelper(TachyonURI path) throws IOException { TachyonFile fd; FileInfo fInfo; try { fd = mTfs.open(path); fInfo = mTfs.getInfo(fd); } catch (TachyonException e) { throw new IOException(e.getMessage()); } if (!fInfo.isFolder) { return new long[] {1L, 0L, fInfo.length}; } long[] rtn = new long[] {0L, 1L, 0L}; List<FileInfo> files = null; try { files = mTfs.listStatus(fd); } catch (TachyonException e) { throw new IOException(e.getMessage()); } Collections.sort(files); for (FileInfo file : files) { long[] toAdd = countHelper(new TachyonURI(file.getPath())); rtn[0] += toAdd[0]; rtn[1] += toAdd[1]; rtn[2] += toAdd[2]; } return rtn; }
/** * Displays a list of hosts that have the file specified in argv stored. * * @param path The TachyonURI path as the input of the command * @return 0 if command is successful, -1 if an error occurred * @throws IOException */ public int location(TachyonURI path) throws IOException { TachyonFile fd; FileInfo fInfo; try { fd = mTfs.open(path); fInfo = mTfs.getInfo(fd); } catch (TachyonException e) { throw new IOException(e.getMessage()); } System.out.println(path + " with file id " + fd.getFileId() + " is on nodes: "); for (long blockId : fInfo.getBlockIds()) { for (BlockLocation location : TachyonBlockStore.get().getInfo(blockId).getLocations()) { System.out.println(location.getWorkerAddress().getHost()); } } return 0; }
/** * Return the {@code InStream} of this file based on the specified read type. If it has no block, * return an {@code EmptyBlockInStream}; if it has only one block, return a {@code BlockInStream} * of the block; otherwise return a {@code FileInStream}. * * @param readType the InStream's read type * @return the <code>InStream</code> * @throws IOException when an event that prevents the operation from completing is encountered */ public FileInStream getInStream(ReadType readType) throws IOException { if (readType == null) { throw new IOException("ReadType can not be null."); } if (!isCompleted()) { throw new IOException("The file " + this + " is not complete."); } if (isDirectory()) { throw new IOException("Cannot open a directory for reading."); } FileInfo info = getUnCachedFileStatus(); TachyonURI uri = new TachyonURI(info.getPath()); InStreamOptions.Builder optionsBuilder = new InStreamOptions.Builder(mTachyonConf); optionsBuilder.setTachyonStorageType(readType.getTachyonStorageType()); try { tachyon.client.file.TachyonFile newFile = mTFS.open(uri); return mTFS.getInStream(newFile, optionsBuilder.build()); } catch (TachyonException e) { throw new IOException(e); } }
/** * Calculates the size of a path (file or folder) specified by a TachyonURI. * * @param tachyonFS A TachyonFileSystem * @param path A TachyonURI denoting the path * @return total size of the specified path in byte * @throws IOException */ private long getFileOrFolderSize(TachyonFileSystem tachyonFS, TachyonURI path) throws IOException { long sizeInBytes = 0; List<FileInfo> files; try { TachyonFile inputFile = tachyonFS.open(path); files = tachyonFS.listStatus(inputFile); } catch (TachyonException e) { throw new IOException(e.getMessage()); } for (FileInfo file : files) { if (file.isFolder) { TachyonURI subFolder = new TachyonURI(file.getPath()); sizeInBytes += getFileOrFolderSize(tachyonFS, subFolder); } else { sizeInBytes += file.getLength(); } } return sizeInBytes; }
return tfs.open(fileURI); } catch (TachyonException e) { throw new IOException(e.getMessage());
/** * Creates a simple file with <code>len</code> bytes. * * @param tfs a TachyonFileSystem handler * @param fileName the name of the file to be created * @param tachyonStorageType TachyonStorageType used to create the file * @param underStorageType UnderStorageType used to create the file * @param len file size * @param blockCapacityByte block size of the file * @return the TachyonFile of the created file * @throws IOException if <code>path</code> is invalid (e.g., illegal URI) */ public static TachyonFile createByteFile(TachyonFileSystem tfs, String fileName, TachyonStorageType tachyonStorageType, UnderStorageType underStorageType, int len, long blockCapacityByte) throws IOException { try { OutStreamOptions options = new OutStreamOptions.Builder(ClientContext.getConf()) .setTachyonStorageType(tachyonStorageType).setUnderStorageType(underStorageType) .setBlockSizeBytes(blockCapacityByte).build(); FileOutStream os = tfs.getOutStream(new TachyonURI(fileName), options); for (int k = 0; k < len; k ++) { os.write((byte) k); } os.close(); return tfs.open(new TachyonURI(fileName)); } catch (TachyonException e) { throw new IOException(e.getMessage()); } }