private boolean readFile(TachyonFileSystem tachyonClient) throws IOException, TachyonException { boolean pass = true; for (int i = 0; i < mNumFiles; i ++) { TachyonURI filePath = new TachyonURI(mFileFolder + "/part-" + i); LOG.debug("Reading data from {}", filePath); TachyonFile file = tachyonClient.open(filePath); FileInStream is = tachyonClient.getInStream(file); FileInfo info = tachyonClient.getInfo(file); ByteBuffer buf = ByteBuffer.allocate((int) info.getBlockSizeBytes()); is.read(buf.array()); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mNumFiles; k ++) { pass = pass && (buf.getInt() == k); } is.close(); } return pass; }
private FileOutStream getOrCreate(TachyonFileSystem tachyonFileSystem, TachyonURI filePath, boolean deleteIfExists, OutStreamOptions clientOptions) throws IOException, TachyonException { TachyonFile file; try { file = tachyonFileSystem.open(filePath); } catch (Exception e) { file = null; } if (file == null) { // file doesn't exist yet, so create it return tachyonFileSystem.getOutStream(filePath, clientOptions); } else if (deleteIfExists) { // file exists, so delete it and recreate tachyonFileSystem.delete(file); return tachyonFileSystem.getOutStream(filePath, clientOptions); } // file exists and deleteIfExists is false throw new FileAlreadyExistsException(" file exists but deleteIfExists is false"); }
/** * Renames a file or directory specified by argv. Will fail if the new path name already exists. * * @param argv [] Array of arguments given by the user's input from the terminal * @return 0 if command is successful, -1 if an error occurred * @throws IOException */ public int rename(String[] argv) throws IOException { TachyonURI srcPath = new TachyonURI(argv[1]); TachyonURI dstPath = new TachyonURI(argv[2]); try { TachyonFile fd = mTfs.open(srcPath); if (mTfs.rename(fd, dstPath)) { System.out.println("Renamed " + srcPath + " to " + dstPath); return 0; } else { System.out.println("mv: Failed to rename " + srcPath + " to " + dstPath); return -1; } } catch (TachyonException e) { throw new IOException(e.getMessage()); } }
private List<FileInfo> listStatusSortedByIncreasingCreationTime(TachyonURI path) throws IOException { List<FileInfo> files = null; try { TachyonFile fd = mTfs.open(path); files = mTfs.listStatus(fd); } catch (TachyonException e) { throw new IOException(e.getMessage()); } Collections.sort(files, new Comparator<FileInfo>() { @Override public int compare(FileInfo fileInfo, FileInfo fileInfo2) { long t1 = fileInfo.creationTimeMs; long t2 = fileInfo2.creationTimeMs; if (t1 < t2) { return -1; } if (t1 == t2) { return 0; } return 1; } }); return files; }
public int report(TachyonURI path) throws IOException { try { TachyonFile fd = mTfs.open(path); mTfs.reportLostFile(fd); System.out .println(path + " with file id " + fd.getFileId() + " has reported been report lost."); listLineages(); return 0; } catch (TachyonException e) { throw new IOException(e.getMessage()); } }
FileInfo fInfo; try { fd = mTfs.open(filePath); fInfo = mTfs.getInfo(fd); } catch (IOException ioe) { return -1; List<FileInfo> files = tachyonClient.listStatus(fd); Collections.sort(files); for (FileInfo file : files) { InStreamOptions op = new InStreamOptions.Builder(mTachyonConf) .setTachyonStorageType(TachyonStorageType.STORE).build(); FileInStream in = closer.register(mTfs.getInStream(fd, op)); byte[] buf = new byte[8 * Constants.MB]; while (in.read(buf) != -1) {
private boolean read(TachyonFileSystem tachyonFileSystem, TachyonURI filePath, ReadType readType) throws IOException, TachyonException { InStreamOptions clientOptions = new InStreamOptions.Builder(ClientContext.getConf()) .setReadType(readType).build(); TachyonFile file = tachyonFileSystem.open(filePath); DataInputStream input = new DataInputStream(tachyonFileSystem.getInStream(file, clientOptions)); boolean passes = true; try { int length = input.readInt(); for (int i = 0; i < length; i ++) { passes &= (input.readInt() == i); } } finally { input.close(); } return passes; }
TachyonURI testURI = new TachyonURI(mWorkDir + mSuccessNum); if (ClientOpType.CREATE_FILE == mOpType) { sTfs.getOutStream(testURI, sOutStreamOptions).close(); } else if (ClientOpType.CREATE_DELETE_FILE == mOpType) { try { sTfs.getOutStream(testURI, sOutStreamOptions).close(); } catch (TachyonException e) { throw e; sTfs.delete(sTfs.open(testURI)); } else if (ClientOpType.CREATE_RENAME_FILE == mOpType) { try { sTfs.getOutStream(testURI, sOutStreamOptions).close(); } catch (TachyonException e) { throw e; sTfs.rename(sTfs.open(testURI), new TachyonURI(testURI + "-rename")); } else if (ClientOpType.CREATE_TABLE == mOpType) { if (sOldTfs.createRawTable(new TachyonURI(mWorkDir + mSuccessNum), 1) == -1) {
private long[] countHelper(TachyonURI path) throws IOException { TachyonFile fd; FileInfo fInfo; try { fd = mTfs.open(path); fInfo = mTfs.getInfo(fd); } catch (TachyonException e) { throw new IOException(e.getMessage()); } if (!fInfo.isFolder) { return new long[] {1L, 0L, fInfo.length}; } long[] rtn = new long[] {0L, 1L, 0L}; List<FileInfo> files = null; try { files = mTfs.listStatus(fd); } catch (TachyonException e) { throw new IOException(e.getMessage()); } Collections.sort(files); for (FileInfo file : files) { long[] toAdd = countHelper(new TachyonURI(file.getPath())); rtn[0] += toAdd[0]; rtn[1] += toAdd[1]; rtn[2] += toAdd[2]; } return rtn; }
/** * Displays the file's all blocks info * * @param path The TachyonURI path as the input of the command * @return 0 if command is successful, -1 if an error occurred * @throws IOException */ public int fileinfo(TachyonURI path) throws IOException { TachyonFile fd; FileInfo fInfo; try { fd = mTfs.open(path); fInfo = mTfs.getInfo(fd); } catch (TachyonException e) { throw new IOException(e.getMessage()); } if (fInfo.isFolder) { System.out.println(path + " is a directory path so does not have file blocks."); return -1; } System.out.println(path + " with file id " + fd.getFileId() + " has the following blocks: "); for (long blockId : fInfo.getBlockIds()) { System.out.println(TachyonBlockStore.get().getInfo(blockId)); } return 0; }
/** * Removes the file specified by argv. * * @param path The TachyonURI path as the input of the command * @return 0 if command is successful, -1 if an error occurred * @throws IOException */ public int rm(TachyonURI path) throws IOException { TachyonFile fd; FileInfo fInfo; try { fd = mTfs.open(path); fInfo = mTfs.getInfo(fd); } catch (TachyonException e) { throw new IOException(e.getMessage()); } if (fInfo.isFolder) { System.out.println("rm: cannot remove a directory, please try rmr <path>"); return -1; } try { mTfs.delete(fd); System.out.println(path + " has been removed"); return 0; } catch (IOException ioe) { return -1; } catch (TachyonException e) { throw new IOException(e.getMessage()); } }
TachyonFile fd = tachyonClient.openIfExists(dstPath); if (fd != null) { FileInfo tFile = tachyonClient.getInfo(fd); if (tFile.isFolder) { dstPath = dstPath.join(src.getName()); try { FileOutStream os = closer.register(tachyonClient.getOutStream(dstPath, OutStreamOptions.defaults())); FileInputStream in = closer.register(new FileInputStream(src)); FileChannel channel = closer.register(in.getChannel()); } else { try { tachyonClient.mkdir(dstPath); } catch (TachyonException e) { throw new IOException(e.getMessage());
/** * Removes the file or directory specified by argv. Will remove all files and directories in the * directory if a directory is specified. * * @param path The TachyonURI path as the input of the command * @return 0 if command is successful, -1 if an error occurred */ public int rmr(TachyonURI path) { try { DeleteOptions options = new DeleteOptions.Builder(mTachyonConf).setRecursive(true).build(); TachyonFile fd = mTfs.open(path); mTfs.delete(fd, options); System.out.println(path + " has been removed"); return 0; } catch (IOException e) { return -1; } catch (TachyonException e) { return -1; } }
.setTachyonStorageType(tachyonStorageType).setUnderStorageType(underStorageType) .build(); FileOutStream os = tfs.getOutStream(fileURI, options); return tfs.open(fileURI); } catch (TachyonException e) { throw new IOException(e.getMessage());
/** * Pins the given file or folder (recursively pinning all children if a folder). Pinned files are * never evicted from memory. * * @param path The TachyonURI path as the input of the command * @return 0 if command is successful, -1 if an error occurred */ public int pin(TachyonURI path) { try { TachyonFile fd = mTfs.open(path); SetStateOptions options = new SetStateOptions.Builder(mTachyonConf).setPinned(true).build(); mTfs.setState(fd, options); System.out.println("File '" + path + "' was successfully pinned."); return 0; } catch (Exception e) { e.printStackTrace(); System.out.println("File '" + path + "' could not be pinned."); return -1; } }
/** * Free the given file or folder from tachyon in-memory (recursively freeing all children if a * folder) * * @param path The TachyonURI path as the input of the command * @return 0 if command if successful, -1 if an error occurred */ public int free(TachyonURI path) throws IOException { try { FreeOptions options = new FreeOptions.Builder(mTachyonConf).setRecursive(true).build(); TachyonFile fd = mTfs.open(path); mTfs.free(fd, options); System.out.println(path + " was successfully freed from memory."); return 0; } catch (TachyonException e) { return -1; } }
/** * Convenience method for {@link #getOutStream(TachyonURI, OutStreamOptions)} with default * options. */ public FileOutStream getOutStream(TachyonURI path) throws IOException, TachyonException, FileAlreadyExistsException, InvalidPathException { return getOutStream(path, OutStreamOptions.defaults()); }
/** * Convenience method for {@link TachyonFileSystemCore#getInfo(TachyonFile, GetInfoOptions)} with * default options. */ public FileInfo getInfo(TachyonFile file) throws FileDoesNotExistException, IOException, TachyonException { return getInfo(file, GetInfoOptions.defaults()); }
/** * Convenience method for {@link #getInStream(TachyonFile, InStreamOptions)} with default options. */ public FileInStream getInStream(TachyonFile file) throws IOException, TachyonException, FileDoesNotExistException { return getInStream(file, InStreamOptions.defaults()); }