@Override public Long getValue() { long ret = 0L; try { String ufsDataFolder = MasterContext.getConf().get(Constants.UNDERFS_ADDRESS); UnderFileSystem ufs = UnderFileSystem.get(ufsDataFolder, MasterContext.getConf()); ret = ufs.getSpace(ufsDataFolder, UnderFileSystem.SpaceType.SPACE_TOTAL); } catch (IOException e) { LOG.error(e.getMessage(), e); } return ret; } });
public LineageDataManager(BlockDataManager blockDataManager) { mBlockDataManager = Preconditions.checkNotNull(blockDataManager); mPersistedFiles = Lists.newArrayList(); mTachyonConf = WorkerContext.getConf(); // Create Under FileSystem Client String ufsAddress = mTachyonConf.get(Constants.UNDERFS_ADDRESS); mUfs = UnderFileSystem.get(ufsAddress, mTachyonConf); }
UnderFileSystem ufs = UnderFileSystem.get(ufsPath, MasterContext.getConf()); List<String> locs; try {
ufsPath = mMountTable.resolve(path); UnderFileSystem ufs = UnderFileSystem.get(ufsPath.toString(), MasterContext.getConf()); try { if (!ufs.exists(ufsPath.getPath())) {
throws TachyonException, IOException { String tmpPath = PathUtils.temporaryFileName(fileId, nonce, ufsPath); UnderFileSystem ufs = UnderFileSystem.get(tmpPath, WorkerContext.getConf()); try { if (!ufs.exists(tmpPath)) {
private void deleteDir(String path) throws IOException { UnderFileSystem ufs = UnderFileSystem.get(path, mMasterConf); if (ufs.exists(path) && !ufs.delete(path, true)) { throw new IOException("Folder " + path + " already exists but can not be deleted."); } }
@Override public Long getValue() { long ret = 0L; try { String ufsDataFolder = MasterContext.getConf().get(Constants.UNDERFS_ADDRESS); UnderFileSystem ufs = UnderFileSystem.get(ufsDataFolder, MasterContext.getConf()); ret = ufs.getSpace(ufsDataFolder, UnderFileSystem.SpaceType.SPACE_USED); } catch (IOException e) { LOG.error(e.getMessage(), e); } return ret; } });
private void setUnderStoreStream(long pos) throws IOException { if (mUnderStoreStream != null) { mUnderStoreStream.close(); } UnderFileSystem ufs = UnderFileSystem.get(mUfsPath, ClientContext.getConf()); mUnderStoreStream = ufs.open(mUfsPath); mPos = 0; if (pos != skip(pos)) { throw new IOException("Failed to skip: " + pos); } } }
@Override public Long getValue() { long ret = 0L; try { String ufsDataFolder = MasterContext.getConf().get(Constants.UNDERFS_ADDRESS); UnderFileSystem ufs = UnderFileSystem.get(ufsDataFolder, MasterContext.getConf()); ret = ufs.getSpace(ufsDataFolder, UnderFileSystem.SpaceType.SPACE_FREE); } catch (IOException e) { LOG.error(e.getMessage(), e); } return ret; } });
private void mkdir(String path) throws IOException { UnderFileSystem ufs = UnderFileSystem.get(path, mMasterConf); if (ufs.exists(path)) { ufs.delete(path, true); } if (!ufs.mkdirs(path, true)) { throw new IOException("Failed to make folder: " + path); } }
UnderFileSystem ufs = UnderFileSystem.get(ufsRoot, conf);
/** * @param journal the handle to the journal */ JournalReader(Journal journal) { mJournal = Preconditions.checkNotNull(journal); TachyonConf conf = MasterContext.getConf(); mUfs = UnderFileSystem.get(mJournal.getDirectory(), conf); mCheckpointPath = mJournal.getCheckpointFilePath(); }
private static boolean formatFolder(String name, String folder, TachyonConf tachyonConf) throws IOException { UnderFileSystem ufs = UnderFileSystem.get(folder, tachyonConf); LOG.info("Formatting {}:{}", name, folder); if (ufs.exists(folder)) { for (String file : ufs.list(folder)) { if (!ufs.delete(PathUtils.concatPath(folder, file), true)) { LOG.info("Failed to remove {}:{}", name, file); return false; } } } else if (!ufs.mkdirs(folder, true)) { LOG.info("Failed to create {}:{}", name, folder); return false; } return true; }
JournalWriter(Journal journal) { mJournal = Preconditions.checkNotNull(journal); mJournalDirectory = mJournal.getDirectory(); mCompletedDirectory = mJournal.getCompletedDirectory(); mTempCheckpointPath = mJournal.getCheckpointFilePath() + ".tmp"; TachyonConf conf = MasterContext.getConf(); mUfs = UnderFileSystem.get(mJournalDirectory, conf); mMaxLogSize = conf.getBytes(Constants.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX); }
private void connectToUFS() throws IOException { TachyonConf conf = MasterContext.getConf(); String ufsAddress = conf.get(Constants.UNDERFS_ADDRESS); UnderFileSystem ufs = UnderFileSystem.get(ufsAddress, conf); ufs.connectFromMaster(conf, NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC, conf)); } }
/** * Checks to see if the journal directory is formatted. * * @param journalDirectory The journal directory to check * @return true if the journal directory was formatted previously, false otherwise * @throws IOException */ private boolean isJournalFormatted(String journalDirectory) throws IOException { TachyonConf conf = MasterContext.getConf(); UnderFileSystem ufs = UnderFileSystem.get(journalDirectory, conf); if (!ufs.providesStorage()) { // TODO(gene): Should the journal really be allowed on a ufs without storage? // This ufs doesn't provide storage. Allow the master to use this ufs for the journal. LOG.info("Journal directory doesn't provide storage: " + journalDirectory); return true; } String[] files = ufs.list(journalDirectory); if (files == null) { return false; } // Search for the format file. String formatFilePrefix = conf.get(Constants.MASTER_FORMAT_FILE_PREFIX); for (String file : files) { if (file.startsWith(formatFilePrefix)) { return true; } } return false; }
for (String dirPath : dirPaths) { String dirWorkerDataFolder = PathUtils.concatPath(dirPath.trim(), workerDataFolder); UnderFileSystem ufs = UnderFileSystem.get(dirWorkerDataFolder, tachyonConf); if (ufs.exists(dirWorkerDataFolder)) { if (!formatFolder(name, dirWorkerDataFolder, tachyonConf)) {
/** * To clean up the test environment over underfs cluster system, so that we can re-use the running * system for the next test round instead of turning on/off it from time to time. This function is * expected to be called either before or after each test case which avoids certain overhead from * the bootstrap. * * @throws IOException when the operation fails */ public void cleanup() throws IOException { if (isStarted()) { String path = getUnderFilesystemAddress() + TachyonURI.SEPARATOR; UnderFileSystem ufs = UnderFileSystem.get(path, mTachyonConf); for (String p : ufs.list(path)) { ufs.delete(PathUtils.concatPath(path, p), true); } } }
mUfsPath = fileInfo.getUfsPath(); String fileName = PathUtils.temporaryFileName(fileId, mNonce, mUfsPath); UnderFileSystem ufs = UnderFileSystem.get(fileName, ClientContext.getConf());
persisted.add(next); String ufsPath = mMountTable.resolve(getPath(next)).toString(); UnderFileSystem ufs = UnderFileSystem.get(ufsPath, MasterContext.getConf()); ufs.mkdirs(ufsPath, false);