/** * To clean up the test environment over underfs cluster system, so that we can re-use the running * system for the next test round instead of turning on/off it from time to time. This function is * expected to be called either before or after each test case which avoids certain overhead from * the bootstrap. * * @throws IOException when the operation fails */ public void cleanup() throws IOException { if (isStarted()) { String path = getUnderFilesystemAddress() + TachyonURI.SEPARATOR; UnderFileSystem ufs = UnderFileSystem.get(path, mTachyonConf); for (String p : ufs.list(path)) { ufs.delete(PathUtils.concatPath(path, p), true); } } }
/** * Returns an output stream for the journal checkpoint. The returned output stream is a singleton * for this writer. * * @param latestSequenceNumber the sequence number of the latest journal entry. This sequence * number will be used to determine the next sequence numbers for the subsequent journal * entries. * @return the output stream for the journal checkpoint * @throws IOException */ public synchronized JournalOutputStream getCheckpointOutputStream(long latestSequenceNumber) throws IOException { if (mCheckpointOutputStream == null) { LOG.info("Creating tmp checkpoint file: " + mTempCheckpointPath); if (!mUfs.exists(mJournalDirectory)) { LOG.info("Creating journal folder: " + mJournalDirectory); mUfs.mkdirs(mJournalDirectory, true); } mNextEntrySequenceNumber = latestSequenceNumber + 1; LOG.info("Latest journal sequence number: " + latestSequenceNumber + " Next journal sequence number: " + mNextEntrySequenceNumber); mCheckpointOutputStream = new CheckpointOutputStream(new DataOutputStream(mUfs.create(mTempCheckpointPath))); } return mCheckpointOutputStream; }
/** * @return the last modified time of the checkpoint file in ms * @throws IOException */ public long getCheckpointLastModifiedTimeMs() throws IOException { if (!mUfs.exists(mCheckpointPath)) { throw new IOException("Checkpoint file " + mCheckpointPath + " does not exist."); } mCheckpointLastModifiedTime = mUfs.getModificationTimeMs(mCheckpointPath); return mCheckpointLastModifiedTime; } }
Pair<String, String> ufsPair = UnderFileSystem.parse(ufsAddrRootPath, tachyonConf); String ufsAddress = ufsPair.getFirst(); String ufsRootPath = ufsPair.getSecond(); UnderFileSystem ufs = UnderFileSystem.get(ufsAddress, tachyonConf); if (!ufs.exists(ufsAddrRootPath.toString())) { throw new FileNotFoundException("ufs path " + ufsAddrRootPath + " not found."); if (ufs.isFile(ufsAddrRootPath.toString())) { if ((ufsRootPath == null) || ufsRootPath.isEmpty() || ufsRootPath.equals("/")) { directoryName = TachyonURI.EMPTY_URI; if (ufs.isFile(ufsPath.toString())) { // TODO(hy): Fix path matching issue. TachyonURI tfsPath = buildTFSPath(directoryName, ufsAddrRootPath, ufsPath); LOG.debug("Loading ufs. tfs path = " + tfsPath + "."); String[] files = ufs.list(ufsPath.toString()); // ufs.list() returns relative path if (files != null) { for (String filePath : files) {
ufsPath = mMountTable.resolve(path); UnderFileSystem ufs = UnderFileSystem.get(ufsPath.toString(), MasterContext.getConf()); try { if (!ufs.exists(ufsPath.getPath())) { throw new FileDoesNotExistException(ufsPath.getPath()); if (ufs.isFile(ufsPath.getPath())) { long ufsBlockSizeByte = ufs.getBlockSizeByte(ufsPath.toString()); long fileSizeByte = ufs.getFileSize(ufsPath.toString());
String ufsSrcPath = mMountTable.resolve(srcPath).toString(); String ufsDstPath = mMountTable.resolve(dstPath).toString(); UnderFileSystem ufs = UnderFileSystem.get(ufsSrcPath, MasterContext.getConf()); String parentPath = new TachyonURI(ufsDstPath).getParent().toString(); if (!ufs.exists(parentPath) && !ufs.mkdirs(parentPath, true)) { LOG.error("Failed to create " + parentPath); return false; if (!ufs.rename(ufsSrcPath, ufsDstPath)) { LOG.error("Failed to rename " + ufsSrcPath + " to " + ufsDstPath); return false;
throws TachyonException, IOException { String tmpPath = PathUtils.temporaryFileName(fileId, nonce, ufsPath); UnderFileSystem ufs = UnderFileSystem.get(tmpPath, WorkerContext.getConf()); try { if (!ufs.exists(tmpPath)) { tmpPath = PathUtils.temporaryFileName(fileId, nonce, ufsPath); if (!ufs.rename(tmpPath, ufsPath)) { throw new FailedToCheckpointException("Failed to rename " + tmpPath + " to " + ufsPath); fileSize = ufs.getFileSize(ufsPath); } catch (IOException ioe) { throw new FailedToCheckpointException("Failed to getFileSize " + ufsPath);
/** * Checks to see if the journal directory is formatted. * * @param journalDirectory The journal directory to check * @return true if the journal directory was formatted previously, false otherwise * @throws IOException */ private boolean isJournalFormatted(String journalDirectory) throws IOException { TachyonConf conf = MasterContext.getConf(); UnderFileSystem ufs = UnderFileSystem.get(journalDirectory, conf); if (!ufs.providesStorage()) { // TODO(gene): Should the journal really be allowed on a ufs without storage? // This ufs doesn't provide storage. Allow the master to use this ufs for the journal. LOG.info("Journal directory doesn't provide storage: " + journalDirectory); return true; } String[] files = ufs.list(journalDirectory); if (files == null) { return false; } // Search for the format file. String formatFilePrefix = conf.get(Constants.MASTER_FORMAT_FILE_PREFIX); for (String file : files) { if (file.startsWith(formatFilePrefix)) { return true; } } return false; }
@Override public Long getValue() { long ret = 0L; try { String ufsDataFolder = MasterContext.getConf().get(Constants.UNDERFS_ADDRESS); UnderFileSystem ufs = UnderFileSystem.get(ufsDataFolder, MasterContext.getConf()); ret = ufs.getSpace(ufsDataFolder, UnderFileSystem.SpaceType.SPACE_TOTAL); } catch (IOException e) { LOG.error(e.getMessage(), e); } return ret; } });
/** * Moves the current log file to the completed folder, marking it as complete. If successful, the * current log file will no longer exist. The current log must already be closed before this call. * * @throws IOException */ private void completeCurrentLog() throws IOException { String currentLog = mJournal.getCurrentLogFilePath(); if (!mUfs.exists(currentLog)) { // All logs are already complete, so nothing to do. return; } if (!mUfs.exists(mCompletedDirectory)) { mUfs.mkdirs(mCompletedDirectory, true); } String completedLog = mJournal.getCompletedLogFilePath(mNextCompleteLogNumber); mUfs.rename(currentLog, completedLog); LOG.info("Completed current log: " + currentLog + " to completed log: " + completedLog); mNextCompleteLogNumber ++; }
/** * Deletes all of the logs in the completed folder. * * @throws IOException */ private void deleteCompletedLogs() throws IOException { LOG.info("Deleting all completed log files..."); // Loop over all complete logs starting from the beginning. // TODO(gpang): should the deletes start from the end? long logNumber = Journal.FIRST_COMPLETED_LOG_NUMBER; String logFilename = mJournal.getCompletedLogFilePath(logNumber); while (mUfs.exists(logFilename)) { LOG.info("Deleting completed log: " + logFilename); mUfs.delete(logFilename, true); logNumber ++; // generate the next completed log filename in the sequence. logFilename = mJournal.getCompletedLogFilePath(logNumber); } LOG.info("Finished deleting all completed log files."); // All complete logs are deleted. Reset the log number counter. mNextCompleteLogNumber = Journal.FIRST_COMPLETED_LOG_NUMBER; }
for (String dirPath : dirPaths) { String dirWorkerDataFolder = PathUtils.concatPath(dirPath.trim(), workerDataFolder); UnderFileSystem ufs = UnderFileSystem.get(dirWorkerDataFolder, tachyonConf); if (ufs.exists(dirWorkerDataFolder)) { if (!formatFolder(name, dirWorkerDataFolder, tachyonConf)) { System.exit(-1);
private void setUnderStoreStream(long pos) throws IOException { if (mUnderStoreStream != null) { mUnderStoreStream.close(); } UnderFileSystem ufs = UnderFileSystem.get(mUfsPath, ClientContext.getConf()); mUnderStoreStream = ufs.open(mUfsPath); mPos = 0; if (pos != skip(pos)) { throw new IOException("Failed to skip: " + pos); } } }
/** * @return the input stream for the next completed log file. Will return null if the next * completed log file does not exist yet. * @throws IOException */ public JournalInputStream getNextInputStream() throws IOException { if (!mCheckpointRead) { throw new IOException("Must read the checkpoint file before getting input stream."); } if (getCheckpointLastModifiedTimeMs() != mCheckpointOpenedTime) { throw new IOException("Checkpoint file has been updated. This reader is no longer valid."); } String currentLogPath = mJournal.getCompletedLogFilePath(mCurrentLogNumber); if (!mUfs.exists(currentLogPath)) { LOG.debug("Journal log file: " + currentLogPath + " does not exist yet."); return null; } // Open input stream from the current log file. LOG.info("Opening journal log file: " + currentLogPath); JournalInputStream jis = mJournal.getJournalFormatter().deserialize(mUfs.open(currentLogPath)); // Increment the log file number. mCurrentLogNumber ++; return jis; }
public LineageDataManager(BlockDataManager blockDataManager) { mBlockDataManager = Preconditions.checkNotNull(blockDataManager); mPersistedFiles = Lists.newArrayList(); mTachyonConf = WorkerContext.getConf(); // Create Under FileSystem Client String ufsAddress = mTachyonConf.get(Constants.UNDERFS_ADDRESS); mUfs = UnderFileSystem.get(ufsAddress, mTachyonConf); }
private void connectToUFS() throws IOException { TachyonConf conf = MasterContext.getConf(); String ufsAddress = conf.get(Constants.UNDERFS_ADDRESS); UnderFileSystem ufs = UnderFileSystem.get(ufsAddress, conf); ufs.connectFromMaster(conf, NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC, conf)); } }