/** * @return the last modified time of the checkpoint file in ms * @throws IOException */ public long getCheckpointLastModifiedTimeMs() throws IOException { if (!mUfs.exists(mCheckpointPath)) { throw new IOException("Checkpoint file " + mCheckpointPath + " does not exist."); } mCheckpointLastModifiedTime = mUfs.getModificationTimeMs(mCheckpointPath); return mCheckpointLastModifiedTime; } }
private void deleteDir(String path) throws IOException { UnderFileSystem ufs = UnderFileSystem.get(path, mMasterConf); if (ufs.exists(path) && !ufs.delete(path, true)) { throw new IOException("Folder " + path + " already exists but can not be deleted."); } }
private void mkdir(String path) throws IOException { UnderFileSystem ufs = UnderFileSystem.get(path, mMasterConf); if (ufs.exists(path)) { ufs.delete(path, true); } if (!ufs.mkdirs(path, true)) { throw new IOException("Failed to make folder: " + path); } }
/** * Moves the current log file to the completed folder, marking it as complete. If successful, the * current log file will no longer exist. The current log must already be closed before this call. * * @throws IOException */ private void completeCurrentLog() throws IOException { String currentLog = mJournal.getCurrentLogFilePath(); if (!mUfs.exists(currentLog)) { // All logs are already complete, so nothing to do. return; } if (!mUfs.exists(mCompletedDirectory)) { mUfs.mkdirs(mCompletedDirectory, true); } String completedLog = mJournal.getCompletedLogFilePath(mNextCompleteLogNumber); mUfs.rename(currentLog, completedLog); LOG.info("Completed current log: " + currentLog + " to completed log: " + completedLog); mNextCompleteLogNumber ++; }
/** * Returns an output stream for the journal checkpoint. The returned output stream is a singleton * for this writer. * * @param latestSequenceNumber the sequence number of the latest journal entry. This sequence * number will be used to determine the next sequence numbers for the subsequent journal * entries. * @return the output stream for the journal checkpoint * @throws IOException */ public synchronized JournalOutputStream getCheckpointOutputStream(long latestSequenceNumber) throws IOException { if (mCheckpointOutputStream == null) { LOG.info("Creating tmp checkpoint file: " + mTempCheckpointPath); if (!mUfs.exists(mJournalDirectory)) { LOG.info("Creating journal folder: " + mJournalDirectory); mUfs.mkdirs(mJournalDirectory, true); } mNextEntrySequenceNumber = latestSequenceNumber + 1; LOG.info("Latest journal sequence number: " + latestSequenceNumber + " Next journal sequence number: " + mNextEntrySequenceNumber); mCheckpointOutputStream = new CheckpointOutputStream(new DataOutputStream(mUfs.create(mTempCheckpointPath))); } return mCheckpointOutputStream; }
/** * Deletes all of the logs in the completed folder. * * @throws IOException */ private void deleteCompletedLogs() throws IOException { LOG.info("Deleting all completed log files..."); // Loop over all complete logs starting from the beginning. // TODO(gpang): should the deletes start from the end? long logNumber = Journal.FIRST_COMPLETED_LOG_NUMBER; String logFilename = mJournal.getCompletedLogFilePath(logNumber); while (mUfs.exists(logFilename)) { LOG.info("Deleting completed log: " + logFilename); mUfs.delete(logFilename, true); logNumber ++; // generate the next completed log filename in the sequence. logFilename = mJournal.getCompletedLogFilePath(logNumber); } LOG.info("Finished deleting all completed log files."); // All complete logs are deleted. Reset the log number counter. mNextCompleteLogNumber = Journal.FIRST_COMPLETED_LOG_NUMBER; }
/** * Marks all logs as completed. */ public synchronized void completeAllLogs() throws IOException { LOG.info("Marking all logs as complete."); // Loop over all complete logs starting from the beginning, to determine the next log number. mNextCompleteLogNumber = Journal.FIRST_COMPLETED_LOG_NUMBER; String logFilename = mJournal.getCompletedLogFilePath(mNextCompleteLogNumber); while (mUfs.exists(logFilename)) { mNextCompleteLogNumber ++; // generate the next completed log filename in the sequence. logFilename = mJournal.getCompletedLogFilePath(mNextCompleteLogNumber); } completeCurrentLog(); }
private static boolean formatFolder(String name, String folder, TachyonConf tachyonConf) throws IOException { UnderFileSystem ufs = UnderFileSystem.get(folder, tachyonConf); LOG.info("Formatting {}:{}", name, folder); if (ufs.exists(folder)) { for (String file : ufs.list(folder)) { if (!ufs.delete(PathUtils.concatPath(folder, file), true)) { LOG.info("Failed to remove {}:{}", name, file); return false; } } } else if (!ufs.mkdirs(folder, true)) { LOG.info("Failed to create {}:{}", name, folder); return false; } return true; }
/** * @return the input stream for the next completed log file. Will return null if the next * completed log file does not exist yet. * @throws IOException */ public JournalInputStream getNextInputStream() throws IOException { if (!mCheckpointRead) { throw new IOException("Must read the checkpoint file before getting input stream."); } if (getCheckpointLastModifiedTimeMs() != mCheckpointOpenedTime) { throw new IOException("Checkpoint file has been updated. This reader is no longer valid."); } String currentLogPath = mJournal.getCompletedLogFilePath(mCurrentLogNumber); if (!mUfs.exists(currentLogPath)) { LOG.debug("Journal log file: " + currentLogPath + " does not exist yet."); return null; } // Open input stream from the current log file. LOG.info("Opening journal log file: " + currentLogPath); JournalInputStream jis = mJournal.getJournalFormatter().deserialize(mUfs.open(currentLogPath)); // Increment the log file number. mCurrentLogNumber ++; return jis; }
String dirWorkerDataFolder = PathUtils.concatPath(dirPath.trim(), workerDataFolder); UnderFileSystem ufs = UnderFileSystem.get(dirWorkerDataFolder, tachyonConf); if (ufs.exists(dirWorkerDataFolder)) { if (!formatFolder(name, dirWorkerDataFolder, tachyonConf)) { System.exit(-1);
if (!ufs.exists(ufsAddrRootPath.toString())) { throw new FileNotFoundException("ufs path " + ufsAddrRootPath + " not found.");
LOG.info("persist file " + fileId + " at " + dstPath); String parentPath = PathUtils.concatPath(ufsRoot, uri.getParent().getPath()); if (!mUfs.exists(parentPath) && !mUfs.mkdirs(parentPath, true)) { throw new IOException("Failed to create " + parentPath);
String ufsPath = mMountTable.resolve(mInodeTree.getPath(delInode)).toString(); UnderFileSystem ufs = UnderFileSystem.get(ufsPath, MasterContext.getConf()); if (!ufs.exists(ufsPath)) { LOG.warn("File does not exist the underfs: " + ufsPath); } else if (!ufs.delete(ufsPath, true)) {
String tmpPath = PathUtils.temporaryFileName(mFileId, mNonce, mUfsPath); UnderFileSystem ufs = UnderFileSystem.get(tmpPath, ClientContext.getConf()); if (!ufs.exists(tmpPath)) { FileInfo fileInfo = getFileInfo(); mUfsPath = fileInfo.getUfsPath();
UnderFileSystem ufs = UnderFileSystem.get(tmpPath, WorkerContext.getConf()); try { if (!ufs.exists(tmpPath)) {
UnderFileSystem ufs = UnderFileSystem.get(ufsSrcPath, MasterContext.getConf()); String parentPath = new TachyonURI(ufsDstPath).getParent().toString(); if (!ufs.exists(parentPath) && !ufs.mkdirs(parentPath, true)) { LOG.error("Failed to create " + parentPath); return false;
if (!ufs.exists(ufsPath.getPath())) { throw new FileDoesNotExistException(ufsPath.getPath());