@Override public boolean supportsPath(String path, TachyonConf conf) { if (path == null) { return false; } return UnderFileSystem.isHadoopUnderFS(path, conf); } }
/** * Create an underfs test bed and register the shutdown hook. * * @param baseDir base directory * @param tachyonConf Tachyon configuration * @throws IOException when the operation fails * @return an instance of the UnderFileSystemCluster class */ public static synchronized UnderFileSystemCluster get(String baseDir, TachyonConf tachyonConf) throws IOException { if (sUnderFSCluster == null) { sUnderFSCluster = getUnderFilesystemCluster(baseDir, tachyonConf); } if (!sUnderFSCluster.isStarted()) { sUnderFSCluster.start(); sUnderFSCluster.registerJVMOnExistHook(); } return sUnderFSCluster; }
/** * To clean up the test environment over underfs cluster system, so that we can re-use the running * system for the next test round instead of turning on/off it from time to time. This function is * expected to be called either before or after each test case which avoids certain overhead from * the bootstrap. * * @throws IOException when the operation fails */ public void cleanup() throws IOException { if (isStarted()) { String path = getUnderFilesystemAddress() + TachyonURI.SEPARATOR; UnderFileSystem ufs = UnderFileSystem.get(path, mTachyonConf); for (String p : ufs.list(path)) { ufs.delete(PathUtils.concatPath(path, p), true); } } }
/** * Moves the current log file to the completed folder, marking it as complete. If successful, the * current log file will no longer exist. The current log must already be closed before this call. * * @throws IOException */ private void completeCurrentLog() throws IOException { String currentLog = mJournal.getCurrentLogFilePath(); if (!mUfs.exists(currentLog)) { // All logs are already complete, so nothing to do. return; } if (!mUfs.exists(mCompletedDirectory)) { mUfs.mkdirs(mCompletedDirectory, true); } String completedLog = mJournal.getCompletedLogFilePath(mNextCompleteLogNumber); mUfs.rename(currentLog, completedLog); LOG.info("Completed current log: " + currentLog + " to completed log: " + completedLog); mNextCompleteLogNumber ++; }
/** * Deletes all of the logs in the completed folder. * * @throws IOException */ private void deleteCompletedLogs() throws IOException { LOG.info("Deleting all completed log files..."); // Loop over all complete logs starting from the beginning. // TODO(gpang): should the deletes start from the end? long logNumber = Journal.FIRST_COMPLETED_LOG_NUMBER; String logFilename = mJournal.getCompletedLogFilePath(logNumber); while (mUfs.exists(logFilename)) { LOG.info("Deleting completed log: " + logFilename); mUfs.delete(logFilename, true); logNumber ++; // generate the next completed log filename in the sequence. logFilename = mJournal.getCompletedLogFilePath(logNumber); } LOG.info("Finished deleting all completed log files."); // All complete logs are deleted. Reset the log number counter. mNextCompleteLogNumber = Journal.FIRST_COMPLETED_LOG_NUMBER; }
private void setUnderStoreStream(long pos) throws IOException { if (mUnderStoreStream != null) { mUnderStoreStream.close(); } UnderFileSystem ufs = UnderFileSystem.get(mUfsPath, ClientContext.getConf()); mUnderStoreStream = ufs.open(mUfsPath); mPos = 0; if (pos != skip(pos)) { throw new IOException("Failed to skip: " + pos); } } }
public LineageDataManager(BlockDataManager blockDataManager) { mBlockDataManager = Preconditions.checkNotNull(blockDataManager); mPersistedFiles = Lists.newArrayList(); mTachyonConf = WorkerContext.getConf(); // Create Under FileSystem Client String ufsAddress = mTachyonConf.get(Constants.UNDERFS_ADDRESS); mUfs = UnderFileSystem.get(ufsAddress, mTachyonConf); }
/** * @return the last modified time of the checkpoint file in ms * @throws IOException */ public long getCheckpointLastModifiedTimeMs() throws IOException { if (!mUfs.exists(mCheckpointPath)) { throw new IOException("Checkpoint file " + mCheckpointPath + " does not exist."); } mCheckpointLastModifiedTime = mUfs.getModificationTimeMs(mCheckpointPath); return mCheckpointLastModifiedTime; } }
private void connectToUFS() throws IOException { TachyonConf conf = MasterContext.getConf(); String ufsAddress = conf.get(Constants.UNDERFS_ADDRESS); UnderFileSystem ufs = UnderFileSystem.get(ufsAddress, conf); ufs.connectFromMaster(conf, NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC, conf)); } }
/** * Marks all logs as completed. */ public synchronized void completeAllLogs() throws IOException { LOG.info("Marking all logs as complete."); // Loop over all complete logs starting from the beginning, to determine the next log number. mNextCompleteLogNumber = Journal.FIRST_COMPLETED_LOG_NUMBER; String logFilename = mJournal.getCompletedLogFilePath(mNextCompleteLogNumber); while (mUfs.exists(logFilename)) { mNextCompleteLogNumber ++; // generate the next completed log filename in the sequence. logFilename = mJournal.getCompletedLogFilePath(mNextCompleteLogNumber); } completeCurrentLog(); }
/** * Returns the current log file output stream * * @return the output stream for the current log file * @throws IOException */ private OutputStream openCurrentLog() throws IOException { String currentLogFile = mJournal.getCurrentLogFilePath(); OutputStream os = mUfs.create(currentLogFile); LOG.info("Opened current log file: " + currentLogFile); return os; }
public void cleanupUnderfs() throws IOException { if (mUfsCluster != null) { mUfsCluster.cleanup(); } System.clearProperty("tachyon.underfs.address"); }
@Override public void run() { if (mUFSCluster != null) { try { mUFSCluster.shutdown(); } catch (IOException e) { System.out.println("Failed to shutdown underfs cluster: " + e); } } } }
/** * Add a shutdown hook. The {@link #shutdown} phase will be automatically called while the process * exists. * * @throws IOException when the operation fails */ public void registerJVMOnExistHook() throws IOException { Runtime.getRuntime().addShutdownHook(new ShutdownHook(this)); }
public void close() throws IOException { if (mCheckpointOutputStream != null) { mCheckpointOutputStream.close(); } if (mEntryOutputStream != null) { mEntryOutputStream.close(); } // Close the ufs. mUfs.close(); }
private static boolean formatFolder(String name, String folder, TachyonConf tachyonConf) throws IOException { UnderFileSystem ufs = UnderFileSystem.get(folder, tachyonConf); LOG.info("Formatting {}:{}", name, folder); if (ufs.exists(folder)) { for (String file : ufs.list(folder)) { if (!ufs.delete(PathUtils.concatPath(folder, file), true)) { LOG.info("Failed to remove {}:{}", name, file); return false; } } } else if (!ufs.mkdirs(folder, true)) { LOG.info("Failed to create {}:{}", name, folder); return false; } return true; }
/** * @param journal the handle to the journal */ JournalReader(Journal journal) { mJournal = Preconditions.checkNotNull(journal); TachyonConf conf = MasterContext.getConf(); mUfs = UnderFileSystem.get(mJournal.getDirectory(), conf); mCheckpointPath = mJournal.getCheckpointFilePath(); }
JournalWriter(Journal journal) { mJournal = Preconditions.checkNotNull(journal); mJournalDirectory = mJournal.getDirectory(); mCompletedDirectory = mJournal.getCompletedDirectory(); mTempCheckpointPath = mJournal.getCheckpointFilePath() + ".tmp"; TachyonConf conf = MasterContext.getConf(); mUfs = UnderFileSystem.get(mJournalDirectory, conf); mMaxLogSize = conf.getBytes(Constants.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX); }