/** * Kill Tachyon Master by 'kill -9' command. */ private static void killMaster() { String[] killMasterCommand = new String[]{"/usr/bin/env", "bash", "-c", "for pid in `ps -Aww -o pid,command | grep -i \"[j]ava\" | grep " + "\"tachyon.master.TachyonMaster\" | awk '{print $1}'`; do kill -9 \"$pid\"; done"}; try { Runtime.getRuntime().exec(killMasterCommand).waitFor(); CommonUtils.sleepMs(LOG, 1000); } catch (Exception e) { LOG.error("Error when killing Master", e); } }
public static DataServer createDataServer(final InetSocketAddress dataAddress, final BlockDataManager blockDataManager, TachyonConf conf) { try { return CommonUtils.createNewClassInstance( conf.<DataServer>getClass(Constants.WORKER_DATA_SERVER), new Class[] { InetSocketAddress.class, BlockDataManager.class, TachyonConf.class }, new Object[] { dataAddress, blockDataManager, conf }); } catch (Exception e) { throw Throwables.propagate(e); } } }
/** * @return Generated {@link WorkerInfo} for this worker */ public synchronized WorkerInfo generateClientWorkerInfo() { WorkerInfo ret = new WorkerInfo(); ret.id = mId; ret.address = mWorkerAddress; ret.lastContactSec = (int) ((CommonUtils.getCurrentMs() - mLastUpdatedTimeMs) / Constants.SECOND_MS); ret.state = "In Service"; ret.capacityBytes = mCapacityBytes; ret.usedBytes = mUsedBytes; ret.startTimeMs = mStartTimeMs; return ret; }
mJournalTailer = new JournalTailer(mMaster, mJournal); while (!mJournalTailer.checkpointExists()) { CommonUtils.sleepMs(LOG, mJournalTailerSleepTimeMs); if (mInitiateShutdown) { LOG.info("Journal tailer has been shutdown while waiting to load the checkpoint file."); if (mInitiateShutdown) { if (waitForShutdownStart == -1) { waitForShutdownStart = CommonUtils.getCurrentMs(); } else if ((CommonUtils.getCurrentMs() - waitForShutdownStart) > mShutdownQuietWaitTimeMs) { CommonUtils.sleepMs(LOG, mJournalTailerSleepTimeMs); CommonUtils.sleepMs(LOG, mJournalTailerSleepTimeMs); } catch (IOException ioe) {
hosts.add(addr.host); blockLocations.add(new BlockLocation(CommonUtils.toStringArray(names), CommonUtils.toStringArray(hosts), offset, info.blockInfo.getLength()));
sFileBytes / Constants.MB, fileBufferBytes / 1024, sBaseFileNumber); CommonUtils.warmUpLoop();
@Override public void run() { long lastCheckMs = System.currentTimeMillis(); while (mRunning) { // Check the time since last check, and wait until it is within check interval long lastIntervalMs = System.currentTimeMillis() - lastCheckMs; long toSleepMs = mCheckIntervalMs - lastIntervalMs; if (toSleepMs > 0) { CommonUtils.sleepMs(LOG, toSleepMs); } else { LOG.warn("Space reserver took: " + lastIntervalMs + ", expected: " + mCheckIntervalMs); } reserveSpace(); } }
/** * @param conf TachyonConf to get the type of {@link JournalFormatter} * @return the created formatter */ public static JournalFormatter createJournalFormatter(TachyonConf conf) { try { return CommonUtils.createNewClassInstance( conf.<JournalFormatter>getClass(Constants.MASTER_JOURNAL_FORMATTER_CLASS), null, null); } catch (Exception e) { throw Throwables.propagate(e); } } }
private void writeFile(long fileId) throws IOException { ByteBuffer buf = ByteBuffer.allocate(mNumbers * 4); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mNumbers; k ++) { buf.putInt(k); } buf.flip(); LOG.debug("Writing data..."); buf.flip(); long startTimeMs = CommonUtils.getCurrentMs(); FileOutStream os = new FileOutStream(fileId, mWriteOptions); os.write(buf.array()); os.close(); LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "writeFile to file " + mFilePath)); }
/** * Start Tachyon Master by executing the launch script. */ private static void startMaster() { String startMasterCommand = new TachyonConf().get(Constants.TACHYON_HOME) + "/bin/tachyon-start.sh master"; try { Runtime.getRuntime().exec(startMasterCommand).waitFor(); CommonUtils.sleepMs(LOG, 1000); } catch (Exception e) { LOG.error("Error when starting Master", e); } }
/** * @param conf Tachyon configuration * @return a new instance of <code>RemoteBlockReader</code> */ public static RemoteBlockReader createRemoteBlockReader(TachyonConf conf) { try { return CommonUtils.createNewClassInstance( conf.<RemoteBlockReader>getClass(Constants.USER_BLOCK_REMOTE_READER), null, null); } catch (Exception e) { throw Throwables.propagate(e); } } }
private boolean readFile(TachyonFileSystem tachyonFileSystem, long fileId) throws IOException, TachyonException { boolean pass = true; LOG.debug("Reading data..."); TachyonFile file = new TachyonFile(fileId); final long startTimeMs = CommonUtils.getCurrentMs(); FileInStream is = tachyonFileSystem.getInStream(file, mReadOptions); ByteBuffer buf = ByteBuffer.allocate((int) is.remaining()); is.read(buf.array()); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mNumbers; k ++) { pass = pass && (buf.getInt() == k); } is.close(); LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "readFile file " + mFilePath)); return pass; }
public static void main(String[] args) throws Exception { LocalTachyonCluster cluster = new LocalTachyonCluster(100, 8 * Constants.MB, Constants.GB); cluster.start(); CommonUtils.sleepMs(Constants.SECOND_MS); cluster.stop(); CommonUtils.sleepMs(Constants.SECOND_MS); cluster = new LocalTachyonCluster(100, 8 * Constants.MB, Constants.GB); cluster.start(); CommonUtils.sleepMs(Constants.SECOND_MS); cluster.stop(); CommonUtils.sleepMs(Constants.SECOND_MS); }
/** * @param conf TachyonConf to determine the scheduler type * @return the generated scheduler */ public static CheckpointScheduler createScheduler(TachyonConf conf, LineageStoreView store) { try { return CommonUtils.createNewClassInstance( conf.<CheckpointScheduler>getClass(Constants.MASTER_LINEAGE_CHECKPOINT_CLASS), new Class[] {LineageStoreView.class}, new Object[] {store}); } catch (Exception e) { throw Throwables.propagate(e); } } }
public static void createFiles() throws TachyonException, IOException { final long startTimeMs = CommonUtils.getCurrentMs(); for (int k = 0; k < sFiles; k ++) { TachyonFile file = sTFS.create(new TachyonURI(sFileName + (k + sBaseFileNumber))); long fileId = file.getFileId(); LOG.info( FormatUtils.formatTimeTakenMs(startTimeMs, "user_createFiles with fileId " + fileId)); } }
/** * Main loop for the cleanup, continuously look for zombie sessions */ @Override public void run() { long lastCheckMs = System.currentTimeMillis(); while (mRunning) { // Check the time since last check, and wait until it is within check interval long lastIntervalMs = System.currentTimeMillis() - lastCheckMs; long toSleepMs = mCheckIntervalMs - lastIntervalMs; if (toSleepMs > 0) { CommonUtils.sleepMs(LOG, toSleepMs); } else { LOG.warn("Session cleanup took: " + lastIntervalMs + ", expected: " + mCheckIntervalMs); } // Check if any sessions have become zombies, if so clean them up mBlockDataManager.cleanupSessions(); lastCheckMs = System.currentTimeMillis(); } }
/** * Creates a new <code>RemoteBlockWriter</code>. * * @param conf Tachyon configuration * @return a new instance of <code>RemoteBlockWriter</code> */ public static RemoteBlockWriter createRemoteBlockWriter(TachyonConf conf) { try { return CommonUtils.createNewClassInstance( conf.<RemoteBlockWriter>getClass(Constants.USER_BLOCK_REMOTE_WRITER), null, null); } catch (Exception e) { throw Throwables.propagate(e); } } }
@Override public void heartbeat() { LOG.debug("System status checking."); TachyonConf conf = MasterContext.getConf(); int masterWorkerTimeoutMs = conf.getInt(Constants.MASTER_WORKER_TIMEOUT_MS); synchronized (mBlocks) { synchronized (mWorkers) { Iterator<MasterWorkerInfo> iter = mWorkers.iterator(); while (iter.hasNext()) { MasterWorkerInfo worker = iter.next(); final long lastUpdate = CommonUtils.getCurrentMs() - worker.getLastUpdatedTimeMs(); if (lastUpdate > masterWorkerTimeoutMs) { LOG.error("The worker {} got timed out!", worker); mLostWorkers.add(worker); iter.remove(); processLostWorker(worker); } } } } }
public static void main(String[] args) throws Exception { LocalTachyonCluster cluster = new LocalTachyonCluster(100, 8 * Constants.MB, Constants.GB); cluster.start(); CommonUtils.sleepMs(Constants.SECOND_MS); cluster.stop(); CommonUtils.sleepMs(Constants.SECOND_MS); cluster = new LocalTachyonCluster(100, 8 * Constants.MB, Constants.GB); cluster.start(); CommonUtils.sleepMs(Constants.SECOND_MS); cluster.stop(); CommonUtils.sleepMs(Constants.SECOND_MS); }
/** * @param conf TachyonConf to determine the evictor type * @param view BlockMetadataManagerView to pass to Evictor * @param allocator an allocation policy * @return the generated Evictor */ public static Evictor createEvictor(TachyonConf conf, BlockMetadataManagerView view, Allocator allocator) { try { return CommonUtils.createNewClassInstance( conf.<Evictor>getClass(Constants.WORKER_EVICTOR_CLASS), new Class[]{BlockMetadataManagerView.class, Allocator.class}, new Object[]{view, allocator}); } catch (Exception e) { throw Throwables.propagate(e); } } }