public String getBlockSizeBytes() { if (mIsDirectory) { return ""; } else { return FormatUtils.getSizeFromBytes(mBlockSizeBytes); } }
private void writeFile(long fileId) throws IOException { ByteBuffer buf = ByteBuffer.allocate(mNumbers * 4); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mNumbers; k ++) { buf.putInt(k); } buf.flip(); LOG.debug("Writing data..."); buf.flip(); long startTimeMs = CommonUtils.getCurrentMs(); FileOutStream os = new FileOutStream(fileId, mWriteOptions); os.write(buf.array()); os.close(); LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "writeFile to file " + mFilePath)); }
public void io() throws IOException { if (sDebugMode) { mBuf.flip(); LOG.info(FormatUtils.byteBufferToString(mBuf));
/** * Marks a block as committed, but without a worker location. This means the block is only in ufs. * Called by internal masters. * * @param blockId the id of the block to commit * @param length the length of the block */ public void commitBlockInUFS(long blockId, long length) { LOG.debug("Commit block to ufs: {}", FormatUtils.parametersToString(blockId, length)); synchronized (mBlocks) { MasterBlockInfo masterBlockInfo = mBlocks.get(blockId); if (masterBlockInfo == null) { // The block has not been committed previously, so add the metadata to commit the block. masterBlockInfo = new MasterBlockInfo(blockId, length); mBlocks.put(blockId, masterBlockInfo); writeJournalEntry( new BlockInfoEntry(masterBlockInfo.getBlockId(), masterBlockInfo.getLength())); flushJournal(); } } }
private void initStorageTier() throws BlockAlreadyExistsException, IOException, WorkerOutOfSpaceException { String workerDataFolder = WorkerContext.getConf().get(Constants.WORKER_DATA_FOLDER); String tierDirPathConf = String.format(Constants.WORKER_TIERED_STORE_LEVEL_DIRS_PATH_FORMAT, mTierLevel); String[] dirPaths = WorkerContext.getConf().get(tierDirPathConf).split(","); // Add the worker data folder path after each storage directory, the final path will be like // /mnt/ramdisk/tachyonworker for (int i = 0; i < dirPaths.length; i ++) { dirPaths[i] = PathUtils.concatPath(dirPaths[i].trim(), workerDataFolder); } String tierDirCapacityConf = String.format(Constants.WORKER_TIERED_STORE_LEVEL_DIRS_QUOTA_FORMAT, mTierLevel); String[] dirQuotas = WorkerContext.getConf().get(tierDirCapacityConf).split(","); mDirs = new ArrayList<StorageDir>(dirPaths.length); long totalCapacity = 0; for (int i = 0; i < dirPaths.length; i ++) { int index = i >= dirQuotas.length ? dirQuotas.length - 1 : i; long capacity = FormatUtils.parseSpaceSize(dirQuotas[index]); totalCapacity += capacity; mDirs.add(StorageDir.newStorageDir(this, i, capacity, dirPaths[i])); } mCapacityBytes = totalCapacity; }
private boolean readFile(TachyonFileSystem tachyonFileSystem, long fileId) throws IOException, TachyonException { boolean pass = true; LOG.debug("Reading data..."); TachyonFile file = new TachyonFile(fileId); final long startTimeMs = CommonUtils.getCurrentMs(); FileInStream is = tachyonFileSystem.getInStream(file, mReadOptions); ByteBuffer buf = ByteBuffer.allocate((int) is.remaining()); is.read(buf.array()); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mNumbers; k ++) { pass = pass && (buf.getInt() == k); } is.close(); LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "readFile file " + mFilePath)); return pass; }
public void writePartition() throws IOException, TachyonException { if (sDebugMode) { mBuf.flip(); LOG.info(FormatUtils.byteBufferToString(mBuf)); } mBuf.flip(); for (int pId = mLeft; pId < mRight; pId ++) { final long startTimeMs = System.currentTimeMillis(); FileOutStream os = mTFS.getOutStream(new TachyonURI(sFileName + (pId + sBaseFileNumber))); for (int k = 0; k < sBlocksPerFile; k ++) { mBuf.putInt(0, k + mWorkerId); os.write(mBuf.array()); } os.close(); logPerIteration(startTimeMs, pId, "th WriteTachyonFile @ Worker ", pId); } }
long length) { LOG.debug("Commit block from worker: {}", FormatUtils.parametersToString(workerId, usedBytesOnTier, blockId, length)); synchronized (mBlocks) { synchronized (mWorkers) {
public String getSize() { if (mIsDirectory) { return ""; } else { return FormatUtils.getSizeFromBytes(mSize); } }
public static void createFiles() throws TachyonException, IOException { final long startTimeMs = CommonUtils.getCurrentMs(); for (int k = 0; k < sFiles; k ++) { TachyonFile file = sTFS.create(new TachyonURI(sFileName + (k + sBaseFileNumber))); long fileId = file.getFileId(); LOG.info( FormatUtils.formatTimeTakenMs(startTimeMs, "user_createFiles with fileId " + fileId)); } }
public void memoryCopyPartition() throws IOException { if (sDebugMode) { mBuf.flip(); LOG.info(FormatUtils.byteBufferToString(mBuf));
LOG.debug("createPath {}", FormatUtils.parametersToString(path));
public String getFreeCapacity() { return FormatUtils.getSizeFromBytes(mFreeBytes); }
private long createFile(TachyonFileSystem tachyonFileSystem) throws IOException, TachyonException { LOG.debug("Creating file..."); long startTimeMs = CommonUtils.getCurrentMs(); CreateOptions createOptions = (new CreateOptions.Builder(ClientContext.getConf())) .setBlockSizeBytes(mWriteOptions.getBlockSizeBytes()).setRecursive(true) .setTTL(mWriteOptions.getTTL()) .setUnderStorageType(mWriteOptions.getUnderStorageType()).build(); TachyonFile tFile = tachyonFileSystem.create(mFilePath, createOptions); long fileId = tFile.getFileId(); LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "createFile with fileId " + fileId)); return fileId; }
mBuf.order(ByteOrder.nativeOrder()); mBuf.flip(); LOG.info(FormatUtils.byteBufferToString(mBuf));
public String getCapacity() { return FormatUtils.getSizeFromBytes(mCapacityBytes); }
public String getUsedCapacity() { return FormatUtils.getSizeFromBytes(mUsedBytes); }
public String getCapacity() { return FormatUtils.getSizeFromBytes(mCapacityBytes); }
public String getUsedMemory() { return FormatUtils.getSizeFromBytes(mUsedBytes); }
request.setAttribute("capacityBytes", FormatUtils.getSizeFromBytes(capacityBytes)); request.setAttribute("usedBytes", FormatUtils.getSizeFromBytes(usedBytes));