/** * @return true if the block is in some worker's memory, false otherwise */ private boolean isInMemory(BlockInfo blockInfo) { for (BlockLocation location : blockInfo.getLocations()) { if (location.getTier() == StorageLevelAlias.MEM.getValue()) { return true; } } return false; }
public UiBlockInfo(FileBlockInfo fileBlockInfo) { Preconditions.checkNotNull(fileBlockInfo); mId = fileBlockInfo.getBlockInfo().getBlockId(); mBlockLength = fileBlockInfo.getBlockInfo().getLength(); mInMemory = isInMemory(fileBlockInfo.getBlockInfo()); mLastAccessTimeMs = -1; addLocations(fileBlockInfo); }
FileBlockInfo info = blocks.get(k); long offset = info.getOffset(); long end = offset + info.blockInfo.getLength(); List<NetAddress> addrs = Lists.newArrayList(); for (tachyon.thrift.BlockLocation location : info.getBlockInfo().getLocations()) { addrs.add(location.getWorkerAddress()); CommonUtils.toStringArray(hosts), offset, info.blockInfo.getLength()));
return new LocalBlockInStream(blockId, blockInfo.getLength(), new InetSocketAddress(workerNetAddress.getHost(), workerNetAddress.getDataPort())); } catch (IOException e) { return new RemoteBlockInStream(blockId, blockInfo.getLength(), new InetSocketAddress(workerNetAddress.getHost(), workerNetAddress.getDataPort())); } finally {
/** * Promotes block back to top layer after access. * * @param blockIndex the index of the block * @return true if success, false otherwise * @throws IOException if the underlying file does not exist or its metadata is corrupted */ public boolean promoteBlock(int blockIndex) throws IOException { FileBlockInfo fileBlockInfo = getClientBlockInfo(blockIndex); return mTachyonFS.promoteBlock(fileBlockInfo.blockInfo.getBlockId()); }
/** * Creates a {@link BlockInfo} form a given {@link MasterBlockInfo}, by populating worker * locations. * * mWorkers should already be locked before calling this method. * * @param masterBlockInfo the {@link MasterBlockInfo} * @return a {@link BlockInfo} from a {@link MasterBlockInfo}. Populates worker locations */ private BlockInfo generateBlockInfo(MasterBlockInfo masterBlockInfo) { // "Join" to get all the addresses of the workers. List<BlockLocation> locations = new ArrayList<BlockLocation>(); for (MasterBlockLocation masterBlockLocation : masterBlockInfo.getBlockLocations()) { MasterWorkerInfo workerInfo = mWorkers.getFirstByField(mIdIndex, masterBlockLocation.getWorkerId()); if (workerInfo != null) { locations.add(new BlockLocation(masterBlockLocation.getWorkerId(), workerInfo.getAddress(), masterBlockLocation.getTier())); } } return new BlockInfo(masterBlockInfo.getBlockId(), masterBlockInfo.getLength(), locations); }
for (int i = 0; i < blockInfoList.size(); i ++) { BlockInfo blockInfo = blockInfoList.get(i); fileLength += blockInfo.getLength(); if (i < blockInfoList.size() - 1 && blockInfo.getLength() != fileBlockSize) { throw new BlockInfoException( "Block index " + i + " has a block size smaller than the file block size ("
/** * Get the block id by the file id and block index. it will check whether the file and the block * exist. * * @param fileId the file id * @param blockIndex The index of the block in the file. * @return the block id if exists * @throws IOException if the file does not exist, or connection issue was encountered */ public synchronized long getBlockId(long fileId, int blockIndex) throws IOException { FileInfo info = getFileStatus(fileId, true); if (info == null) { throw new IOException("File " + fileId + " does not exist."); } if (info.blockIds.size() > blockIndex) { return info.blockIds.get(blockIndex); } try { return mFSMasterClient.getFileBlockInfo(fileId, blockIndex).blockInfo.getBlockId(); } catch (TachyonException e) { throw new IOException(e); } }
/** * @return true if the given block is in some worker's memory, false otherwise */ private boolean isInMemory(BlockInfo blockInfo) { for (BlockLocation location : blockInfo.getLocations()) { if (location.getTier() == StorageLevelAlias.MEM.getValue()) { return true; } } return false; }
/** * Get the in-memory percentage of an Inode. For a file that has all blocks in memory, it returns * 100; for a file that has no block in memory, it returns 0. Returns 0 for a directory. * * @param inode the inode * @return the in memory percentage */ private int getInMemoryPercentage(Inode inode) { if (!inode.isFile()) { return 0; } InodeFile inodeFile = (InodeFile) inode; long length = inodeFile.getLength(); if (length == 0) { return 100; } long inMemoryLength = 0; for (BlockInfo info : mBlockMaster.getBlockInfoList(inodeFile.getBlockIds())) { if (isInMemory(info)) { inMemoryLength += info.getLength(); } } return (int) (inMemoryLength * 100 / length); }
/** * Returns the local filename for the block if that file exists on the local file system. This is * an alpha power-api feature for applications that want short-circuit-read files directly. There * is no guarantee that the file still exists after this call returns, as Tachyon may evict blocks * from memory at any time. * * @param blockIndex The index of the block in the file * @return filename on local file system or null if file not present on local file system * @throws IOException if the underlying file does not exist or its metadata is corrupted */ public String getLocalFilename(int blockIndex) throws IOException { FileBlockInfo fileBlockInfo = getClientBlockInfo(blockIndex); long blockId = fileBlockInfo.blockInfo.getBlockId(); int blockLockId = mTachyonFS.getBlockLockId(); String filename = mTachyonFS.lockBlock(blockId, blockLockId); if (filename != null) { mTachyonFS.unlockBlock(blockId, blockLockId); } return filename; }
private void addLocations(FileBlockInfo fileBlockInfo) { Set<String> locations = Sets.newHashSet(); // add tachyon locations for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { locations.add(location.getWorkerAddress().getHost()); } // add underFS locations for (NetAddress address : fileBlockInfo.getUfsLocations()) { locations.add(address.getHost()); } mLocations.addAll(locations); }
BlockMasterClient blockMasterClient = mContext.acquireMasterClient(); try { blockSize = blockMasterClient.getBlockInfo(blockId).getLength(); } finally { mContext.releaseMasterClient(blockMasterClient);
/** * Returns the net address of all the location hosts * * @return the list of those net address, in String * @throws IOException if the underlying file does not exist or its metadata is corrupted */ public List<String> getLocationHosts() throws IOException { List<String> ret = Lists.newArrayList(); if (getNumberOfBlocks() > 0) { // add tachyon locations first List<BlockLocation> blockLocations = getClientBlockInfo(0).getBlockInfo().getLocations(); if (blockLocations != null) { for (BlockLocation location : blockLocations) { ret.add(location.workerAddress.host); } } // under FS locations List<NetAddress> underFsLocations = getClientBlockInfo(0).getUfsLocations(); if (underFsLocations != null) { for (NetAddress location : underFsLocations) { ret.add(location.host); } } } return ret; }
/** * Attempts to promote a block in Tachyon space. If the block is not present, this method will * return without an error. If the block is present in multiple workers, only one worker will * receive the promotion request. * * @param blockId the id of the block to promote * @throws IOException if the block does not exist */ public void promote(long blockId) throws IOException { BlockMasterClient blockMasterClient = mContext.acquireMasterClient(); try { BlockInfo info = blockMasterClient.getBlockInfo(blockId); if (info.getLocations().isEmpty()) { // Nothing to promote return; } // Get the first worker address for now, as this will likely be the location being read from // TODO: Get this location via a policy (possibly location is a parameter to promote) NetAddress workerAddr = info.getLocations().get(0).getWorkerAddress(); WorkerClient workerClient = mContext.acquireWorkerClient(workerAddr.getHost()); try { workerClient.promoteBlock(blockId); } finally { mContext.releaseWorkerClient(workerClient); } } finally { mContext.releaseMasterClient(blockMasterClient); } } }
/** * Displays a list of hosts that have the file specified in argv stored. * * @param path The TachyonURI path as the input of the command * @return 0 if command is successful, -1 if an error occurred * @throws IOException */ public int location(TachyonURI path) throws IOException { TachyonFile fd; FileInfo fInfo; try { fd = mTfs.open(path); fInfo = mTfs.getInfo(fd); } catch (TachyonException e) { throw new IOException(e.getMessage()); } System.out.println(path + " with file id " + fd.getFileId() + " is on nodes: "); for (long blockId : fInfo.getBlockIds()) { for (BlockLocation location : TachyonBlockStore.get().getInfo(blockId).getLocations()) { System.out.println(location.getWorkerAddress().getHost()); } } return 0; }
List<NetAddress> addrs = Lists.newArrayList(); for (BlockLocation location : blockInfo.getBlockInfo().getLocations()) { addrs.add(location.getWorkerAddress());