/** * @param blockWorkers a list of block workers * @param blockInfo information about a block * @return the block worker hosts which are not storing the specified block */ private List<String> getWorkersWithoutBlock(List<BlockWorkerInfo> blockWorkers, FileBlockInfo blockInfo) { List<String> blockLocations = blockInfo.getBlockInfo().getLocations().stream() .map(location -> location.getWorkerAddress().getHost()) .collect(Collectors.toList()); return blockWorkers.stream() .filter(worker -> !blockLocations.contains(worker.getNetAddress().getHost())) .map(worker -> worker.getNetAddress().getHost()) .collect(Collectors.toList()); }
private void addLocations(FileBlockInfo fileBlockInfo) { Set<String> locations = new HashSet<>(); // add alluxio locations for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { locations.add(location.getWorkerAddress().getHost()); } // add underFS locations for (String location : fileBlockInfo.getUfsLocations()) { locations.add(HostAndPort.fromString(location).getHost()); } mLocations.addAll(locations); }
/** * @return true if the given block is in the top storage level in some worker, false otherwise */ private boolean isInTopStorageTier(BlockInfo blockInfo) { for (BlockLocation location : blockInfo.getLocations()) { if (mBlockMaster.getGlobalStorageTierAssoc().getOrdinal(location.getTierAlias()) == 0) { return true; } } return false; }
@Override public Map<WorkerInfo, SerializableVoid> selectExecutors(ReplicateConfig config, List<WorkerInfo> jobWorkerInfoList, JobMasterContext jobMasterContext) throws Exception { Preconditions.checkArgument(!jobWorkerInfoList.isEmpty(), "No worker is available"); long blockId = config.getBlockId(); int numReplicas = config.getReplicas(); Preconditions.checkArgument(numReplicas > 0); AlluxioBlockStore blockStore = AlluxioBlockStore.create(mFsContext); BlockInfo blockInfo = blockStore.getInfo(blockId); Set<String> hosts = new HashSet<>(); for (BlockLocation blockLocation : blockInfo.getLocations()) { hosts.add(blockLocation.getWorkerAddress().getHost()); } Map<WorkerInfo, SerializableVoid> result = Maps.newHashMap(); Collections.shuffle(jobWorkerInfoList); for (WorkerInfo workerInfo : jobWorkerInfoList) { // Select job workers that don't have this block locally to replicate if (!hosts.contains(workerInfo.getAddress().getHost())) { result.put(workerInfo, null); if (result.size() >= numReplicas) { break; } } } return result; }
@Override public Map<WorkerInfo, SerializableVoid> selectExecutors(EvictConfig config, List<WorkerInfo> jobWorkerInfoList, JobMasterContext jobMasterContext) throws Exception { Preconditions.checkArgument(!jobWorkerInfoList.isEmpty(), "No worker is available"); long blockId = config.getBlockId(); int numReplicas = config.getReplicas(); AlluxioBlockStore blockStore = AlluxioBlockStore.create(mFsContext); BlockInfo blockInfo = blockStore.getInfo(blockId); Set<String> hosts = new HashSet<>(); for (BlockLocation blockLocation : blockInfo.getLocations()) { hosts.add(blockLocation.getWorkerAddress().getHost()); } Map<WorkerInfo, SerializableVoid> result = Maps.newHashMap(); Collections.shuffle(jobWorkerInfoList); for (WorkerInfo workerInfo : jobWorkerInfoList) { // Select job workers that have this block locally to evict if (hosts.contains(workerInfo.getAddress().getHost())) { result.put(workerInfo, null); if (result.size() >= numReplicas) { break; } } } return result; }
@Override protected void runPlainPath(AlluxioURI plainPath, CommandLine cl) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(plainPath); System.out.println(plainPath + " with file id " + status.getFileId() + " is on nodes: "); AlluxioBlockStore blockStore = AlluxioBlockStore.create(mFsContext); for (long blockId : status.getBlockIds()) { for (BlockLocation location : blockStore.getInfo(blockId).getLocations()) { System.out.println(location.getWorkerAddress().getHost()); } } }
/** * Creates a new instance of {@link alluxio.util.webui.UIFileBlockInfo}. * * @param fileBlockInfo underlying {@link FileBlockInfo} */ public UIFileBlockInfo(FileBlockInfo fileBlockInfo) { Preconditions.checkNotNull(fileBlockInfo, "fileBlockInfo"); mId = fileBlockInfo.getBlockInfo().getBlockId(); mBlockLength = fileBlockInfo.getBlockInfo().getLength(); mLastAccessTimeMs = -1; addLocations(fileBlockInfo); for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) { mTierAliases.add(location.getTierAlias()); } }
/** * Converts wire type to proto type. * * @param blockInfo the wire type to convert * @return the converted proto type */ public static alluxio.grpc.BlockInfo toProto(BlockInfo blockInfo) { List<alluxio.grpc.BlockLocation> locations = new ArrayList<>(); for (BlockLocation location : blockInfo.getLocations()) { locations.add(toProto(location)); } return alluxio.grpc.BlockInfo.newBuilder().setBlockId(blockInfo.getBlockId()) .setLength(blockInfo.getLength()).addAllLocations(locations).build(); }
/** * Gets the in-Alluxio percentage of an Inode. For a file that has all blocks in Alluxio, it * returns 100; for a file that has no block in Alluxio, it returns 0. Returns 0 for a directory. * * @param inode the inode * @return the in alluxio percentage */ private int getInAlluxioPercentage(Inode inode) throws UnavailableException { if (!inode.isFile()) { return 0; } InodeFile inodeFile = inode.asFile(); long length = inodeFile.getLength(); if (length == 0) { return 100; } long inAlluxioLength = 0; for (BlockInfo info : mBlockMaster.getBlockInfoList(inodeFile.getBlockIds())) { if (!info.getLocations().isEmpty()) { inAlluxioLength += info.getLength(); } } return (int) (inAlluxioLength * 100 / length); }
@Test public void workerHeartbeatUpdatesRemovedBlocks() throws Exception { // Create a worker. long worker = mBlockMaster.getWorkerId(NET_ADDRESS_1); mBlockMaster.workerRegister(worker, Arrays.asList("MEM"), ImmutableMap.of("MEM", 100L), ImmutableMap.of("MEM", 0L), NO_BLOCKS_ON_TIERS, RegisterWorkerPOptions.getDefaultInstance()); long blockId = 1L; mBlockMaster.commitBlock(worker, 50L, "MEM", blockId, 20L); // Indicate that blockId is removed on the worker. mBlockMaster.workerHeartbeat(worker, null, ImmutableMap.of("MEM", 0L), ImmutableList.of(blockId), NO_BLOCKS_ON_TIERS, mMetrics); assertTrue(mBlockMaster.getBlockInfo(blockId).getLocations().isEmpty()); }
@Test public void workerHeartbeatUpdatesAddedBlocks() throws Exception { // Create two workers. long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1); mBlockMaster.workerRegister(worker1, Arrays.asList("MEM"), ImmutableMap.of("MEM", 100L), ImmutableMap.of("MEM", 0L), NO_BLOCKS_ON_TIERS, RegisterWorkerPOptions.getDefaultInstance()); long worker2 = mBlockMaster.getWorkerId(NET_ADDRESS_2); mBlockMaster.workerRegister(worker2, Arrays.asList("MEM"), ImmutableMap.of("MEM", 100L), ImmutableMap.of("MEM", 0L), NO_BLOCKS_ON_TIERS, RegisterWorkerPOptions.getDefaultInstance()); // Commit blockId to worker1. long blockId = 1L; mBlockMaster.commitBlock(worker1, 50L, "MEM", blockId, 20L); // Send a heartbeat from worker2 saying that it's added blockId. List<Long> addedBlocks = ImmutableList.of(blockId); mBlockMaster.workerHeartbeat(worker2, null, ImmutableMap.of("MEM", 0L), NO_BLOCKS, ImmutableMap.of("MEM", addedBlocks), mMetrics); // The block now has two locations. assertEquals(2, mBlockMaster.getBlockInfo(blockId).getLocations().size()); }
/** * Check if the two BlockInfo object are equal. * * @param a the first BlockInfo object to be checked * @param b the second BlockInfo object to be checked */ public void checkEquality(BlockInfo a, BlockInfo b) { Assert.assertEquals(a.getBlockId(), b.getBlockId()); Assert.assertEquals(a.getLength(), b.getLength()); Assert.assertEquals(a.getLocations(), b.getLocations()); Assert.assertEquals(a, b); }
/** * Tests that file information is still present after it has been freed after the TTL has been set * to 0. */ @Test public void ttlFileFree() throws Exception { long blockId = createFileWithSingleBlock(NESTED_FILE_URI); assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size()); // Set ttl & operation. mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.defaults( SetAttributePOptions.newBuilder().setTtl(0).setTtlAction(alluxio.grpc.TtlAction.FREE))); Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of("MEM", (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.<String, List<Long>>of(), mMetrics); // Verify the muted Free command on worker1. assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat); assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size()); }
/** * Tests {@link FileSystemMaster#free} on persisted file. */ @Test public void free() throws Exception { mNestedFileContext.setPersisted(true); long blockId = createFileWithSingleBlock(NESTED_FILE_URI); assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size()); // free the file mFileSystemMaster.free(NESTED_FILE_URI, FreeContext.defaults(FreePOptions.newBuilder() .setForced(false).setRecursive(false))); // Update the heartbeat of removedBlockId received from worker 1. Command heartbeat2 = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of("MEM", (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.<String, List<Long>>of(), mMetrics); // Verify the muted Free command on worker1. assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat2); assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size()); }
/** * Tests the {@link FileSystemMaster#free} method with a directory. */ @Test public void freeDir() throws Exception { mNestedFileContext.setPersisted(true); long blockId = createFileWithSingleBlock(NESTED_FILE_URI); assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size()); // free the dir mFileSystemMaster.free(NESTED_FILE_URI.getParent(), FreeContext.defaults(FreePOptions.newBuilder().setForced(true).setRecursive(true))); // Update the heartbeat of removedBlockId received from worker 1. Command heartbeat3 = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of("MEM", (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.<String, List<Long>>of(), mMetrics); // Verify the muted Free command on worker1. assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat3); assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size()); }
/** * Tests that TTL free of a file is not forgotten across restarts. */ @Test public void ttlFileFreeReplay() throws Exception { long blockId = createFileWithSingleBlock(NESTED_FILE_URI); assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size()); // Set ttl & operation. mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.defaults( SetAttributePOptions.newBuilder().setTtl(0).setTtlAction(alluxio.grpc.TtlAction.FREE))); // Simulate restart. stopServices(); startServices(); Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of("MEM", (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.<String, List<Long>>of(), mMetrics); // Verify the muted Free command on worker1. assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat); assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size()); }
/** * Tests that file information is still present after it has been freed after the parent * directory's TTL has been set to 0. */ @Test public void ttlDirectoryFree() throws Exception { CreateDirectoryContext directoryContext = CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true)); mFileSystemMaster.createDirectory(NESTED_URI, directoryContext); long blockId = createFileWithSingleBlock(NESTED_FILE_URI); assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size()); // Set ttl & operation. mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext.defaults( SetAttributePOptions.newBuilder().setTtl(0).setTtlAction(alluxio.grpc.TtlAction.FREE))); Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of("MEM", (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.<String, List<Long>>of(), mMetrics); // Verify the muted Free command on worker1. assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat); assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size()); }
/** * Tests {@link FileSystemMaster#free} on pinned file when forced flag is true. */ @Test public void freePinnedFileWithForce() throws Exception { mNestedFileContext.setPersisted(true); long blockId = createFileWithSingleBlock(NESTED_FILE_URI); mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setPinned(true))); assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size()); // free the file mFileSystemMaster.free(NESTED_FILE_URI, FreeContext.defaults(FreePOptions.newBuilder().setForced(true))); // Update the heartbeat of removedBlockId received from worker 1. Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of("MEM", (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.<String, List<Long>>of(), mMetrics); // Verify the muted Free command on worker1. assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat); assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size()); }
/** * Tests that TTL free of a directory is not forgotten across restarts. */ @Test public void ttlDirectoryFreeReplay() throws Exception { CreateDirectoryContext directoryContext = CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true)); mFileSystemMaster.createDirectory(NESTED_URI, directoryContext); long blockId = createFileWithSingleBlock(NESTED_FILE_URI); assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size()); // Set ttl & operation. mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext.defaults( SetAttributePOptions.newBuilder().setTtl(0).setTtlAction(alluxio.grpc.TtlAction.FREE))); // Simulate restart. stopServices(); startServices(); Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of("MEM", (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.<String, List<Long>>of(), mMetrics); // Verify the muted Free command on worker1. assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat); assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size()); }
/** * Tests the {@link FileSystemMaster#free} method with a directory with a file pinned when * forced flag is true. */ @Test public void freeDirWithPinnedFileAndForced() throws Exception { mNestedFileContext.setPersisted(true); long blockId = createFileWithSingleBlock(NESTED_FILE_URI); mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setPinned(true))); // free the parent dir of a pinned file with "forced" mFileSystemMaster.free(NESTED_FILE_URI.getParent(), FreeContext.defaults(FreePOptions.newBuilder().setForced(true).setRecursive(true))); // Update the heartbeat of removedBlockId received from worker 1. Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of("MEM", (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.<String, List<Long>>of(), mMetrics); // Verify the muted Free command on worker1. assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat); assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size()); }