/** * Randomly create a BlockInfo object. * * @return the created BlockInfo object */ public static BlockInfo createRandom() { BlockInfo result = new BlockInfo(); Random random = new Random(); long blockId = random.nextLong(); long length = random.nextLong(); List<BlockLocation> locations = new ArrayList<>(); long numLocations = random.nextInt(10); for (int i = 0; i < numLocations; i++) { locations.add(BlockLocationTest.createRandom()); } result.setBlockId(blockId); result.setLength(length); result.setLocations(locations); return result; } }
/** * Converts a proto type to a wire type. * * @param blockPInfo the proto type to convert * @return the converted wire type */ public static BlockInfo fromProto(alluxio.grpc.BlockInfo blockPInfo) { BlockInfo blockInfo = new BlockInfo(); blockInfo.setBlockId(blockPInfo.getBlockId()); blockInfo.setLength(blockPInfo.getLength()); blockInfo.setLocations(map(GrpcUtils::fromProto, blockPInfo.getLocationsList())); return blockInfo; }
new BlockInfo().setBlockId(blockId).setLength(block.getLength()).setLocations(locations));
private void testGetInStreamFallback(int workerCount, boolean isPersisted, int[] blockLocations, Map<Integer, Long> failedWorkers, int expectedWorker) throws Exception { WorkerNetAddress[] workers = new WorkerNetAddress[workerCount]; Arrays.setAll(workers, i -> new WorkerNetAddress().setHost(String.format("worker-%d", i))); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID) .setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList())); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(isPersisted) .setBlockIds(Collections.singletonList(BLOCK_ID)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class); when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg .getArgumentAt(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next() .getNetAddress()); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); options.setUfsReadLocationPolicy(mockPolicy); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); when(mMasterClient.getWorkerInfoList()).thenReturn(Arrays.stream(workers) .map(x -> new WorkerInfo().setAddress(x)).collect((Collectors.toList()))); Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream() .map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); BlockInStream inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses); assertEquals(workers[expectedWorker], inStream.getAddress()); } }
/** * Helper function to select executors. * * @param blockLocations where the block is store currently * @param numReplicas how many replicas to replicate or evict * @param workerInfoList a list of current available job workers * @return the selection result */ private Map<WorkerInfo, SerializableVoid> selectExecutorsTestHelper( List<BlockLocation> blockLocations, int numReplicas, List<WorkerInfo> workerInfoList) throws Exception { BlockInfo blockInfo = new BlockInfo().setBlockId(TEST_BLOCK_ID); blockInfo.setLocations(blockLocations); when(mMockBlockStore.getInfo(TEST_BLOCK_ID)).thenReturn(blockInfo); PowerMockito.mockStatic(AlluxioBlockStore.class); when(AlluxioBlockStore.create(mMockFileSystemContext)).thenReturn(mMockBlockStore); String path = "/test"; ReplicateConfig config = new ReplicateConfig(path, TEST_BLOCK_ID, numReplicas); ReplicateDefinition definition = new ReplicateDefinition(mMockFileSystemContext, mMockFileSystem); return definition.selectExecutors(config, workerInfoList, mMockJobMasterContext); }
new FileInfo().setPath(path).setBlockIds(Lists.newArrayList(TEST_BLOCK_ID)) .setFileBlockInfos(Lists.newArrayList( new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(TEST_BLOCK_ID))))); when(mMockFileSystem.getStatus(any(AlluxioURI.class))).thenReturn(status); any(OutStreamOptions.class))).thenReturn(mockOutStream); when(mMockBlockStore.getInfo(TEST_BLOCK_ID)) .thenReturn(new BlockInfo().setBlockId(TEST_BLOCK_ID) .setLocations(Lists.newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1)))); PowerMockito.mockStatic(AlluxioBlockStore.class);
@Test public void scheduleAsyncPersist() throws Exception { DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster)); AlluxioURI path = new AlluxioURI("/test"); long blockId = 0; long workerId = 1; long fileId = 2; List<FileBlockInfo> blockInfoList = new ArrayList<>(); BlockLocation location = new BlockLocation().setWorkerId(workerId); blockInfoList.add(new FileBlockInfo().setBlockInfo( new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location)))); when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList); when(mFileSystemMaster.getFileId(path)).thenReturn(fileId); when(mFileSystemMaster.getPath(fileId)).thenReturn(path); when(mFileSystemMaster.getFileInfo(fileId)) .thenReturn(new FileInfo().setLength(1).setCompleted(true)); handler.scheduleAsyncPersistence(path); List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId); assertEquals(1, persistFiles.size()); assertEquals(Lists.newArrayList(blockId), persistFiles.get(0).getBlockIds()); }
/** * Tests persistence after deletion of files. */ @Test public void persistenceFileAfterDeletion() throws Exception { DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster)); AlluxioURI path = new AlluxioURI("/test"); long blockId = 0; long workerId = 1; long fileId = 2; List<FileBlockInfo> blockInfoList = new ArrayList<>(); BlockLocation location = new BlockLocation().setWorkerId(workerId); blockInfoList.add(new FileBlockInfo().setBlockInfo( new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location)))); when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList); when(mFileSystemMaster.getFileId(path)).thenReturn(fileId); when(mFileSystemMaster.getPath(fileId)).thenReturn(path); when(mFileSystemMaster.getFileInfo(fileId)) .thenReturn(new FileInfo().setLength(1).setCompleted(true)); handler.scheduleAsyncPersistence(path); when(mFileSystemMaster.getFileInfo(fileId)) .thenThrow(new FileDoesNotExistException("no file")); List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId); assertEquals(0, persistFiles.size()); } }
/** * Helper function to select executors. * * @param blockLocations where the block is stored currently * @param replicas how many replicas to evict * @param workerInfoList a list of currently available job workers * @return the selection result */ private Map<WorkerInfo, SerializableVoid> selectExecutorsTestHelper( List<BlockLocation> blockLocations, int replicas, List<WorkerInfo> workerInfoList) throws Exception { BlockInfo blockInfo = new BlockInfo().setBlockId(TEST_BLOCK_ID); blockInfo.setLocations(blockLocations); Mockito.when(mMockBlockStore.getInfo(TEST_BLOCK_ID)).thenReturn(blockInfo); PowerMockito.mockStatic(AlluxioBlockStore.class); PowerMockito.when(AlluxioBlockStore.create(mMockFileSystemContext)).thenReturn(mMockBlockStore); EvictConfig config = new EvictConfig(TEST_BLOCK_ID, replicas); EvictDefinition definition = new EvictDefinition(mMockFileSystemContext); return definition.selectExecutors(config, workerInfoList, mMockJobMasterContext); }
.thenThrow(new UnavailableException("failed to connect to " + WORKER_NET_ADDRESS_LOCAL.getHost())); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID) .setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList()));
@Test public void selectExecutorsMissingLocationTest() throws Exception { AlluxioURI uri = new AlluxioURI("/test"); PersistConfig config = new PersistConfig(uri.getPath(), -1, true, ""); long blockId = 1; BlockInfo blockInfo = new BlockInfo().setBlockId(blockId); FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo); FileInfo testFileInfo = new FileInfo(); testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo)); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); try { new PersistDefinition(mMockFileSystemContext, mMockFileSystem).selectExecutors(config, Lists.newArrayList(new WorkerInfo()), mMockJobMasterContext); } catch (Exception e) { Assert.assertEquals("Block " + blockId + " does not exist", e.getMessage()); } } }
@Before public void before() throws Exception { BlockWorkerClient workerClient = PowerMockito.mock(BlockWorkerClient.class); ClientCallStreamObserver requestObserver = PowerMockito.mock(ClientCallStreamObserver.class); when(requestObserver.isReady()).thenReturn(true); when(workerClient.openLocalBlock(any(StreamObserver.class))) .thenAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { StreamObserver<OpenLocalBlockResponse> observer = invocation.getArgumentAt(0, StreamObserver.class); observer.onNext(OpenLocalBlockResponse.newBuilder().setPath("/tmp").build()); observer.onCompleted(); return requestObserver; } }); mMockContext = PowerMockito.mock(FileSystemContext.class); PowerMockito.when(mMockContext.acquireBlockWorkerClient(Matchers.any(WorkerNetAddress.class))) .thenReturn(workerClient); PowerMockito.when(mMockContext.getClientContext()).thenReturn(ClientContext.create(mConf)); PowerMockito.when(mMockContext.getConf()).thenReturn(mConf); PowerMockito.doNothing().when(mMockContext) .releaseBlockWorkerClient(Matchers.any(WorkerNetAddress.class), Matchers.any(BlockWorkerClient.class)); mInfo = new BlockInfo().setBlockId(1); mOptions = new InStreamOptions(new URIStatus(new FileInfo().setBlockIds(Collections .singletonList(1L))), mConf); }
@Test public void selectExecutorsTest() throws Exception { AlluxioURI uri = new AlluxioURI("/test"); PersistConfig config = new PersistConfig(uri.getPath(), -1, true, ""); WorkerNetAddress workerNetAddress = new WorkerNetAddress().setDataPort(10); WorkerInfo workerInfo = new WorkerInfo().setAddress(workerNetAddress); long blockId = 1; BlockInfo blockInfo = new BlockInfo().setBlockId(blockId); FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo); BlockLocation location = new BlockLocation(); location.setWorkerAddress(workerNetAddress); blockInfo.setLocations(Lists.newArrayList(location)); FileInfo testFileInfo = new FileInfo(); testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo)); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); Map<WorkerInfo, SerializableVoid> result = new PersistDefinition(mMockFileSystemContext, mMockFileSystem).selectExecutors(config, Lists.newArrayList(workerInfo), mMockJobMasterContext); Assert.assertEquals(1, result.size()); Assert.assertEquals(workerInfo, result.keySet().iterator().next()); }
@Test public void getInStreamRemote() throws Exception { WorkerNetAddress remote1 = new WorkerNetAddress().setHost("remote1"); WorkerNetAddress remote2 = new WorkerNetAddress().setHost("remote2"); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays .asList(new BlockLocation().setWorkerAddress(remote1), new BlockLocation().setWorkerAddress(remote2))); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); // We should sometimes get remote1 and sometimes get remote2. Set<WorkerNetAddress> results = new HashSet<>(); for (int i = 0; i < 40; i++) { results.add(mBlockStore.getInStream(BLOCK_ID, new InStreamOptions( new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf)) .getAddress()); } assertEquals(Sets.newHashSet(remote1, remote2), results); }
@Test public void getBlockInfo() throws Exception { // Create a worker with a block. long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1); long blockId = 1L; long blockLength = 20L; mBlockMaster.workerRegister(worker1, Arrays.asList("MEM"), ImmutableMap.of("MEM", 100L), ImmutableMap.of("MEM", 0L), NO_BLOCKS_ON_TIERS, RegisterWorkerPOptions.getDefaultInstance()); mBlockMaster.commitBlock(worker1, 50L, "MEM", blockId, blockLength); BlockLocation blockLocation = new BlockLocation() .setTierAlias("MEM") .setWorkerAddress(NET_ADDRESS_1) .setWorkerId(worker1); BlockInfo expectedBlockInfo = new BlockInfo() .setBlockId(1L) .setLength(20L) .setLocations(ImmutableList.of(blockLocation)); assertEquals(expectedBlockInfo, mBlockMaster.getBlockInfo(blockId)); }
for (int i = 0; i < NUM_STREAMS; i++) { blockIds.add((long) i); FileBlockInfo fbInfo = new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(i)); fileBlockInfos.add(fbInfo); final byte[] input = BufferUtils
@Test public void getInStreamLocal() throws Exception { WorkerNetAddress remote = new WorkerNetAddress().setHost("remote"); WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL); // Mock away gRPC usage. OpenLocalBlockResponse response = OpenLocalBlockResponse.newBuilder().setPath("/tmp").build(); when(mWorkerClient.openLocalBlock(any(StreamObserver.class))) .thenAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { StreamObserver<OpenLocalBlockResponse> observer = invocation.getArgumentAt(0, StreamObserver.class); observer.onNext(response); observer.onCompleted(); return mStreamObserver; } }); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays .asList(new BlockLocation().setWorkerAddress(remote), new BlockLocation().setWorkerAddress(local))); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); assertEquals(local, mBlockStore.getInStream(BLOCK_ID, new InStreamOptions( new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf)) .getAddress()); }
@Test public void getInStreamUfs() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1"); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2"); BlockInfo info = new BlockInfo().setBlockId(0); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); OpenFilePOptions readOptions = OpenFilePOptions.newBuilder() .setFileReadLocationPolicy(MockFileWriteLocationPolicy.class.getTypeName()).build(); InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, sConf); ((MockFileWriteLocationPolicy) options.getUfsReadLocationPolicy()) .setHosts(Arrays.asList(worker1, worker2)); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn( Arrays.asList(new WorkerInfo().setAddress(worker1), new WorkerInfo().setAddress(worker2))); // Location policy chooses worker1 first. assertEquals(worker1, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); // Location policy chooses worker2 second. assertEquals(worker2, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); }
/** * Creates a new instance of {@link BlockInfo} from a thrift representation. * * @param blockInfo the thrift representation of a block information * @return the instance */ public static BlockInfo fromThrift(alluxio.thrift.BlockInfo blockInfo) { return new BlockInfo() .setBlockId(blockInfo.getBlockId()) .setLength(blockInfo.getLength()) .setLocations(map(BlockLocation::fromThrift, blockInfo.getLocations())); }
return new BlockInfo().setBlockId(masterBlockInfo.getBlockId()) .setLength(masterBlockInfo.getLength()).setLocations(locations);