@Test public void getInStreamMissingBlock() throws Exception { URIStatus dummyStatus = new URIStatus( new FileInfo().setPersisted(false).setBlockIds(Collections.singletonList(0L))); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn(Collections.emptyList()); mException.expect(NotFoundException.class); mException.expectMessage("unavailable in both Alluxio and UFS"); mBlockStore.getInStream(BLOCK_ID, options).getAddress(); }
private void testGetInStreamFallback(int workerCount, boolean isPersisted, int[] blockLocations, Map<Integer, Long> failedWorkers, int expectedWorker) throws Exception { WorkerNetAddress[] workers = new WorkerNetAddress[workerCount]; Arrays.setAll(workers, i -> new WorkerNetAddress().setHost(String.format("worker-%d", i))); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID) .setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList())); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(isPersisted) .setBlockIds(Collections.singletonList(BLOCK_ID)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class); when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg .getArgumentAt(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next() .getNetAddress()); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); options.setUfsReadLocationPolicy(mockPolicy); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); when(mMasterClient.getWorkerInfoList()).thenReturn(Arrays.stream(workers) .map(x -> new WorkerInfo().setAddress(x)).collect((Collectors.toList()))); Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream() .map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); BlockInStream inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses); assertEquals(workers[expectedWorker], inStream.getAddress()); } }
@Test public void getInStreamNoWorkers() throws Exception { URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L))); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn(Collections.emptyList()); mException.expect(UnavailableException.class); mException.expectMessage("No Alluxio worker available"); mBlockStore.getInStream(BLOCK_ID, options).getAddress(); }
String path = "/test"; URIStatus status = new URIStatus( new FileInfo().setPath(path).setBlockIds(Lists.newArrayList(TEST_BLOCK_ID)) .setFileBlockInfos(Lists.newArrayList( new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(TEST_BLOCK_ID)))));
URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(persisted) .setBlockIds(Collections.singletonList(BLOCK_ID)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class);
result.setBlockIds(blockIds); result.setBlockSizeBytes(blockSizeBytes); result.setCacheable(cacheable);
@Before public void before() throws Exception { BlockWorkerClient workerClient = PowerMockito.mock(BlockWorkerClient.class); ClientCallStreamObserver requestObserver = PowerMockito.mock(ClientCallStreamObserver.class); when(requestObserver.isReady()).thenReturn(true); when(workerClient.openLocalBlock(any(StreamObserver.class))) .thenAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { StreamObserver<OpenLocalBlockResponse> observer = invocation.getArgumentAt(0, StreamObserver.class); observer.onNext(OpenLocalBlockResponse.newBuilder().setPath("/tmp").build()); observer.onCompleted(); return requestObserver; } }); mMockContext = PowerMockito.mock(FileSystemContext.class); PowerMockito.when(mMockContext.acquireBlockWorkerClient(Matchers.any(WorkerNetAddress.class))) .thenReturn(workerClient); PowerMockito.when(mMockContext.getClientContext()).thenReturn(ClientContext.create(mConf)); PowerMockito.when(mMockContext.getConf()).thenReturn(mConf); PowerMockito.doNothing().when(mMockContext) .releaseBlockWorkerClient(Matchers.any(WorkerNetAddress.class), Matchers.any(BlockWorkerClient.class)); mInfo = new BlockInfo().setBlockId(1); mOptions = new InStreamOptions(new URIStatus(new FileInfo().setBlockIds(Collections .singletonList(1L))), mConf); }
@Test public void getInStreamRemote() throws Exception { WorkerNetAddress remote1 = new WorkerNetAddress().setHost("remote1"); WorkerNetAddress remote2 = new WorkerNetAddress().setHost("remote2"); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays .asList(new BlockLocation().setWorkerAddress(remote1), new BlockLocation().setWorkerAddress(remote2))); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); // We should sometimes get remote1 and sometimes get remote2. Set<WorkerNetAddress> results = new HashSet<>(); for (int i = 0; i < 40; i++) { results.add(mBlockStore.getInStream(BLOCK_ID, new InStreamOptions( new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf)) .getAddress()); } assertEquals(Sets.newHashSet(remote1, remote2), results); }
mInfo.setBlockIds(blockIds); mInfo.setFileBlockInfos(fileBlockInfos); mStatus = new URIStatus(mInfo);
@Test public void getInStreamLocal() throws Exception { WorkerNetAddress remote = new WorkerNetAddress().setHost("remote"); WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL); // Mock away gRPC usage. OpenLocalBlockResponse response = OpenLocalBlockResponse.newBuilder().setPath("/tmp").build(); when(mWorkerClient.openLocalBlock(any(StreamObserver.class))) .thenAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { StreamObserver<OpenLocalBlockResponse> observer = invocation.getArgumentAt(0, StreamObserver.class); observer.onNext(response); observer.onCompleted(); return mStreamObserver; } }); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays .asList(new BlockLocation().setWorkerAddress(remote), new BlockLocation().setWorkerAddress(local))); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); assertEquals(local, mBlockStore.getInStream(BLOCK_ID, new InStreamOptions( new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf)) .getAddress()); }
@Test public void getInStreamUfs() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1"); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2"); BlockInfo info = new BlockInfo().setBlockId(0); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); OpenFilePOptions readOptions = OpenFilePOptions.newBuilder() .setFileReadLocationPolicy(MockFileWriteLocationPolicy.class.getTypeName()).build(); InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, sConf); ((MockFileWriteLocationPolicy) options.getUfsReadLocationPolicy()) .setHosts(Arrays.asList(worker1, worker2)); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn( Arrays.asList(new WorkerInfo().setAddress(worker1), new WorkerInfo().setAddress(worker2))); // Location policy chooses worker1 first. assertEquals(worker1, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); // Location policy chooses worker2 second. assertEquals(worker2, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); }
ret.setCompleted(isCompleted()); ret.setPersisted(isPersisted()); ret.setBlockIds(getBlockIds()); ret.setLastModificationTimeMs(getLastModificationTimeMs()); ret.setTtl(mTtl);
/** * Converts a proto type to a wire type. * * @param pInfo the proto representation of a file information * @return wire representation of the file information */ public static FileInfo fromProto(alluxio.grpc.FileInfo pInfo) { return new FileInfo().setFileId(pInfo.getFileId()).setName(pInfo.getName()) .setPath(pInfo.getPath()).setUfsPath(pInfo.getUfsPath()).setLength(pInfo.getLength()) .setBlockSizeBytes(pInfo.getBlockSizeBytes()).setCreationTimeMs(pInfo.getCreationTimeMs()) .setCompleted(pInfo.getCompleted()).setFolder(pInfo.getFolder()) .setPinned(pInfo.getPinned()).setCacheable(pInfo.getCacheable()) .setPersisted(pInfo.getPersisted()).setBlockIds(pInfo.getBlockIdsList()) .setLastModificationTimeMs(pInfo.getLastModificationTimeMs()).setTtl(pInfo.getTtl()) .setTtlAction(pInfo.getTtlAction()).setOwner(pInfo.getOwner()) .setGroup(pInfo.getGroup()).setMode(pInfo.getMode()) .setPersistenceState(pInfo.getPersistenceState()).setMountPoint(pInfo.getMountPoint()) .setFileBlockInfos(map(GrpcUtils::fromProto, pInfo.getFileBlockInfosList())) .setMountId(pInfo.getMountId()).setInAlluxioPercentage(pInfo.getInAlluxioPercentage()) .setInMemoryPercentage(pInfo.getInMemoryPercentage()) .setUfsFingerprint(pInfo.hasUfsFingerprint() ? pInfo.getUfsFingerprint() : Constants.INVALID_UFS_FINGERPRINT) .setAcl(pInfo.hasAcl() ? (fromProto(pInfo.getAcl())) : AccessControlList.EMPTY_ACL) .setDefaultAcl( pInfo.hasDefaultAcl() ? ((DefaultAccessControlList) fromProto(pInfo.getDefaultAcl())) : DefaultAccessControlList.EMPTY_DEFAULT_ACL) .setReplicationMax(pInfo.getReplicationMax()).setReplicationMin(pInfo.getReplicationMin()); }
@Override public FileInfo generateClientFileInfo(String path) { FileInfo ret = new FileInfo(); // note: in-Alluxio percentage is NOT calculated here, because it needs blocks info stored in // block master ret.setFileId(getId()); ret.setName(getName()); ret.setPath(path); ret.setLength(getLength()); ret.setBlockSizeBytes(getBlockSizeBytes()); ret.setCreationTimeMs(getCreationTimeMs()); ret.setCacheable(isCacheable()); ret.setFolder(isDirectory()); ret.setPinned(isPinned()); ret.setCompleted(isCompleted()); ret.setPersisted(isPersisted()); ret.setBlockIds(getBlockIds()); ret.setLastModificationTimeMs(getLastModificationTimeMs()); ret.setTtl(mTtl); ret.setTtlAction(mTtlAction); ret.setOwner(getOwner()); ret.setGroup(getGroup()); ret.setMode(getMode()); ret.setPersistenceState(getPersistenceState().toString()); ret.setMountPoint(false); ret.setUfsFingerprint(getUfsFingerprint()); return ret; }
.setCacheable(info.isCacheable()) .setPersisted(info.isPersisted()) .setBlockIds(info.getBlockIds()) .setInMemoryPercentage(info.getInMemoryPercentage()) .setLastModificationTimeMs(info.getLastModificationTimeMs())