/** * @return generated {@link WorkerInfo} for this worker */ public synchronized WorkerInfo generateClientWorkerInfo() { return new WorkerInfo().setId(mId).setAddress(mWorkerAddress).setLastContactSec( (int) ((CommonUtils.getCurrentMs() - mLastUpdatedTimeMs) / Constants.SECOND_MS)) .setState("In Service").setStartTimeMs(mStartTimeMs); }
@Before public void before() throws Exception { mCommandManager = new CommandManager(); // Create mock job info. mJobconfig = Mockito.mock(JobConfig.class, Mockito.withSettings().serializable()); Mockito.when(mJobconfig.getName()).thenReturn("mock"); mJobId = 1; // Create mock job definition. @SuppressWarnings("unchecked") JobDefinition<JobConfig, Serializable, Serializable> mockJobDefinition = Mockito.mock(JobDefinition.class); JobDefinitionRegistry singleton = PowerMockito.mock(JobDefinitionRegistry.class); Whitebox.setInternalState(JobDefinitionRegistry.class, "INSTANCE", singleton); Mockito.when(singleton.getJobDefinition(mJobconfig)).thenReturn(mockJobDefinition); mJobDefinition = mockJobDefinition; // Create test worker. mWorkerInfo = new WorkerInfo(); mWorkerInfo.setId(0); mWorkerInfoList = Lists.newArrayList(mWorkerInfo); mUfsManager = Mockito.mock(UfsManager.class); }
public static WorkerInfo createRandom() { WorkerInfo result = new WorkerInfo(); Random random = new Random(); long id = random.nextLong(); WorkerNetAddress address = WorkerNetAddressTest.createRandom(); int lastContactSec = random.nextInt(); long capacityBytes = random.nextLong(); long usedBytes = random.nextLong(); long startTimeMs = random.nextLong(); Map<String, Long> capacityBytesOnTiers = new HashMap<>(); capacityBytesOnTiers.put("MEM", capacityBytes); Map<String, Long> usedBytesOnTiers = new HashMap<>(); usedBytesOnTiers.put("MEM", usedBytes); String state = random.nextInt(1) == 1 ? "In Service" : "Out of Service"; result.setId(id); result.setAddress(address); result.setLastContactSec(lastContactSec); result.setCapacityBytes(capacityBytes); result.setUsedBytes(usedBytes); result.setStartTimeMs(startTimeMs); result.setState(state); result.setCapacityBytesOnTiers(capacityBytesOnTiers); result.setUsedBytesOnTiers(usedBytesOnTiers); return result; } }
@Test public void getOutStreamWithReplicated() throws Exception { File file = File.createTempFile("test", ".tmp"); CreateLocalBlockResponse response = CreateLocalBlockResponse.newBuilder() .setPath(file.getAbsolutePath()).build(); when(mWorkerClient.createLocalBlock(any(StreamObserver.class))) .thenAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { StreamObserver<CreateLocalBlockResponse> observer = invocation.getArgumentAt(0, StreamObserver.class); observer.onNext(response); return mStreamObserver; } }); when(mMasterClient.getWorkerInfoList()).thenReturn(Lists .newArrayList(new alluxio.wire.WorkerInfo().setAddress(WORKER_NET_ADDRESS_LOCAL), new alluxio.wire.WorkerInfo().setAddress(WORKER_NET_ADDRESS_REMOTE))); OutStreamOptions options = OutStreamOptions.defaults(sConf).setBlockSizeBytes(BLOCK_LENGTH) .setLocationPolicy(new MockFileWriteLocationPolicy( Lists.newArrayList(WORKER_NET_ADDRESS_LOCAL, WORKER_NET_ADDRESS_REMOTE))) .setWriteType(WriteType.MUST_CACHE).setReplicationMin(2); BlockOutStream stream = mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options); assertEquals(alluxio.client.block.stream.BlockOutStream.class, stream.getClass()); }
WorkerInfo info = new WorkerInfo(); Set<WorkerInfoField> checkedFieldRange = fieldRange != null ? fieldRange : new HashSet<>(Arrays.asList(WorkerInfoField.values()));
private void testGetInStreamFallback(int workerCount, boolean isPersisted, int[] blockLocations, Map<Integer, Long> failedWorkers, int expectedWorker) throws Exception { WorkerNetAddress[] workers = new WorkerNetAddress[workerCount]; Arrays.setAll(workers, i -> new WorkerNetAddress().setHost(String.format("worker-%d", i))); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID) .setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList())); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(isPersisted) .setBlockIds(Collections.singletonList(BLOCK_ID)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class); when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg .getArgumentAt(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next() .getNetAddress()); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); options.setUfsReadLocationPolicy(mockPolicy); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); when(mMasterClient.getWorkerInfoList()).thenReturn(Arrays.stream(workers) .map(x -> new WorkerInfo().setAddress(x)).collect((Collectors.toList()))); Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream() .map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); BlockInStream inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses); assertEquals(workers[expectedWorker], inStream.getAddress()); } }
when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); when(mMasterClient.getWorkerInfoList()).thenReturn(Arrays.stream(workers) .map(x -> new WorkerInfo().setAddress(x)).collect((Collectors.toList()))); Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream() .map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue()))
@Test public void selectExecutorsMissingLocationTest() throws Exception { AlluxioURI uri = new AlluxioURI("/test"); PersistConfig config = new PersistConfig(uri.getPath(), -1, true, ""); long blockId = 1; BlockInfo blockInfo = new BlockInfo().setBlockId(blockId); FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo); FileInfo testFileInfo = new FileInfo(); testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo)); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); try { new PersistDefinition(mMockFileSystemContext, mMockFileSystem).selectExecutors(config, Lists.newArrayList(new WorkerInfo()), mMockJobMasterContext); } catch (Exception e) { Assert.assertEquals("Block " + blockId + " does not exist", e.getMessage()); } } }
@Test public void selectExecutorsTest() throws Exception { AlluxioURI uri = new AlluxioURI("/test"); PersistConfig config = new PersistConfig(uri.getPath(), -1, true, ""); WorkerNetAddress workerNetAddress = new WorkerNetAddress().setDataPort(10); WorkerInfo workerInfo = new WorkerInfo().setAddress(workerNetAddress); long blockId = 1; BlockInfo blockInfo = new BlockInfo().setBlockId(blockId); FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo); BlockLocation location = new BlockLocation(); location.setWorkerAddress(workerNetAddress); blockInfo.setLocations(Lists.newArrayList(location)); FileInfo testFileInfo = new FileInfo(); testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo)); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); Map<WorkerInfo, SerializableVoid> result = new PersistDefinition(mMockFileSystemContext, mMockFileSystem).selectExecutors(config, Lists.newArrayList(workerInfo), mMockJobMasterContext); Assert.assertEquals(1, result.size()); Assert.assertEquals(workerInfo, result.keySet().iterator().next()); }
@Test public void getInStreamUfs() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1"); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2"); BlockInfo info = new BlockInfo().setBlockId(0); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); OpenFilePOptions readOptions = OpenFilePOptions.newBuilder() .setFileReadLocationPolicy(MockFileWriteLocationPolicy.class.getTypeName()).build(); InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, sConf); ((MockFileWriteLocationPolicy) options.getUfsReadLocationPolicy()) .setHosts(Arrays.asList(worker1, worker2)); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn( Arrays.asList(new WorkerInfo().setAddress(worker1), new WorkerInfo().setAddress(worker2))); // Location policy chooses worker1 first. assertEquals(worker1, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); // Location policy chooses worker2 second. assertEquals(worker2, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); }
/** * Converts a proto type to a wire type. * * @param workerInfo the proto type to convert * @return the converted wire type */ public static WorkerInfo fromProto(alluxio.grpc.WorkerInfo workerInfo) { return new WorkerInfo().setAddress(fromProto(workerInfo.getAddress())) .setCapacityBytes(workerInfo.getCapacityBytes()) .setCapacityBytesOnTiers(workerInfo.getCapacityBytesOnTiers()).setId(workerInfo.getId()) .setLastContactSec(workerInfo.getLastContactSec()) .setStartTimeMs(workerInfo.getStartTimeMs()).setState(workerInfo.getState()) .setUsedBytes(workerInfo.getUsedBytes()) .setUsedBytesOnTiers(workerInfo.getUsedBytesOnTiersMap()); }
WorkerInfo info = new WorkerInfo(); Set<WorkerInfoField> checkedFieldRange = fieldRange != null ? fieldRange : new HashSet<>(Arrays.asList(WorkerInfoField.values()));
/** * Creates a new instance of {@link WorkerInfo} from a thrift representation. * * @param workerInfo the thrift representation of a worker information * @return the instance */ public static WorkerInfo fromThrift(alluxio.thrift.WorkerInfo workerInfo) { return new WorkerInfo() .setAddress(WorkerNetAddress.fromThrift(workerInfo.getAddress())) .setCapacityBytes(workerInfo.getCapacityBytes()) .setCapacityBytesOnTiers(workerInfo.getCapacityBytesOnTiers()) .setId(workerInfo.getId()) .setLastContactSec(workerInfo.getLastContactSec()) .setStartTimeMs(workerInfo.getStartTimeMs()) .setState(workerInfo.getState()) .setUsedBytes(workerInfo.getUsedBytes()) .setUsedBytesOnTiers(workerInfo.getUsedBytesOnTiers()); }