public LocationId getLocationId() { return new LocationId(locationId); }
public void writeTaskLocality(TaskName taskName, LocationId locationId) { String taskLocalityPath = String.format("%s/%s", keyBuilder.getTaskLocalityPath(), taskName); validatePaths(new String[] {taskLocalityPath}); writeData(taskLocalityPath, locationId.getId()); }
public LocationId getLocationId() { return new LocationId(locationId); }
public LocationId getLocationId() { return new LocationId(locationId); }
public LocationId getLocationId() { return new LocationId(locationId); }
public LocationId getLocationId() { return new LocationId(locationId); }
@Override public LocationIdProvider getLocationIdProvider(Config config) { return () -> new LocationId(Util.getLocalHost().getHostName()); } }
@Override public LocationIdProvider getLocationIdProvider(Config config) { return () -> new LocationId(Util.getLocalHost().getHostName()); } }
@Override public LocationIdProvider getLocationIdProvider(Config config) { return () -> new LocationId(Util.getLocalHost().getHostName()); } }
@Override public LocationIdProvider getLocationIdProvider(Config config) { return () -> new LocationId(Util.getLocalHost().getHostName()); } }
@Override public LocationIdProvider getLocationIdProvider(Config config) { return () -> new LocationId(Util.getLocalHost().getHostName()); } }
public Map<TaskName, LocationId> readTaskLocality() { Map<TaskName, LocationId> taskLocality = new HashMap<>(); String taskLocalityPath = keyBuilder.getTaskLocalityPath(); List<String> tasks = new ArrayList<>(); if (zkClient.exists(taskLocalityPath)) { tasks = zkClient.getChildren(taskLocalityPath); } for (String taskName : tasks) { String taskPath = String.format("%s/%s", keyBuilder.getTaskLocalityPath(), taskName); String locationId = zkClient.readData(taskPath, true); if (locationId != null) { taskLocality.put(new TaskName(taskName), new LocationId(locationId)); } } return taskLocality; }
@Test public void testGetProcessorLocality() { // Mock the dependencies. LocalityManager mockLocalityManager = mock(LocalityManager.class); Map<String, Map<String, String>> localityMappings = new HashMap<>(); localityMappings.put("0", ImmutableMap.of(SetContainerHostMapping.HOST_KEY, "abc-affinity")); // Mock the container locality assignment. when(mockLocalityManager.readContainerLocality()).thenReturn(localityMappings); Map<String, LocationId> processorLocality = JobModelManager.getProcessorLocality(new MapConfig(), mockLocalityManager); Mockito.verify(mockLocalityManager).readContainerLocality(); Assert.assertEquals(ImmutableMap.of("0", new LocationId("abc-affinity"), "1", new LocationId("ANY_HOST")), processorLocality); }
@Test public void testReadTaskLocalityShouldReturnAllTheExistingLocalityValue() { zkUtils.writeTaskLocality(new TaskName("task-1"), new LocationId("LocationId-1")); zkUtils.writeTaskLocality(new TaskName("task-2"), new LocationId("LocationId-2")); zkUtils.writeTaskLocality(new TaskName("task-3"), new LocationId("LocationId-3")); zkUtils.writeTaskLocality(new TaskName("task-4"), new LocationId("LocationId-4")); zkUtils.writeTaskLocality(new TaskName("task-5"), new LocationId("LocationId-5")); Map<TaskName, LocationId> taskLocality = ImmutableMap.of(new TaskName("task-1"), new LocationId("LocationId-1"), new TaskName("task-2"), new LocationId("LocationId-2"), new TaskName("task-3"), new LocationId("LocationId-3"), new TaskName("task-4"), new LocationId("LocationId-4"), new TaskName("task-5"), new LocationId("LocationId-5")); Assert.assertEquals(taskLocality, zkUtils.readTaskLocality()); }
@Test public void testReadAfterWriteTaskLocality() { zkUtils.writeTaskLocality(new TaskName("task-1"), new LocationId("LocationId-1")); zkUtils.writeTaskLocality(new TaskName("task-2"), new LocationId("LocationId-2")); Map<TaskName, LocationId> taskLocality = ImmutableMap.of(new TaskName("task-1"), new LocationId("LocationId-1"), new TaskName("task-2"), new LocationId("LocationId-2")); Assert.assertEquals(taskLocality, zkUtils.readTaskLocality()); }
@Test public void testWriteTaskLocalityShouldUpdateTheExistingValue() { zkUtils.writeTaskLocality(new TaskName("task-1"), new LocationId("LocationId-1")); Map<TaskName, LocationId> taskLocality = ImmutableMap.of(new TaskName("task-1"), new LocationId("LocationId-1")); Assert.assertEquals(taskLocality, zkUtils.readTaskLocality()); zkUtils.writeTaskLocality(new TaskName("task-1"), new LocationId("LocationId-2")); taskLocality = ImmutableMap.of(new TaskName("task-1"), new LocationId("LocationId-2")); Assert.assertEquals(taskLocality, zkUtils.readTaskLocality()); }
@Test public void testGetGrouperMetadata() { // Mocking setup. LocalityManager mockLocalityManager = mock(LocalityManager.class); TaskAssignmentManager mockTaskAssignmentManager = Mockito.mock(TaskAssignmentManager.class); Map<String, Map<String, String>> localityMappings = new HashMap<>(); localityMappings.put("0", ImmutableMap.of(SetContainerHostMapping.HOST_KEY, "abc-affinity")); Map<String, String> taskAssignment = ImmutableMap.of("task-0", "0"); // Mock the container locality assignment. when(mockLocalityManager.readContainerLocality()).thenReturn(localityMappings); // Mock the container to task assignment. when(mockTaskAssignmentManager.readTaskAssignment()).thenReturn(taskAssignment); when(mockTaskAssignmentManager.readTaskModes()).thenReturn(Collections.singletonMap(new TaskName("task-0"), TaskMode.Active)); GrouperMetadataImpl grouperMetadata = JobModelManager.getGrouperMetadata(new MapConfig(), mockLocalityManager, mockTaskAssignmentManager); Mockito.verify(mockLocalityManager).readContainerLocality(); Mockito.verify(mockTaskAssignmentManager).readTaskAssignment(); Assert.assertEquals(ImmutableMap.of("0", new LocationId("abc-affinity"), "1", new LocationId("ANY_HOST")), grouperMetadata.getProcessorLocality()); Assert.assertEquals(ImmutableMap.of(new TaskName("task-0"), new LocationId("abc-affinity")), grouperMetadata.getTaskLocality()); }
String testProcessorId2 = "testProcessorId2"; LocationId testLocationId1 = new LocationId("testLocationId1"); LocationId testLocationId2 = new LocationId("testLocationId2"); LocationId testLocationId3 = new LocationId("testLocationId3");
@Test public void testGenerateContainerModelForSingleContainer() { TaskNameGrouper taskNameGrouper = buildSimpleGrouper(1); String testProcessorId1 = "testProcessorId1"; LocationId testLocationId1 = new LocationId("testLocationId1"); LocationId testLocationId2 = new LocationId("testLocationId2"); LocationId testLocationId3 = new LocationId("testLocationId3"); TaskName testTaskName1 = new TaskName("testTasKId1"); TaskName testTaskName2 = new TaskName("testTaskId2"); TaskName testTaskName3 = new TaskName("testTaskId3"); TaskModel testTaskModel1 = new TaskModel(testTaskName1, new HashSet<>(), new Partition(0)); TaskModel testTaskModel2 = new TaskModel(testTaskName2, new HashSet<>(), new Partition(1)); TaskModel testTaskModel3 = new TaskModel(testTaskName3, new HashSet<>(), new Partition(2)); Map<String, LocationId> processorLocality = ImmutableMap.of(testProcessorId1, testLocationId1); Map<TaskName, LocationId> taskLocality = ImmutableMap.of(testTaskName1, testLocationId1, testTaskName2, testLocationId2, testTaskName3, testLocationId3); GrouperMetadataImpl grouperMetadata = new GrouperMetadataImpl(processorLocality, taskLocality, new HashMap<>(), new HashMap<>()); Set<TaskModel> taskModels = ImmutableSet.of(testTaskModel1, testTaskModel2, testTaskModel3); Set<ContainerModel> expectedContainerModels = ImmutableSet.of(new ContainerModel(testProcessorId1, ImmutableMap.of(testTaskName1, testTaskModel1, testTaskName2, testTaskModel2, testTaskName3, testTaskModel3))); Set<ContainerModel> actualContainerModels = taskNameGrouper.group(taskModels, grouperMetadata); assertEquals(expectedContainerModels, actualContainerModels); }
String testProcessorId3 = "testProcessorId3"; LocationId testLocationId1 = new LocationId("testLocationId1"); LocationId testLocationId2 = new LocationId("testLocationId2"); LocationId testLocationId3 = new LocationId("testLocationId3");