public static ContainerRequestEvent createContainerRequestEventForFailedContainer( TaskAttemptId attemptID, Resource capability) { //ContainerRequest for failed events does not consider rack / node locality? return new ContainerRequestEvent(attemptID, capability); }
public ContainerRequest(ContainerRequestEvent event, Priority priority, long requestTimeMs) { this(event.getAttemptID(), event.getCapability(), event.getHosts(), event.getRacks(), priority, requestTimeMs,null); }
if (event.getEarlierAttemptFailed()) { earlierFailedMaps.add(event.getAttemptID()); request = new ContainerRequest(event, PRIORITY_FAST_FAIL_MAP); LOG.info("Added "+event.getAttemptID()+" to list of failed maps"); } else { for (String host : event.getHosts()) { LinkedList<TaskAttemptId> list = mapsHostMapping.get(host); if (list == null) { mapsHostMapping.put(host, list); list.add(event.getAttemptID()); if (LOG.isDebugEnabled()) { LOG.debug("Added attempt req to host " + host); for (String rack: event.getRacks()) { LinkedList<TaskAttemptId> list = mapsRackMapping.get(rack); if (list == null) { mapsRackMapping.put(rack, list); list.add(event.getAttemptID()); if (LOG.isDebugEnabled()) { LOG.debug("Added attempt req to rack " + rack); maps.put(event.getAttemptID(), request); addContainerReq(request);
@SuppressWarnings({ "unchecked" }) private void handleReduceContainerRequest(ContainerRequestEvent reqEvent) { assert(reqEvent.getAttemptID().getTaskId().getTaskType().equals( TaskType.REDUCE)); reduceResourceRequest = reqEvent.getCapability(); eventHandler.handle(new JobHistoryEvent(jobId, new NormalizedResourceEvent( reqEvent.getCapability().setVirtualCores( reduceResourceRequest.getVirtualCores()); reqEvent.getCapability().setMemorySize( reduceResourceRequest.getMemorySize()); if (reqEvent.getEarlierAttemptFailed()) {
@SuppressWarnings("unchecked") @Override public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) { // Tell any speculator that we're requesting a container taskAttempt.eventHandler.handle (new SpeculatorEvent(taskAttempt.getID().getTaskId(), +1)); //request for container if (rescheduled) { taskAttempt.eventHandler.handle( ContainerRequestEvent.createContainerRequestEventForFailedContainer( taskAttempt.attemptId, taskAttempt.resourceCapability)); } else { taskAttempt.eventHandler.handle(new ContainerRequestEvent( taskAttempt.attemptId, taskAttempt.resourceCapability, taskAttempt.dataLocalHosts.toArray( new String[taskAttempt.dataLocalHosts.size()]), taskAttempt.dataLocalRacks.toArray( new String[taskAttempt.dataLocalRacks.size()]))); } } }
private void checkAssignment(ContainerRequestEvent request, TaskAttemptContainerAssignedEvent assigned, boolean checkHostMatch) { Assert.assertNotNull("Nothing assigned to attempt " + request.getAttemptID(), assigned); Assert.assertEquals("assigned to wrong attempt", request.getAttemptID(), assigned.getTaskAttemptID()); if (checkHostMatch) { Assert.assertTrue("Not assigned to requested host", Arrays.asList( request.getHosts()).contains( assigned.getContainer().getNodeId().getHost())); } }
@SuppressWarnings({ "unchecked" }) private void handleMapContainerRequest(ContainerRequestEvent reqEvent) { assert(reqEvent.getAttemptID().getTaskId().getTaskType().equals( TaskType.MAP)); mapResourceRequest = reqEvent.getCapability(); eventHandler.handle(new JobHistoryEvent(jobId, new NormalizedResourceEvent( reqEvent.getCapability().setMemorySize( mapResourceRequest.getMemorySize()); reqEvent.getCapability().setVirtualCores( mapResourceRequest.getVirtualCores());
private void checkAssignments(ContainerRequestEvent[] requests, List<TaskAttemptContainerAssignedEvent> assignments, boolean checkHostMatch) { Assert.assertNotNull("Container not assigned", assignments); Assert.assertEquals("Assigned count not correct", requests.length, assignments.size()); // check for uniqueness of containerIDs Set<ContainerId> containerIds = new HashSet<ContainerId>(); for (TaskAttemptContainerAssignedEvent assigned : assignments) { containerIds.add(assigned.getContainer().getId()); } Assert.assertEquals("Assigned containers must be different", assignments .size(), containerIds.size()); // check for all assignment for (ContainerRequestEvent req : requests) { TaskAttemptContainerAssignedEvent assigned = null; for (TaskAttemptContainerAssignedEvent ass : assignments) { if (ass.getTaskAttemptID().equals(req.getAttemptID())) { assigned = ass; break; } } checkAssignment(req, assigned, checkHostMatch); } }
@Override public void handle(ContainerAllocatorEvent event) { if (event.getType() == ContainerAllocator.EventType.CONTAINER_REQ && ((ContainerRequestEvent)event).getEarlierAttemptFailed()) { failedMapContainerReqEventCnt.incrementAndGet(); } } public void waitForFailedMapContainerReqEvents(int count)
private Resource getResourceInfoFromContainerRequest( TaskAttemptImpl taImpl, EventHandler eventHandler) { taImpl.handle(new TaskAttemptEvent(taImpl.getID(), TaskAttemptEventType.TA_SCHEDULE)); assertEquals("Task attempt is not in STARTING state", taImpl.getState(), TaskAttemptState.STARTING); ArgumentCaptor<Event> captor = ArgumentCaptor.forClass(Event.class); verify(eventHandler, times(2)).handle(captor.capture()); List<ContainerRequestEvent> containerRequestEvents = new ArrayList<>(); for (Event e : captor.getAllValues()) { if (e instanceof ContainerRequestEvent) { containerRequestEvents.add((ContainerRequestEvent) e); } } assertEquals("Expected one ContainerRequestEvent after scheduling " + "task attempt", 1, containerRequestEvents.size()); return containerRequestEvents.get(0).getCapability(); }
ContainerRequestEvent cre = (ContainerRequestEvent) arg.getAllValues().get(1); String[] requestedHosts = cre.getHosts(); for (String h : requestedHosts) { expected.remove(h);
@Test public void testSingleRackRequest() throws Exception { TaskAttemptImpl.RequestContainerTransition rct = new TaskAttemptImpl.RequestContainerTransition(false); EventHandler eventHandler = mock(EventHandler.class); String[] hosts = new String[3]; hosts[0] = "host1"; hosts[1] = "host2"; hosts[2] = "host3"; TaskSplitMetaInfo splitInfo = new TaskSplitMetaInfo(hosts, 0, 128 * 1024 * 1024l); TaskAttemptImpl mockTaskAttempt = createMapTaskAttemptImplForTest(eventHandler, splitInfo); TaskAttemptEvent mockTAEvent = mock(TaskAttemptEvent.class); rct.transition(mockTaskAttempt, mockTAEvent); ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class); verify(eventHandler, times(2)).handle(arg.capture()); if (!(arg.getAllValues().get(1) instanceof ContainerRequestEvent)) { Assert.fail("Second Event not of type ContainerRequestEvent"); } ContainerRequestEvent cre = (ContainerRequestEvent) arg.getAllValues().get(1); String[] requestedRacks = cre.getRacks(); //Only a single occurrence of /DefaultRack assertEquals(1, requestedRacks.length); }
if (event.getEarlierAttemptFailed()) { earlierFailedMaps.add(event.getAttemptID()); request = new ContainerRequest(event, PRIORITY_FAST_FAIL_MAP, mapNodeLabelExpression); LOG.info("Added "+event.getAttemptID()+" to list of failed maps"); } else { for (String host : event.getHosts()) { LinkedList<TaskAttemptId> list = mapsHostMapping.get(host); if (list == null) { mapsHostMapping.put(host, list); list.add(event.getAttemptID()); if (LOG.isDebugEnabled()) { LOG.debug("Added attempt req to host " + host); for (String rack: event.getRacks()) { LinkedList<TaskAttemptId> list = mapsRackMapping.get(rack); if (list == null) { list.add(event.getAttemptID()); if (LOG.isDebugEnabled()) { LOG.debug("Added attempt req to rack " + rack); maps.put(event.getAttemptID(), request); addContainerReq(request);
JobId jobId = getJob().getID(); Resource supportedMaxContainerCapability = getMaxContainerCapability(); if (reqEvent.getAttemptID().getTaskId().getTaskType().equals(TaskType.MAP)) { if (mapResourceRequest.equals(Resources.none())) { mapResourceRequest = reqEvent.getCapability(); eventHandler.handle(new JobHistoryEvent(jobId, new NormalizedResourceEvent( reqEvent.getCapability().setMemory(mapResourceRequest.getMemory()); reqEvent.getCapability().setVirtualCores( mapResourceRequest.getVirtualCores()); reduceResourceRequest = reqEvent.getCapability(); eventHandler.handle(new JobHistoryEvent(jobId, new NormalizedResourceEvent( reqEvent.getCapability().setMemory(reduceResourceRequest.getMemory()); reqEvent.getCapability().setVirtualCores( reduceResourceRequest.getVirtualCores()); if (reqEvent.getEarlierAttemptFailed()) {
@SuppressWarnings("unchecked") @Override public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) { // Tell any speculator that we're requesting a container taskAttempt.eventHandler.handle (new SpeculatorEvent(taskAttempt.getID().getTaskId(), +1)); //request for container if (rescheduled) { taskAttempt.eventHandler.handle( ContainerRequestEvent.createContainerRequestEventForFailedContainer( taskAttempt.attemptId, taskAttempt.resourceCapability)); } else { taskAttempt.eventHandler.handle(new ContainerRequestEvent( taskAttempt.attemptId, taskAttempt.resourceCapability, taskAttempt.dataLocalHosts.toArray( new String[taskAttempt.dataLocalHosts.size()]), taskAttempt.dataLocalRacks.toArray( new String[taskAttempt.dataLocalRacks.size()]))); } } }
if (event.getType() == ContainerAllocator.EventType.CONTAINER_REQ) { ContainerRequestEvent reqEvent = (ContainerRequestEvent) event; boolean isMap = reqEvent.getAttemptID().getTaskId().getTaskType(). equals(TaskType.MAP); if (isMap) {
public ContainerRequest(ContainerRequestEvent event, Priority priority, String nodeLabelExpression) { this(event.getAttemptID(), event.getCapability(), event.getHosts(), event.getRacks(), priority, nodeLabelExpression); }
if (event.getEarlierAttemptFailed()) { earlierFailedMaps.add(event.getAttemptID()); request = new ContainerRequest(event, PRIORITY_FAST_FAIL_MAP, mapNodeLabelExpression); LOG.info("Added "+event.getAttemptID()+" to list of failed maps"); maps.put(event.getAttemptID(), request); addContainerReq(request); } else { new ContainerRequest(event, PRIORITY_OPPORTUNISTIC_MAP, mapNodeLabelExpression); maps.put(event.getAttemptID(), request); addOpportunisticResourceRequest(request.priority, request.capability); } else { request = new ContainerRequest(event, PRIORITY_MAP, mapNodeLabelExpression); for (String host : event.getHosts()) { LinkedList<TaskAttemptId> list = mapsHostMapping.get(host); if (list == null) { list.add(event.getAttemptID()); if (LOG.isDebugEnabled()) { LOG.debug("Added attempt req to host " + host); for (String rack : event.getRacks()) { LinkedList<TaskAttemptId> list = mapsRackMapping.get(rack); if (list == null) {
JobId jobId = getJob().getID(); Resource supportedMaxContainerCapability = getMaxContainerCapability(); if (reqEvent.getAttemptID().getTaskId().getTaskType().equals(TaskType.MAP)) { if (mapResourceRequest.equals(Resources.none())) { mapResourceRequest = reqEvent.getCapability(); eventHandler.handle(new JobHistoryEvent(jobId, new NormalizedResourceEvent( reqEvent.getCapability().setMemorySize(mapResourceRequest.getMemorySize()); reqEvent.getCapability().setVirtualCores( mapResourceRequest.getVirtualCores()); reduceResourceRequest = reqEvent.getCapability(); eventHandler.handle(new JobHistoryEvent(jobId, new NormalizedResourceEvent( reqEvent.getCapability().setMemorySize(reduceResourceRequest.getMemorySize()); reqEvent.getCapability().setVirtualCores( reduceResourceRequest.getVirtualCores()); if (reqEvent.getEarlierAttemptFailed()) {
@SuppressWarnings("unchecked") @Override public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) { // Tell any speculator that we're requesting a container taskAttempt.eventHandler.handle (new SpeculatorEvent(taskAttempt.getID().getTaskId(), +1)); //request for container if (rescheduled) { taskAttempt.eventHandler.handle( ContainerRequestEvent.createContainerRequestEventForFailedContainer( taskAttempt.attemptId, taskAttempt.resourceCapability)); } else { taskAttempt.eventHandler.handle(new ContainerRequestEvent( taskAttempt.attemptId, taskAttempt.resourceCapability, taskAttempt.dataLocalHosts.toArray( new String[taskAttempt.dataLocalHosts.size()]), taskAttempt.dataLocalRacks.toArray( new String[taskAttempt.dataLocalRacks.size()]))); } } }