public void updateAndLogIfChanged(String msgPrefix) { boolean changed = false; // synchronized to fix findbug warnings synchronized (RMContainerAllocator.this) { changed |= (numPendingReduces != pendingReduces.size()); numPendingReduces = pendingReduces.size(); changed |= (numScheduledMaps != scheduledRequests.maps.size()); numScheduledMaps = scheduledRequests.maps.size(); changed |= (numScheduledReduces != scheduledRequests.reduces.size()); numScheduledReduces = scheduledRequests.reduces.size(); changed |= (numAssignedMaps != assignedRequests.maps.size()); numAssignedMaps = assignedRequests.maps.size(); changed |= (numAssignedReduces != assignedRequests.reduces.size()); numAssignedReduces = assignedRequests.reduces.size(); changed |= (numCompletedMaps != getJob().getCompletedMaps()); numCompletedMaps = getJob().getCompletedMaps(); changed |= (numCompletedReduces != getJob().getCompletedReduces()); numCompletedReduces = getJob().getCompletedReduces(); changed |= (numContainersAllocated != containersAllocated); numContainersAllocated = containersAllocated; changed |= (numContainersReleased != containersReleased); numContainersReleased = containersReleased; } if (changed) { log(msgPrefix); } }
public void updateAndLogIfChanged(String msgPrefix) { boolean changed = false; // synchronized to fix findbug warnings synchronized (RMContainerAllocator.this) { changed |= (numPendingReduces != pendingReduces.size()); numPendingReduces = pendingReduces.size(); changed |= (numScheduledMaps != scheduledRequests.maps.size()); numScheduledMaps = scheduledRequests.maps.size(); changed |= (numScheduledReduces != scheduledRequests.reduces.size()); numScheduledReduces = scheduledRequests.reduces.size(); changed |= (numAssignedMaps != assignedRequests.maps.size()); numAssignedMaps = assignedRequests.maps.size(); changed |= (numAssignedReduces != assignedRequests.reduces.size()); numAssignedReduces = assignedRequests.reduces.size(); changed |= (numCompletedMaps != getJob().getCompletedMaps()); numCompletedMaps = getJob().getCompletedMaps(); changed |= (numCompletedReduces != getJob().getCompletedReduces()); numCompletedReduces = getJob().getCompletedReduces(); changed |= (numContainersAllocated != containersAllocated); numContainersAllocated = containersAllocated; changed |= (numContainersReleased != containersReleased); numContainersReleased = containersReleased; } if (changed) { log(msgPrefix); } }
public void updateAndLogIfChanged(String msgPrefix) { boolean changed = false; // synchronized to fix findbug warnings synchronized (RMContainerAllocator.this) { changed |= (numPendingReduces != pendingReduces.size()); numPendingReduces = pendingReduces.size(); changed |= (numScheduledMaps != scheduledRequests.maps.size()); numScheduledMaps = scheduledRequests.maps.size(); changed |= (numScheduledReduces != scheduledRequests.reduces.size()); numScheduledReduces = scheduledRequests.reduces.size(); changed |= (numAssignedMaps != assignedRequests.maps.size()); numAssignedMaps = assignedRequests.maps.size(); changed |= (numAssignedReduces != assignedRequests.reduces.size()); numAssignedReduces = assignedRequests.reduces.size(); changed |= (numCompletedMaps != getJob().getCompletedMaps()); numCompletedMaps = getJob().getCompletedMaps(); changed |= (numCompletedReduces != getJob().getCompletedReduces()); numCompletedReduces = getJob().getCompletedReduces(); changed |= (numContainersAllocated != containersAllocated); numContainersAllocated = containersAllocated; changed |= (numContainersReleased != containersReleased); numContainersReleased = containersReleased; } if (changed) { log(msgPrefix); } }
private void applyConcurrentTaskLimits() { int numScheduledMaps = scheduledRequests.maps.size(); if (maxRunningMaps > 0 && numScheduledMaps > 0 && getJob().getTotalMaps() > maxRunningMaps) { int maxRequestedMaps = Math.max(0, maxRunningMaps - assignedRequests.maps.size()); int numScheduledFailMaps = scheduledRequests.earlierFailedMaps.size(); int failedMapRequestLimit = Math.min(maxRequestedMaps, numScheduledFailMaps); int normalMapRequestLimit = Math.min( maxRequestedMaps - failedMapRequestLimit, numScheduledMaps - numScheduledFailMaps); setRequestLimit(PRIORITY_FAST_FAIL_MAP, mapResourceRequest, failedMapRequestLimit); setRequestLimit(PRIORITY_MAP, mapResourceRequest, normalMapRequestLimit); setRequestLimit(PRIORITY_OPPORTUNISTIC_MAP, mapResourceRequest, normalMapRequestLimit); } int numScheduledReduces = scheduledRequests.reduces.size(); if (maxRunningReduces > 0 && numScheduledReduces > 0 && getJob().getTotalReduces() > maxRunningReduces) { int maxRequestedReduces = Math.max(0, maxRunningReduces - assignedRequests.reduces.size()); int reduceRequestLimit = Math.min(maxRequestedReduces, numScheduledReduces); setRequestLimit(PRIORITY_REDUCE, reduceResourceRequest, reduceRequestLimit); } }
@Override protected synchronized void heartbeat() throws Exception { scheduleStats.updateAndLogIfChanged("Before Scheduling: "); List<Container> allocatedContainers = getResources(); if (allocatedContainers != null && allocatedContainers.size() > 0) { scheduledRequests.assign(allocatedContainers); } int completedMaps = getJob().getCompletedMaps(); int completedTasks = completedMaps + getJob().getCompletedReduces(); if ((lastCompletedTasks != completedTasks) || (scheduledRequests.maps.size() > 0)) { lastCompletedTasks = completedTasks; recalculateReduceSchedule = true; } if (recalculateReduceSchedule) { boolean reducerPreempted = preemptReducesIfNeeded(); if (!reducerPreempted) { // Only schedule new reducers if no reducer preemption happens for // this heartbeat scheduleReduces(getJob().getTotalMaps(), completedMaps, scheduledRequests.maps.size(), scheduledRequests.reduces.size(), assignedRequests.maps.size(), assignedRequests.reduces.size(), mapResourceRequest, reduceResourceRequest, pendingReduces.size(), maxReduceRampupLimit, reduceSlowStart); } recalculateReduceSchedule = false; } scheduleStats.updateAndLogIfChanged("After Scheduling: "); }
@SuppressWarnings("unchecked") @Override public void run() { ContainerAllocatorEvent event; while (!stopped.get() && !Thread.currentThread().isInterrupted()) { try { event = RMContainerAllocator.this.eventQueue.take(); } catch (InterruptedException e) { if (!stopped.get()) { LOG.error("Returning, interrupted : " + e); } return; } try { handleEvent(event); } catch (Throwable t) { LOG.error("Error in handling event type " + event.getType() + " to the ContainreAllocator", t); // Kill the AM eventHandler.handle(new JobEvent(getJob().getID(), JobEventType.INTERNAL_ERROR)); return; } } } };
@Override protected synchronized void heartbeat() throws Exception { scheduleStats.updateAndLogIfChanged("Before Scheduling: "); List<Container> allocatedContainers = getResources(); if (allocatedContainers != null && allocatedContainers.size() > 0) { scheduledRequests.assign(allocatedContainers); } int completedMaps = getJob().getCompletedMaps(); int completedTasks = completedMaps + getJob().getCompletedReduces(); if ((lastCompletedTasks != completedTasks) || (scheduledRequests.maps.size() > 0)) { lastCompletedTasks = completedTasks; recalculateReduceSchedule = true; } if (recalculateReduceSchedule) { boolean reducerPreempted = preemptReducesIfNeeded(); if (!reducerPreempted) { // Only schedule new reducers if no reducer preemption happens for // this heartbeat scheduleReduces(getJob().getTotalMaps(), completedMaps, scheduledRequests.maps.size(), scheduledRequests.reduces.size(), assignedRequests.maps.size(), assignedRequests.reduces.size(), mapResourceRequest, reduceResourceRequest, pendingReduces.size(), maxReduceRampupLimit, reduceSlowStart); } recalculateReduceSchedule = false; } scheduleStats.updateAndLogIfChanged("After Scheduling: "); }
@Override protected synchronized void heartbeat() throws Exception { scheduleStats.updateAndLogIfChanged("Before Scheduling: "); List<Container> allocatedContainers = getResources(); if (allocatedContainers != null && allocatedContainers.size() > 0) { scheduledRequests.assign(allocatedContainers); } int completedMaps = getJob().getCompletedMaps(); int completedTasks = completedMaps + getJob().getCompletedReduces(); if ((lastCompletedTasks != completedTasks) || (scheduledRequests.maps.size() > 0)) { lastCompletedTasks = completedTasks; recalculateReduceSchedule = true; } if (recalculateReduceSchedule) { boolean reducerPreempted = preemptReducesIfNeeded(); if (!reducerPreempted) { // Only schedule new reducers if no reducer preemption happens for // this heartbeat scheduleReduces(getJob().getTotalMaps(), completedMaps, scheduledRequests.maps.size(), scheduledRequests.reduces.size(), assignedRequests.maps.size(), assignedRequests.reduces.size(), mapResourceRequest, reduceResourceRequest, pendingReduces.size(), maxReduceRampupLimit, reduceSlowStart); } recalculateReduceSchedule = false; } scheduleStats.updateAndLogIfChanged("After Scheduling: "); }
@SuppressWarnings("unchecked") @Override public void run() { ContainerAllocatorEvent event; while (!stopped.get() && !Thread.currentThread().isInterrupted()) { try { event = RMContainerAllocator.this.eventQueue.take(); } catch (InterruptedException e) { if (!stopped.get()) { LOG.error("Returning, interrupted : " + e); } return; } try { handleEvent(event); } catch (Throwable t) { LOG.error("Error in handling event type " + event.getType() + " to the ContainreAllocator", t); // Kill the AM eventHandler.handle(new JobEvent(getJob().getID(), JobEventType.INTERNAL_ERROR)); return; } } } };
@SuppressWarnings("unchecked") @Override public void run() { ContainerAllocatorEvent event; while (!stopped.get() && !Thread.currentThread().isInterrupted()) { try { event = RMContainerAllocator.this.eventQueue.take(); } catch (InterruptedException e) { if (!stopped.get()) { LOG.error("Returning, interrupted : " + e); } return; } try { handleEvent(event); } catch (Throwable t) { LOG.error("Error in handling event type " + event.getType() + " to the ContainreAllocator", t); // Kill the AM eventHandler.handle(new JobEvent(getJob().getID(), JobEventType.INTERNAL_ERROR)); return; } } } };
private void handleJobPriorityChange(AllocateResponse response) { Priority priorityFromResponse = Priority.newInstance(response .getApplicationPriority().getPriority()); // Update the job priority to Job directly. getJob().setJobPriority(priorityFromResponse); }
private void handleJobPriorityChange(AllocateResponse response) { Priority priorityFromResponse = Priority.newInstance(response .getApplicationPriority().getPriority()); // Update the job priority to Job directly. getJob().setJobPriority(priorityFromResponse); }
eventHandler.handle(new JobUpdatedNodesEvent(getJob().getID(), updatedNodes));
eventHandler.handle(new JobUpdatedNodesEvent(getJob().getID(), updatedNodes));
eventHandler.handle(new JobUpdatedNodesEvent(getJob().getID(), updatedNodes));
JobId jobId = getJob().getID();
JobId jobId = getJob().getID();