@Override protected void serviceStop() throws Exception { if (stopped.getAndSet(true)) { // return if already stopped return; } if (eventHandlingThread != null) { eventHandlingThread.interrupt(); } super.serviceStop(); scheduleStats.log("Final Stats: "); }
@Override protected synchronized void heartbeat() throws Exception { scheduleStats.updateAndLogIfChanged("Before Scheduling: "); List<Container> allocatedContainers = getResources(); if (allocatedContainers != null && allocatedContainers.size() > 0) { scheduledRequests.assign(allocatedContainers); } int completedMaps = getJob().getCompletedMaps(); int completedTasks = completedMaps + getJob().getCompletedReduces(); if ((lastCompletedTasks != completedTasks) || (scheduledRequests.maps.size() > 0)) { lastCompletedTasks = completedTasks; recalculateReduceSchedule = true; } if (recalculateReduceSchedule) { boolean reducerPreempted = preemptReducesIfNeeded(); if (!reducerPreempted) { // Only schedule new reducers if no reducer preemption happens for // this heartbeat scheduleReduces(getJob().getTotalMaps(), completedMaps, scheduledRequests.maps.size(), scheduledRequests.reduces.size(), assignedRequests.maps.size(), assignedRequests.reduces.size(), mapResourceRequest, reduceResourceRequest, pendingReduces.size(), maxReduceRampupLimit, reduceSlowStart); } recalculateReduceSchedule = false; } scheduleStats.updateAndLogIfChanged("After Scheduling: "); }
public void updateAndLogIfChanged(String msgPrefix) { boolean changed = false; // synchronized to fix findbug warnings synchronized (RMContainerAllocator.this) { changed |= (numPendingReduces != pendingReduces.size()); numPendingReduces = pendingReduces.size(); changed |= (numScheduledMaps != scheduledRequests.maps.size()); numScheduledMaps = scheduledRequests.maps.size(); changed |= (numScheduledReduces != scheduledRequests.reduces.size()); numScheduledReduces = scheduledRequests.reduces.size(); changed |= (numAssignedMaps != assignedRequests.maps.size()); numAssignedMaps = assignedRequests.maps.size(); changed |= (numAssignedReduces != assignedRequests.reduces.size()); numAssignedReduces = assignedRequests.reduces.size(); changed |= (numCompletedMaps != getJob().getCompletedMaps()); numCompletedMaps = getJob().getCompletedMaps(); changed |= (numCompletedReduces != getJob().getCompletedReduces()); numCompletedReduces = getJob().getCompletedReduces(); changed |= (numContainersAllocated != containersAllocated); numContainersAllocated = containersAllocated; changed |= (numContainersReleased != containersReleased); numContainersReleased = containersReleased; } if (changed) { log(msgPrefix); } }
public void updateAndLogIfChanged(String msgPrefix) { boolean changed = false; // synchronized to fix findbug warnings synchronized (RMContainerAllocator.this) { changed |= (numPendingReduces != pendingReduces.size()); numPendingReduces = pendingReduces.size(); changed |= (numScheduledMaps != scheduledRequests.maps.size()); numScheduledMaps = scheduledRequests.maps.size(); changed |= (numScheduledReduces != scheduledRequests.reduces.size()); numScheduledReduces = scheduledRequests.reduces.size(); changed |= (numAssignedMaps != assignedRequests.maps.size()); numAssignedMaps = assignedRequests.maps.size(); changed |= (numAssignedReduces != assignedRequests.reduces.size()); numAssignedReduces = assignedRequests.reduces.size(); changed |= (numCompletedMaps != getJob().getCompletedMaps()); numCompletedMaps = getJob().getCompletedMaps(); changed |= (numCompletedReduces != getJob().getCompletedReduces()); numCompletedReduces = getJob().getCompletedReduces(); changed |= (numContainersAllocated != containersAllocated); numContainersAllocated = containersAllocated; changed |= (numContainersReleased != containersReleased); numContainersReleased = containersReleased; } if (changed) { log(msgPrefix); } }
public void updateAndLogIfChanged(String msgPrefix) { boolean changed = false; // synchronized to fix findbug warnings synchronized (RMContainerAllocator.this) { changed |= (numPendingReduces != pendingReduces.size()); numPendingReduces = pendingReduces.size(); changed |= (numScheduledMaps != scheduledRequests.maps.size()); numScheduledMaps = scheduledRequests.maps.size(); changed |= (numScheduledReduces != scheduledRequests.reduces.size()); numScheduledReduces = scheduledRequests.reduces.size(); changed |= (numAssignedMaps != assignedRequests.maps.size()); numAssignedMaps = assignedRequests.maps.size(); changed |= (numAssignedReduces != assignedRequests.reduces.size()); numAssignedReduces = assignedRequests.reduces.size(); changed |= (numCompletedMaps != getJob().getCompletedMaps()); numCompletedMaps = getJob().getCompletedMaps(); changed |= (numCompletedReduces != getJob().getCompletedReduces()); numCompletedReduces = getJob().getCompletedReduces(); changed |= (numContainersAllocated != containersAllocated); numContainersAllocated = containersAllocated; changed |= (numContainersReleased != containersReleased); numContainersReleased = containersReleased; } if (changed) { log(msgPrefix); } }
@Override protected synchronized void heartbeat() throws Exception { scheduleStats.updateAndLogIfChanged("Before Scheduling: "); List<Container> allocatedContainers = getResources(); if (allocatedContainers != null && allocatedContainers.size() > 0) { scheduledRequests.assign(allocatedContainers); } int completedMaps = getJob().getCompletedMaps(); int completedTasks = completedMaps + getJob().getCompletedReduces(); if ((lastCompletedTasks != completedTasks) || (scheduledRequests.maps.size() > 0)) { lastCompletedTasks = completedTasks; recalculateReduceSchedule = true; } if (recalculateReduceSchedule) { boolean reducerPreempted = preemptReducesIfNeeded(); if (!reducerPreempted) { // Only schedule new reducers if no reducer preemption happens for // this heartbeat scheduleReduces(getJob().getTotalMaps(), completedMaps, scheduledRequests.maps.size(), scheduledRequests.reduces.size(), assignedRequests.maps.size(), assignedRequests.reduces.size(), mapResourceRequest, reduceResourceRequest, pendingReduces.size(), maxReduceRampupLimit, reduceSlowStart); } recalculateReduceSchedule = false; } scheduleStats.updateAndLogIfChanged("After Scheduling: "); }
@Override protected synchronized void heartbeat() throws Exception { scheduleStats.updateAndLogIfChanged("Before Scheduling: "); List<Container> allocatedContainers = getResources(); if (allocatedContainers != null && allocatedContainers.size() > 0) { scheduledRequests.assign(allocatedContainers); } int completedMaps = getJob().getCompletedMaps(); int completedTasks = completedMaps + getJob().getCompletedReduces(); if ((lastCompletedTasks != completedTasks) || (scheduledRequests.maps.size() > 0)) { lastCompletedTasks = completedTasks; recalculateReduceSchedule = true; } if (recalculateReduceSchedule) { boolean reducerPreempted = preemptReducesIfNeeded(); if (!reducerPreempted) { // Only schedule new reducers if no reducer preemption happens for // this heartbeat scheduleReduces(getJob().getTotalMaps(), completedMaps, scheduledRequests.maps.size(), scheduledRequests.reduces.size(), assignedRequests.maps.size(), assignedRequests.reduces.size(), mapResourceRequest, reduceResourceRequest, pendingReduces.size(), maxReduceRampupLimit, reduceSlowStart); } recalculateReduceSchedule = false; } scheduleStats.updateAndLogIfChanged("After Scheduling: "); }
@Override protected void serviceStop() throws Exception { if (stopped.getAndSet(true)) { // return if already stopped return; } if (eventHandlingThread != null) { eventHandlingThread.interrupt(); } super.serviceStop(); scheduleStats.log("Final Stats: "); }
@Override protected void serviceStop() throws Exception { if (stopped.getAndSet(true)) { // return if already stopped return; } if (eventHandlingThread != null) { eventHandlingThread.interrupt(); } super.serviceStop(); scheduleStats.log("Final Stats: "); }