private void preemptReducer(int hangingMapRequests) { clearAllPendingReduceRequests(); // preempt for making space for at least one map int preemptionReduceNumForOneMap = ResourceCalculatorUtils.divideAndCeilContainers(mapResourceRequest, reduceResourceRequest, getSchedulerResourceTypes()); int preemptionReduceNumForPreemptionLimit = ResourceCalculatorUtils.divideAndCeilContainers( Resources.multiply(getResourceLimit(), maxReducePreemptionLimit), reduceResourceRequest, getSchedulerResourceTypes()); int preemptionReduceNumForAllMaps = ResourceCalculatorUtils.divideAndCeilContainers( Resources.multiply(mapResourceRequest, hangingMapRequests), reduceResourceRequest, getSchedulerResourceTypes()); int toPreempt = Math.min(Math.max(preemptionReduceNumForOneMap, preemptionReduceNumForPreemptionLimit), preemptionReduceNumForAllMaps); LOG.info("Going to preempt " + toPreempt + " due to lack of space for maps"); assignedRequests.preemptReduce(toPreempt); }
private void preemptReducer(int hangingMapRequests) { clearAllPendingReduceRequests(); // preempt for making space for at least one map int preemptionReduceNumForOneMap = ResourceCalculatorUtils.divideAndCeilContainers(mapResourceRequest, reduceResourceRequest, getSchedulerResourceTypes()); int preemptionReduceNumForPreemptionLimit = ResourceCalculatorUtils.divideAndCeilContainers( Resources.multiply(getResourceLimit(), maxReducePreemptionLimit), reduceResourceRequest, getSchedulerResourceTypes()); int preemptionReduceNumForAllMaps = ResourceCalculatorUtils.divideAndCeilContainers( Resources.multiply(mapResourceRequest, hangingMapRequests), reduceResourceRequest, getSchedulerResourceTypes()); int toPreempt = Math.min(Math.max(preemptionReduceNumForOneMap, preemptionReduceNumForPreemptionLimit), preemptionReduceNumForAllMaps); LOG.info("Going to preempt " + toPreempt + " due to lack of space for maps"); assignedRequests.preemptReduce(toPreempt); }
private void preemptReducer(int hangingMapRequests) { clearAllPendingReduceRequests(); // preempt for making space for at least one map int preemptionReduceNumForOneMap = ResourceCalculatorUtils.divideAndCeilContainers(mapResourceRequest, reduceResourceRequest, getSchedulerResourceTypes()); int preemptionReduceNumForPreemptionLimit = ResourceCalculatorUtils.divideAndCeilContainers( Resources.multiply(getResourceLimit(), maxReducePreemptionLimit), reduceResourceRequest, getSchedulerResourceTypes()); int preemptionReduceNumForAllMaps = ResourceCalculatorUtils.divideAndCeilContainers( Resources.multiply(mapResourceRequest, hangingMapRequests), reduceResourceRequest, getSchedulerResourceTypes()); int toPreempt = Math.min(Math.max(preemptionReduceNumForOneMap, preemptionReduceNumForPreemptionLimit), preemptionReduceNumForAllMaps); LOG.info("Going to preempt " + toPreempt + " due to lack of space for maps"); assignedRequests.preemptReduce(toPreempt); }
|| PRIORITY_MAP.equals(priority)) { if (ResourceCalculatorUtils.computeAvailableContainers(allocatedResource, mapResourceRequest, getSchedulerResourceTypes()) <= 0 || maps.isEmpty()) { LOG.info("Cannot assign container " + allocated reduceResourceRequest, getSchedulerResourceTypes()) <= 0 || (reducePending <= 0)) { LOG.info("Cannot assign container " + allocated
|| PRIORITY_OPPORTUNISTIC_MAP.equals(priority)) { if (ResourceCalculatorUtils.computeAvailableContainers(allocatedResource, mapResourceRequest, getSchedulerResourceTypes()) <= 0 || maps.isEmpty()) { LOG.info("Cannot assign container " + allocated reduceResourceRequest, getSchedulerResourceTypes()) <= 0 || (reducePending <= 0)) { LOG.info("Cannot assign container " + allocated
|| PRIORITY_MAP.equals(priority)) { if (ResourceCalculatorUtils.computeAvailableContainers(allocatedResource, mapResourceRequest, getSchedulerResourceTypes()) <= 0 || maps.isEmpty()) { LOG.info("Cannot assign container " + allocated reduceResourceRequest, getSchedulerResourceTypes()) <= 0 || reduces.isEmpty()) { LOG.info("Cannot assign container " + allocated
mapResourceRequest, getSchedulerResourceTypes()) > 0) {
Resources.subtract(getAvailableResources(), scheduledReducesResource); if (ResourceCalculatorUtils.computeAvailableContainers(availableResourceForMap, mapResourceRequest, getSchedulerResourceTypes()) > 0) {
Resources.subtract(getAvailableResources(), scheduledReducesResource); if (ResourceCalculatorUtils.computeAvailableContainers(availableResourceForMap, mapResourceRequest, getSchedulerResourceTypes()) > 0) {
mapResourceReqt, getSchedulerResourceTypes()) >= (scheduledMaps + assignedMaps)) { ResourceCalculatorUtils.computeAvailableContainers(Resources.subtract( finalReduceResourceLimit, netScheduledReduceResource), reduceResourceReqt, getSchedulerResourceTypes());
mapResourceReqt, getSchedulerResourceTypes()) >= (scheduledMaps + assignedMaps)) { ResourceCalculatorUtils.computeAvailableContainers(Resources.subtract( finalReduceResourceLimit, netScheduledReduceResource), reduceResourceReqt, getSchedulerResourceTypes());
mapResourceReqt, getSchedulerResourceTypes()) >= (scheduledMaps + assignedMaps)) { ResourceCalculatorUtils.computeAvailableContainers(Resources.subtract( finalReduceResourceLimit, netScheduledReduceResource), reduceResourceReqt, getSchedulerResourceTypes());
any(Resource.class), anyInt(), anyFloat(), anyFloat()); doReturn(EnumSet.of(SchedulerResourceTypes.MEMORY)).when(allocator) .getSchedulerResourceTypes(); .when(allocator).getSchedulerResourceTypes();