protected void addContainerReq(ContainerRequest req) { // Create resource requests for (String host : req.hosts) { // Data-local if (!isNodeBlacklisted(host)) { addResourceRequest(req.priority, host, req.capability); } } // Nothing Rack-local for now for (String rack : req.racks) { addResourceRequest(req.priority, rack, req.capability); } // Off-switch addResourceRequest(req.priority, ResourceRequest.ANY, req.capability); }
protected void decContainerReq(ContainerRequest req) { // Update resource requests for (String hostName : req.hosts) { decResourceRequest(req.priority, hostName, req.capability); } for (String rack : req.racks) { decResourceRequest(req.priority, rack, req.capability); } decResourceRequest(req.priority, ResourceRequest.ANY, req.capability); }
@Override public Set<String> getBlacklistedNodes() { return ((RMContainerRequestor) containerAllocator).getBlacklistedNodes(); }
protected void addOutstandingRequestOnResync() { for (Map<String, Map<Resource, ResourceRequest>> rr : remoteRequestsTable .values()) { for (Map<Resource, ResourceRequest> capabalities : rr.values()) { for (ResourceRequest request : capabalities.values()) { addResourceRequestToAsk(request); } } } if (!ignoreBlacklisting.get()) { blacklistAdditions.addAll(blacklistedNodes); } if (!pendingRelease.isEmpty()) { release.addAll(pendingRelease); } requestLimitsToUpdate.addAll(requestLimits.keySet()); }
protected ContainerRequest getFilteredContainerRequest(ContainerRequest orig) { ArrayList<String> newHosts = new ArrayList<String>(); for (String host : orig.hosts) { if (!isNodeBlacklisted(host)) { newHosts.add(host); } } String[] hosts = newHosts.toArray(new String[newHosts.size()]); ContainerRequest newReq = new ContainerRequest(orig.attemptID, orig.capability, hosts, orig.racks, orig.priority, orig.nodeLabelExpression); return newReq; }
@Override protected void serviceStop() throws Exception { if (stopped.getAndSet(true)) { // return if already stopped return; } if (eventHandlingThread != null) { eventHandlingThread.interrupt(); } super.serviceStop(); scheduleStats.log("Final Stats: "); }
@Override protected void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); reduceSlowStart = conf.getFloat( MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART,
protected AllocateResponse makeRemoteRequest() throws YarnException, IOException { applyRequestLimits(); ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<String>(blacklistAdditions),
private void addResourceRequest(Priority priority, String resourceName, Resource capability, String nodeLabelExpression) { addResourceRequest(priority, resourceName, capability, nodeLabelExpression, ExecutionType.GUARANTEED); }
protected void addOutstandingRequestOnResync() { for (Map<String, Map<Resource, ResourceRequest>> rr : remoteRequestsTable .values()) { for (Map<Resource, ResourceRequest> capabalities : rr.values()) { for (ResourceRequest request : capabalities.values()) { addResourceRequestToAsk(request); } } } if (!ignoreBlacklisting.get()) { blacklistAdditions.addAll(blacklistedNodes); } if (!pendingRelease.isEmpty()) { release.addAll(pendingRelease); } requestLimitsToUpdate.addAll(requestLimits.keySet()); }
protected ContainerRequest getFilteredContainerRequest(ContainerRequest orig) { ArrayList<String> newHosts = new ArrayList<String>(); for (String host : orig.hosts) { if (!isNodeBlacklisted(host)) { newHosts.add(host); } } String[] hosts = newHosts.toArray(new String[newHosts.size()]); ContainerRequest newReq = new ContainerRequest(orig.attemptID, orig.capability, hosts, orig.racks, orig.priority); return newReq; }
@Override protected void serviceStop() throws Exception { if (stopped.getAndSet(true)) { // return if already stopped return; } if (eventHandlingThread != null) { eventHandlingThread.interrupt(); } super.serviceStop(); scheduleStats.log("Final Stats: "); }
@Override protected void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); reduceSlowStart = conf.getFloat( MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART); maxReduceRampupLimit = conf.getFloat( MRJobConfig.MR_AM_JOB_REDUCE_RAMPUP_UP_LIMIT, MRJobConfig.DEFAULT_MR_AM_JOB_REDUCE_RAMP_UP_LIMIT); maxReducePreemptionLimit = conf.getFloat( MRJobConfig.MR_AM_JOB_REDUCE_PREEMPTION_LIMIT, MRJobConfig.DEFAULT_MR_AM_JOB_REDUCE_PREEMPTION_LIMIT); reducerUnconditionalPreemptionDelayMs = 1000 * conf.getInt( MRJobConfig.MR_JOB_REDUCER_UNCONDITIONAL_PREEMPT_DELAY_SEC, MRJobConfig.DEFAULT_MR_JOB_REDUCER_UNCONDITIONAL_PREEMPT_DELAY_SEC); reducerNoHeadroomPreemptionDelayMs = conf.getInt( MRJobConfig.MR_JOB_REDUCER_PREEMPT_DELAY_SEC, MRJobConfig.DEFAULT_MR_JOB_REDUCER_PREEMPT_DELAY_SEC) * 1000;//sec -> ms maxRunningMaps = conf.getInt(MRJobConfig.JOB_RUNNING_MAP_LIMIT, MRJobConfig.DEFAULT_JOB_RUNNING_MAP_LIMIT); maxRunningReduces = conf.getInt(MRJobConfig.JOB_RUNNING_REDUCE_LIMIT, MRJobConfig.DEFAULT_JOB_RUNNING_REDUCE_LIMIT); RackResolver.init(conf); retryInterval = getConfig().getLong(MRJobConfig.MR_AM_TO_RM_WAIT_INTERVAL_MS, MRJobConfig.DEFAULT_MR_AM_TO_RM_WAIT_INTERVAL_MS); // Init startTime to current time. If all goes well, it will be reset after // first attempt to contact RM. retrystartTime = System.currentTimeMillis(); }
protected AllocateResponse makeRemoteRequest() throws YarnException, IOException { applyRequestLimits(); ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<String>(blacklistAdditions),
protected void addOpportunisticResourceRequest(Priority priority, Resource capability) { addResourceRequest(priority, ResourceRequest.ANY, capability, null, ExecutionType.OPPORTUNISTIC); }
protected void addContainerReq(ContainerRequest req) { // Create resource requests for (String host : req.hosts) { // Data-local if (!isNodeBlacklisted(host)) { addResourceRequest(req.priority, host, req.capability, null); } } // Nothing Rack-local for now for (String rack : req.racks) { addResourceRequest(req.priority, rack, req.capability, null); } // Off-switch addResourceRequest(req.priority, ResourceRequest.ANY, req.capability, req.nodeLabelExpression); }
protected void addOutstandingRequestOnResync() { for (Map<String, Map<Resource, ResourceRequest>> rr : remoteRequestsTable .values()) { for (Map<Resource, ResourceRequest> capabalities : rr.values()) { for (ResourceRequest request : capabalities.values()) { addResourceRequestToAsk(request); } } } if (!ignoreBlacklisting.get()) { blacklistAdditions.addAll(blacklistedNodes); } if (!pendingRelease.isEmpty()) { release.addAll(pendingRelease); } requestLimitsToUpdate.addAll(requestLimits.keySet()); }
protected ContainerRequest getFilteredContainerRequest(ContainerRequest orig) { ArrayList<String> newHosts = new ArrayList<String>(); for (String host : orig.hosts) { if (!isNodeBlacklisted(host)) { newHosts.add(host); } } String[] hosts = newHosts.toArray(new String[newHosts.size()]); ContainerRequest newReq = new ContainerRequest(orig.attemptID, orig.capability, hosts, orig.racks, orig.priority, orig.nodeLabelExpression); return newReq; }