@Override public boolean equals(Object o) { if (! (o instanceof SchedulerApplicationAttempt)) { return false; } SchedulerApplicationAttempt other = (SchedulerApplicationAttempt) o; return (this == other || this.getApplicationAttemptId().equals(other.getApplicationAttemptId())); }
@Override public int hashCode() { return getApplicationAttemptId().hashCode(); }
@Override public String toString() { return "(Application=" + schedulerApplicationAttempt .getApplicationAttemptId() + "; Node=" + schedulerNode.getNodeID() + "; Resource=" + rmContainer.getAllocatedOrReservedResource() + ")"; } }
private void updateNMToken(Container container) { NMToken nmToken = rmContext.getNMTokenSecretManager().createAndGetNMToken(getUser(), getApplicationAttemptId(), container); if (nmToken != null) { updatedNMTokens.add(nmToken); } }
private boolean commonReserve(SchedulerNode node, SchedulerRequestKey schedulerKey, RMContainer rmContainer, Resource reservedResource) { try { rmContainer.handle(new RMContainerReservedEvent(rmContainer .getContainerId(), reservedResource, node.getNodeID(), schedulerKey)); } catch (InvalidStateTransitionException e) { // We reach here could be caused by container already finished, return // false indicate it fails return false; } Map<NodeId, RMContainer> reservedContainers = this.reservedContainers.get(schedulerKey); if (reservedContainers == null) { reservedContainers = new HashMap<NodeId, RMContainer>(); this.reservedContainers.put(schedulerKey, reservedContainers); } reservedContainers.put(node.getNodeID(), rmContainer); if (LOG.isDebugEnabled()) { LOG.debug("Application attempt " + getApplicationAttemptId() + " reserved container " + rmContainer + " on node " + node + ". This attempt currently has " + reservedContainers.size() + " reserved containers at priority " + schedulerKey.getPriority() + "; currentReservation " + reservedResource); } return true; }
public void addRMContainer( ContainerId id, RMContainer rmContainer) { try { writeLock.lock(); if (!getApplicationAttemptId().equals( rmContainer.getApplicationAttemptId()) && !liveContainers.containsKey(id)) { LOG.info("recovered container " + id + " from previous attempt " + rmContainer.getApplicationAttemptId()); recoveredPreviousAttemptContainers.add(rmContainer.getContainer()); } liveContainers.put(id, rmContainer); if (rmContainer.getExecutionType() == ExecutionType.OPPORTUNISTIC) { this.attemptOpportunisticResourceUsage.incUsed( rmContainer.getAllocatedResource()); } if (rmContainer.isRemotelyAllocated()) { this.attemptResourceUsageAllocatedRemotely.incUsed( rmContainer.getAllocatedResource()); } } finally { writeLock.unlock(); } }
new RMContainerImpl(container, getApplicationAttemptId(), node.getNodeID(), appSchedulingInfo.getUser(), rmContext); LOG.debug("Application attempt " + getApplicationAttemptId() + " reserved container " + rmContainer + " on node " + node + ". This attempt currently has " + reservedContainers.size()
public synchronized void recoverContainer(RMContainer rmContainer) { // recover app scheduling info appSchedulingInfo.recoverContainer(rmContainer); if (rmContainer.getState().equals(RMContainerState.COMPLETED)) { return; } LOG.info("SchedulerAttempt " + getApplicationAttemptId() + " is recovering container " + rmContainer.getContainerId()); liveContainers.put(rmContainer.getContainerId(), rmContainer); Resources.addTo(currentConsumption, rmContainer.getContainer() .getResource()); // resourceLimit: updated when LeafQueue#recoverContainer#allocateResource // is called. // newlyAllocatedContainers.add(rmContainer); // schedulingOpportunities // lastScheduledContainer }
public synchronized void recoverContainer(RMContainer rmContainer) { // recover app scheduling info appSchedulingInfo.recoverContainer(rmContainer); if (rmContainer.getState().equals(RMContainerState.COMPLETED)) { return; } LOG.info("SchedulerAttempt " + getApplicationAttemptId() + " is recovering container " + rmContainer.getContainerId()); liveContainers.put(rmContainer.getContainerId(), rmContainer); Resources.addTo(currentConsumption, rmContainer.getContainer() .getResource()); // resourceLimit: updated when LeafQueue#recoverContainer#allocateResource // is called. // newlyAllocatedContainers.add(rmContainer); // schedulingOpportunities // lastScheduledContainer }
@Override public synchronized void unreserveResource( SchedulerApplicationAttempt application) { // Cannot unreserve for wrong application... ApplicationAttemptId reservedApplication = getReservedContainer().getContainer().getId().getApplicationAttemptId(); if (!reservedApplication.equals( application.getApplicationAttemptId())) { throw new IllegalStateException("Trying to unreserve " + " for application " + application.getApplicationId() + " when currently reserved " + " for application " + reservedApplication.getApplicationId() + " on node " + this); } setReservedContainer(null); this.reservedAppSchedulable = null; }
@Override public synchronized void unreserveResource( SchedulerApplicationAttempt application) { // Cannot unreserve for wrong application... ApplicationAttemptId reservedApplication = getReservedContainer().getContainer().getId() .getApplicationAttemptId(); if (!reservedApplication.equals( application.getApplicationAttemptId())) { throw new IllegalStateException("Trying to unreserve " + " for application " + application.getApplicationId() + " when currently reserved " + " for application " + reservedApplication.getApplicationId() + " on node " + this); } setReservedContainer(null); this.reservedAppSchedulable = null; }
@Override public synchronized void unreserveResource( SchedulerApplicationAttempt application) { // Cannot unreserve for wrong application... ApplicationAttemptId reservedApplication = getReservedContainer().getContainer().getId().getApplicationAttemptId(); if (!reservedApplication.equals( application.getApplicationAttemptId())) { throw new IllegalStateException("Trying to unreserve " + " for application " + application.getApplicationId() + " when currently reserved " + " for application " + reservedApplication.getApplicationId() + " on node " + this); } setReservedContainer(null); this.reservedAppSchedulable = null; }
public void recoverContainer(SchedulerNode node, RMContainer rmContainer) { try { writeLock.lock(); // recover app scheduling info appSchedulingInfo.recoverContainer(rmContainer, node.getPartition()); if (rmContainer.getState().equals(RMContainerState.COMPLETED)) { return; } LOG.info("SchedulerAttempt " + getApplicationAttemptId() + " is recovering container " + rmContainer.getContainerId()); addRMContainer(rmContainer.getContainerId(), rmContainer); if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) { attemptResourceUsage.incUsed(node.getPartition(), rmContainer.getContainer().getResource()); } // resourceLimit: updated when LeafQueue#recoverContainer#allocateResource // is called. // newlyAllocatedContainers.add(rmContainer); // schedulingOpportunities // lastScheduledContainer } finally { writeLock.unlock(); } }
@Override public synchronized void unreserveResource( SchedulerApplicationAttempt application) { // adding NP checks as this can now be called for preemption if (getReservedContainer() != null && getReservedContainer().getContainer() != null && getReservedContainer().getContainer().getId() != null && getReservedContainer().getContainer().getId() .getApplicationAttemptId() != null) { // Cannot unreserve for wrong application... ApplicationAttemptId reservedApplication = getReservedContainer().getContainer().getId() .getApplicationAttemptId(); if (!reservedApplication.equals( application.getApplicationAttemptId())) { throw new IllegalStateException("Trying to unreserve " + " for application " + application.getApplicationAttemptId() + " when currently reserved " + " for application " + reservedApplication.getApplicationId() + " on node " + this); } } setReservedContainer(null); }
@Override public synchronized void unreserveResource( SchedulerApplicationAttempt application) { // adding NP checks as this can now be called for preemption if (getReservedContainer() != null && getReservedContainer().getContainer() != null && getReservedContainer().getContainer().getId() != null && getReservedContainer().getContainer().getId() .getApplicationAttemptId() != null) { // Cannot unreserve for wrong application... ApplicationAttemptId reservedApplication = getReservedContainer().getContainer().getId() .getApplicationAttemptId(); if (!reservedApplication.equals( application.getApplicationAttemptId())) { throw new IllegalStateException("Trying to unreserve " + " for application " + application.getApplicationAttemptId() + " when currently reserved " + " for application " + reservedApplication.getApplicationId() + " on node " + this); } } setReservedContainer(null); } }
@Override public synchronized void unreserveResource( SchedulerApplicationAttempt application) { // adding NP checks as this can now be called for preemption if (getReservedContainer() != null && getReservedContainer().getContainer() != null && getReservedContainer().getContainer().getId() != null && getReservedContainer().getContainer().getId() .getApplicationAttemptId() != null) { // Cannot unreserve for wrong application... ApplicationAttemptId reservedApplication = getReservedContainer().getContainer().getId() .getApplicationAttemptId(); if (!reservedApplication.equals( application.getApplicationAttemptId())) { throw new IllegalStateException("Trying to unreserve " + " for application " + application.getApplicationAttemptId() + " when currently reserved " + " for application " + reservedApplication.getApplicationId() + " on node " + this); } } setReservedContainer(null); } }
public RMContainer reserve(SchedulerNode node, SchedulerRequestKey schedulerKey, RMContainer rmContainer, Container container) { try { writeLock.lock(); // Create RMContainer if necessary if (rmContainer == null) { rmContainer = new RMContainerImpl(container, schedulerKey, getApplicationAttemptId(), node.getNodeID(), appSchedulingInfo.getUser(), rmContext); } if (rmContainer.getState() == RMContainerState.NEW) { attemptResourceUsage.incReserved(node.getPartition(), container.getResource()); ((RMContainerImpl) rmContainer).setQueueName(this.getQueueName()); // Reset the re-reservation count resetReReservations(schedulerKey); } else{ // Note down the re-reservation addReReservation(schedulerKey); } commonReserve(node, schedulerKey, rmContainer, container.getResource()); return rmContainer; } finally { writeLock.unlock(); } }
public static RMContainer createOpportunisticRmContainer(RMContext rmContext, Container container, boolean isRemotelyAllocated) { SchedulerApplicationAttempt appAttempt = ((AbstractYarnScheduler) rmContext.getScheduler()) .getCurrentAttemptForContainer(container.getId()); RMContainer rmContainer = new RMContainerImpl(container, SchedulerRequestKey.extractFrom(container), appAttempt.getApplicationAttemptId(), container.getNodeId(), appAttempt.getUser(), rmContext, isRemotelyAllocated); appAttempt.addRMContainer(container.getId(), rmContainer); ((AbstractYarnScheduler) rmContext.getScheduler()).getNode( container.getNodeId()).allocateContainer(rmContainer); return rmContainer; } }
private RMContainer createDecreasedRMContainer( SchedulerApplicationAttempt appAttempt, UpdateContainerRequest uReq, RMContainer rmContainer) { SchedulerRequestKey sk = SchedulerRequestKey.extractFrom(rmContainer.getContainer()); Container decreasedContainer = BuilderUtils.newContainer( ContainerId.newContainerId(appAttempt.getApplicationAttemptId(), appAttempt.getNewContainerId()), rmContainer.getContainer().getNodeId(), rmContainer.getContainer().getNodeHttpAddress(), Resources.none(), sk.getPriority(), null, rmContainer.getExecutionType(), sk.getAllocationRequestId()); decreasedContainer.setVersion(rmContainer.getContainer().getVersion()); RMContainer newRmContainer = new RMContainerImpl(decreasedContainer, sk, appAttempt.getApplicationAttemptId(), decreasedContainer.getNodeId(), appAttempt.getUser(), rmContext, rmContainer.isRemotelyAllocated()); appAttempt.addRMContainer(decreasedContainer.getId(), rmContainer); ((AbstractYarnScheduler) rmContext.getScheduler()).getNode( decreasedContainer.getNodeId()).allocateContainer(newRmContainer); return newRmContainer; }
private RMContainer createDemotedRMContainer( SchedulerApplicationAttempt appAttempt, OpportunisticContainerContext oppCntxt, RMContainer rmContainer) { SchedulerRequestKey sk = SchedulerRequestKey.extractFrom(rmContainer.getContainer()); Container demotedContainer = BuilderUtils.newContainer( ContainerId.newContainerId(appAttempt.getApplicationAttemptId(), oppCntxt.getContainerIdGenerator().generateContainerId()), rmContainer.getContainer().getNodeId(), rmContainer.getContainer().getNodeHttpAddress(), rmContainer.getContainer().getResource(), sk.getPriority(), null, ExecutionType.OPPORTUNISTIC, sk.getAllocationRequestId()); demotedContainer.setVersion(rmContainer.getContainer().getVersion()); return SchedulerUtils.createOpportunisticRmContainer( rmContext, demotedContainer, false); }