@Nonnull private AMRMClient.ContainerRequest createContainerRequest(Resource capability) { return new AMRMClient.ContainerRequest(capability, null, null, RM_REQUEST_PRIORITY); }
workerCapability.setMemory(workerMemory); workerCapability.setVirtualCores(workerVCores); workerContainerRequest = new ContainerRequest(workerCapability, hostLocals, null, priority); LOG.info("Create worker container request: " + workerContainerRequest.toString()); psCapability.setMemory(psMemory); psCapability.setVirtualCores(psVCores); psContainerRequest = new ContainerRequest(psCapability, hostLocals, null, priority); LOG.info("Create ps container request: " + psContainerRequest.toString());
@Nonnull @VisibleForTesting AMRMClient.ContainerRequest getContainerRequest() { return new AMRMClient.ContainerRequest( getContainerResource(), null, null, RM_REQUEST_PRIORITY); }
@Override public synchronized void onNext(final ResourceRequestEvent resourceRequestEvent) { LOG.log(Level.FINEST, "Got ResourceRequestEvent in YarnResourceRequestHandler: memory = {0}, cores = {1}.", new Object[]{resourceRequestEvent.getMemorySize(), resourceRequestEvent.getVirtualCores()}); final String[] nodes = resourceRequestEvent.getNodeNameList().size() == 0 ? null : resourceRequestEvent.getNodeNameList().toArray(new String[resourceRequestEvent.getNodeNameList().size()]); final String[] racks = resourceRequestEvent.getRackNameList().size() == 0 ? null : resourceRequestEvent.getRackNameList().toArray(new String[resourceRequestEvent.getRackNameList().size()]); // set the priority for the request final Priority pri = getPriority(resourceRequestEvent); final Resource resource = getResource(resourceRequestEvent); final boolean relaxLocality = resourceRequestEvent.getRelaxLocality().orElse(true); final AMRMClient.ContainerRequest[] containerRequests = new AMRMClient.ContainerRequest[resourceRequestEvent.getResourceCount()]; for (int i = 0; i < resourceRequestEvent.getResourceCount(); i++) { containerRequests[i] = new AMRMClient.ContainerRequest(resource, nodes, racks, pri, relaxLocality); } this.yarnContainerRequestHandler.onContainerRequest(containerRequests); }
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM */ private ContainerRequest setupContainerAskForRM(int memory, int vcores, int priority, String nodeLabel) { Priority pri = Records.newRecord(Priority.class); pri.setPriority(priority); // Set up resource type requirements // For now, memory and CPU are supported so we set memory and cpu requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(memory); capability.setVirtualCores(vcores); return new ContainerRequest(capability, null, null, pri, true, nodeLabel); }
private void requestContainer(Optional<String> preferredNode) { Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); Resource capability = Records.newRecord(Resource.class); int maxMemoryCapacity = this.maxResourceCapacity.get().getMemory(); capability.setMemory(this.requestedContainerMemoryMbs <= maxMemoryCapacity ? this.requestedContainerMemoryMbs : maxMemoryCapacity); int maxCoreCapacity = this.maxResourceCapacity.get().getVirtualCores(); capability.setVirtualCores(this.requestedContainerCores <= maxCoreCapacity ? this.requestedContainerCores : maxCoreCapacity); String[] preferredNodes = preferredNode.isPresent() ? new String[] {preferredNode.get()} : null; this.amrmClientAsync.addContainerRequest( new AMRMClient.ContainerRequest(capability, preferredNodes, null, priority)); }
@Nonnull @VisibleForTesting AMRMClient.ContainerRequest getContainerRequest() { return new AMRMClient.ContainerRequest( getContainerResource(), null, null, RM_REQUEST_PRIORITY); }
@Override protected AMRMClient.ContainerRequest createContainerRequest(Priority priority, Resource capability, @Nullable String[] hosts, @Nullable String[] racks, boolean relaxLocality) { return new AMRMClient.ContainerRequest(capability, hosts, racks, priority, relaxLocality); }
@Nonnull private AMRMClient.ContainerRequest createContainerRequest(Resource capability) { return new AMRMClient.ContainerRequest(capability, null, null, RM_REQUEST_PRIORITY); }
@Override protected AMRMClient.ContainerRequest createContainerRequest(Priority priority, Resource capability, @Nullable String[] hosts, @Nullable String[] racks, boolean relaxLocality) { return new AMRMClient.ContainerRequest(capability, hosts, racks, priority, relaxLocality); }
public synchronized boolean addAllocatedContainers(List<Container> containers) { for (int i = 0; i < containers.size(); i++) { Container c = containers.get(i); ContainerRequest req = new ContainerRequest(this.maxResourceCapability, new String[]{c.getNodeId().getHost()}, // String[] nodes, null, // String[] racks, DEFAULT_PRIORITY, false); super.removeContainerRequest(req); requestingNodes.remove(c.getNodeId().getHost()); } return this.containers.addAll(containers); }
private void requestYarnContainer(Resource resource, Priority priority) { resourceManagerClient.addContainerRequest(new AMRMClient.ContainerRequest(resource, null, null, priority)); // make sure we transmit the request fast and receive fast news of granted allocations resourceManagerClient.setHeartbeatInterval(FAST_YARN_HEARTBEAT_INTERVAL_MS); numPendingContainerRequests++; log.info("Requesting new TaskExecutor container with resources {}. Number pending requests {}.", resource, numPendingContainerRequests); }
private AMRMClient.ContainerRequest setupContainerRequestForRM(TensorFlowContainerRequest request) { Priority priority = Priority.newInstance(request.getPriority()); Resource capability = Resource.newInstance(request.getMemory(), request.getVCores()); Utils.setCapabilityGPU(capability, request.getGPU()); session.addAllocationId(request.getJobName(), lastAllocationRequestId); AMRMClient.ContainerRequest containerRequest = new AMRMClient.ContainerRequest(capability, null, null, priority, lastAllocationRequestId++); LOG.info("Requested container ask: " + containerRequest.toString()); return containerRequest; }
public void requestContainers(int number, Resource characteristic) { Priority pri = Priority.newInstance(0); state.getQueue(characteristic); AMRMClient.ContainerRequest request = new AMRMClient.ContainerRequest(characteristic, null, null, pri, true); for(int i = 0;i < number;++i) { amRMClient.addContainerRequest(request); } }
return new ContainerRequest(capability, nodeArr, rackArr, priorityRec, relaxLocality, nodeExpr);
final YarnResourceManagerCallbackHandler callbackHandler = new YarnResourceManagerCallbackHandler(); AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient = mock(AMRMClientAsync.class); doReturn(Collections.singletonList(Collections.nCopies(numInitialTaskManagers, new AMRMClient.ContainerRequest(Resource.newInstance(1024 * 1024, 1), null, null, Priority.newInstance(0))))) .when(resourceManagerClient).getMatchingRequests(any(Priority.class), anyString(), any(Resource.class));
new AMRMClient.ContainerRequest(capability, null, null, priority);
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM */ public ContainerRequest setupContainerAskForRM(int containerMemory, int containerVirtualCores, int priority, String host) { // setup requirements for hosts // using * as any host will do for the jstorm app // set the priority for the request Priority pri = Priority.newInstance(priority); // Set up resource type requirements // For now, memory and CPU are supported so we set memory and cpu requirements Resource capability = Resource.newInstance(containerMemory, containerVirtualCores); ContainerRequest request = new ContainerRequest(capability, null, null, pri); LOG.info("By Thrift Server Requested container ask: " + request.toString()); return request; }
private void requestContainer(Optional<String> preferredNode) { Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); Resource capability = Records.newRecord(Resource.class); int maxMemoryCapacity = this.maxResourceCapacity.get().getMemory(); capability.setMemory(this.requestedContainerMemoryMbs <= maxMemoryCapacity ? this.requestedContainerMemoryMbs : maxMemoryCapacity); int maxCoreCapacity = this.maxResourceCapacity.get().getVirtualCores(); capability.setVirtualCores(this.requestedContainerCores <= maxCoreCapacity ? this.requestedContainerCores : maxCoreCapacity); String[] preferredNodes = preferredNode.isPresent() ? new String[] {preferredNode.get()} : null; this.amrmClientAsync.addContainerRequest( new AMRMClient.ContainerRequest(capability, preferredNodes, null, priority)); }
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM */ public ContainerRequest setupContainerAskForRM(int containerMemory, int containerVirtualCores, int priority, String[] racks, String[] hosts) { Priority pri = Priority.newInstance(priority); Resource capability = Resource.newInstance(containerMemory, containerVirtualCores); ContainerRequest request = new ContainerRequest(capability, hosts, racks, pri, false); LOG.info("By Thrift Server Requested container ask: " + request.toString()); return request; }