workerCapability.setVirtualCores(workerVCores); workerContainerRequest = new ContainerRequest(workerCapability, hostLocals, null, priority); LOG.info("Create worker container request: " + workerContainerRequest.toString()); psCapability.setVirtualCores(psVCores); psContainerRequest = new ContainerRequest(psCapability, hostLocals, null, priority); LOG.info("Create ps container request: " + psContainerRequest.toString());
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM */ public ContainerRequest setupContainerAskForRM(int containerMemory, int containerVirtualCores, int priority, String[] racks, String[] hosts) { Priority pri = Priority.newInstance(priority); Resource capability = Resource.newInstance(containerMemory, containerVirtualCores); ContainerRequest request = new ContainerRequest(capability, hosts, racks, pri, false); LOG.info("By Thrift Server Requested container ask: " + request.toString()); return request; }
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM */ public ContainerRequest setupContainerAskForRM(int containerMemory, int containerVirtualCores, int priority, String host) { // setup requirements for hosts // using * as any host will do for the jstorm app // set the priority for the request Priority pri = Priority.newInstance(priority); // Set up resource type requirements // For now, memory and CPU are supported so we set memory and cpu requirements Resource capability = Resource.newInstance(containerMemory, containerVirtualCores); ContainerRequest request = new ContainerRequest(capability, null, null, pri); LOG.info("By Thrift Server Requested container ask: " + request.toString()); return request; }
ContainerRequest nnContainerRequest = setupContainerAskForRM(amOptions.getNameNodeMemoryMB(), amOptions.getNameNodeVirtualCores(), 0, amOptions.getNameNodeNodeLabelExpression()); LOG.info("Requested NameNode ask: " + nnContainerRequest.toString()); amRMClient.addContainerRequest(nnContainerRequest); amOptions.getDataNodeVirtualCores(), 1, amOptions.getDataNodeNodeLabelExpression()); amRMClient.addContainerRequest(datanodeAsk); LOG.debug("Requested datanode ask: " + datanodeAsk.toString());
public static String requestToString(AMRMClient.ContainerRequest request) { Preconditions.checkArgument(request != null, "Null request"); StringBuilder buffer = new StringBuilder(request.toString()); buffer.append("; "); buffer.append("relaxLocality=").append(request.getRelaxLocality()).append("; "); String labels = request.getNodeLabelExpression(); if (labels != null) { buffer.append("nodeLabels=").append(labels).append("; "); } List<String> nodes = request.getNodes(); if (nodes != null) { buffer.append("Nodes = [ "); int size = nodes.size(); for (int i = 0; i < Math.min(NODE_LIST_LIMIT, size); i++) { buffer.append(nodes.get(i)).append(' '); } if (size > NODE_LIST_LIMIT) { buffer.append(String.format("...(total %d entries)", size)); } buffer.append("]; "); } List<String> racks = request.getRacks(); if (racks != null) { buffer.append("racks = [") .append(join(racks, ", ", false)) .append("]; "); } return buffer.toString(); }
public static String requestToString(AMRMClient.ContainerRequest request) { Preconditions.checkArgument(request != null, "Null request"); StringBuilder buffer = new StringBuilder(request.toString()); buffer.append("; "); buffer.append("relaxLocality=").append(request.getRelaxLocality()).append("; "); String labels = request.getNodeLabelExpression(); if (labels != null) { buffer.append("nodeLabels=").append(labels).append("; "); } List<String> nodes = request.getNodes(); if (nodes != null) { buffer.append("Nodes = [ "); int size = nodes.size(); for (int i = 0; i < Math.min(NODE_LIST_LIMIT, size); i++) { buffer.append(nodes.get(i)).append(' '); } if (size > NODE_LIST_LIMIT) { buffer.append(String.format("...(total %d entries)", size)); } buffer.append("]; "); } List<String> racks = request.getRacks(); if (racks != null) { buffer.append("racks = [") .append(join(racks, ", ", false)) .append("]; "); } return buffer.toString(); }
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM */ private ContainerRequest setupContainerAskForRM() { // setup requirements for hosts // using * as any host will do for the distributed shell app // set the priority for the request Priority pri = Records.newRecord(Priority.class); // TODO - what is the range for priority? how to decide? pri.setPriority(requestPriority); // Set up resource type requirements // For now, memory and CPU are supported so we set memory and cpu // requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(containerMemory); capability.setVirtualCores(1); ContainerRequest request = new ContainerRequest(capability, null, null, pri); LOG.info("Requested container ask: " + request.toString()); return request; }
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM */ private AMRMClient.ContainerRequest setupContainerAskForRM() { // setup requirements for hosts // using * as any host will do for the distributed shell app // set the priority for the request // TODO - what is the range for priority? how to decide? Priority pri = Priority.newInstance(requestPriority); // Set up resource type requirements // For now, memory and CPU are supported so we set memory and cpu // requirements Resource capability = Resource.newInstance(containerMemory, containerVirtualCores); AMRMClient.ContainerRequest request = new AMRMClient.ContainerRequest( capability, null, null, pri); LOG.info("Requested container ask: " + request.toString()); return request; }
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM */ private ContainerRequest setupContainerAskForRM() { // setup requirements for hosts // using * as any host will do for the distributed shell app // set the priority for the request // TODO - what is the range for priority? how to decide? Priority pri = Priority.newInstance(requestPriority); // Set up resource type requirements ContainerRequest request = new ContainerRequest( getTaskResourceCapability(), null, null, pri, 0, true, null, ExecutionTypeRequest.newInstance(containerType), containerResourceProfile); LOG.info("Requested container ask: " + request.toString()); return request; }
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM */ private ContainerRequest setupContainerAskForRM() { // setup requirements for hosts, request containers firstly and then check allocated containers and splits to // get data locality. // TODO, better here to requests according to hosts of splits. Priority pri = Records.newRecord(Priority.class); pri.setPriority(GuaguaYarnConstants.GUAGUA_YARN_DEFAULT_PRIORITY); Resource capability = Records.newRecord(Resource.class); capability.setMemory(getHeapPerContainer()); capability.setVirtualCores(getYarnConf().getInt(GuaguaYarnConstants.GUAGUA_YARN_TASK_VCORES, GuaguaYarnConstants.GUAGUA_YARN_TASK_DEFAULT_VCORES)); ContainerRequest request = new ContainerRequest(capability, null, null, pri); LOG.info("Requested container ask: {}", request.toString()); return request; }
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM */ private ContainerRequest setupContainerAskForRM() { // setup requirements for hosts, request containers firstly and then check allocated containers and splits to // get data locality. // TODO, better here to requests according to hosts of splits. Priority pri = Records.newRecord(Priority.class); pri.setPriority(GuaguaYarnConstants.GUAGUA_YARN_DEFAULT_PRIORITY); Resource capability = Records.newRecord(Resource.class); capability.setMemory(getHeapPerContainer()); capability.setVirtualCores(getYarnConf().getInt(GuaguaYarnConstants.GUAGUA_YARN_TASK_VCORES, GuaguaYarnConstants.GUAGUA_YARN_TASK_DEFAULT_VCORES)); ContainerRequest request = new ContainerRequest(capability, null, null, pri); LOG.info("Requested container ask: {}", request.toString()); return request; }
private AMRMClient.ContainerRequest setupContainerRequestForRM(TensorFlowContainerRequest request) { Priority priority = Priority.newInstance(request.getPriority()); Resource capability = Resource.newInstance(request.getMemory(), request.getVCores()); Utils.setCapabilityGPU(capability, request.getGPU()); session.addAllocationId(request.getJobName(), lastAllocationRequestId); AMRMClient.ContainerRequest containerRequest = new AMRMClient.ContainerRequest(capability, null, null, priority, lastAllocationRequestId++); LOG.info("Requested container ask: " + containerRequest.toString()); return containerRequest; }