@Override public Resource getResource() { return Resource.newInstance(memory, vcores); }
private Resource getContainerResource() { final long mem = taskManagerParameters.taskManagerTotalMemoryMB(); final int containerMemorySizeMB; if (mem <= Integer.MAX_VALUE) { containerMemorySizeMB = (int) mem; } else { containerMemorySizeMB = Integer.MAX_VALUE; LOG.error("Decreasing container size from {} MB to {} MB (integer value overflow)", mem, containerMemorySizeMB); } // Resource requirements for worker containers int taskManagerSlots = taskManagerParameters.numSlots(); int vcores = config.getInteger(YarnConfigOptions.VCORES, Math.max(taskManagerSlots, 1)); return Resource.newInstance(containerMemorySizeMB, vcores); }
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM */ public ContainerRequest setupContainerAskForRM(int containerMemory, int containerVirtualCores, int priority, String[] racks, String[] hosts) { Priority pri = Priority.newInstance(priority); Resource capability = Resource.newInstance(containerMemory, containerVirtualCores); ContainerRequest request = new ContainerRequest(capability, hosts, racks, pri, false); LOG.info("By Thrift Server Requested container ask: " + request.toString()); return request; }
public static Resource getContainerResource(Configuration conf) { int memory = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) > 0 ? HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) : conf.getInt(MRJobConfig.MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_MEMORY_MB); int cpus = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCPUVCORES) > 0 ? HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCPUVCORES) : conf.getInt(MRJobConfig.MAP_CPU_VCORES, MRJobConfig.DEFAULT_MAP_CPU_VCORES); return Resource.newInstance(memory, cpus); }
private Resource prepareContainerResource(GetNewApplicationResponse newApplicationResponse) { int memoryMbs = this.config.getInt(GobblinYarnConfigurationKeys.APP_MASTER_MEMORY_MBS_KEY); int maximumMemoryCapacity = newApplicationResponse.getMaximumResourceCapability().getMemory(); if (memoryMbs > maximumMemoryCapacity) { LOGGER.info(String.format("Specified AM memory [%d] is above the maximum memory capacity [%d] of the " + "cluster, using the maximum memory capacity instead.", memoryMbs, maximumMemoryCapacity)); memoryMbs = maximumMemoryCapacity; } int vCores = this.config.getInt(GobblinYarnConfigurationKeys.APP_MASTER_CORES_KEY); int maximumVirtualCoreCapacity = newApplicationResponse.getMaximumResourceCapability().getVirtualCores(); if (vCores > maximumVirtualCoreCapacity) { LOGGER.info(String.format("Specified AM vcores [%d] is above the maximum vcore capacity [%d] of the " + "cluster, using the maximum vcore capacity instead.", memoryMbs, maximumMemoryCapacity)); vCores = maximumVirtualCoreCapacity; } // Set up resource type requirements for ApplicationMaster return Resource.newInstance(memoryMbs, vCores); }
public static Resource getContainerResource(Configuration conf) { int memory = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) > 0 ? HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) : conf.getInt(MRJobConfig.MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_MEMORY_MB); int cpus = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCPUVCORES) > 0 ? HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCPUVCORES) : conf.getInt(MRJobConfig.MAP_CPU_VCORES, MRJobConfig.DEFAULT_MAP_CPU_VCORES); return Resource.newInstance(memory, cpus); }
@Override public Resource getTotalResources() { int memory = 0; int vcores = 0; readLock.lock(); try { int numInstancesFound = 0; for (LlapServiceInstance inst : activeInstances.getAll()) { Resource r = inst.getResource(); memory += r.getMemory(); vcores += r.getVirtualCores(); numInstancesFound++; } if (LOG.isDebugEnabled()) { LOG.debug("GetTotalResources: numInstancesFound={}, totalMem={}, totalVcores={}", numInstancesFound, memory, vcores); } } finally { readLock.unlock(); } return Resource.newInstance(memory, vcores); }
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM */ public ContainerRequest setupContainerAskForRM(int containerMemory, int containerVirtualCores, int priority, String host) { // setup requirements for hosts // using * as any host will do for the jstorm app // set the priority for the request Priority pri = Priority.newInstance(priority); // Set up resource type requirements // For now, memory and CPU are supported so we set memory and cpu requirements Resource capability = Resource.newInstance(containerMemory, containerVirtualCores); ContainerRequest request = new ContainerRequest(capability, null, null, pri); LOG.info("By Thrift Server Requested container ask: " + request.toString()); return request; }
public DynamicServiceInstance(ServiceRecord srv) throws IOException { super(srv, IPC_LLAP); final Endpoint shuffle = srv.getInternalEndpoint(IPC_SHUFFLE); final Endpoint mng = srv.getInternalEndpoint(IPC_MNG); final Endpoint outputFormat = srv.getInternalEndpoint(IPC_OUTPUTFORMAT); final Endpoint services = srv.getExternalEndpoint(IPC_SERVICES); this.mngPort = Integer.parseInt(RegistryTypeUtils.getAddressField(mng.addresses.get(0), AddressTypes.ADDRESS_PORT_FIELD)); this.shufflePort = Integer.parseInt(RegistryTypeUtils.getAddressField(shuffle.addresses.get(0), AddressTypes.ADDRESS_PORT_FIELD)); this.outputFormatPort = Integer.valueOf(RegistryTypeUtils.getAddressField(outputFormat.addresses.get(0), AddressTypes.ADDRESS_PORT_FIELD)); this.serviceAddress = RegistryTypeUtils.getAddressField(services.addresses.get(0), AddressTypes.ADDRESS_URI); String memStr = srv.get(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, ""); String coreStr = srv.get(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.varname, ""); try { this.resource = Resource.newInstance(Integer.parseInt(memStr), Integer.parseInt(coreStr)); } catch (NumberFormatException ex) { throw new IOException("Invalid resource configuration for a LLAP node: memory " + memStr + ", vcores " + coreStr); } }
readLock.unlock(); return Resource.newInstance(memory, vcores);
this.defaultTaskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(flinkConfig).getMebiBytes(); this.defaultCpus = flinkConfig.getInteger(YarnConfigOptions.VCORES, numberOfTaskSlots); this.resource = Resource.newInstance(defaultTaskManagerMemoryMB, defaultCpus);
Resource capability = Resource.newInstance(jstormClientContext.amMemory, jstormClientContext.amVCores); appContext.setResource(capability);
int memoryPerInstance = serviceInstance.getResource().getMemory(); int memoryPerExecutor = (int)(memoryPerInstance / (double) numVcores); resourcePerExecutor = Resource.newInstance(memoryPerExecutor, 1);
public static Resource toResource(EnumMap<Resources, Integer> resourceMap) { return Resource.newInstance(resourceMap.get(Resources.MEMORY), resourceMap.get(Resources.V_CORE)); } }
final YarnResourceManagerCallbackHandler callbackHandler = new YarnResourceManagerCallbackHandler(); AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient = mock(AMRMClientAsync.class); doReturn(Collections.singletonList(Collections.nCopies(numInitialTaskManagers, new AMRMClient.ContainerRequest(Resource.newInstance(1024 * 1024, 1), null, null, Priority.newInstance(0))))) .when(resourceManagerClient).getMatchingRequests(any(Priority.class), anyString(), any(Resource.class));
Resource capability = Resource.newInstance(amMemory, amVCores); appContext.setResource(capability);
protected Node(NodeId nodeid) { labels = null; resource = Resource.newInstance(0, 0); running = false; nodeId = nodeid; }
/** * Get a Resource object with for the maximum allocation possible. * @return a Resource object with the maximum allocation for the scheduler */ public static Resource getResourceTypesMaximumAllocation() { Resource ret = Resource.newInstance(0, 0); for (ResourceInformation entry : resourceTypesArray) { ret.setResourceValue(entry.getName(), entry.getMaximumAllocation()); } return ret; }
public Resource divideAndCeil(Resource numerator, long denominator) { Resource ret = Resource.newInstance(numerator); int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation resourceInformation = ret.getResourceInformation(i); resourceInformation .setValue(divideAndCeil(resourceInformation.getValue(), denominator)); } return ret; }
private AMRMClient.ContainerRequest setupContainerRequestForRM(TensorFlowContainerRequest request) { Priority priority = Priority.newInstance(request.getPriority()); Resource capability = Resource.newInstance(request.getMemory(), request.getVCores()); Utils.setCapabilityGPU(capability, request.getGPU()); session.addAllocationId(request.getJobName(), lastAllocationRequestId); AMRMClient.ContainerRequest containerRequest = new AMRMClient.ContainerRequest(capability, null, null, priority, lastAllocationRequestId++); LOG.info("Requested container ask: " + containerRequest.toString()); return containerRequest; }