@Override public void limitContainerSize(Resource maxResource) throws AMException { if (taskSpec.containerSpec.memoryMb > maxResource.getMemory()) { LOG.warn(taskSpec.name + " requires " + taskSpec.containerSpec.memoryMb + " MB but the maximum YARN container size is " + maxResource.getMemory() + " MB"); taskSpec.containerSpec.memoryMb = maxResource.getMemory(); } if (taskSpec.containerSpec.vCores > maxResource.getVirtualCores()) { LOG.warn(taskSpec.name + " requires " + taskSpec.containerSpec.vCores + " vcores but the maximum YARN container size is " + maxResource.getVirtualCores() + " vcores"); taskSpec.containerSpec.vCores = maxResource.getVirtualCores(); } }
private String buildContainerCommand(Container container, String helixInstanceName) { String containerProcessName = GobblinYarnTaskRunner.class.getSimpleName(); return new StringBuilder() .append(ApplicationConstants.Environment.JAVA_HOME.$()).append("/bin/java") .append(" -Xmx").append(container.getResource().getMemory()).append("M") .append(" ").append(JvmUtils.formatJvmArguments(this.containerJvmArgs)) .append(" ").append(GobblinYarnTaskRunner.class.getName()) .append(" --").append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME) .append(" ").append(this.applicationName) .append(" --").append(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_OPTION_NAME) .append(" ").append(helixInstanceName) .append(" 1>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator).append( containerProcessName).append(".").append(ApplicationConstants.STDOUT) .append(" 2>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator).append( containerProcessName).append(".").append(ApplicationConstants.STDERR) .toString(); }
private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException { List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING); int totalFreeMemory = 0; int containerLimit = 0; int[] nodeManagersFree = new int[nodes.size()]; for (int i = 0; i < nodes.size(); i++) { NodeReport rep = nodes.get(i); int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0); nodeManagersFree[i] = free; totalFreeMemory += free; if (free > containerLimit) { containerLimit = free; } } return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree); }
@Override protected void requestNewWorkers(int numWorkers) { final Resource capability = getContainerResource(); for (int i = 0; i < numWorkers; i++) { numPendingContainerRequests++; LOG.info("Requesting new TaskManager container with {} megabytes memory. Pending requests: {}", capability.getMemory(), numPendingContainerRequests); resourceManagerClient.addContainerRequest(createContainerRequest(capability)); } // make sure we transmit the request fast and receive fast news of granted allocations resourceManagerClient.setHeartbeatInterval(FAST_YARN_HEARTBEAT_INTERVAL_MS); }
private Resource prepareContainerResource(GetNewApplicationResponse newApplicationResponse) { int memoryMbs = this.config.getInt(GobblinYarnConfigurationKeys.APP_MASTER_MEMORY_MBS_KEY); int maximumMemoryCapacity = newApplicationResponse.getMaximumResourceCapability().getMemory(); if (memoryMbs > maximumMemoryCapacity) { LOGGER.info(String.format("Specified AM memory [%d] is above the maximum memory capacity [%d] of the " + "cluster, using the maximum memory capacity instead.", memoryMbs, maximumMemoryCapacity)); memoryMbs = maximumMemoryCapacity; } int vCores = this.config.getInt(GobblinYarnConfigurationKeys.APP_MASTER_CORES_KEY); int maximumVirtualCoreCapacity = newApplicationResponse.getMaximumResourceCapability().getVirtualCores(); if (vCores > maximumVirtualCoreCapacity) { LOGGER.info(String.format("Specified AM vcores [%d] is above the maximum vcore capacity [%d] of the " + "cluster, using the maximum vcore capacity instead.", memoryMbs, maximumMemoryCapacity)); vCores = maximumVirtualCoreCapacity; } // Set up resource type requirements for ApplicationMaster return Resource.newInstance(memoryMbs, vCores); }
@Override public Resource getTotalResources() { int memory = 0; int vcores = 0; readLock.lock(); try { int numInstancesFound = 0; for (LlapServiceInstance inst : activeInstances.getAll()) { Resource r = inst.getResource(); memory += r.getMemory(); vcores += r.getVirtualCores(); numInstancesFound++; } if (LOG.isDebugEnabled()) { LOG.debug("GetTotalResources: numInstancesFound={}, totalMem={}, totalVcores={}", numInstancesFound, memory, vcores); } } finally { readLock.unlock(); } return Resource.newInstance(memory, vcores); }
@Override public String info() throws TException { StringBuffer sbRet = new StringBuffer(); sbRet.append("JstormOnYarn\n"); sbRet.append("Instance Name:" + jstormMasterContext.instanceName + "\n"); sbRet.append("Jstorm's location on hdfs:" + jstormMasterContext.deployPath + "\n"); if (jstormMasterContext.user != null) { sbRet.append("Jstorm's data path:" + jstormMasterContext.nimbusDataDirPrefix + jstormMasterContext.instanceName + "\n"); sbRet.append("Cluster userName:" + jstormMasterContext.user + "\n"); } sbRet.append("Nimbus Count:" + jstormMasterContext.nimbusContainers.size() + "\n"); sbRet.append("Supervisor Count:" + jstormMasterContext.supervisorContainers.size() + "\n"); sbRet.append("detail :\n"); sbRet.append("Type \tContainerId \tHost \tContainerMemory\tContainerVCores\n"); for (Container container : jstormMasterContext.nimbusContainers) { sbRet.append("Nimbus \t" + container.getId().toString() + "\t" + container.getNodeId().getHost() + "\t" + container.getResource().getMemory() + "\t " + container.getResource().getVirtualCores() + "\n"); } for (Container container : jstormMasterContext.supervisorContainers) { sbRet.append("Supervisor\t" + container.getId().toString() + "\t" + container.getNodeId().getHost() + "\t" + container.getResource().getMemory() + "\t " + container.getResource().getVirtualCores() + "\n"); } LOG.info("info is: " + sbRet.toString()); return sbRet.toString(); }
for (NodeReport rep : nodes) { final Resource res = rep.getCapability(); totalMemory += res.getMemory(); totalCores += res.getVirtualCores(); ps.format(format, "NodeID", rep.getNodeId()); ps.format(format, "Memory", res.getMemory() + " MB"); ps.format(format, "vCores", res.getVirtualCores()); ps.format(format, "HealthReport", rep.getHealthReport());
/** * Utility method to display YARN container information in a useful way for * log messages. * * @param container * @return */ public static String describeContainer(Container container) { StringBuilder buf = new StringBuilder() .append("[id: ") .append(container.getId()) .append(", host: ") .append(container.getNodeId().getHost()) .append(", priority: ") .append(container.getPriority()) .append(", memory: ") .append(container.getResource().getMemory()) .append(" MB, vcores: ") .append(container.getResource().getVirtualCores()) .append("]"); return buf.toString(); }
private void yarnReport() { RegisterApplicationMasterResponse response = yarn.getRegistrationResponse(); LOG.info("YARN queue: " + response.getQueue()); Resource resource = response.getMaximumResourceCapability(); LOG.info("YARN max resource: " + resource.getMemory() + " MB, " + resource.getVirtualCores() + " cores"); EnumSet<SchedulerResourceTypes> types = response .getSchedulerResourceTypes(); StringBuilder buf = new StringBuilder(); String sep = ""; for (SchedulerResourceTypes type : types) { buf.append(sep); buf.append(type.toString()); sep = ", "; } LOG.info("YARN scheduler resource types: " + buf.toString()); }
if (jobManagerMemoryMb > maximumResourceCapability.getMemory()) { throw new YarnDeploymentException("The cluster does not have the requested resources for the JobManager available!\n" + "Maximum Memory: " + maximumResourceCapability.getMemory() + "MB Requested: " + jobManagerMemoryMb + "MB. " + note); if (taskManagerMemoryMb > maximumResourceCapability.getMemory()) { throw new YarnDeploymentException("The cluster does not have the requested resources for the TaskManagers available!\n" + "Maximum Memory: " + maximumResourceCapability.getMemory() + " Requested: " + taskManagerMemoryMb + "MB. " + note);
/** * @return {@code True} if cluster contains available resources. */ private boolean checkAvailableResource() { Resource availableRes = rmClient.getAvailableResources(); return availableRes == null || availableRes.getMemory() >= props.totalMemoryPerNode() && availableRes.getVirtualCores() >= props.cpusPerNode(); }
private void requestContainer(Optional<String> preferredNode) { Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); Resource capability = Records.newRecord(Resource.class); int maxMemoryCapacity = this.maxResourceCapacity.get().getMemory(); capability.setMemory(this.requestedContainerMemoryMbs <= maxMemoryCapacity ? this.requestedContainerMemoryMbs : maxMemoryCapacity); int maxCoreCapacity = this.maxResourceCapacity.get().getVirtualCores(); capability.setVirtualCores(this.requestedContainerCores <= maxCoreCapacity ? this.requestedContainerCores : maxCoreCapacity); String[] preferredNodes = preferredNode.isPresent() ? new String[] {preferredNode.get()} : null; this.amrmClientAsync.addContainerRequest( new AMRMClient.ContainerRequest(capability, preferredNodes, null, priority)); }
if (nodeInfo != null && !nodeInfo.isDisabled()) { Resource r = inst.getResource(); memory += r.getMemory(); vcores += r.getVirtualCores(); numInstancesFound++;
ContaineredTaskManagerParameters.create(flinkConfig, resource.getMemory(), numberOfTaskSlots);
private void validateResources( AppSpec master ) throws ClientException { // Memory and core checks per YARN app specs. int maxMemory = appResponse.getMaximumResourceCapability().getMemory(); int maxCores = appResponse.getMaximumResourceCapability().getVirtualCores(); if (verbose) { System.out.println("Max Memory: " + maxMemory); System.out.println("Max Cores: " + maxCores); } // YARN behaves very badly if we request a container larger than the // maximum. if (master.memoryMb > maxMemory) { throw new ClientException( "YARN maximum memory is " + maxMemory + " but the application master requests " + master.memoryMb ); } if (master.vCores > maxCores) { throw new ClientException("YARN maximum vcores is " + maxCores + " but the application master requests " + master.vCores); } // Verify the limits for the Drillbit as well. if (config.getInt(DrillOnYarnConfig.DRILLBIT_MEMORY) > maxMemory) { throw new ClientException( "YARN maximum memory is " + maxMemory + " but the Drillbit requests " + config.getInt(DrillOnYarnConfig.DRILLBIT_MEMORY)); } if (config.getInt(DrillOnYarnConfig.DRILLBIT_VCORES) > maxCores) { throw new ClientException("YARN maximum vcores is " + maxCores + " but the Drillbit requests " + config.getInt(DrillOnYarnConfig.DRILLBIT_VCORES)); } }
private void buildNodeMap() throws YarnFacadeException { List<NodeReport> nodes = yarn.getNodeReports(); for (NodeReport node : nodes) { String hostName = node.getNodeId().getHost(); nodeMap.put(hostName, node.getHttpAddress()); yarnNodes.put(hostName, node); } if (LOG.isInfoEnabled()) { LOG.info("YARN Node report"); for (NodeReport node : nodes) { LOG.info("Node: " + node.getHttpAddress() + ", Rack: " + node.getRackName() + " has " + node.getCapability().getMemory() + " MB, " + node.getCapability().getVirtualCores() + " vcores, labels: " + node.getNodeLabels()); } } }
int memoryPerInstance = serviceInstance.getResource().getMemory(); int memoryPerExecutor = (int)(memoryPerInstance / (double) numVcores); resourcePerExecutor = Resource.newInstance(memoryPerExecutor, 1); LOG.info("Setting up node: {} with available capacity={}, pendingQueueSize={}, memory={}", serviceInstance, serviceInstance.getResource().getVirtualCores(), pendingQueueCapacityString, serviceInstance.getResource().getMemory()); if (pendingQueueCapacityString != null) { pendingQueueuCapacity = Integer.parseInt(pendingQueueCapacityString);
/** * @param cont Container. * @return {@code True} if container satisfies requirements. */ private boolean checkContainer(Container cont) { // Check limit on running nodes. if (props.instances() <= containers.size()) return false; // Check host name if (props.hostnameConstraint() != null && props.hostnameConstraint().matcher(cont.getNodeId().getHost()).matches()) return false; // Check that slave satisfies min requirements. if (cont.getResource().getVirtualCores() < props.cpusPerNode() || cont.getResource().getMemory() < props.totalMemoryPerNode()) { log.log(Level.FINE, "Container resources not sufficient requirements. Host: {0}, cpu: {1}, mem: {2}", new Object[]{cont.getNodeId().getHost(), cont.getResource().getVirtualCores(), cont.getResource().getMemory()}); return false; } return true; }
for (LlapServiceInstance instance : llapRegistryService.getInstances().getAll()) { ss.out.println(Joiner.on("\t").join(appId, instance.getWorkerIdentity(), instance.getHost(), instance.getRpcPort(), instance.getResource().getMemory() * 1024L * 1024L, instance.getResource().getVirtualCores()));