/** * Queries RM for the list of applications with the given tag that have started * after the given timestamp. */ private Set<ApplicationId> getYarnChildJobs(String tag, long timestamp) { Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>(); LOG.info(String.format("Querying RM for tag = %s, starting with ts = %s", tag, timestamp)); GetApplicationsRequest gar = GetApplicationsRequest.newInstance(); gar.setScope(ApplicationsRequestScope.OWN); gar.setStartRange(timestamp, System.currentTimeMillis()); gar.setApplicationTags(Collections.singleton(tag)); try { ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class); GetApplicationsResponse apps = proxy.getApplications(gar); List<ApplicationReport> appsList = apps.getApplicationList(); for(ApplicationReport appReport : appsList) { childYarnJobs.add(appReport.getApplicationId()); } } catch (IOException ioe) { throw new RuntimeException("Exception occurred while finding child jobs", ioe); } catch (YarnException ye) { throw new RuntimeException("Exception occurred while finding child jobs", ye); } return childYarnJobs; } }
/** * Kills YARN application and stops YARN client. * * <p>Use this method to kill the App before it has been properly deployed */ private void failSessionDuringDeployment(YarnClient yarnClient, YarnClientApplication yarnApplication) { LOG.info("Killing YARN application"); try { yarnClient.killApplication(yarnApplication.getNewApplicationResponse().getApplicationId()); } catch (Exception e) { // we only log a debug message here because the "killApplication" call is a best-effort // call (we don't know if the application has been deployed when the error occured). LOG.debug("Error while killing YARN application", e); } yarnClient.stop(); }
private Resource prepareContainerResource(GetNewApplicationResponse newApplicationResponse) { int memoryMbs = this.config.getInt(GobblinYarnConfigurationKeys.APP_MASTER_MEMORY_MBS_KEY); int maximumMemoryCapacity = newApplicationResponse.getMaximumResourceCapability().getMemory(); if (memoryMbs > maximumMemoryCapacity) { LOGGER.info(String.format("Specified AM memory [%d] is above the maximum memory capacity [%d] of the " + "cluster, using the maximum memory capacity instead.", memoryMbs, maximumMemoryCapacity)); memoryMbs = maximumMemoryCapacity; } int vCores = this.config.getInt(GobblinYarnConfigurationKeys.APP_MASTER_CORES_KEY); int maximumVirtualCoreCapacity = newApplicationResponse.getMaximumResourceCapability().getVirtualCores(); if (vCores > maximumVirtualCoreCapacity) { LOGGER.info(String.format("Specified AM vcores [%d] is above the maximum vcore capacity [%d] of the " + "cluster, using the maximum vcore capacity instead.", memoryMbs, maximumMemoryCapacity)); vCores = maximumVirtualCoreCapacity; } // Set up resource type requirements for ApplicationMaster return Resource.newInstance(memoryMbs, vCores); }
private GetApplicationsRequest getDefaultRequest() { GetApplicationsRequest req = GetApplicationsRequest.newInstance(); req.setStartRange(0, Long.MAX_VALUE); req.setFinishRange(0, Long.MAX_VALUE); return req; }
applicationId = newAppResponse.getApplicationId(); LOG.info("Got new Application: " + applicationId.toString()); appMasterEnv.put(XLearningConstants.Environment.XLEARNING_CONTAINER_MAX_MEMORY.toString(), String.valueOf(newAppResponse.getMaximumResourceCapability().getMemory()));
private void yarnReport() { RegisterApplicationMasterResponse response = yarn.getRegistrationResponse(); LOG.info("YARN queue: " + response.getQueue()); Resource resource = response.getMaximumResourceCapability(); LOG.info("YARN max resource: " + resource.getMemory() + " MB, " + resource.getVirtualCores() + " cores"); EnumSet<SchedulerResourceTypes> types = response .getSchedulerResourceTypes(); StringBuilder buf = new StringBuilder(); String sep = ""; for (SchedulerResourceTypes type : types) { buf.append(sep); buf.append(type.toString()); sep = ", "; } LOG.info("YARN scheduler resource types: " + buf.toString()); }
@Override protected void startUp() throws Exception { LOGGER.info("Starting the YarnService"); // Register itself with the EventBus for container-related requests this.eventBus.register(this); this.amrmClientAsync.start(); this.nmClientAsync.start(); // The ApplicationMaster registration response is used to determine the maximum resource capacity of the cluster RegisterApplicationMasterResponse response = this.amrmClientAsync.registerApplicationMaster( GobblinClusterUtils.getHostname(), -1, ""); LOGGER.info("ApplicationMaster registration response: " + response); this.maxResourceCapacity = Optional.of(response.getMaximumResourceCapability()); LOGGER.info("Requesting initial containers"); requestInitialContainers(this.initialContainers); }
jstormMasterContext.maxMemory = response.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capability of resources in this cluster " + jstormMasterContext.maxMemory); jstormMasterContext.maxVcores = response.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max vcores capability of resources in this cluster " + jstormMasterContext.maxVcores); response.getContainersFromPreviousAttempts(); LOG.info(jstormMasterContext.appAttemptID + " received " + previousAMRunningContainers.size() + " previous attempts' running containers on AM registration.");
@Override public List<ApplicationAttemptReport> getApplicationAttempts( ApplicationId appId) throws YarnException, IOException { GetApplicationAttemptsRequest request = GetApplicationAttemptsRequest .newInstance(appId); GetApplicationAttemptsResponse response = ahsClient .getApplicationAttempts(request); return response.getApplicationAttemptList(); }
assertNotNull("Illegal state", nodeManager); StopContainersRequest scr = StopContainersRequest.newInstance(Collections.singletonList(taskManagerContainer));
@Override public void register(String trackingUrl) throws YarnFacadeException { String thisHostName = NetUtils.getHostname(); LOG.debug("Host Name from YARN: " + thisHostName); if (trackingUrl != null) { // YARN seems to provide multiple names: MACHNAME.local/10.250.56.235 // The second seems to be the IP address, which is what we want. String names[] = thisHostName.split("/"); amHost = names[names.length - 1]; appMasterTrackingUrl = trackingUrl.replace("<host>", amHost); LOG.info("Tracking URL: " + appMasterTrackingUrl); } try { LOG.trace("Registering with YARN"); registration = resourceMgr.registerApplicationMaster(thisHostName, 0, appMasterTrackingUrl); } catch (YarnException | IOException e) { throw new YarnFacadeException("Register AM failed", e); } // Some distributions (but not the stock YARN) support Disk // resources. Since Drill compiles against Apache YARN, without disk // resources, we have to use an indirect mechnanism to look for the // disk enum at runtime when we don't have that enum value at compile time. for (SchedulerResourceTypes type : registration .getSchedulerResourceTypes()) { if (type.name().equals("DISK")) { supportsDisks = true; } } }
public static Set<ApplicationId> getChildYarnJobs(Configuration conf, String tag) throws IOException, YarnException { Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>(); GetApplicationsRequest gar = GetApplicationsRequest.newInstance(); gar.setScope(ApplicationsRequestScope.OWN); gar.setApplicationTags(Collections.singleton(tag)); ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class); GetApplicationsResponse apps = proxy.getApplications(gar); List<ApplicationReport> appsList = apps.getApplicationList(); for(ApplicationReport appReport : appsList) { if (isAdmin() || appReport.getApplicationTags().contains(QueryState.USERID_TAG + "=" + SessionState.get() .getUserName())) { childYarnJobs.add(appReport.getApplicationId()); } } if (childYarnJobs.isEmpty()) { LOG.info("No child applications found"); } else { LOG.info("Found child YARN applications: " + StringUtils.join(childYarnJobs, ",")); } return childYarnJobs; }
private void validateResources( AppSpec master ) throws ClientException { // Memory and core checks per YARN app specs. int maxMemory = appResponse.getMaximumResourceCapability().getMemory(); int maxCores = appResponse.getMaximumResourceCapability().getVirtualCores(); if (verbose) { System.out.println("Max Memory: " + maxMemory); System.out.println("Max Cores: " + maxCores); } // YARN behaves very badly if we request a container larger than the // maximum. if (master.memoryMb > maxMemory) { throw new ClientException( "YARN maximum memory is " + maxMemory + " but the application master requests " + master.memoryMb ); } if (master.vCores > maxCores) { throw new ClientException("YARN maximum vcores is " + maxCores + " but the application master requests " + master.vCores); } // Verify the limits for the Drillbit as well. if (config.getInt(DrillOnYarnConfig.DRILLBIT_MEMORY) > maxMemory) { throw new ClientException( "YARN maximum memory is " + maxMemory + " but the Drillbit requests " + config.getInt(DrillOnYarnConfig.DRILLBIT_MEMORY)); } if (config.getInt(DrillOnYarnConfig.DRILLBIT_VCORES) > maxCores) { throw new ClientException("YARN maximum vcores is " + maxCores + " but the Drillbit requests " + config.getInt(DrillOnYarnConfig.DRILLBIT_VCORES)); } }
private void createApp() throws ClientException { try { appResponse = client.createAppMaster(); } catch (YarnClientException e) { throw new ClientException("Failed to allocate Drill application master", e); } appId = appResponse.getApplicationId(); System.out.println("Application ID: " + appId.toString()); }
/** * Called when the caller has completed start-up and the controller should * become live. */ @Override public synchronized void started() throws YarnFacadeException, AMException { nodeInventory = new NodeInventory(yarn); // Verify that no resource seeks a container larger than // what YARN can provide. Ensures a graceful exit in this // case. Resource maxResource = yarn.getRegistrationResponse() .getMaximumResourceCapability(); for (SchedulerStateActions group : prioritizedGroups) { group.getScheduler().limitContainerSize(maxResource); } state = State.LIVE; }
private void checkArguments(XLearningConfiguration conf, GetNewApplicationResponse newApplication) { int maxMem = newApplication.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capability of resources in this cluster " + maxMem); int maxVCores = newApplication.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max vcores capability of resources in this cluster " + maxVCores);
public GetNewApplicationResponse createAppMaster() throws YarnClientException { // Create application via yarnClient // Response is a new application ID along with cluster capacity info try { app = yarnClient.createApplication(); } catch (YarnException | IOException e) { throw new YarnClientException("Create application failed", e); } GetNewApplicationResponse response = app.getNewApplicationResponse(); appId = response.getApplicationId(); return response; }
int maxMem = response.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = response.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores); maasHandler = new MaaSHandler(zkQuorum, zkRoot);
int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores);
final GetNewApplicationResponse appResponse = yarnApplication.getNewApplicationResponse(); Resource maxRes = appResponse.getMaximumResourceCapability();