/** * Queries RM for the list of applications with the given tag that have started * after the given timestamp. */ private Set<ApplicationId> getYarnChildJobs(String tag, long timestamp) { Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>(); LOG.info(String.format("Querying RM for tag = %s, starting with ts = %s", tag, timestamp)); GetApplicationsRequest gar = GetApplicationsRequest.newInstance(); gar.setScope(ApplicationsRequestScope.OWN); gar.setStartRange(timestamp, System.currentTimeMillis()); gar.setApplicationTags(Collections.singleton(tag)); try { ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class); GetApplicationsResponse apps = proxy.getApplications(gar); List<ApplicationReport> appsList = apps.getApplicationList(); for(ApplicationReport appReport : appsList) { childYarnJobs.add(appReport.getApplicationId()); } } catch (IOException ioe) { throw new RuntimeException("Exception occurred while finding child jobs", ioe); } catch (YarnException ye) { throw new RuntimeException("Exception occurred while finding child jobs", ye); } return childYarnJobs; } }
public static Set<ApplicationId> getChildYarnJobs(Configuration conf, String tag) throws IOException, YarnException { Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>(); GetApplicationsRequest gar = GetApplicationsRequest.newInstance(); gar.setScope(ApplicationsRequestScope.OWN); gar.setApplicationTags(Collections.singleton(tag)); ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class); GetApplicationsResponse apps = proxy.getApplications(gar); List<ApplicationReport> appsList = apps.getApplicationList(); for(ApplicationReport appReport : appsList) { if (isAdmin() || appReport.getApplicationTags().contains(QueryState.USERID_TAG + "=" + SessionState.get() .getUserName())) { childYarnJobs.add(appReport.getApplicationId()); } } if (childYarnJobs.isEmpty()) { LOG.info("No child applications found"); } else { LOG.info("Found child YARN applications: " + StringUtils.join(childYarnJobs, ",")); } return childYarnJobs; }
public GetApplicationsRequest build() { validate(); GetApplicationsRequest request = GetApplicationsRequest.newInstance(); Set<String> appStates = parseQueries(statesQuery, true); if (!appStates.isEmpty()) { request.setApplicationStates(appStates); } if (!users.isEmpty()) { request.setUsers(users); } if (!queues.isEmpty()) { request.setQueues(queues); } if (limitNumber != null) { request.setLimit(limitNumber); } request.setStartRange(startedTimeBegin, startedTimeEnd); request.setFinishRange(finishTimeBegin, finishTimeEnd); if (!appTypes.isEmpty()) { request.setApplicationTypes(appTypes); } if (!appTags.isEmpty()) { request.setApplicationTags(appTags); } return request; } }
@Test public void testGetApplicationsRequest(){ GetApplicationsRequest request = GetApplicationsRequest.newInstance(); request.setApplicationStates(appStates); request.setApplicationTags(tags); request.setApplicationTypes(types); request.setStartRange(startBegin, startEnd); long finishBegin = System.currentTimeMillis() + 2; long finishEnd = System.currentTimeMillis() + 3; request.setFinishRange(finishBegin, finishEnd); request.setLimit(limit); request.setQueues(queues); request.setUsers(users); request.setScope(scope); requestFromProto.getApplicationStates(), appStates); requestFromProto.getApplicationTags(), tags); requestFromProto.getApplicationTypes(), types); requestFromProto.getStartRange(), Range.between(startBegin, startEnd));
GetApplicationsRequest request = GetApplicationsRequest.newInstance(); assertEquals("Incorrect total number of apps", 6, rmService.getApplications(request).getApplicationList().size()); request.setLimit(1L); assertEquals("Failed to limit applications", 1, rmService.getApplications(request).getApplicationList().size()); request = GetApplicationsRequest.newInstance(); request.setStartRange(submitTimeMillis[0], System.currentTimeMillis()); request.setStartRange(submitTimeMillis[1], System.currentTimeMillis()); assertEquals("Incorrect number of matching start range", 1, rmService.getApplications(request).getApplicationList().size()); request.setStartRange(submitTimeMillis[2], System.currentTimeMillis()); assertEquals("Incorrect number of matching start range", 0, rmService.getApplications(request).getApplicationList().size()); request = GetApplicationsRequest.newInstance(); Set<String> queueSet = new HashSet<String>(); request.setQueues(queueSet); request = GetApplicationsRequest.newInstance(); Set<String> userSet = new HashSet<String>(); request.setUsers(userSet); request = GetApplicationsRequest.newInstance( ApplicationsRequestScope.ALL, null, null, null, null, null, null,
@Override public List<ApplicationReport> getApplications(Set<String> queues, Set<String> users, Set<String> applicationTypes, EnumSet<YarnApplicationState> applicationStates) throws YarnException, IOException { GetApplicationsRequest request = GetApplicationsRequest.newInstance(applicationTypes, applicationStates); request.setQueues(queues); request.setUsers(users); GetApplicationsResponse response = rmClient.getApplications(request); return response.getApplicationList(); }
@Override public List<ApplicationReport> getApplications() throws YarnException, IOException { GetApplicationsRequest request = GetApplicationsRequest.newInstance(null, null); GetApplicationsResponse response = ahsClient.getApplications(request); return response.getApplicationList(); }
GetApplicationsRequest.newInstance(reqAppStates); String appsNumStr = $(APPS_NUM); if (appsNumStr != null && !appsNumStr.isEmpty()) { long appsNum = Long.parseLong(appsNumStr); request.setLimit(appsNum); "app.started-time.end must be greater than app.started-time.begin"); request.setStartRange( new LongRange(appStartedTimeBegain, appStartedTimeEnd));
GetApplicationsRequest.newInstance(reqAppStates); String appsNumStr = $(APPS_NUM); if (appsNumStr != null && !appsNumStr.isEmpty()) { long appsNum = Long.parseLong(appsNumStr); request.setLimit(appsNum);
@Override public List<ApplicationReport> doInYarn(ApplicationClientProtocol proxy) throws YarnException, IOException { GetApplicationsRequest request = Records.newRecord(GetApplicationsRequest.class); request.setApplicationStates(states); request.setApplicationTypes(types); GetApplicationsResponse response = proxy.getApplications(request); return response.getApplicationList(); } });
private GetApplicationsRequest getDefaultRequest() { GetApplicationsRequest req = GetApplicationsRequest.newInstance(); req.setStartRange(0, Long.MAX_VALUE); req.setFinishRange(0, Long.MAX_VALUE); return req; }
@Override public List<ApplicationReport> getApplications(Set<String> applicationTypes, EnumSet<YarnApplicationState> applicationStates, Set<String> applicationTags) throws YarnException, IOException { GetApplicationsRequest request = GetApplicationsRequest.newInstance(applicationTypes, applicationStates); request.setApplicationTags(applicationTags); GetApplicationsResponse response = rmClient.getApplications(request); return response.getApplicationList(); }
@Test public void testRequestWithValidStartedTimeBeginQuery() { GetApplicationsRequest request = ApplicationsRequestBuilder.create() .withStartedTimeBegin("999").build(); GetApplicationsRequest expectedRequest = getDefaultRequest(); expectedRequest.setStartRange(999L, Long.MAX_VALUE); assertEquals(expectedRequest, request); }
/** * <p> * The request from clients to get a report of Applications matching the * giving application types in the cluster from the * <code>ResourceManager</code>. * </p> * * * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) */ @Public @Stable public static GetApplicationsRequest newInstance(Set<String> applicationTypes) { GetApplicationsRequest request = Records.newRecord(GetApplicationsRequest.class); request.setApplicationTypes(applicationTypes); return request; }
/** * <p> * The request from clients to get a report of Applications matching the * giving application states in the cluster from the * <code>ResourceManager</code>. * </p> * * * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) */ @Public @Stable public static GetApplicationsRequest newInstance( EnumSet<YarnApplicationState> applicationStates) { GetApplicationsRequest request = Records.newRecord(GetApplicationsRequest.class); request.setApplicationStates(applicationStates); return request; }
@Test public void testRequestWithValidApplicationTagsQuery() { GetApplicationsRequest request = ApplicationsRequestBuilder.create() .withApplicationTags(Sets.newHashSet("tag1")).build(); GetApplicationsRequest expectedRequest = getDefaultRequest(); expectedRequest.setApplicationTags(Sets.newHashSet("tag1")); assertEquals(expectedRequest, request); } }
/** * <p> * The request from clients to get a report of Applications matching the * giving application types in the cluster from the * <code>ResourceManager</code>. * </p> * * @param scope {@link ApplicationsRequestScope} to filter by * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) */ @Public @Stable public static GetApplicationsRequest newInstance( ApplicationsRequestScope scope) { GetApplicationsRequest request = Records.newRecord(GetApplicationsRequest.class); request.setScope(scope); return request; }
@Test(expected = BadRequestException.class) public void testRequestWithQueueQueryNotExistingQueue() throws IOException { CapacityScheduler cs = mock(CapacityScheduler.class); when(cs.getQueueInfo(eq("queue1"), anyBoolean(), anyBoolean())) .thenThrow(new IOException()); ResourceManager rm = mock(ResourceManager.class); when(rm.getResourceScheduler()).thenReturn(cs); GetApplicationsRequest request = ApplicationsRequestBuilder.create() .withQueueQuery(rm, "queue1").build(); GetApplicationsRequest expectedRequest = getDefaultRequest(); expectedRequest.setQueues(Sets.newHashSet("queue1")); assertEquals(expectedRequest, request); }
@Test public void testRequestWithValidFinishedTimeBeginQuery() { GetApplicationsRequest request = ApplicationsRequestBuilder.create() .withFinishTimeBegin("999").build(); GetApplicationsRequest expectedRequest = getDefaultRequest(); expectedRequest.setFinishRange(999L, Long.MAX_VALUE); assertEquals(expectedRequest, request); }
@Test public void testRequestWithValidLimitQuery() { GetApplicationsRequest request = ApplicationsRequestBuilder.create().withLimit("999").build(); GetApplicationsRequest expectedRequest = getDefaultRequest(); expectedRequest.setLimit(999L); assertEquals(expectedRequest, request); }