aclManager = new TimelineACLsManager(conf); aclManager.setTimelineStore(summaryStore); summaryTdm = new TimelineDataManager(summaryStore, aclManager); summaryTdm.init(conf); summaryTdm.start(); activeRootPath = new Path(conf.get(TIMELINE_SERVICE_ENTITYFILE_ACTIVE_DIR, TIMELINE_SERVICE_ENTITYFILE_ACTIVE_DIR_DEFAULT));
/** * Store the given entities into the timeline store, and return the errors * that happen during storing. */ @POST @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) public TimelinePutResponse postEntities( @Context HttpServletRequest req, @Context HttpServletResponse res, TimelineEntities entities) { init(res); UserGroupInformation callerUGI = getUser(req); if (callerUGI == null) { String msg = "The owner of the posted timeline entities is not set"; LOG.error(msg); throw new ForbiddenException(msg); } try { return timelineDataManager.postEntities(entities, callerUGI); } catch (Exception e) { LOG.error("Error putting entities", e); throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); } }
@Override public Map<ApplicationId, ApplicationReport> getApplications(long appsNum) throws YarnException, IOException { TimelineEntities entities = timelineDataManager.getEntities( ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, null, null, appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum, EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser()); Map<ApplicationId, ApplicationReport> apps = new LinkedHashMap<ApplicationId, ApplicationReport>(); if (entities != null && entities.getEntities() != null) { for (TimelineEntity entity : entities.getEntities()) { try { ApplicationReportExt app = generateApplicationReport(entity, ApplicationReportField.ALL); apps.put(app.appReport.getApplicationId(), app.appReport); } catch (Exception e) { LOG.error("Error on generating application report for " + entity.getEntityId(), e); } } } return apps; }
static TimelineDataManager getTdmWithStore(Configuration config, TimelineStore store) { TimelineACLsManager aclManager = new TimelineACLsManager(config); TimelineDataManager tdm = new TimelineDataManager(store, aclManager); tdm.init(config); return tdm; }
public synchronized TimelineStore refreshCache() throws IOException { //TODO: make a config for cache freshness if (!cacheCompleted && Time.monotonicNow() - cacheRefreshTime > 10000) { if (!isDone()) { parseSummaryLogs(); } else if (detailLogs.isEmpty()) { scanForLogs(); } if (!detailLogs.isEmpty()) { if (cacheStore == null) { cacheStore = new MemoryTimelineStore(); cacheStore.init(getConfig()); cacheStore.start(); } TimelineDataManager tdm = new TimelineDataManager(cacheStore, aclManager); tdm.init(getConfig()); tdm.start(); for (LogInfo log : detailLogs) { log.parseForCache(tdm, dirPath, isDone()); } tdm.close(); } cacheRefreshTime = Time.monotonicNow(); cacheCompleted = isDone(); } return cacheStore; }
private ApplicationReportExt getApplication(ApplicationId appId, ApplicationReportField field) throws YarnException, IOException { TimelineEntity entity = timelineDataManager.getEntity( ApplicationMetricsConstants.ENTITY_TYPE, appId.toString(), EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser()); if (entity == null) { throw new ApplicationNotFoundException("The entity for application " + appId + " doesn't exist in the timeline store"); } else { return generateApplicationReport(entity, field); } }
TimelineEntity entity3 = tdm.getEntity("type_3", mainTestAppId.toString(), EnumSet.allOf(TimelineReader.Field.class), UserGroupInformation.getLoginUser()); TimelineEntities entities = tdm.getEntities("type_3", primaryFilter, null, null, null, null, null, null, EnumSet.allOf(TimelineReader.Field.class), UserGroupInformation.getLoginUser());
timelineDataManager.putDomain(domain, callerUGI); } catch (YarnException e) {
private TimelineDataManager createTimelineDataManager(Configuration conf) { TimelineACLsManager aclsMgr = new TimelineACLsManager(conf); aclsMgr.setTimelineStore(timelineStore); return new TimelineDataManager(timelineStore, aclsMgr); }
@Test public void testParseEntity() throws Exception { // Load test data TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config); EntityLogInfo testLogInfo = new EntityLogInfo(TEST_ATTEMPT_DIR_NAME, TEST_ENTITY_FILE_NAME, UserGroupInformation.getLoginUser().getUserName()); testLogInfo.parseForStore(tdm, getTestRootPath(), true, jsonFactory, objMapper, fs); // Verify for the first batch PluginStoreTestUtils.verifyTestEntities(tdm); // Load new data TimelineEntity entityNew = PluginStoreTestUtils .createEntity("id_3", "type_3", 789l, null, null, null, null, "domain_id_1"); TimelineEntities entityList = new TimelineEntities(); entityList.addEntity(entityNew); writeEntitiesLeaveOpen(entityList, new Path(getTestRootPath(TEST_ATTEMPT_DIR_NAME), TEST_ENTITY_FILE_NAME)); testLogInfo.parseForStore(tdm, getTestRootPath(), true, jsonFactory, objMapper, fs); // Verify the newly added data TimelineEntity entity3 = tdm.getEntity(entityNew.getEntityType(), entityNew.getEntityId(), EnumSet.allOf(TimelineReader.Field.class), UserGroupInformation.getLoginUser()); assertNotNull(entity3); assertEquals("Failed to read out entity new", entityNew.getStartTime(), entity3.getStartTime()); tdm.close(); }
/** * Get the single timeline entity that the given user has access to. The * meaning of each argument has been documented with * {@link TimelineReader#getEntity}. * * @see TimelineReader#getEntity */ public TimelineEntity getEntity( String entityType, String entityId, EnumSet<Field> fields, UserGroupInformation callerUGI) throws YarnException, IOException { TimelineEntity entity = null; entity = store.getEntity(entityId, entityType, fields); if (entity != null) { addDefaultDomainIdIfAbsent(entity); // check ACLs if (!timelineACLsManager.checkAccess( callerUGI, ApplicationAccessType.VIEW_APP, entity)) { entity = null; } } return entity; }
/** * Return a single domain of the given domain Id. */ @GET @Path("/domain/{domainId}") @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) public TimelineDomain getDomain( @Context HttpServletRequest req, @Context HttpServletResponse res, @PathParam("domainId") String domainId) { init(res); domainId = parseStr(domainId); if (domainId == null || domainId.length() == 0) { throw new BadRequestException("Domain ID is not specified."); } TimelineDomain domain = null; try { domain = timelineDataManager.getDomain( parseStr(domainId), getUser(req)); } catch (Exception e) { LOG.error("Error getting domain", e); throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); } if (domain == null) { throw new NotFoundException("Timeline domain [" + domainId + "] is not found"); } return domain; }
/** * Return a list of domains of the given owner. */ @GET @Path("/domain") @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8 /* , MediaType.APPLICATION_XML */}) public TimelineDomains getDomains( @Context HttpServletRequest req, @Context HttpServletResponse res, @QueryParam("owner") String owner) { init(res); owner = parseStr(owner); UserGroupInformation callerUGI = getUser(req); if (owner == null || owner.length() == 0) { if (callerUGI == null) { throw new BadRequestException("Domain owner is not specified."); } else { // By default it's going to list the caller's domains owner = callerUGI.getShortUserName(); } } try { return timelineDataManager.getDomains(owner, callerUGI); } catch (Exception e) { LOG.error("Error getting domains", e); throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); } }
init(res); try { return timelineDataManager.getEvents( parseStr(entityType), parseArrayStr(entityId, ","),
aclManager = new TimelineACLsManager(conf); aclManager.setTimelineStore(summaryStore); summaryTdm = new TimelineDataManager(summaryStore, aclManager); summaryTdm.init(conf); addService(summaryTdm);
store.start(); TimelineDataManager tdm = new TimelineDataManager(store, aclManager); tdm.init(config); tdm.start(); if (appLogs.getDetailLogs().isEmpty()) { LOG.debug("cache id {}'s detail log is empty! ", groupId); tdm.close();
private ApplicationReportExt getApplication(ApplicationId appId, ApplicationReportField field) throws YarnException, IOException { TimelineEntity entity = timelineDataManager.getEntity( ApplicationMetricsConstants.ENTITY_TYPE, appId.toString(), EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser()); if (entity == null) { throw new ApplicationNotFoundException("The entity for application " + appId + " doesn't exist in the timeline store"); } else { return generateApplicationReport(entity, field); } }
timelineDataManager.putDomain(domain, callerUGI); } catch (YarnException e) {
private TimelineDataManager createTimelineDataManager(Configuration conf) { return new TimelineDataManager( timelineStore, new TimelineACLsManager(conf)); }
/** * Get the single timeline entity that the given user has access to. The * meaning of each argument has been documented with * {@link TimelineReader#getEntity}. * * @see TimelineReader#getEntity */ public TimelineEntity getEntity( String entityType, String entityId, EnumSet<Field> fields, UserGroupInformation callerUGI) throws YarnException, IOException { TimelineEntity entity = null; entity = store.getEntity(entityId, entityType, fields); if (entity != null) { addDefaultDomainIdIfAbsent(entity); // check ACLs if (!timelineACLsManager.checkAccess( callerUGI, ApplicationAccessType.VIEW_APP, entity)) { entity = null; } } return entity; }