private TimelineStore getCachedStore(ApplicationId appId) throws IOException { AppLogs appLogs = null; synchronized (cachedApps) { appLogs = cachedApps.get(appId); if (appLogs == null) { appLogs = findAppLogs(appId); if (appLogs != null) { cachedApps.put(appId, appLogs); } } } TimelineStore store = null; if (appLogs != null) { store = appLogs.refreshCache(); } return store; }
private void scanActiveLogs() throws IOException { RemoteIterator<FileStatus> iter = fs.listStatusIterator(activeRootPath); while (iter.hasNext()) { FileStatus stat = iter.next(); ApplicationId appId = toAppId(stat.getPath().getName()); if (appId != null) { AppLogs logs = getActiveApp(appId, stat.getPath()); executor.execute(new ActiveLogParser(logs)); } } }
@Override public TimelineEntity getEntity(String entityId, String entityType, EnumSet<Field> fieldsToRetrieve) throws IOException { LOG.debug("getEntity type={} id={}", entityType, entityId); TimelineStore store = getTimelineStoreForRead(entityType, entityId); return store.getEntity(entityId, entityType, fieldsToRetrieve); }
private AppLogs findAppLogs(ApplicationId appId) throws IOException { AppLogs appLogs = appLogMap.get(appId); if (appLogs == null) { AppState appState = AppState.UNKNOWN; Path dirPath = getDonePath(appId); if (fs.exists(dirPath)) { appState = AppState.COMPLETED; } else { dirPath = new Path(activeRootPath, appId.toString()); if (fs.exists(dirPath)) { appState = AppState.ACTIVE; } } if (appState != AppState.UNKNOWN) { appLogs = new AppLogs(appId, dirPath, appState); AppLogs oldAppLogs = appLogMap.putIfAbsent(appId, appLogs); if (oldAppLogs != null) { appLogs = oldAppLogs; } } } return appLogs; }
store = getCachedStore(appId); if (store != null) { LOG.debug("Using cache store from {} for {}", appId, entityType);
yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); summaryStore = createSummaryStore(); summaryStore.init(conf); long logRetainSecs = conf.getLong(
@Override protected void serviceStop() throws Exception { LOG.info("Stopping {}", getName()); if (executor != null) { executor.shutdown(); if (executor.isTerminating()) { LOG.info("Waiting for executor to terminate"); boolean terminated = executor.awaitTermination(10, TimeUnit.SECONDS); if (terminated) { LOG.info("Executor terminated"); } else { LOG.warn("Executor did not terminate"); executor.shutdownNow(); } } } if (summaryTdm != null) { summaryTdm.stop(); } if (summaryStore != null) { summaryStore.stop(); } if (yarnClient != null) { yarnClient.stop(); } super.serviceStop(); }
@Override public TimelineEntities getEntities(String entityType, Long limit, Long windowStart, Long windowEnd, String fromId, Long fromTs, NameValuePair primaryFilter, Collection<NameValuePair> secondaryFilters, EnumSet<Field> fieldsToRetrieve, CheckAcl checkAcl) throws IOException { LOG.debug("getEntities type={} primary={}", entityType, primaryFilter); Object arg = (primaryFilter == null) ? null : primaryFilter.getValue(); TimelineStore store = getTimelineStoreForRead(entityType, arg); return store.getEntities(entityType, limit, windowStart, windowEnd, fromId, fromTs, primaryFilter, secondaryFilters, fieldsToRetrieve, checkAcl); }
public synchronized TimelineStore refreshCache() throws IOException { //TODO: make a config for cache freshness if (!cacheCompleted && Time.monotonicNow() - cacheRefreshTime > 10000) { if (!isDone()) { parseSummaryLogs(); } else if (detailLogs.isEmpty()) { scanForLogs(); } if (!detailLogs.isEmpty()) { if (cacheStore == null) { cacheStore = new MemoryTimelineStore(); cacheStore.init(getConfig()); cacheStore.start(); } TimelineDataManager tdm = new TimelineDataManager(cacheStore, aclManager); tdm.init(getConfig()); tdm.start(); for (LogInfo log : detailLogs) { log.parseForCache(tdm, dirPath, isDone()); } tdm.close(); } cacheRefreshTime = Time.monotonicNow(); cacheCompleted = isDone(); } return cacheStore; }