private AppLogs getAndSetAppLogs(ApplicationId applicationId) throws IOException { LOG.debug("Looking for app logs mapped for app id {}", applicationId); AppLogs appLogs = appIdLogMap.get(applicationId); if (appLogs == null) { AppState appState = AppState.UNKNOWN; Path appDirPath = getDoneAppPath(applicationId); if (fs.exists(appDirPath)) { appState = AppState.COMPLETED; } else { appDirPath = getActiveAppPath(applicationId); if (fs.exists(appDirPath)) { appState = AppState.ACTIVE; } } if (appState != AppState.UNKNOWN) { appLogs = new AppLogs(applicationId, appDirPath, appState); LOG.debug("Create and try to add new appLogs to appIdLogMap for {}", applicationId); AppLogs oldAppLogs = appIdLogMap.putIfAbsent(applicationId, appLogs); if (oldAppLogs != null) { appLogs = oldAppLogs; } } } return appLogs; }
synchronized void loadDetailLog(TimelineDataManager tdm, TimelineEntityGroupId groupId) throws IOException { List<LogInfo> removeList = new ArrayList<>(); for (LogInfo log : detailLogs) { LOG.debug("Try refresh logs for {}", log.getFilename()); // Only refresh the log that matches the cache id if (log.matchesGroupId(groupId)) { Path dirPath = getAppDirPath(); if (fs.exists(log.getPath(dirPath))) { LOG.debug("Refresh logs for cache id {}", groupId); log.parseForStore(tdm, dirPath, isDone(), jsonFactory, objMapper, fs); } else { // The log may have been removed, remove the log removeList.add(log); LOG.info( "File {} no longer exists, removing it from log list", log.getPath(dirPath)); } } } detailLogs.removeAll(removeList); }
/** * Force releasing the cache item for the given group id, even though there * may be active references. */ public synchronized void forceRelease() { try { if (store != null) { store.close(); } } catch (IOException e) { LOG.warn("Error closing timeline store", e); } store = null; // reset offsets so next time logs are re-parsed for (LogInfo log : appLogs.getDetailLogs()) { if (log.getFilename().contains(groupId.toString())) { log.setOffset(0); } } LOG.debug("Cache for group {} released. ", groupId); }
store.new AppLogs(mainTestAppId, mainTestAppDirPath, AppState.COMPLETED); EntityCacheItem cacheItem = new EntityCacheItem(
LOG.debug("scan for log file: {}", filename); if (filename.startsWith(DOMAIN_LOG_PREFIX)) { addSummaryLog(attemptDirName, filename, statCache.getOwner(), true); } else if (filename.startsWith(SUMMARY_LOG_PREFIX)) { addSummaryLog(attemptDirName, filename, statCache.getOwner(), false); } else if (filename.startsWith(ENTITY_LOG_PREFIX)) { addDetailLog(attemptDirName, filename, statCache.getOwner()); } else { shouldSetTime = false;
if (!appLogs.isDone()) { appLogs.parseSummaryLogs(); } else if (appLogs.getDetailLogs().isEmpty()) { appLogs.scanForLogs(); if (!appLogs.getDetailLogs().isEmpty()) { if (store == null) { store = ReflectionUtils.newInstance(config.getClass( tdm.start(); appLogs.loadDetailLog(tdm, groupId);
ObjectMapper objMapper) throws IOException { if (needRefresh()) { if (!appLogs.isDone()) { appLogs.parseSummaryLogs(); } else if (appLogs.getDetailLogs().isEmpty()) { appLogs.scanForLogs(); if (!appLogs.getDetailLogs().isEmpty()) { if (store == null) { store = new LevelDBCacheTimelineStore(groupId.toString(), tdm.init(config); tdm.start(); if (appLogs.getDetailLogs().isEmpty()) { LOG.debug("cache id {}'s detail log is empty! ", groupId); for (LogInfo log : appLogs.getDetailLogs()) { LOG.debug("Try refresh logs for {}", log.getFilename()); Path appDirPath = appLogs.getAppDirPath(); if (fs.exists(log.getPath(appDirPath))) { LOG.debug("Refresh logs for cache id {}", groupId); log.parseForStore(tdm, appDirPath, appLogs.isDone(), jsonFactory, objMapper, fs); } else { appLogs.getDetailLogs().removeAll(removeList); tdm.close();
LOG.debug("scan for log file: {}", filename); if (filename.startsWith(DOMAIN_LOG_PREFIX)) { addSummaryLog(attemptDirName, filename, statCache.getOwner(), true); } else if (filename.startsWith(SUMMARY_LOG_PREFIX)) { addSummaryLog(attemptDirName, filename, statCache.getOwner(), false); } else if (filename.startsWith(ENTITY_LOG_PREFIX)) { addDetailLog(attemptDirName, filename, statCache.getOwner());
throws IOException { long startTime = Time.monotonicNow(); if (!isDone()) { LOG.debug("Try to parse summary log for log {} in {}", appId, appDirPath); appState = getAppState(appId); long recentLogModTime = scanForLogs(); if (appState == AppState.UNKNOWN) { if (Time.now() - recentLogModTime > unknownActiveMillis) { if (fs.exists(log.getPath(appDirPath))) { long summaryEntityParsed = log.parseForStore(tdm, appDirPath, isDone(), jsonFactory, objMapper, fs); metrics.incrEntitiesReadToSummary(summaryEntityParsed);
synchronized void parseSummaryLogs(TimelineDataManager tdm) throws IOException { if (!isDone()) { LOG.debug("Try to parse summary log for log {} in {}", appId, appDirPath); appState = getAppState(appId); long recentLogModTime = scanForLogs(); if (appState == AppState.UNKNOWN) { if (Time.now() - recentLogModTime > unknownActiveMillis) { for (LogInfo log : summaryLogs) { if (fs.exists(log.getPath(appDirPath))) { log.parseForStore(tdm, appDirPath, isDone(), jsonFactory, objMapper, fs); } else {
private TimelineStore getCachedStore(TimelineEntityGroupId groupId, List<EntityCacheItem> cacheItems) throws IOException { EntityCacheItem cacheItem; synchronized (this.cachedLogs) { // Note that the content in the cache log storage may be stale. cacheItem = this.cachedLogs.get(groupId); if (cacheItem == null) { LOG.debug("Set up new cache item for id {}", groupId); cacheItem = new EntityCacheItem(groupId, getConfig()); AppLogs appLogs = getAndSetAppLogs(groupId.getApplicationId()); if (appLogs != null) { LOG.debug("Set applogs {} for group id {}", appLogs, groupId); cacheItem.setAppLogs(appLogs); this.cachedLogs.put(groupId, cacheItem); } else { LOG.warn("AppLogs for groupId {} is set to null!", groupId); } } } TimelineStore store = null; if (cacheItem.getAppLogs() != null) { AppLogs appLogs = cacheItem.getAppLogs(); LOG.debug("try refresh cache {} {}", groupId, appLogs.getAppId()); cacheItems.add(cacheItem); store = cacheItem.refreshCache(aclManager, metrics); } else { LOG.warn("AppLogs for group id {} is null", groupId); } return store; }
private TimelineStore getCachedStore(TimelineEntityGroupId groupId) throws IOException { EntityCacheItem cacheItem; synchronized (this.cachedLogs) { // Note that the content in the cache log storage may be stale. cacheItem = this.cachedLogs.get(groupId); if (cacheItem == null) { LOG.debug("Set up new cache item for id {}", groupId); cacheItem = new EntityCacheItem(getConfig(), fs); AppLogs appLogs = getAndSetAppLogs(groupId.getApplicationId()); if (appLogs != null) { LOG.debug("Set applogs {} for group id {}", appLogs, groupId); cacheItem.setAppLogs(appLogs); this.cachedLogs.put(groupId, cacheItem); } else { LOG.warn("AppLogs for groupId {} is set to null!", groupId); } } } TimelineStore store = null; if (cacheItem.getAppLogs() != null) { AppLogs appLogs = cacheItem.getAppLogs(); LOG.debug("try refresh cache {} {}", groupId, appLogs.getAppId()); store = cacheItem.refreshCache(groupId, aclManager, jsonFactory, objMapper); } else { LOG.warn("AppLogs for group id {} is null", groupId); } return store; }
@Test public void testSummaryRead() throws Exception { // Load data EntityGroupFSTimelineStore.AppLogs appLogs = store.new AppLogs(mainTestAppId, mainTestAppDirPath, AppState.COMPLETED); MutableCounterLong summaryLogEntityRead = store.metrics.getGetEntityToSummaryOps(); long numEntityReadBefore = summaryLogEntityRead.value(); TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithStore(config, store); appLogs.scanForLogs(); appLogs.parseSummaryLogs(tdm); // Verify single entity read PluginStoreTestUtils.verifyTestEntities(tdm); // Verify multiple entities read TimelineEntities entities = tdm.getEntities("type_1", null, null, null, null, null, null, null, EnumSet.allOf(TimelineReader.Field.class), UserGroupInformation.getLoginUser()); assertEquals(entities.getEntities().size(), 1); for (TimelineEntity entity : entities.getEntities()) { assertEquals((Long) 123L, entity.getStartTime()); } // Verify metrics assertEquals(numEntityReadBefore + 5L, summaryLogEntityRead.value()); }
@Test public void testAppLogsScanLogs() throws Exception { EntityGroupFSTimelineStore.AppLogs appLogs = store.new AppLogs(mainTestAppId, mainTestAppDirPath, AppState.COMPLETED); appLogs.scanForLogs(); List<LogInfo> summaryLogs = appLogs.getSummaryLogs(); List<LogInfo> detailLogs = appLogs.getDetailLogs(); assertEquals(2, summaryLogs.size()); assertEquals(1, detailLogs.size()); for (LogInfo log : summaryLogs) { String fileName = log.getFilename(); assertTrue(fileName.equals(TEST_SUMMARY_LOG_FILE_NAME) || fileName.equals(TEST_DOMAIN_LOG_FILE_NAME)); } for (LogInfo log : detailLogs) { String fileName = log.getFilename(); assertEquals(fileName, mainEntityLogFileName); } }
@Override public void run() { try { LOG.debug("Begin parsing summary logs. "); appLogs.parseSummaryLogs(); if (appLogs.isDone()) { appLogs.moveToDone(); appIdLogMap.remove(appLogs.getAppId()); } LOG.debug("End parsing summary logs. "); } catch (Exception e) { Throwable t = extract(e); if (t instanceof InterruptedException) { LOG.info("Log parser interrupted"); } else { LOG.error("Error processing logs for " + appLogs.getAppId(), t); } } } }
@Override protected boolean removeEldestEntry( Map.Entry<TimelineEntityGroupId, EntityCacheItem> eldest) { if (super.size() > appCacheMaxSize) { TimelineEntityGroupId groupId = eldest.getKey(); LOG.debug("Evicting {} due to space limitations", groupId); EntityCacheItem cacheItem = eldest.getValue(); LOG.debug("Force release cache {}.", groupId); cacheItem.forceRelease(); if (cacheItem.getAppLogs().isDone()) { appIdLogMap.remove(groupId.getApplicationId()); } metrics.incrCacheEvicts(); return true; } return false; } });
public synchronized void releaseCache(TimelineEntityGroupId groupId) { try { if (store != null) { store.close(); } } catch (IOException e) { LOG.warn("Error closing datamanager", e); } store = null; // reset offsets so next time logs are re-parsed for (LogInfo log : appLogs.getDetailLogs()) { if (log.getFilename().contains(groupId.toString())) { log.offset = 0; } } } }
@Override protected boolean removeEldestEntry( Map.Entry<TimelineEntityGroupId, EntityCacheItem> eldest) { if (super.size() > appCacheMaxSize) { TimelineEntityGroupId groupId = eldest.getKey(); LOG.debug("Evicting {} due to space limitations", groupId); EntityCacheItem cacheItem = eldest.getValue(); cacheItem.releaseCache(groupId); if (cacheItem.getAppLogs().isDone()) { appIdLogMap.remove(groupId.getApplicationId()); } return true; } return false; } });
@Override public void run() { try { LOG.debug("Begin parsing summary logs. "); appLogs.parseSummaryLogs(); if (appLogs.isDone()) { appLogs.moveToDone(); appIdLogMap.remove(appLogs.getAppId()); } LOG.debug("End parsing summary logs. "); } catch (Exception e) { LOG.error("Error processing logs for " + appLogs.getAppId(), e); } } }
public synchronized void parseSummaryLogs() throws IOException { parseSummaryLogs(summaryTdm); }