public synchronized TimelineStore refreshCache() throws IOException { //TODO: make a config for cache freshness if (!cacheCompleted && Time.monotonicNow() - cacheRefreshTime > 10000) { if (!isDone()) { parseSummaryLogs(); } else if (detailLogs.isEmpty()) { scanForLogs(); } if (!detailLogs.isEmpty()) { if (cacheStore == null) { cacheStore = new MemoryTimelineStore(); cacheStore.init(getConfig()); cacheStore.start(); } TimelineDataManager tdm = new TimelineDataManager(cacheStore, aclManager); tdm.init(getConfig()); tdm.start(); for (LogInfo log : detailLogs) { log.parseForCache(tdm, dirPath, isDone()); } tdm.close(); } cacheRefreshTime = Time.monotonicNow(); cacheCompleted = isDone(); } return cacheStore; }
@Test public void testParseBrokenEntity() throws Exception { // Load test data TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config); EntityLogInfo testLogInfo = new EntityLogInfo(TEST_ATTEMPT_DIR_NAME, TEST_BROKEN_FILE_NAME, UserGroupInformation.getLoginUser().getUserName()); DomainLogInfo domainLogInfo = new DomainLogInfo(TEST_ATTEMPT_DIR_NAME, TEST_BROKEN_FILE_NAME, UserGroupInformation.getLoginUser().getUserName()); // Try parse, should not fail testLogInfo.parseForStore(tdm, getTestRootPath(), true, jsonFactory, objMapper, fs); domainLogInfo.parseForStore(tdm, getTestRootPath(), true, jsonFactory, objMapper, fs); tdm.close(); }
@Test public void testParseEntity() throws Exception { // Load test data TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config); EntityLogInfo testLogInfo = new EntityLogInfo(TEST_ATTEMPT_DIR_NAME, TEST_ENTITY_FILE_NAME, UserGroupInformation.getLoginUser().getUserName()); testLogInfo.parseForStore(tdm, getTestRootPath(), true, jsonFactory, objMapper, fs); // Verify for the first batch PluginStoreTestUtils.verifyTestEntities(tdm); // Load new data TimelineEntity entityNew = PluginStoreTestUtils .createEntity("id_3", "type_3", 789l, null, null, null, null, "domain_id_1"); TimelineEntities entityList = new TimelineEntities(); entityList.addEntity(entityNew); writeEntitiesLeaveOpen(entityList, new Path(getTestRootPath(TEST_ATTEMPT_DIR_NAME), TEST_ENTITY_FILE_NAME)); testLogInfo.parseForStore(tdm, getTestRootPath(), true, jsonFactory, objMapper, fs); // Verify the newly added data TimelineEntity entity3 = tdm.getEntity(entityNew.getEntityType(), entityNew.getEntityId(), EnumSet.allOf(TimelineReader.Field.class), UserGroupInformation.getLoginUser()); assertNotNull(entity3); assertEquals("Failed to read out entity new", entityNew.getStartTime(), entity3.getStartTime()); tdm.close(); }