@Override public void run() { LOG.debug("Active scan starting"); try { scanActiveLogs(); } catch (Exception e) { LOG.error("Error scanning active files", e); } LOG.debug("Active scan complete"); } }
@Override public void run() { LOG.debug("Active scan starting"); try { int scanned = scanActiveLogs(); LOG.debug("Scanned {} active applications", scanned); } catch (Exception e) { Throwable t = extract(e); if (t instanceof InterruptedException) { LOG.info("File scanner interrupted"); } else { LOG.error("Error scanning active files", t); } } LOG.debug("Active scan complete"); } }
@InterfaceAudience.Private @VisibleForTesting int scanActiveLogs() throws IOException { long startTime = Time.monotonicNow(); int logsToScanCount = scanActiveLogs(activeRootPath); metrics.addActiveLogDirScanTime(Time.monotonicNow() - startTime); return logsToScanCount; }
int scanActiveLogs(Path dir) throws IOException { RemoteIterator<FileStatus> iter = list(dir); int logsToScanCount = 0; while (iter.hasNext()) { FileStatus stat = iter.next(); String name = stat.getPath().getName(); ApplicationId appId = parseApplicationId(name); if (appId != null) { LOG.debug("scan logs for {} in {}", appId, stat.getPath()); logsToScanCount++; AppLogs logs = getAndSetActiveLog(appId, stat.getPath()); executor.execute(new ActiveLogParser(logs)); } else { logsToScanCount += scanActiveLogs(stat.getPath()); } } return logsToScanCount; }
@Test public void testScanActiveLogsAndMoveToDonePluginRead() throws Exception { EntityGroupFSTimelineStore store = null; ApplicationId appId = ApplicationId.fromString("application_1501509265053_0002"); String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path userBase = new Path(testActiveDirPath, user); Path userAppRoot = new Path(userBase, appId.toString()); Path attemotDirPath = new Path(userAppRoot, getAttemptDirName(appId)); try { store = createAndStartTimelineStore(AppState.COMPLETED); String logFileName = EntityGroupFSTimelineStore.ENTITY_LOG_PREFIX + EntityGroupPlugInForTest.getStandardTimelineGroupId(appId); createTestFiles(appId, attemotDirPath, logFileName); store.scanActiveLogs(); TimelineEntity entity = store.getEntity(entityNew.getEntityId(), entityNew.getEntityType(), EnumSet.allOf(Field.class)); assertNotNull(entity); assertEquals(entityNew.getEntityId(), entity.getEntityId()); assertEquals(entityNew.getEntityType(), entity.getEntityType()); } finally { if (store != null) { store.stop(); } fs.delete(userBase, true); } }