static EventLogger getInstance(HiveConf conf) { if (instance == null) { synchronized (EventLogger.class) { if (instance == null) { instance = new EventLogger(conf, SystemClock.getInstance()); ShutdownHookManager.addShutdownHook(instance::shutdown); } } } return instance; } }
private ProtoMessageReader<HiveHookEventProto> getTestReader(HiveConf conf, String tmpFolder) throws IOException { Path path = new Path(tmpFolder); FileSystem fs = path.getFileSystem(conf); FileStatus[] status = fs.listStatus(path); Assert.assertEquals(1, status.length); status = fs.listStatus(status[0].getPath()); Assert.assertEquals(1, status.length); DatePartitionedLogger<HiveHookEventProto> logger = new DatePartitionedLogger<>( HiveHookEventProto.PARSER, path, conf, SystemClock.getInstance()); return logger.getReader(status[0].getPath()); }
private ApplicationReport getAppReport(String appName, ServiceClient serviceClient, long timeoutMs) throws LlapStatusCliException { Clock clock = SystemClock.getInstance(); long startTime = clock.getTime(); long timeoutTime = timeoutMs < 0 ? Long.MAX_VALUE : (startTime + timeoutMs);
Clock clock = SystemClock.getInstance(); long lastSummaryLogTime = -1;
@Test public void testPostEventLog() throws Exception { context.setHookType(HookType.POST_EXEC_HOOK); context.getPerfLogger().PerfLogBegin("test", "LogTest"); context.getPerfLogger().PerfLogEnd("test", "LogTest"); EventLogger evtLogger = new EventLogger(conf, SystemClock.getInstance()); evtLogger.handle(context); evtLogger.shutdown(); HiveHookEventProto event = loadEvent(conf, tmpFolder); Assert.assertEquals(EventType.QUERY_COMPLETED.name(), event.getEventType()); Assert.assertEquals(System.getProperty("user.name"), event.getUser()); Assert.assertEquals("test_user", event.getRequestUser()); Assert.assertEquals("test_queryId", event.getHiveQueryId()); Assert.assertEquals("test_op_id", event.getOperationId()); assertOtherInfo(event, OtherInfoType.STATUS, Boolean.TRUE.toString()); String val = findOtherInfo(event, OtherInfoType.PERF); Map<String, Long> map = new ObjectMapper().readValue(val, new TypeReference<Map<String, Long>>() {}); // This should be really close to zero. Assert.assertTrue("Expected LogTest in PERF", map.get("LogTest") < 100); }
@Test public void testFailureEventLog() throws Exception { context.setHookType(HookType.ON_FAILURE_HOOK); EventLogger evtLogger = new EventLogger(conf, SystemClock.getInstance()); evtLogger.handle(context); evtLogger.shutdown(); HiveHookEventProto event = loadEvent(conf, tmpFolder); Assert.assertEquals(EventType.QUERY_COMPLETED.name(), event.getEventType()); Assert.assertEquals(System.getProperty("user.name"), event.getUser()); Assert.assertEquals("test_user", event.getRequestUser()); Assert.assertEquals("test_queryId", event.getHiveQueryId()); Assert.assertEquals("test_op_id", event.getOperationId()); assertOtherInfo(event, OtherInfoType.STATUS, Boolean.FALSE.toString()); assertOtherInfo(event, OtherInfoType.PERF, null); }
@Test public void testPreEventLog() throws Exception { context.setHookType(HookType.PRE_EXEC_HOOK); EventLogger evtLogger = new EventLogger(conf, SystemClock.getInstance()); evtLogger.handle(context); evtLogger.shutdown(); HiveHookEventProto event = loadEvent(conf, tmpFolder); Assert.assertEquals(EventType.QUERY_SUBMITTED.name(), event.getEventType()); Assert.assertEquals(1234L, event.getTimestamp()); Assert.assertEquals(System.getProperty("user.name"), event.getUser()); Assert.assertEquals("test_user", event.getRequestUser()); Assert.assertEquals("test_queryId", event.getHiveQueryId()); Assert.assertEquals("test_op_id", event.getOperationId()); Assert.assertEquals(ExecutionMode.NONE.name(), event.getExecutionMode()); Assert.assertFalse(event.hasQueue()); assertOtherInfo(event, OtherInfoType.TEZ, Boolean.FALSE.toString()); assertOtherInfo(event, OtherInfoType.MAPRED, Boolean.FALSE.toString()); assertOtherInfo(event, OtherInfoType.CLIENT_IP_ADDRESS, "192.168.10.10"); assertOtherInfo(event, OtherInfoType.SESSION_ID, "test_session_id"); assertOtherInfo(event, OtherInfoType.THREAD_NAME, "test_thread_id"); assertOtherInfo(event, OtherInfoType.HIVE_INSTANCE_TYPE, "HS2"); assertOtherInfo(event, OtherInfoType.HIVE_ADDRESS, "hive_addr"); assertOtherInfo(event, OtherInfoType.CONF, null); assertOtherInfo(event, OtherInfoType.QUERY, null); }
@Test public void testPreAndPostEventBoth() throws Exception { context.setHookType(HookType.PRE_EXEC_HOOK); EventLogger evtLogger = new EventLogger(conf, SystemClock.getInstance()); evtLogger.handle(context); context.setHookType(HookType.POST_EXEC_HOOK); evtLogger.handle(context); evtLogger.shutdown(); ProtoMessageReader<HiveHookEventProto> reader = getTestReader(conf, tmpFolder); HiveHookEventProto event = reader.readEvent(); Assert.assertNotNull("Pre hook event not found", event); Assert.assertEquals(EventType.QUERY_SUBMITTED.name(), event.getEventType()); event = reader.readEvent(); Assert.assertNotNull("Post hook event not found", event); Assert.assertEquals(EventType.QUERY_COMPLETED.name(), event.getEventType()); }
@Test public void testQueueLogs() throws Exception { context.setHookType(HookType.PRE_EXEC_HOOK); EventLogger evtLogger = new EventLogger(conf, SystemClock.getInstance());
/** * Create a monitor for a Windows process tree. * @param pid Identifier of the job object. */ public WindowsBasedProcessTree(final String pid) { this(pid, SystemClock.getInstance()); }
@Private @VisibleForTesting public Clock getSystemClock() { return SystemClock.getInstance(); }
/** * Create resource calculator for all Yarn containers. */ public CGroupsResourceCalculator() throws YarnException { this(null, PROCFS, ResourceHandlerModule.getCGroupsHandler(), SystemClock.getInstance(), SysInfoLinux.JIFFY_LENGTH_IN_MILLIS); }
public ContainerImpl(Configuration conf, Dispatcher dispatcher, ContainerLaunchContext launchContext, Credentials creds, NodeManagerMetrics metrics, ContainerTokenIdentifier containerTokenIdentifier, Context context) { this(conf, dispatcher, launchContext, creds, metrics, containerTokenIdentifier, context, SystemClock.getInstance().getTime()); }
public void turnOnAppActivitiesRecording(ApplicationId applicationId, double maxTime) { long startTS = SystemClock.getInstance().getTime(); long endTS = startTS + (long) (maxTime * 1000); recordingAppActivitiesUntilSpecifiedTime.put(applicationId, endTS); }
public AppActivitiesInfo(String errorMessage, String applicationId) { this.diagnostic = errorMessage; this.applicationId = applicationId; Date date = new Date(); date.setTime(SystemClock.getInstance().getTime()); this.timeStamp = date.toString(); }
public MyAppMaster(Clock clock) { super(MyAppMaster.class.getName()); if (clock == null) { clock = SystemClock.getInstance(); } this.clock = clock; LOG.info("Created MyAppMaster"); } }
public ControlledClock() { this(SystemClock.getInstance()); setTime(0); } public ControlledClock(Clock actualClock) {
@Test public void testReducerMemoryRequestDefaultMemory() { EventHandler eventHandler = mock(EventHandler.class); Clock clock = SystemClock.getInstance(); TaskAttemptImpl taImpl = createReduceTaskAttemptImplForTest(eventHandler, clock, new JobConf()); long memorySize = getResourceInfoFromContainerRequest(taImpl, eventHandler). getMemorySize(); assertEquals(MRJobConfig.DEFAULT_REDUCE_MEMORY_MB, memorySize); }
@Test public void testReducerMemoryRequestViaMapreduceReduceMemoryMb() { EventHandler eventHandler = mock(EventHandler.class); Clock clock = SystemClock.getInstance(); JobConf jobConf = new JobConf(); jobConf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 2048); TaskAttemptImpl taImpl = createReduceTaskAttemptImplForTest(eventHandler, clock, jobConf); long memorySize = getResourceInfoFromContainerRequest(taImpl, eventHandler). getMemorySize(); assertEquals(2048, memorySize); }
@Test public void testReducerCpuRequestViaMapreduceReduceCpuVcores() { EventHandler eventHandler = mock(EventHandler.class); Clock clock = SystemClock.getInstance(); JobConf jobConf = new JobConf(); jobConf.setInt(MRJobConfig.REDUCE_CPU_VCORES, 3); TaskAttemptImpl taImpl = createReduceTaskAttemptImplForTest(eventHandler, clock, jobConf); int vCores = getResourceInfoFromContainerRequest(taImpl, eventHandler). getVirtualCores(); assertEquals(3, vCores); }