private static void publishApplicationAttemptEvent( final TimelineClient timelineClient, String appAttemptId, DSEvent appEvent, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId(appAttemptId); entity.setEntityType(DSEntity.DS_APP_ATTEMPT.toString()); entity.setDomainId(domainId); entity.addPrimaryFilter(JOYConstants.USER, ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setEventType(appEvent.toString()); event.setTimestamp(System.currentTimeMillis()); entity.addEvent(event); try { timelineClient.putEntities(entity); } catch (YarnException e) { LOG.error("App Attempt " + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? JOYConstants.START : JOYConstants.END) + " event could not be published for " + appAttemptId.toString(), e); } catch (IOException e) { LOG.error("App Attempt " + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? JOYConstants.START : JOYConstants.END) + " event could not be published for " + appAttemptId.toString(), e); } }
TimelineEntity createPostHookEvent(String queryId, long stopTime, String user, String requestuser, boolean success, String opId, Map<String, Long> durations, String domainId) throws Exception { LOG.info("Received post-hook notification for :" + queryId); TimelineEntity atsEntity = new TimelineEntity(); atsEntity.setEntityId(queryId); atsEntity.setEntityType(EntityTypes.HIVE_QUERY_ID.name()); atsEntity.addPrimaryFilter(PrimaryFilterTypes.user.name(), user); atsEntity.addPrimaryFilter(PrimaryFilterTypes.requestuser.name(), requestuser); if (opId != null) { atsEntity.addPrimaryFilter(PrimaryFilterTypes.operationid.name(), opId); } TimelineEvent stopEvt = new TimelineEvent(); stopEvt.setEventType(EventTypes.QUERY_COMPLETED.name()); stopEvt.setTimestamp(stopTime); atsEntity.addEvent(stopEvt); atsEntity.addOtherInfo(OtherInfoTypes.STATUS.name(), success); // Perf times JSONObject perfObj = new JSONObject(new LinkedHashMap<>()); for (Map.Entry<String, Long> entry : durations.entrySet()) { perfObj.put(entry.getKey(), entry.getValue()); } atsEntity.addOtherInfo(OtherInfoTypes.PERF.name(), perfObj.toString()); atsEntity.setDomainId(domainId); return atsEntity; }
TimelineEntity createPostHookEvent(String queryId, long stopTime, String user, String requestuser, boolean success, String opId, Map<String, Long> durations, String domainId) throws Exception { LOG.info("Received post-hook notification for :" + queryId); TimelineEntity atsEntity = new TimelineEntity(); atsEntity.setEntityId(queryId); atsEntity.setEntityType(EntityTypes.HIVE_QUERY_ID.name()); atsEntity.addPrimaryFilter(PrimaryFilterTypes.user.name(), user); atsEntity.addPrimaryFilter(PrimaryFilterTypes.requestuser.name(), requestuser); if (opId != null) { atsEntity.addPrimaryFilter(PrimaryFilterTypes.operationid.name(), opId); } TimelineEvent stopEvt = new TimelineEvent(); stopEvt.setEventType(EventTypes.QUERY_COMPLETED.name()); stopEvt.setTimestamp(stopTime); atsEntity.addEvent(stopEvt); atsEntity.addOtherInfo(OtherInfoTypes.STATUS.name(), success); // Perf times JSONObject perfObj = new JSONObject(new LinkedHashMap<>()); for (Map.Entry<String, Long> entry : durations.entrySet()) { perfObj.put(entry.getKey(), entry.getValue()); } atsEntity.addOtherInfo(OtherInfoTypes.PERF.name(), perfObj.toString()); atsEntity.setDomainId(domainId); return atsEntity; }
private static void publishContainerStartEvent( final TimelineClient timelineClient, Container container, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId(container.getId().toString()); entity.setEntityType(DSEntity.DS_CONTAINER.toString()); entity.setDomainId(domainId); entity.addPrimaryFilter(JOYConstants.USER, ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(DSEvent.DS_CONTAINER_START.toString()); event.addEventInfo(JOYConstants.NODE, container.getNodeId().toString()); event.addEventInfo(JOYConstants.RESOURCES, container.getResource().toString()); entity.addEvent(event); try { ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() { @Override public TimelinePutResponse run() throws Exception { return timelineClient.putEntities(entity); } }); } catch (Exception e) { LOG.error("Container start event could not be published for " + container.getId().toString(), e instanceof UndeclaredThrowableException ? e.getCause() : e); } }
private static void publishContainerEndEvent( final TimelineClient timelineClient, ContainerStatus container, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId(container.getContainerId().toString()); entity.setEntityType(DSEntity.DS_CONTAINER.toString()); entity.setDomainId(domainId); entity.addPrimaryFilter(JOYConstants.USER, ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(DSEvent.DS_CONTAINER_END.toString()); event.addEventInfo(JOYConstants.STATE, container.getState().name()); event.addEventInfo(JOYConstants.EXIT_STATE, container.getExitStatus()); entity.addEvent(event); try { timelineClient.putEntities(entity); } catch (YarnException e) { LOG.error("Container end event could not be published for " + container.getContainerId().toString(), e); } catch (IOException e) { LOG.error("Container end event could not be published for " + container.getContainerId().toString(), e); } }
startEvt.setEventType(EventTypes.QUERY_SUBMITTED.name()); startEvt.setTimestamp(startTime); atsEntity.addEvent(startEvt);
startEvt.setEventType(EventTypes.QUERY_SUBMITTED.name()); startEvt.setTimestamp(startTime); atsEntity.addEvent(startEvt);
public void publishApplicationAttemptEvent( final TimelineClient timelineClient, String appAttemptId, ContainerEvents appEvent, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId(appAttemptId); entity.setEntityType(ApplicationMaster.DSEntity.DS_APP_ATTEMPT.toString()); entity.setDomainId(domainId); entity.addPrimaryFilter("user", ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setEventType(appEvent.toString()); event.setTimestamp(System.currentTimeMillis()); entity.addEvent(event); try { timelineClient.putEntities(entity); } catch (YarnException | IOException e) { LOG.error("App Attempt " + (appEvent.equals(ContainerEvents.APP_ATTEMPT_START) ? "start" : "end") + " event could not be published for " + appAttemptId.toString(), e); } } public void publishContainerStartEvent(
public void publishContainerStartEvent( final TimelineClient timelineClient, Container container, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId("" + container.getId()); entity.setEntityType(ApplicationMaster.DSEntity.DS_CONTAINER.toString()); entity.setDomainId(domainId); entity.addPrimaryFilter("user", ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(ContainerEvents.CONTAINER_START.toString()); event.addEventInfo("Node", container.getNodeId().toString()); event.addEventInfo("Resources", container.getResource().toString()); entity.addEvent(event); try { ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() { @Override public TimelinePutResponse run() throws Exception { return timelineClient.putEntities(entity); } }); } catch (Exception e) { LOG.error("Container start event could not be published for " + container.getId().toString(), e instanceof UndeclaredThrowableException ? e.getCause() : e); } } }
public void publishContainerEndEvent( final TimelineClient timelineClient, ContainerStatus container, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId(container.getContainerId().toString()); entity.setEntityType(ApplicationMaster.DSEntity.DS_CONTAINER.toString()); entity.setDomainId(domainId); entity.addPrimaryFilter("user", ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(ContainerEvents.CONTAINER_END.toString()); event.addEventInfo("State", container.getState().name()); event.addEventInfo("Exit Status", container.getExitStatus()); entity.addEvent(event); try { timelineClient.putEntities(entity); } catch (YarnException | IOException e) { LOG.error("Container end event could not be published for " + container.getContainerId().toString(), e); } } public void publishApplicationAttemptEvent(
private static void createAppModifiedEvent(ApplicationId appId, TimelineEvent tEvent, long updatedTimeIndex, String queue, int priority) { tEvent.setEventType(ApplicationMetricsConstants.UPDATED_EVENT_TYPE); tEvent.setTimestamp(Integer.MAX_VALUE + updatedTimeIndex + appId.getId()); Map<String, Object> eventInfo = new HashMap<String, Object>(); eventInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, queue); eventInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, priority); tEvent.setEventInfo(eventInfo); }
private void publishApplicationACLsUpdatedEvent( ApplicationACLsUpdatedEvent event) { TimelineEntity entity = createApplicationEntity(event.getApplicationId()); TimelineEvent tEvent = new TimelineEvent(); Map<String, Object> entityInfo = new HashMap<String, Object>(); entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, event.getViewAppACLs()); entity.setOtherInfo(entityInfo); tEvent.setEventType( ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE); tEvent.setTimestamp(event.getTimestamp()); entity.addEvent(tEvent); putEntity(entity); }
private void publishApplicationACLsUpdatedEvent( ApplicationACLsUpdatedEvent event) { TimelineEntity entity = createApplicationEntity(event.getApplicationId()); TimelineEvent tEvent = new TimelineEvent(); Map<String, Object> entityInfo = new HashMap<String, Object>(); entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, event.getViewAppACLs()); entity.setOtherInfo(entityInfo); tEvent.setEventType( ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE); tEvent.setTimestamp(event.getTimestamp()); entity.addEvent(tEvent); putEntity(entity); }
@SuppressWarnings("unchecked") @Override public void appACLsUpdated(RMApp app, String appViewACLs, long updatedTime) { TimelineEntity entity = createApplicationEntity(app.getApplicationId()); TimelineEvent tEvent = new TimelineEvent(); Map<String, Object> entityInfo = new HashMap<String, Object>(); entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, (appViewACLs == null) ? "" : appViewACLs); entity.setOtherInfo(entityInfo); tEvent.setEventType(ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE); tEvent.setTimestamp(updatedTime); entity.addEvent(tEvent); getDispatcher().getEventHandler().handle(new TimelineV1PublishEvent( SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId())); }
@SuppressWarnings("unchecked") @Override public void appStateUpdated(RMApp app, YarnApplicationState appState, long updatedTime) { TimelineEntity entity = createApplicationEntity(app.getApplicationId()); Map<String, Object> eventInfo = new HashMap<String, Object>(); eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, appState); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType(ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE); tEvent.setTimestamp(updatedTime); tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); getDispatcher().getEventHandler().handle(new TimelineV1PublishEvent( SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId())); }
/** * Create a test event */ private static TimelineEvent createEvent(long timestamp, String type, Map<String, Object> info) { TimelineEvent event = new TimelineEvent(); event.setTimestamp(timestamp); event.setEventType(type); event.setEventInfo(info); return event; }
/** * Create a test event */ static TimelineEvent createEvent(long timestamp, String type, Map<String, Object> info) { TimelineEvent event = new TimelineEvent(); event.setTimestamp(timestamp); event.setEventType(type); event.setEventInfo(info); return event; }
private static TimelineEntity convertAMStartedEvent(AMStartedEvent event) { TimelineEntity atsEntity = new TimelineEntity(); atsEntity.setEntityId("tez_" + event.getApplicationAttemptId().toString()); atsEntity.setEntityType(EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser()); atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID, event.getApplicationAttemptId().getApplicationId().toString()); TimelineEvent startEvt = new TimelineEvent(); startEvt.setEventType(HistoryEventType.AM_STARTED.name()); startEvt.setTimestamp(event.getStartTime()); atsEntity.addEvent(startEvt); return atsEntity; }
private static TimelineEntity convertDAGFinishedToDAGExtraInfoEntity(DAGFinishedEvent event) { TimelineEntity atsEntity = new TimelineEntity(); atsEntity.setEntityId(event.getDagID().toString()); atsEntity.setEntityType(EntityTypes.TEZ_DAG_EXTRA_INFO.name()); atsEntity.addRelatedEntity(EntityTypes.TEZ_DAG_ID.name(), event.getDagID().toString()); TimelineEvent submitEvt = new TimelineEvent(); submitEvt.setEventType(HistoryEventType.DAG_FINISHED.name()); submitEvt.setTimestamp(event.getFinishTime()); atsEntity.addEvent(submitEvt); atsEntity.addOtherInfo(ATSConstants.COUNTERS, DAGUtils.convertCountersToATSMap(event.getTezCounters())); return atsEntity; }
private static TimelineEntity convertDAGStartedEvent(DAGStartedEvent event) { TimelineEntity atsEntity = new TimelineEntity(); atsEntity.setEntityId(event.getDagID().toString()); atsEntity.setEntityType(EntityTypes.TEZ_DAG_ID.name()); TimelineEvent startEvt = new TimelineEvent(); startEvt.setEventType(HistoryEventType.DAG_STARTED.name()); startEvt.setTimestamp(event.getStartTime()); atsEntity.addEvent(startEvt); atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser()); atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID, event.getDagID().getApplicationId().toString()); atsEntity.addPrimaryFilter(ATSConstants.DAG_NAME, event.getDagName()); atsEntity.addOtherInfo(ATSConstants.START_TIME, event.getStartTime()); atsEntity.addOtherInfo(ATSConstants.STATUS, event.getDagState().toString()); return atsEntity; }