private static TimelineEntity createEntity(String id, String type) { TimelineEntity entityToStore = new TimelineEntity(); entityToStore.setId(id); entityToStore.setType(type); entityToStore.setCreatedTime(0L); return entityToStore; }
private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity createBaseEntity(HistoryEvent event, long timestamp, String entityType, boolean setCreatedTime) { org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent tEvent = event.toTimelineEvent(); tEvent.setTimestamp(timestamp); org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity(); entity.addEvent(tEvent); entity.setType(entityType); if (setCreatedTime) { entity.setCreatedTime(timestamp); } Set<TimelineMetric> timelineMetrics = event.getTimelineMetrics(); if (timelineMetrics != null) { entity.setMetrics(timelineMetrics); } return entity; }
private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity createTaskEntity(HistoryEvent event, long timestamp, String taskId, String entityType, String relatedJobEntity, JobId jobId, boolean setCreatedTime, long taskIdPrefix) { org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = createBaseEntity(event, timestamp, entityType, setCreatedTime); entity.setId(taskId); if (event.getEventType() == EventType.TASK_STARTED) { entity.addInfo("TASK_TYPE", ((TaskStartedEvent)event).getTaskType().toString()); } entity.addIsRelatedToEntity(relatedJobEntity, jobId.toString()); entity.setIdPrefix(taskIdPrefix); return entity; }
@Override public int compareTo(TimelineEntity other) { int comparison = getType().compareTo(other.getType()); if (comparison == 0) { if (getIdPrefix() > other.getIdPrefix()) { // Descending order by entity id prefix return -1; } else if (getIdPrefix() < other.getIdPrefix()) { return 1; } else { return getId().compareTo(other.getId()); } } else { return comparison; } }
private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity createTaskAttemptEntity(HistoryEvent event, long timestamp, String taskAttemptId, String entityType, String relatedTaskEntity, String taskId, boolean setCreatedTime, long taskAttemptIdPrefix) { org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = createBaseEntity(event, timestamp, entityType, setCreatedTime); entity.setId(taskAttemptId); entity.addIsRelatedToEntity(relatedTaskEntity, taskId); entity.setIdPrefix(taskAttemptIdPrefix); return entity; }
private void publishContainerStartFailedEventOnTimelineServiceV2( final ContainerId containerId, String diagnostics) { final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = new org.apache.hadoop.yarn.api.records.timelineservice. TimelineEntity(); entity.setId(containerId.toString()); entity.setType(DSEntity.DS_CONTAINER.toString()); entity.addInfo("user", appSubmitterUgi.getShortUserName()); org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event = new org.apache.hadoop.yarn.api.records.timelineservice .TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setId(DSEvent.DS_CONTAINER_END.toString()); event.addInfo(DIAGNOSTICS, diagnostics); entity.addEvent(event); try { appSubmitterUgi.doAs((PrivilegedExceptionAction<Object>) () -> { timelineV2Client.putEntitiesAsync(entity); return null; }); } catch (Exception e) { LOG.error("Container start failed event could not be published for {}", containerId, e instanceof UndeclaredThrowableException ? e.getCause() : e); } }
return null; TimelineEntity entity = new TimelineEntity(); EntityRowKey parseRowKey = EntityRowKey.parseRowKey(result.getRow()); entity.setType(parseRowKey.getEntityType()); entity.setId(parseRowKey.getEntityId()); entity.setIdPrefix(parseRowKey.getEntityIdPrefix().longValue()); entity.setCreatedTime(createdTime); entity.getIsRelatedToEntities().clear(); entity.getRelatesToEntities().clear(); entity.getEvents().clear(); entity.getInfo().put(TimelineReaderUtils.FROMID_KEY, parseRowKey.getRowKeyAsString()); return entity;
@Test public void testTimelineEntities() throws Exception { TimelineEntity entity = new TimelineEntity(); entity.setType("test type 1"); entity.setId("test id 1"); entity.addInfo("test info key 1", "test info value 1"); entity.addInfo("test info key 2", Arrays.asList("test info value 2", "test info value 3")); entity.addInfo("test info key 3", true); Assert.assertTrue( entity.getInfo().get("test info key 3") instanceof Boolean); entity.addConfig("test config key 1", "test config value 1"); entity.addConfig("test config key 2", "test config value 2"); Assert.assertEquals(1.0F, entry.getValue()); Assert.assertFalse(itr.hasNext()); entity.addMetric(metric1); entity.addMetric(metric2); event1.getInfo().get("test info key 3") instanceof Boolean); event1.setTimestamp(1L); entity.addEvent(event1); event2.getInfo().get("test info key 3") instanceof Boolean); event2.setTimestamp(2L); entity.addEvent(event2); entity.setCreatedTime(0L); entity.addRelatesToEntity("test type 2", "test id 2");
private static void loadEntityData(String rootDir) throws Exception { File appDir = getAppDir(rootDir, "cluster1", "user1", "flow1", "1", "app1", "app"); TimelineEntity entity11 = new TimelineEntity(); entity11.setId("id_1"); entity11.setType("app"); entity11.setCreatedTime(1425016502000L); Map<String, Object> info1 = new HashMap<String, Object>(); info1.put("info1", "val1"); info1.put("info2", "val5"); entity11.addInfo(info1); TimelineEvent event = new TimelineEvent(); event.setId("event_1"); event.setTimestamp(1425016502003L); entity11.addEvent(event); Set<TimelineMetric> metrics = new HashSet<TimelineMetric>(); TimelineMetric metric1 = new TimelineMetric(); metric2.addValue(1425016502016L, 34); metrics.add(metric2); entity11.setMetrics(metrics); Map<String, String> configs = new HashMap<String, String>(); configs.put("config_1", "127"); entity11.setConfigs(configs); entity11.addRelatesToEntity("flow", "flow1"); entity11.addIsRelatedToEntity("type1", "tid1_1"); writeEntityFile(entity11, appDir); TimelineEntity entity12 = new TimelineEntity(); entity12.setId("id_1"); entity12.setType("app");
case APPLICATION_TABLE: ColumnRWHelper.store(rowKey, applicationTable, ApplicationColumn.ID, null, te.getId()); ColumnRWHelper.store(rowKey, applicationTable, ApplicationColumn.CREATED_TIME, null, te.getCreatedTime()); ColumnRWHelper.store(rowKey, applicationTable, ApplicationColumn.FLOW_VERSION, null, flowVersion); storeInfo(rowKey, te.getInfo(), flowVersion, ApplicationColumnPrefix.INFO, applicationTable); storeMetrics(rowKey, te.getMetrics(), ApplicationColumnPrefix.METRIC, applicationTable); storeEvents(rowKey, te.getEvents(), ApplicationColumnPrefix.EVENT, applicationTable); storeConfig(rowKey, te.getConfigs(), ApplicationColumnPrefix.CONFIG, applicationTable); storeRelations(rowKey, te.getIsRelatedToEntities(), ApplicationColumnPrefix.IS_RELATED_TO, applicationTable); storeRelations(rowKey, te.getRelatesToEntities(), ApplicationColumnPrefix.RELATES_TO, applicationTable); break; case ENTITY_TABLE: ColumnRWHelper.store(rowKey, entityTable, EntityColumn.ID, null, te.getId()); ColumnRWHelper.store(rowKey, entityTable, EntityColumn.TYPE, null, te.getType()); ColumnRWHelper.store(rowKey, entityTable, EntityColumn.CREATED_TIME, null, te.getCreatedTime()); ColumnRWHelper.store(rowKey, entityTable, EntityColumn.FLOW_VERSION, null, flowVersion);
private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity createJobEntity(JobId jobId) { org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity(); entity.setId(jobId.toString()); entity.setType(MAPREDUCE_JOB_ENTITY_TYPE); return entity; }
@Test public void testGetEntityCustomFields() throws Exception { Client client = createClient(); try { // Fields are case insensitive. URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/app1/entities/app/id_1?" + "fields=CONFIGS,Metrics,info"); ClientResponse resp = getResponse(client, uri); TimelineEntity entity = resp.getEntity(TimelineEntity.class); assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, resp.getType().toString()); assertNotNull(entity); assertEquals("id_1", entity.getId()); assertEquals("app", entity.getType()); assertEquals(3, entity.getConfigs().size()); assertEquals(3, entity.getMetrics().size()); assertTrue("UID should be present", entity.getInfo().containsKey(TimelineReaderUtils.UID_KEY)); // Includes UID. assertEquals(3, entity.getInfo().size()); // No events will be returned as events are not part of fields. assertEquals(0, entity.getEvents().size()); } finally { client.destroy(); } }
Set<TimelineEntity> createEntities(int noOfUsers, String userKey) { Set<TimelineEntity> entities = new LinkedHashSet<>(); for (int i = 0; i < noOfUsers; i++) { TimelineEntity e = new TimelineEntity(); e.setType("user" + i); e.setId("user" + i); e.getInfo().put(userKey, "user" + i); entities.add(e); } return entities; }
ApplicationAttemptId attemptId = appAttempt.getAppAttemptId(); TimelineEntity entity = createAppAttemptEntity(attemptId); entity.setCreatedTime(registeredTime); entity.addEvent(tEvent); appAttempt.getMasterContainer().getNodeId().toString()); entity.setInfo(entityInfo); entity.setIdPrefix( TimelineServiceHelper.invertLong(attemptId.getAttemptId()));
@Test public void testGetApplicationEvent() { TimelineEntity te = null; TimelineEvent tEvent = ApplicationEntity.getApplicationEvent(te, "no event"); assertEquals(null, tEvent); te = new TimelineEntity(); te.setType(TimelineEntityType.YARN_APPLICATION.toString()); TimelineEvent event = new TimelineEvent(); event.setId("start_event"); event.setTimestamp(System.currentTimeMillis()); te.addEvent(event); tEvent = ApplicationEntity.getApplicationEvent(te, "start_event"); assertEquals(event, tEvent); te = new TimelineEntity(); te.setType(TimelineEntityType.YARN_CLUSTER.toString()); event = new TimelineEvent(); event.setId("start_event_cluster"); event.setTimestamp(System.currentTimeMillis()); te.addEvent(event); tEvent = ApplicationEntity.getApplicationEvent(te, "start_event_cluster"); assertEquals(null, tEvent); } }
private TimelineEntity parseEntityForType(Result result) throws IOException { if (result == null || result.isEmpty()) { return null; } TimelineEntity entity = new TimelineEntity(); EntityRowKey newRowKey = EntityRowKey.parseRowKey(result.getRow()); entity.setType(newRowKey.getEntityType()); return entity; }
@Test public void testGetEntityDefaultView() throws Exception { Client client = createClient(); try { URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/app1/entities/app/id_1"); ClientResponse resp = getResponse(client, uri); TimelineEntity entity = resp.getEntity(TimelineEntity.class); assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, resp.getType().toString()); assertNotNull(entity); assertEquals("id_1", entity.getId()); assertEquals("app", entity.getType()); assertEquals((Long)1425016502000L, entity.getCreatedTime()); // Default view i.e. when no fields are specified, entity contains only // entity id, entity type and created time. assertEquals(0, entity.getConfigs().size()); assertEquals(0, entity.getMetrics().size()); } finally { client.destroy(); } }
if (publishContainerEvents) { TimelineEntity entity = createContainerEntity(container.getContainerId()); entity.setCreatedTime(createdTime); entity.addEvent(tEvent); ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_INFO, container.getNodeHttpAddress()); entity.setInfo(entityInfo);
@SuppressWarnings("unchecked") private void publishContainerFinishedEvent(ContainerStatus containerStatus, long containerFinishTime, long containerStartTime) { ContainerId containerId = containerStatus.getContainerId(); TimelineEntity entity = createContainerEntity(containerId); Map<String, Object> entityInfo = new HashMap<String, Object>(); entityInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO, containerStatus.getDiagnostics()); entityInfo.put(ContainerMetricsConstants.EXIT_STATUS_INFO, containerStatus.getExitStatus()); entityInfo.put(ContainerMetricsConstants.STATE_INFO, ContainerState.COMPLETE.toString()); entityInfo.put(ContainerMetricsConstants.CONTAINER_FINISHED_TIME, containerFinishTime); entity.setInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(ContainerMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(containerFinishTime); entity.addEvent(tEvent); entity.setIdPrefix(TimelineServiceHelper.invertLong(containerStartTime)); dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity, containerId.getApplicationAttemptId().getApplicationId())); }