/** * Create a {@link LineageEventBuilder} from a {@link GobblinEventBuilder}. An inverse function * to {@link LineageEventBuilder#build()} */ public static LineageEventBuilder fromEvent(GobblinTrackingEvent event) { Map<String, String> metadata = event.getMetadata(); LineageEventBuilder lineageEvent = new LineageEventBuilder(event.getName()); metadata.forEach((key, value) -> { switch (key) { case SOURCE: lineageEvent.setSource(Descriptor.fromJson(value)); break; case DESTINATION: lineageEvent.setDestination(Descriptor.fromJson(value)); break; default: lineageEvent.addMetadata(key, value); break; } }); return lineageEvent; }
private void verify(LineageEventBuilder event, String name, Descriptor source, Descriptor destination) { Assert.assertEquals(event.getName(), name); Assert.assertEquals(event.getNamespace(), LineageEventBuilder.LIENAGE_EVENT_NAMESPACE); Assert.assertEquals(event.getMetadata().get(GobblinEventBuilder.EVENT_TYPE), LineageEventBuilder.LINEAGE_EVENT_TYPE); Assert.assertTrue(event.getSource().equals(source)); Assert.assertTrue(event.getDestination().equals(destination)); }
/** * Prefix all keys with {@link LineageEventBuilder#LIENAGE_EVENT_NAMESPACE} */ private static String getKey(Object... objects) { Object[] args = new Object[objects.length + 1]; args[0] = LineageEventBuilder.LIENAGE_EVENT_NAMESPACE; System.arraycopy(objects, 0, args, 1, objects.length); return LineageEventBuilder.getKey(args); } }
List<Descriptor> descriptors = Descriptor.fromJsonList(entry.getValue().toString()); for (Descriptor descriptor : descriptors) { LineageEventBuilder event = new LineageEventBuilder(name); event.setSource(source); event.setDestination(descriptor); branchEvents.add(event);
LineageEventBuilder event = find(events, "partition" + i + j); Assert.assertTrue(null != event); Assert.assertEquals(event.getSource(), source); Assert.assertEquals(event.getDestination(),
@Test public void testEventForPartitionedDataset() { final String topic = "testTopic"; final String kafka = "kafka"; final String hdfs = "hdfs"; final String path = "/data/tracking/PageViewEvent"; final String partitionName = "hourly/2018/08/15/15"; State state = new State(); LineageInfo lineageInfo = getLineageInfo(); DatasetDescriptor source = new DatasetDescriptor(kafka, topic); lineageInfo.setSource(source, state); DatasetDescriptor destinationDataset = new DatasetDescriptor(hdfs, path); PartitionDescriptor destination = new PartitionDescriptor(partitionName, destinationDataset); lineageInfo.putDestination(destination, 0, state); Map<String, Set<LineageEventBuilder>> events = LineageInfo.load(state); LineageEventBuilder event = first(events.get("0")); verify(event, topic, source, destination); // Verify gobblin tracking event GobblinTrackingEvent trackingEvent = event.build(); Assert.assertEquals(LineageEventBuilder.isLineageEvent(trackingEvent), true); Assert.assertEquals(LineageEventBuilder.fromEvent(trackingEvent), event); }
private void submitLineageEvent(String dataset, Collection<TaskState> states) { Collection<LineageEventBuilder> events = LineageInfo.load(states); // Send events events.forEach(event -> event.submit(metricContext)); log.info(String.format("Submitted %d lineage events for dataset %s", events.size(), dataset)); }
private static LineageEventBuilder find(Collection<LineageEventBuilder> events, String partitionName) { for (LineageEventBuilder event : events) { if (event.getDestination().getName().equals(partitionName)) { return event; } } return null; }
public LineageEventBuilder(String name) { super(name, LIENAGE_EVENT_NAMESPACE); addMetadata(EVENT_TYPE, LINEAGE_EVENT_TYPE); }
List<Descriptor> descriptors = Descriptor.fromJsonList(entry.getValue().toString()); for (Descriptor descriptor : descriptors) { LineageEventBuilder event = new LineageEventBuilder(name); event.setSource(source); event.setDestination(descriptor); branchEvents.add(event);
private void submitLineageEvent(String dataset, Collection<TaskState> states) { Collection<LineageEventBuilder> events = LineageInfo.load(states); // Send events events.forEach(event -> event.submit(metricContext)); log.info(String.format("Submitted %d lineage events for dataset %s", events.size(), dataset)); }
private LineageEventBuilder getLineageEvent(Collection<LineageEventBuilder> events, int branchId, String destinationPlatform) { for (LineageEventBuilder event : events) { DatasetDescriptor descriptor = (DatasetDescriptor) event.getDestination(); if (descriptor.getPlatform().equals(destinationPlatform) && descriptor.getMetadata().get(DatasetConstants.BRANCH).equals(String.valueOf(branchId))) { return event; } } return null; }
public LineageEventBuilder(String name) { super(name, LIENAGE_EVENT_NAMESPACE); addMetadata(EVENT_TYPE, LINEAGE_EVENT_TYPE); }
/** * Create a {@link LineageEventBuilder} from a {@link GobblinEventBuilder}. An inverse function * to {@link LineageEventBuilder#build()} */ public static LineageEventBuilder fromEvent(GobblinTrackingEvent event) { Map<String, String> metadata = event.getMetadata(); LineageEventBuilder lineageEvent = new LineageEventBuilder(event.getName()); metadata.forEach((key, value) -> { switch (key) { case SOURCE: lineageEvent.setSource(Descriptor.fromJson(value)); break; case DESTINATION: lineageEvent.setDestination(Descriptor.fromJson(value)); break; default: lineageEvent.addMetadata(key, value); break; } }); return lineageEvent; }
/** * Prefix all keys with {@link LineageEventBuilder#LIENAGE_EVENT_NAMESPACE} */ private static String getKey(Object... objects) { Object[] args = new Object[objects.length + 1]; args[0] = LineageEventBuilder.LIENAGE_EVENT_NAMESPACE; System.arraycopy(objects, 0, args, 1, objects.length); return LineageEventBuilder.getKey(args); } }