private void addLineageInfo(WorkUnitState state, int branchId) { if (!this.lineageInfo.isPresent()) { LOG.info("Will not add lineage info"); return; } // Final dataset descriptor DatasetDescriptor datasetDescriptor = createDestinationDescriptor(state, branchId); List<PartitionDescriptor> partitions = PartitionedDataWriter.getPartitionInfoAndClean(state, branchId); List<Descriptor> descriptors = new ArrayList<>(); if (partitions.size() == 0) { // Report as dataset level lineage descriptors.add(datasetDescriptor); } else { // Report as partition level lineage for (PartitionDescriptor partition : partitions) { descriptors.add(partition.copyWithNewDataset(datasetDescriptor)); } } this.lineageInfo.get().putDestination(descriptors, branchId, state); }
@Test public void testPartitionDescriptor() { DatasetDescriptor dataset = new DatasetDescriptor("hdfs", "/data/tracking/PageViewEvent"); String partitionName = "hourly/2018/08/14/18"; PartitionDescriptor partition = new PartitionDescriptor(partitionName, dataset); // Test copy with new dataset DatasetDescriptor dataset2 = new DatasetDescriptor("hive", "/data/tracking/PageViewEvent"); Descriptor partition2 = partition.copyWithNewDataset(dataset2); Assert.assertEquals(partition2.getName(), partition.getName()); Assert.assertEquals(((PartitionDescriptor)partition2).getDataset(), dataset2); // Test copy PartitionDescriptor partition3 = partition.copy(); Assert.assertEquals(partition3.getDataset(), dataset); Assert.assertEquals(partition3.getName(), partitionName); } }
private void addLineageInfo(WorkUnitState state, int branchId) { if (!this.lineageInfo.isPresent()) { LOG.info("Will not add lineage info"); return; } // Final dataset descriptor DatasetDescriptor datasetDescriptor = createDestinationDescriptor(state, branchId); List<PartitionDescriptor> partitions = PartitionedDataWriter.getPartitionInfoAndClean(state, branchId); List<Descriptor> descriptors = new ArrayList<>(); if (partitions.size() == 0) { // Report as dataset level lineage descriptors.add(datasetDescriptor); } else { // Report as partition level lineage for (PartitionDescriptor partition : partitions) { descriptors.add(partition.copyWithNewDataset(datasetDescriptor)); } } this.lineageInfo.get().putDestination(descriptors, branchId, state); }