private void addLineageInfo(CopyEntity copyEntity, WorkUnit workUnit) { if (copyEntity instanceof CopyableFile) { CopyableFile copyableFile = (CopyableFile) copyEntity; /* * In Gobblin Distcp, the source and target path info of a CopyableFile are determined by its dataset found by * a DatasetFinder. Consequently, the source and destination dataset for the CopyableFile lineage are expected * to be set by the same logic */ if (lineageInfo.isPresent() && copyableFile.getSourceData() != null && copyableFile.getDestinationData() != null) { lineageInfo.get().setSource(copyableFile.getSourceData(), workUnit); } } } }
lineageInfo.get().putDestination(copyableFile.getDestinationData(), 0, wus);
Assert.assertEquals(source.getPlatform(), "hdfs"); Assert.assertEquals(source.getMetadata().get("fsUri"), originFsUri); DatasetDescriptor destination = (DatasetDescriptor) copyableFile.getDestinationData(); Assert.assertEquals(destination.getName(), "/data/databases/destination"); Assert.assertEquals(destination.getPlatform(), "file"); Assert.assertEquals(source.getPlatform(), "hdfs"); Assert.assertEquals(source.getMetadata().get("fsUri"), originFsUri); destination = (DatasetDescriptor) copyableFile.getDestinationData(); Assert.assertEquals(destination.getName(), "/data/databases/destination/profile"); Assert.assertEquals(destination.getPlatform(), "file");
private void addLineageInfo(CopyEntity copyEntity, WorkUnit workUnit) { if (copyEntity instanceof CopyableFile) { CopyableFile copyableFile = (CopyableFile) copyEntity; /* * In Gobblin Distcp, the source and target path info of a CopyableFile are determined by its dataset found by * a DatasetFinder. Consequently, the source and destination dataset for the CopyableFile lineage are expected * to be set by the same logic */ if (lineageInfo.isPresent() && copyableFile.getSourceData() != null && copyableFile.getDestinationData() != null) { lineageInfo.get().setSource(copyableFile.getSourceData(), workUnit); } } } }
lineageInfo.get().putDestination(copyableFile.getDestinationData(), 0, wus);