private void addLineageInfo(CopyEntity copyEntity, WorkUnit workUnit) { if (copyEntity instanceof CopyableFile) { CopyableFile copyableFile = (CopyableFile) copyEntity; /* * In Gobblin Distcp, the source and target path info of a CopyableFile are determined by its dataset found by * a DatasetFinder. Consequently, the source and destination dataset for the CopyableFile lineage are expected * to be set by the same logic */ if (lineageInfo.isPresent() && copyableFile.getSourceData() != null && copyableFile.getDestinationData() != null) { lineageInfo.get().setSource(copyableFile.getSourceData(), workUnit); } } } }
PreserveAttributes.fromMnemonicString(""), "", 0, 0, Maps.<String, String>newHashMap(), ""); copyableFile.setFsDatasets(originFs, targetFs); DatasetDescriptor source = (DatasetDescriptor) copyableFile.getSourceData(); Assert.assertEquals(source.getName(), "/data/databases/source"); Assert.assertEquals(source.getPlatform(), "hdfs"); PreserveAttributes.fromMnemonicString(""), "", 0, 0, Maps.<String, String>newHashMap(), ""); copyableFile.setFsDatasets(originFs, targetFs); source = (DatasetDescriptor) copyableFile.getSourceData(); Assert.assertEquals(source.getName(), "/data/databases/source/profile"); Assert.assertEquals(source.getPlatform(), "hdfs");
private void addLineageInfo(CopyEntity copyEntity, WorkUnit workUnit) { if (copyEntity instanceof CopyableFile) { CopyableFile copyableFile = (CopyableFile) copyEntity; /* * In Gobblin Distcp, the source and target path info of a CopyableFile are determined by its dataset found by * a DatasetFinder. Consequently, the source and destination dataset for the CopyableFile lineage are expected * to be set by the same logic */ if (lineageInfo.isPresent() && copyableFile.getSourceData() != null && copyableFile.getDestinationData() != null) { lineageInfo.get().setSource(copyableFile.getSourceData(), workUnit); } } } }