/** * @deprecated use {@link #copy()} */ @Deprecated public DatasetDescriptor(DatasetDescriptor copy) { super(copy.getName()); platform = copy.getPlatform(); metadata.putAll(copy.getMetadata()); }
private LineageEventBuilder getLineageEvent(Collection<LineageEventBuilder> events, int branchId, String destinationPlatform) { for (LineageEventBuilder event : events) { DatasetDescriptor descriptor = (DatasetDescriptor) event.getDestination(); if (descriptor.getPlatform().equals(destinationPlatform) && descriptor.getMetadata().get(DatasetConstants.BRANCH).equals(String.valueOf(branchId))) { return event; } } return null; }
@Test public void testDatasetDescriptor() { DatasetDescriptor dataset = new DatasetDescriptor("hdfs", "/data/tracking/PageViewEvent"); dataset.addMetadata("fsUri", "hdfs://test.com:2018"); DatasetDescriptor copy = dataset.copy(); Assert.assertEquals(copy.getName(), dataset.getName()); Assert.assertEquals(copy.getPlatform(), dataset.getPlatform()); Assert.assertEquals(copy.getMetadata(), dataset.getMetadata()); Assert.assertEquals(dataset, copy); Assert.assertEquals(dataset.hashCode(), copy.hashCode()); }
@Override public DatasetDescriptor resolve(DatasetDescriptor raw, State state) { DatasetDescriptor descriptor = new DatasetDescriptor(raw.getPlatform(), DATASET_NAME); raw.getMetadata().forEach(descriptor::addMetadata); return descriptor; } }
DatasetDescriptor source = (DatasetDescriptor) copyableFile.getSourceData(); Assert.assertEquals(source.getName(), "/data/databases/source"); Assert.assertEquals(source.getPlatform(), "hdfs"); Assert.assertEquals(source.getMetadata().get("fsUri"), originFsUri); DatasetDescriptor destination = (DatasetDescriptor) copyableFile.getDestinationData(); Assert.assertEquals(destination.getName(), "/data/databases/destination"); Assert.assertEquals(destination.getPlatform(), "file"); Assert.assertEquals(destination.getMetadata().get("fsUri"), targetFsUri); source = (DatasetDescriptor) copyableFile.getSourceData(); Assert.assertEquals(source.getName(), "/data/databases/source/profile"); Assert.assertEquals(source.getPlatform(), "hdfs"); Assert.assertEquals(source.getMetadata().get("fsUri"), originFsUri); destination = (DatasetDescriptor) copyableFile.getDestinationData(); Assert.assertEquals(destination.getName(), "/data/databases/destination/profile"); Assert.assertEquals(destination.getPlatform(), "file"); Assert.assertEquals(destination.getMetadata().get("fsUri"), targetFsUri);
DatasetDescriptor sourceDD = GSON.fromJson(props.getProperty("gobblin.event.lineage.source"), DatasetDescriptor.class); Assert.assertEquals(sourceDD.getPlatform(), "file"); Assert.assertEquals(sourceDD.getName(), "/tmp/test"); Assert.assertEquals(sourceDD.getMetadata().get(HiveToHdfsDatasetResolver.HIVE_TABLE), "db1.tb1"); DatasetDescriptor destDD1 = (DatasetDescriptor) firstDescriptor(props, "gobblin.event.lineage.branch.1.destination"); Assert.assertEquals(destDD1.getPlatform(), "file"); Assert.assertEquals(destDD1.getName(), "/tmp/data_nestedOrc/db1/tb1/final"); Assert.assertEquals(destDD1.getMetadata().get(HiveToHdfsDatasetResolver.HIVE_TABLE), DatasetDescriptor destDD2 = (DatasetDescriptor) firstDescriptor(props, "gobblin.event.lineage.branch.2.destination"); Assert.assertEquals(destDD2.getPlatform(), "file"); Assert.assertEquals(destDD2.getName(), "/tmp/data_flattenedOrc/db1/tb1/final"); Assert.assertEquals(destDD2.getMetadata().get(HiveToHdfsDatasetResolver.HIVE_TABLE),
/** * @deprecated use {@link #copy()} */ @Deprecated public DatasetDescriptor(DatasetDescriptor copy) { super(copy.getName()); platform = copy.getPlatform(); metadata.putAll(copy.getMetadata()); }