@Override public PartitionDescriptor copy() { return new PartitionDescriptor(getName(), dataset); }
public PartitionDescriptor copyWithNewDataset(DatasetDescriptor dataset) { return new PartitionDescriptor(getName(), dataset); }
@Override public Descriptor getDataDescriptor() { // Dataset is resulted from WriterUtils.getWriterOutputDir(properties, this.numBranches, this.branchId) // The writer dataset might not be same as the published dataset DatasetDescriptor datasetDescriptor = new DatasetDescriptor(fs.getScheme(), outputFile.getParent().toString()); if (partitionKey == null) { return datasetDescriptor; } return new PartitionDescriptor(partitionKey, datasetDescriptor); }
@Override public Descriptor getDataDescriptor() { DatasetDescriptor dataset = new DatasetDescriptor("testPlatform", "testDataset"); return new PartitionDescriptor(this.partition, dataset); } }
@Test public void testAsDescriptorResolver() { DescriptorResolver resolver = new TestDatasetResolver(); State state = new State(); // Test dataset descriptor DatasetDescriptor dataset = new DatasetDescriptor("hdfs", "/data/tracking/PageViewEvent"); Descriptor descriptor = resolver.resolve(dataset, state); Assert.assertTrue(descriptor.getClass().isAssignableFrom(DatasetDescriptor.class)); Assert.assertEquals(descriptor.getName(), TestDatasetResolver.DATASET_NAME); // Test partition descriptor String partitionName = "hourly/2018/08/14/18"; PartitionDescriptor partition = new PartitionDescriptor(partitionName, dataset); descriptor = resolver.resolve(partition, state); Assert.assertTrue(descriptor.getClass().isAssignableFrom(DatasetDescriptor.class)); Assert.assertEquals(descriptor.getName(), TestDatasetResolver.DATASET_NAME); // Test unsupported descriptor Assert.assertEquals(resolver.resolve(new MockDescriptor("test"), state), null); }
PartitionDescriptor source = new PartitionDescriptor(partition.getName(), sourceDataset); fileEntity.setSourceData(source); this.existingTargetPartition.isPresent() ? this.existingTargetPartition.get() : partition; PartitionDescriptor destination = new PartitionDescriptor(destinationPartition.getName(), destinationDataset); fileEntity.setDestinationData(destination);
@Test public void testSerializeDeserialze() throws Exception { CopyableFile copyableFile = new CopyableFile(new FileStatus(10, false, 12, 100, 12345, new Path("/path")), new Path("/destination"), new OwnerAndPermission("owner", "group", FsPermission.getDefault()), Lists.newArrayList(new OwnerAndPermission("owner2", "group2", FsPermission.getDefault())), "checksum".getBytes(), PreserveAttributes.fromMnemonicString(""), "", 0, 0, Maps .<String, String>newHashMap(), ""); DatasetDescriptor dataset = new DatasetDescriptor("hive", "db.table"); PartitionDescriptor descriptor = new PartitionDescriptor("datepartition=2018/09/05", dataset); copyableFile.setDestinationData(descriptor); String s = CopyEntity.serialize(copyableFile); CopyEntity de = CopyEntity.deserialize(s); Assert.assertEquals(de, copyableFile); }
@Test public void testPartitionDescriptor() { DatasetDescriptor dataset = new DatasetDescriptor("hdfs", "/data/tracking/PageViewEvent"); String partitionName = "hourly/2018/08/14/18"; PartitionDescriptor partition = new PartitionDescriptor(partitionName, dataset); // Test copy with new dataset DatasetDescriptor dataset2 = new DatasetDescriptor("hive", "/data/tracking/PageViewEvent"); Descriptor partition2 = partition.copyWithNewDataset(dataset2); Assert.assertEquals(partition2.getName(), partition.getName()); Assert.assertEquals(((PartitionDescriptor)partition2).getDataset(), dataset2); // Test copy PartitionDescriptor partition3 = partition.copy(); Assert.assertEquals(partition3.getDataset(), dataset); Assert.assertEquals(partition3.getName(), partitionName); } }
Assert.assertEquals(partitions.get(0), new PartitionDescriptor("a", dataset)); Assert.assertEquals(partitions.get(1), new PartitionDescriptor("1", dataset));
for (int j = 0; j < numPartitionsPerBranch; j++) { partitions.add(new PartitionDescriptor("partition" + i + j, datasetAtWriter)); Assert.assertEquals(event.getDestination(), new PartitionDescriptor("partition" + i + j, destinationDataset));
@Test public void testEventForPartitionedDataset() { final String topic = "testTopic"; final String kafka = "kafka"; final String hdfs = "hdfs"; final String path = "/data/tracking/PageViewEvent"; final String partitionName = "hourly/2018/08/15/15"; State state = new State(); LineageInfo lineageInfo = getLineageInfo(); DatasetDescriptor source = new DatasetDescriptor(kafka, topic); lineageInfo.setSource(source, state); DatasetDescriptor destinationDataset = new DatasetDescriptor(hdfs, path); PartitionDescriptor destination = new PartitionDescriptor(partitionName, destinationDataset); lineageInfo.putDestination(destination, 0, state); Map<String, Set<LineageEventBuilder>> events = LineageInfo.load(state); LineageEventBuilder event = first(events.get("0")); verify(event, topic, source, destination); // Verify gobblin tracking event GobblinTrackingEvent trackingEvent = event.build(); Assert.assertEquals(LineageEventBuilder.isLineageEvent(trackingEvent), true); Assert.assertEquals(LineageEventBuilder.fromEvent(trackingEvent), event); }
@Override public PartitionDescriptor copy() { return new PartitionDescriptor(getName(), dataset); }
public PartitionDescriptor copyWithNewDataset(DatasetDescriptor dataset) { return new PartitionDescriptor(getName(), dataset); }
@Override public Descriptor getDataDescriptor() { // Dataset is resulted from WriterUtils.getWriterOutputDir(properties, this.numBranches, this.branchId) // The writer dataset might not be same as the published dataset DatasetDescriptor datasetDescriptor = new DatasetDescriptor(fs.getScheme(), outputFile.getParent().toString()); if (partitionKey == null) { return datasetDescriptor; } return new PartitionDescriptor(partitionKey, datasetDescriptor); }
PartitionDescriptor source = new PartitionDescriptor(partition.getName(), sourceDataset); fileEntity.setSourceData(source); this.existingTargetPartition.isPresent() ? this.existingTargetPartition.get() : partition; PartitionDescriptor destination = new PartitionDescriptor(destinationPartition.getName(), destinationDataset); fileEntity.setDestinationData(destination);