@Override public String toString() { return this.dataset.datasetURN() + "@" + this.name; } }
/** * Deserialize the json string to the a {@link Descriptor} object */ public static Descriptor fromJson(String json) { return fromJson(json, Descriptor.class); }
@Override default Descriptor resolve(Descriptor raw, State state) { DatasetDescriptor rawDataset; if (raw instanceof DatasetDescriptor) { rawDataset = (DatasetDescriptor) raw; } else if (raw instanceof PartitionDescriptor) { rawDataset = ((PartitionDescriptor) raw).getDataset(); } else { // type not supported return null; } return resolve(rawDataset, state); } }
@Test public void testPartitionDescriptor() { DatasetDescriptor dataset = new DatasetDescriptor("hdfs", "/data/tracking/PageViewEvent"); String partitionName = "hourly/2018/08/14/18"; PartitionDescriptor partition = new PartitionDescriptor(partitionName, dataset); // Test copy with new dataset DatasetDescriptor dataset2 = new DatasetDescriptor("hive", "/data/tracking/PageViewEvent"); Descriptor partition2 = partition.copyWithNewDataset(dataset2); Assert.assertEquals(partition2.getName(), partition.getName()); Assert.assertEquals(((PartitionDescriptor)partition2).getDataset(), dataset2); // Test copy PartitionDescriptor partition3 = partition.copy(); Assert.assertEquals(partition3.getDataset(), dataset); Assert.assertEquals(partition3.getName(), partitionName); } }
@Override public Descriptor getDataDescriptor() { DatasetDescriptor dataset = new DatasetDescriptor("testPlatform", "testDataset"); return new PartitionDescriptor(this.partition, dataset); } }
@Override public DatasetDescriptor copy() { return new DatasetDescriptor(this); }
@Override public PartitionDescriptor copy() { return new PartitionDescriptor(getName(), dataset); }
@Override public Iterator<T> getDatasetsIterator() throws IOException { return this.datasetFinder.findDatasets().iterator(); } }
@Override protected WorkUnit workUnitForDatasetPartition(PartitionableDataset.DatasetPartition partition) { WorkUnit workUnit = new WorkUnit(); workUnit.setProp(DATASET_URN, partition.getDataset().getUrn()); workUnit.setProp(PARTITION_URN, partition.getUrn()); return workUnit; }
public static <T extends org.apache.gobblin.dataset.Dataset> IterableDatasetFinder<T> instantiateIterableDatasetFinder( Properties props, FileSystem fs, String default_class, Object... additionalArgs) throws IOException { DatasetsFinder<T> datasetsFinder = instantiateDatasetFinder(props, fs, default_class, additionalArgs); return datasetsFinder instanceof IterableDatasetFinder ? (IterableDatasetFinder<T>) datasetsFinder : new IterableDatasetFinderImpl<>(datasetsFinder); }
public Descriptor copy() { return new Descriptor(name); }
/** * Compare against a raw URN. */ public int compare(String urn1, URNIdentified o2) { return urn1.compareTo(o2.getUrn()); } }
@Override public Descriptor getDataDescriptor() { // Dataset is resulted from WriterUtils.getWriterOutputDir(properties, this.numBranches, this.branchId) // The writer dataset might not be same as the published dataset DatasetDescriptor datasetDescriptor = new DatasetDescriptor(fs.getScheme(), outputFile.getParent().toString()); if (partitionKey == null) { return datasetDescriptor; } return new PartitionDescriptor(partitionKey, datasetDescriptor); }
public PartitionDescriptor copyWithNewDataset(DatasetDescriptor dataset) { return new PartitionDescriptor(getName(), dataset); }
@Override default String getUrn() { return datasetURN(); } }
/** * Find all datasets on the basis on whitelist and blacklist, and then add them in a list in lexicographical order. */ protected void populateDatasets() throws IOException { this.datasets = this.datasetFinder.findDatasets(); sortHiveDatasets(datasets); }
public DatasetVerificationException (Dataset dataset, Throwable cause) { super ("Dataset:" + dataset.datasetURN() + " Exception:" + cause); this.dataset = dataset; this.cause = cause; } }
@Override public String getUrn() { return this.dataset.datasetURN(); } }