private Stream<WorkUnit> createWorkUnitStream(SourceState state) throws IOException { IterableDatasetFinder datasetsFinder = createDatasetsFinder(state); Stream<Dataset> datasetStream = datasetsFinder.getDatasetsStream(0, null); if (this.drilldownIntoPartitions) { return datasetStream.flatMap(dataset -> { if (dataset instanceof PartitionableDataset) { try { return (Stream<PartitionableDataset.DatasetPartition>) ((PartitionableDataset) dataset).getPartitions(0, null); } catch (IOException ioe) { log.error("Failed to get partitions for dataset " + dataset.getUrn()); return Stream.empty(); } } else { return Stream.of(new DatasetWrapper(dataset)); } }).map(this::workUnitForPartitionInternal).filter(Objects::nonNull); } else { return datasetStream.map(this::workUnitForDataset).filter(Objects::nonNull); } }
private Stream<WorkUnit> createWorkUnitStream(SourceState state) throws IOException { IterableDatasetFinder datasetsFinder = createDatasetsFinder(state); Stream<Dataset> datasetStream = datasetsFinder.getDatasetsStream(0, null); if (this.drilldownIntoPartitions) { return datasetStream.flatMap(dataset -> { if (dataset instanceof PartitionableDataset) { try { return (Stream<PartitionableDataset.DatasetPartition>) ((PartitionableDataset) dataset).getPartitions(0, null); } catch (IOException ioe) { log.error("Failed to get partitions for dataset " + dataset.getUrn()); return Stream.empty(); } } else { return Stream.of(new DatasetWrapper(dataset)); } }).map(this::workUnitForPartitionInternal).filter(Objects::nonNull); } else { return datasetStream.map(this::workUnitForDataset).filter(Objects::nonNull); } }