@Override public Iterator<T> getDatasetsIterator() throws IOException { return this.datasetFinder.findDatasets().iterator(); } }
@Override public List<Dataset> findDatasets() throws IOException { List<Dataset> datasets = Lists.newArrayList(); for (DatasetsFinder<Dataset> df : this.datasetFinders) { datasets.addAll(df.findDatasets()); } return datasets; }
List<Dataset> dataSets = this.datasetFinder.findDatasets(); this.finishCleanSignal = Optional.of(new CountDownLatch(dataSets.size())); for (final Dataset dataset : dataSets) {
ImmutableList.of(fs, jobProps, client)); for (CleanableDataset dataset : finder.findDatasets()) { dataset.clean();
@Override public WorkUnitStream getWorkunitStream(SourceState state) { try { fs = getSourceFileSystem(state); suite = CompactionSuiteUtils.getCompactionSuiteFactory(state).createSuite(state); initRequestAllocator(state); initJobDir(state); copyJarDependencies(state); DatasetsFinder finder = DatasetUtils.instantiateDatasetFinder(state.getProperties(), getSourceFileSystem(state), DefaultFileSystemGlobFinder.class.getName()); List<Dataset> datasets = finder.findDatasets(); CompactionWorkUnitIterator workUnitIterator = new CompactionWorkUnitIterator (); // Spawn a single thread to create work units new Thread(new SingleWorkUnitGeneratorService (state, prioritize(datasets, state), workUnitIterator), "SingleWorkUnitGeneratorService").start(); return new BasicWorkUnitStream.Builder (workUnitIterator).build(); } catch (IOException e) { throw new RuntimeException(e); } }