@Override public Iterator<T> getDatasetsIterator() throws IOException { return this.datasetFinder.findDatasets().iterator(); } }
@Override public List<Dataset> findDatasets() throws IOException { List<Dataset> datasets = Lists.newArrayList(); for (DatasetsFinder<Dataset> df : this.datasetFinders) { datasets.addAll(df.findDatasets()); } return datasets; }
/** * Find all datasets on the basis on whitelist and blacklist, and then add them in a list in lexicographical order. */ protected void populateDatasets() throws IOException { this.datasets = this.datasetFinder.findDatasets(); sortHiveDatasets(datasets); }
public void run() throws IOException { Preconditions.checkNotNull(this.finder, "Dataset finder class is not set"); List<Dataset> datasets = this.finder.findDatasets(); this.finishCleanSignal = Optional.of(new CountDownLatch(datasets.size())); for (final Dataset dataset : datasets) {
List<Dataset> dataSets = this.datasetFinder.findDatasets(); this.finishCleanSignal = Optional.of(new CountDownLatch(dataSets.size())); for (final Dataset dataset : dataSets) {
public void run() throws IOException { Preconditions.checkNotNull(this.finder, "Dataset finder class is not set"); List<Dataset> datasets = this.finder.findDatasets(); this.finishCleanSignal = Optional.of(new CountDownLatch(datasets.size())); for (final Dataset dataset : datasets) {
List<Dataset> datasets = this.finder.findDatasets(); this.finishCleanSignal = Optional.of(new CountDownLatch(datasets.size())); for (final Dataset dataset : datasets) {
ImmutableList.of(fs, jobProps, client)); for (CleanableDataset dataset : finder.findDatasets()) { dataset.clean();
@Override public WorkUnitStream getWorkunitStream(SourceState state) { try { fs = getSourceFileSystem(state); state.setProp(COMPACTION_INIT_TIME, DateTimeUtils.currentTimeMillis()); suite = CompactionSuiteUtils.getCompactionSuiteFactory(state).createSuite(state); initRequestAllocator(state); initJobDir(state); copyJarDependencies(state); DatasetsFinder finder = DatasetUtils.instantiateDatasetFinder(state.getProperties(), getSourceFileSystem(state), DefaultFileSystemGlobFinder.class.getName()); List<Dataset> datasets = finder.findDatasets(); CompactionWorkUnitIterator workUnitIterator = new CompactionWorkUnitIterator (); // Spawn a single thread to create work units new Thread(new SingleWorkUnitGeneratorService (state, prioritize(datasets, state), workUnitIterator), "SingleWorkUnitGeneratorService").start(); return new BasicWorkUnitStream.Builder (workUnitIterator).build(); } catch (IOException e) { throw new RuntimeException(e); } }
@Override public Iterator<T> getDatasetsIterator() throws IOException { return this.datasetFinder.findDatasets().iterator(); } }
@Override public List<Dataset> findDatasets() throws IOException { List<Dataset> datasets = Lists.newArrayList(); for (DatasetsFinder<Dataset> df : this.datasetFinders) { datasets.addAll(df.findDatasets()); } return datasets; }
List<Dataset> dataSets = this.datasetFinder.findDatasets(); this.finishCleanSignal = Optional.of(new CountDownLatch(dataSets.size())); for (final Dataset dataset : dataSets) {
ImmutableList.of(fs, jobProps, client)); for (CleanableDataset dataset : finder.findDatasets()) { dataset.clean();
@Override public WorkUnitStream getWorkunitStream(SourceState state) { try { fs = getSourceFileSystem(state); state.setProp(COMPACTION_INIT_TIME, DateTimeUtils.currentTimeMillis()); suite = CompactionSuiteUtils.getCompactionSuiteFactory(state).createSuite(state); initRequestAllocator(state); initJobDir(state); copyJarDependencies(state); DatasetsFinder finder = DatasetUtils.instantiateDatasetFinder(state.getProperties(), getSourceFileSystem(state), DefaultFileSystemGlobFinder.class.getName()); List<Dataset> datasets = finder.findDatasets(); CompactionWorkUnitIterator workUnitIterator = new CompactionWorkUnitIterator (); // Spawn a single thread to create work units new Thread(new SingleWorkUnitGeneratorService (state, prioritize(datasets, state), workUnitIterator), "SingleWorkUnitGeneratorService").start(); return new BasicWorkUnitStream.Builder (workUnitIterator).build(); } catch (IOException e) { throw new RuntimeException(e); } }