@Override public Collection<CopyableFile> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration) throws IOException { log.info(String.format("Getting copyable files at root path: %s", this.datasetRoot)); List<TimestampedDatasetVersion> versions = Lists.newArrayList(this.datasetVersionFinder.findDatasetVersions(this)); if (versions.isEmpty()) { log.warn("No dataset version can be found. Ignoring."); return Lists.newArrayList(); } Collection<TimestampedDatasetVersion> copyableVersions = this.versionSelectionPolicy.listSelectedVersions(versions); ConcurrentLinkedQueue<CopyableFile> copyableFileList = new ConcurrentLinkedQueue<>(); List<Future<?>> futures = Lists.newArrayList(); for (TimestampedDatasetVersion copyableVersion : copyableVersions) { futures.add(this.executor.submit(this.getCopyableFileGenetator(targetFs, configuration, copyableVersion, copyableFileList))); } try { for (Future<?> future : futures) { future.get(); } } catch (ExecutionException | InterruptedException e) { throw new IOException("Failed to generate copyable files.", e); } finally { ExecutorsUtils.shutdownExecutorService(executor, Optional.of(log)); } return copyableFileList; }
/** * Test {@link TimestampBasedCopyableDataset.CopyableFileGenerator} when src location is empty and also when it is null. */ @Test(expectedExceptions = RuntimeException.class) public void testCopyableFileGenerator() { Properties props = new Properties(); props.put(TimestampBasedCopyableDataset.COPY_POLICY, TimeBasedCopyPolicyForTest.class.getName()); props.put(TimestampBasedCopyableDataset.DATASET_VERSION_FINDER, TimestampedDatasetVersionFinderForTest.class.getName()); TimestampBasedCopyableDataset copyabledataset = new TimestampBasedCopyableDataset(localFs, props, new Path("dummy")); CopyConfiguration configuration = mock(CopyConfiguration.class); when(configuration.getPublishDir()).thenReturn(new Path("publishDir")); ConcurrentLinkedQueue<CopyableFile> copyableFileList = new ConcurrentLinkedQueue<>(); // The src path is empty. TimestampedDatasetVersion emptyVersion = new TimestampedDatasetVersion(new DateTime(), new Path("dummy2")); TimestampBasedCopyableDataset.CopyableFileGenerator emptyGenerator = copyabledataset.getCopyableFileGenetator(localFs, configuration, emptyVersion, copyableFileList); emptyGenerator.run(); Assert.assertEquals(copyableFileList.size(), 0); // The src path is null. TimestampedDatasetVersion versionHasNullPath = new TimestampedDatasetVersion(new DateTime(), null); TimestampBasedCopyableDataset.CopyableFileGenerator exceptionGenerator = copyabledataset.getCopyableFileGenetator(localFs, configuration, versionHasNullPath, copyableFileList); exceptionGenerator.run(); }
@Override public Collection<CopyableFile> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration) throws IOException { log.info(String.format("Getting copyable files at root path: %s", this.datasetRoot)); List<TimestampedDatasetVersion> versions = Lists.newArrayList(this.datasetVersionFinder.findDatasetVersions(this)); if (versions.isEmpty()) { log.warn("No dataset version can be found. Ignoring."); return Lists.newArrayList(); } Collection<TimestampedDatasetVersion> copyableVersions = this.versionSelectionPolicy.listSelectedVersions(versions); ConcurrentLinkedQueue<CopyableFile> copyableFileList = new ConcurrentLinkedQueue<>(); List<Future<?>> futures = Lists.newArrayList(); for (TimestampedDatasetVersion copyableVersion : copyableVersions) { futures.add(this.executor.submit(this.getCopyableFileGenetator(targetFs, configuration, copyableVersion, copyableFileList))); } try { for (Future<?> future : futures) { future.get(); } } catch (ExecutionException | InterruptedException e) { throw new IOException("Failed to generate copyable files.", e); } finally { ExecutorsUtils.shutdownExecutorService(executor, Optional.of(log)); } return copyableFileList; }