@Override public void run() { for (Path locationToCopy : locationsToCopy) { long timestampFromPath = this.versionDatetime.getMillis(); try { for (FileStatus singleFile : this.srcFs.listStatus(locationToCopy, this.filter)) { Path singleFilePath = singleFile.getPath(); log.debug("Checking if it is a copyable file: " + singleFilePath); Path relativePath = PathUtils.relativizePath(singleFilePath, datasetRoot); Path targetPath = new Path(targetRoot, relativePath); if (this.isCopyableFile(singleFile, targetPath)) { log.debug("Will create workunit for: " + singleFilePath); copyableFileList .add(this.generateCopyableFile(singleFile, targetPath, timestampFromPath, locationToCopy)); } } } catch (IOException e) { e.printStackTrace(); throw new RuntimeException("Failed to get copyable files for " + locationToCopy, e); } } }
new SimpleCopyableFileGenerator(copyabledataset, localFs, localFs, configuration1, srcVersion, copyableFileList1); copyFileGenerator1.run(); Assert.assertEquals(copyableFileList1.size(), 1); Assert.assertEquals(copyableFileList1.poll().getFileSet(), localFs.getFileStatus(srcfile).getPath().toString()); Path newTargetfile = new Path(targetDir, new Path(versionDir, noNeedToCopyFile)); this.localFs.create(newTargetfile); copyFileGenerator1.run(); Assert.assertEquals(copyableFileList1.size(), 1); Assert.assertEquals(copyableFileList1.poll().getFileSet(), localFs.getFileStatus(srcfile).getPath().toString()); new SimpleCopyableFileGenerator(copyabledataset, localFs, localFs, configuration2, srcVersion, copyableFileList2); copyFileGenerator2.run(); Assert.assertEquals(copyableFileList2.size(), 0); this.localFs.delete(testRoot, true);
@Override public void run() { for (Path locationToCopy : locationsToCopy) { long timestampFromPath = this.versionDatetime.getMillis(); try { for (FileStatus singleFile : this.srcFs.listStatus(locationToCopy, this.filter)) { Path singleFilePath = singleFile.getPath(); log.debug("Checking if it is a copyable file: " + singleFilePath); Path relativePath = PathUtils.relativizePath(singleFilePath, datasetRoot); Path targetPath = new Path(targetRoot, relativePath); if (this.isCopyableFile(singleFile, targetPath)) { log.debug("Will create workunit for: " + singleFilePath); copyableFileList .add(this.generateCopyableFile(singleFile, targetPath, timestampFromPath, locationToCopy)); } } } catch (IOException e) { e.printStackTrace(); throw new RuntimeException("Failed to get copyable files for " + locationToCopy, e); } } }
@VisibleForTesting protected CopyableFileGenerator getCopyableFileGenetator(FileSystem targetFs, CopyConfiguration configuration, TimestampedDatasetVersion copyableVersion, ConcurrentLinkedQueue<CopyableFile> copyableFileList) { return new CopyableFileGenerator(this.srcFs, targetFs, configuration, this.datasetRoot, this.getTargetRoot(configuration.getPublishDir()), copyableVersion.getDateTime(), copyableVersion.getPaths(), copyableFileList, this.copyableFileFilter()); }
/** * Test {@link TimestampBasedCopyableDataset.CopyableFileGenerator} when src location is empty and also when it is null. */ @Test(expectedExceptions = RuntimeException.class) public void testCopyableFileGenerator() { Properties props = new Properties(); props.put(TimestampBasedCopyableDataset.COPY_POLICY, TimeBasedCopyPolicyForTest.class.getName()); props.put(TimestampBasedCopyableDataset.DATASET_VERSION_FINDER, TimestampedDatasetVersionFinderForTest.class.getName()); TimestampBasedCopyableDataset copyabledataset = new TimestampBasedCopyableDataset(localFs, props, new Path("dummy")); CopyConfiguration configuration = mock(CopyConfiguration.class); when(configuration.getPublishDir()).thenReturn(new Path("publishDir")); ConcurrentLinkedQueue<CopyableFile> copyableFileList = new ConcurrentLinkedQueue<>(); // The src path is empty. TimestampedDatasetVersion emptyVersion = new TimestampedDatasetVersion(new DateTime(), new Path("dummy2")); TimestampBasedCopyableDataset.CopyableFileGenerator emptyGenerator = copyabledataset.getCopyableFileGenetator(localFs, configuration, emptyVersion, copyableFileList); emptyGenerator.run(); Assert.assertEquals(copyableFileList.size(), 0); // The src path is null. TimestampedDatasetVersion versionHasNullPath = new TimestampedDatasetVersion(new DateTime(), null); TimestampBasedCopyableDataset.CopyableFileGenerator exceptionGenerator = copyabledataset.getCopyableFileGenetator(localFs, configuration, versionHasNullPath, copyableFileList); exceptionGenerator.run(); }
@VisibleForTesting protected CopyableFileGenerator getCopyableFileGenetator(FileSystem targetFs, CopyConfiguration configuration, TimestampedDatasetVersion copyableVersion, ConcurrentLinkedQueue<CopyableFile> copyableFileList) { return new CopyableFileGenerator(this.srcFs, targetFs, configuration, this.datasetRoot, this.getTargetRoot(configuration.getPublishDir()), copyableVersion.getDateTime(), copyableVersion.getPaths(), copyableFileList, this.copyableFileFilter()); }