int maxThreads = state.getPropAsInt(MAX_CONCURRENT_LISTING_SERVICES, DEFAULT_MAX_CONCURRENT_LISTING_SERVICES); final CopyConfiguration copyConfiguration = CopyConfiguration.builder(targetFs, state.getProperties()).build();
.publishDir(new Path(destinationDir)) .preserve(PreserveAttributes.fromMnemonicString("ugp")) .build();
int maxThreads = state.getPropAsInt(MAX_CONCURRENT_LISTING_SERVICES, DEFAULT_MAX_CONCURRENT_LISTING_SERVICES); final CopyConfiguration copyConfiguration = CopyConfiguration.builder(targetFs, state.getProperties()).build();
public static CopyConfigurationBuilder builder(FileSystem targetFs, Properties properties) { return new CopyConfigurationBuilder(targetFs, properties); }
CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).publishDir(new Path("/target")) .preserve(PreserveAttributes.fromMnemonicString("")).build()) .destinationOwnerAndPermission(ownerAndPermission) .ancestorsOwnerAndPermission(ancestorOwnerAndPermissions)
.getPropAsInt(CopySource.MAX_CONCURRENT_LISTING_SERVICES, CopySource.DEFAULT_MAX_CONCURRENT_LISTING_SERVICES); final CopyConfiguration copyConfiguration = CopyConfiguration.builder(targetFs, state.getProperties()).build();
new CopyConfiguration.CopyConfigurationBuilder(fs, properties).build()); Assert.assertEquals(ownerAndPermission.getOwner(), null); Assert.assertEquals(ownerAndPermission.getGroup(), null); new CopyConfiguration.CopyConfigurationBuilder(fs, properties).targetGroup(Optional.of("target")).build()); Assert.assertEquals(ownerAndPermission.getOwner(), null); Assert.assertEquals(ownerAndPermission.getGroup(), "target"); new CopyConfiguration.CopyConfigurationBuilder(fs, properties).targetGroup(Optional.of("target")). preserve(PreserveAttributes.fromMnemonicString("ug")).build()); Assert.assertEquals(ownerAndPermission.getOwner(), "owner"); Assert.assertEquals(ownerAndPermission.getGroup(), "target"); new CopyConfiguration.CopyConfigurationBuilder(fs, properties).preserve(PreserveAttributes.fromMnemonicString("ug")).build()); Assert.assertEquals(ownerAndPermission.getOwner(), "owner"); Assert.assertEquals(ownerAndPermission.getGroup(), "group"); new CopyConfiguration.CopyConfigurationBuilder(fs, properties).preserve(PreserveAttributes.fromMnemonicString("ugp")).build()); Assert.assertEquals(ownerAndPermission.getOwner(), "owner"); Assert.assertEquals(ownerAndPermission.getGroup(), "group");
@Test public void testCopyWithDeleteTargetAndDeleteParentDirectories() throws Exception { Path source = new Path("/source"); Path target = new Path("/target"); List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1")); List<FileStatus> targetFiles = Lists.newArrayList(createFileStatus(target, "file3")); Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString()); properties.setProperty(RecursiveCopyableDataset.DELETE_EMPTY_DIRECTORIES_KEY, "true"); properties.setProperty(RecursiveCopyableDataset.DELETE_KEY, "true"); RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, target, sourceFiles, targetFiles, properties); Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(FileSystem.getLocal(new Configuration()), CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build()); Assert.assertEquals(copyableFiles.size(), 2); ClassifiedFiles classifiedFiles = classifyFiles(copyableFiles); Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file1"))); Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file1")), new Path(target, "file1")); Assert.assertEquals(classifiedFiles.getPathsToDelete().size(), 1); Assert.assertTrue(classifiedFiles.getPathsToDelete().contains(new Path(target, "file3"))); CommitStepCopyEntity entity = (CommitStepCopyEntity) Iterables.filter(copyableFiles, new Predicate<CopyEntity>() { @Override public boolean apply(@Nullable CopyEntity copyEntity) { return copyEntity instanceof CommitStepCopyEntity; } }).iterator().next(); DeleteFileCommitStep step = (DeleteFileCommitStep) entity.getStep(); Assert.assertTrue(step.getParentDeletionLimit().isPresent()); Assert.assertEquals(step.getParentDeletionLimit().get(), target); }
@Test public void testCopyWithDeleteTarget() throws Exception { Path source = new Path("/source"); Path target = new Path("/target"); List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1")); List<FileStatus> targetFiles = Lists.newArrayList(createFileStatus(target, "file3")); Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString()); properties.setProperty(RecursiveCopyableDataset.DELETE_KEY, "true"); RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, target, sourceFiles, targetFiles, properties); Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(FileSystem.getLocal(new Configuration()), CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build()); Assert.assertEquals(copyableFiles.size(), 2); ClassifiedFiles classifiedFiles = classifyFiles(copyableFiles); Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file1"))); Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file1")), new Path(target, "file1")); Assert.assertEquals(classifiedFiles.getPathsToDelete().size(), 1); Assert.assertTrue(classifiedFiles.getPathsToDelete().contains(new Path(target, "file3"))); CommitStepCopyEntity entity = (CommitStepCopyEntity) Iterables.filter(copyableFiles, new Predicate<CopyEntity>() { @Override public boolean apply(@Nullable CopyEntity copyEntity) { return copyEntity instanceof CommitStepCopyEntity; } }).iterator().next(); DeleteFileCommitStep step = (DeleteFileCommitStep) entity.getStep(); Assert.assertFalse(step.getParentDeletionLimit().isPresent()); }
@Test public void testCopyWithUpdate() throws Exception { Path source = new Path("/source"); Path target = new Path("/target"); FileStatus targetFile1 = createFileStatus(target, "file1", 2); List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1", 1), createFileStatus(source, "file2")); List<FileStatus> targetFiles = Lists.newArrayList(targetFile1); Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString()); properties.setProperty(RecursiveCopyableDataset.UPDATE_KEY, "true"); RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, target, sourceFiles, targetFiles, properties); FileSystem targetFsUnderlying = FileSystem.getLocal(new Configuration()); FileSystem targetFs = Mockito.spy(targetFsUnderlying); Mockito.doReturn(targetFile1).when(targetFs).getFileStatus(new Path(target, "file1")); Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(targetFs, CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build()); Assert.assertEquals(copyableFiles.size(), 3); ClassifiedFiles classifiedFiles = classifyFiles(copyableFiles); Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file1"))); Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file1")), new Path(target, "file1")); Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file2"))); Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file2")), new Path(target, "file2")); Assert.assertEquals(classifiedFiles.getPathsToDelete().size(), 1); Assert.assertTrue(classifiedFiles.getPathsToDelete().contains(new Path(target, "file1"))); }
@Test public void testCorrectComputationOfTargetPathsWhenUsingGlob() throws Exception { Path source = new Path("/source/directory"); Path target = new Path("/target"); List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1")); List<FileStatus> targetFiles = Lists.newArrayList(); Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString()); FileSystem sourceUnderlyingFS = FileSystem.getLocal(new Configuration()); FileSystem sourceFs = Mockito.spy(sourceUnderlyingFS); Mockito.doReturn(new FileStatus(0, true, 0, 0, 0, source)).when(sourceFs).getFileStatus(source); RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, new Path(target, "directory"), sourceFiles, targetFiles, properties, new Path("/source/*"), sourceFs); Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(FileSystem.get(new Configuration()), CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build()); Assert.assertEquals(copyableFiles.size(), 1); ClassifiedFiles classifiedFiles = classifyFiles(copyableFiles); Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file1"))); Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file1")), new Path(target, "directory/file1")); }
@Test public void testSimpleCopy() throws Exception { Path source = new Path("/source"); Path target = new Path("/target"); List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1"), createFileStatus(source, "file2")); List<FileStatus> targetFiles = Lists.newArrayList(createFileStatus(target, "file3")); Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString()); RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, target, sourceFiles, targetFiles, properties); Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(FileSystem.getLocal(new Configuration()), CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build()); Assert.assertEquals(copyableFiles.size(), 2); ClassifiedFiles classifiedFiles = classifyFiles(copyableFiles); Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file1"))); Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file1")), new Path(target, "file1")); Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file2"))); Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file2")), new Path(target, "file2")); Assert.assertEquals(classifiedFiles.getPathsToDelete().size(), 0); }
@Test public void testCopyWithConflictingCollisionDueToSize() throws Exception { Path source = new Path("/source"); Path target = new Path("/target"); List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1", 1), createFileStatus(source, "file2")); List<FileStatus> targetFiles = Lists.newArrayList(createFileStatus(target, "file1", 2)); Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString()); RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, target, sourceFiles, targetFiles, properties); try { Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(FileSystem.getLocal(new Configuration()), CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build()); Assert.fail(); } catch (IOException ioe) { // should throw exception due to collision } }
@Test public void testCopyWithNonConflictingCollision() throws Exception { Path source = new Path("/source"); Path target = new Path("/target"); List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1", 1), createFileStatus(source, "file2")); List<FileStatus> targetFiles = Lists.newArrayList(createFileStatus(target, "file1", 1)); Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString()); RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, target, sourceFiles, targetFiles, properties); Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(FileSystem.getLocal(new Configuration()), CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build()); Assert.assertEquals(copyableFiles.size(), 1); ClassifiedFiles classifiedFiles = classifyFiles(copyableFiles); Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file2"))); Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file2")), new Path(target, "file2")); Assert.assertEquals(classifiedFiles.getPathsToDelete().size(), 0); }
public TestDatasetManager(Path testMethodTempPath, State state, String datasetTargetPath, List<String> relativeFilePaths) throws IOException { this.fs = FileSystem.getLocal(new Configuration()); this.copyableDataset = new TestCopyableDataset(new Path("origin")); this.metadata = new CopyableDatasetMetadata(this.copyableDataset); this.relativeFilePaths = relativeFilePaths; this.writerOutputPath = new Path(state.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR)); this.targetPath = new Path(testMethodTempPath, datasetTargetPath); FileStatus file = new FileStatus(0, false, 0, 0, 0, new Path("/file")); FileSystem fs = FileSystem.getLocal(new Configuration()); this.copyEntity = CopyableFile.fromOriginAndDestination(fs, file, new Path("/destination"), CopyConfiguration.builder(fs, state.getProperties()).preserve(PreserveAttributes.fromMnemonicString("")) .build()).build(); fs.mkdirs(testMethodTempPath); log.info("Created a temp directory for test at " + testMethodTempPath); }
@Test public void testCopyWithConflictingCollisionDueToModtime() throws Exception { Path source = new Path("/source"); Path target = new Path("/target"); List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1", 1, 10), createFileStatus(source, "file2")); List<FileStatus> targetFiles = Lists.newArrayList(createFileStatus(target, "file1", 1, 9)); Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString()); RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, target, sourceFiles, targetFiles, properties); try { Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(FileSystem.getLocal(new Configuration()), CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build()); Assert.fail(); } catch (IOException ioe) { // should throw exception due to collision } }
public static CopyConfigurationBuilder builder(FileSystem targetFs, Properties properties) { return new CopyConfigurationBuilder(targetFs, properties); }