/** * * @param fsSpecStoreDirPath The directory path for specs. * @param uri Uri as the identifier of JobSpec * @return */ protected Path getPathForURI(Path fsSpecStoreDirPath, URI uri, String version) { return PathUtils.addExtension(PathUtils.mergePaths(fsSpecStoreDirPath, new Path(uri)), version); } }
Path readyFilePath = PathUtils.addExtension(cf.getOrigin().getPath(), READY_EXTENSION); try { if (sourceFs.exists(readyFilePath)) {
@Test public void testAddExtension() throws Exception { Path path = PathUtils.addExtension(new Path("file"), ".txt"); Assert.assertEquals(path, new Path("file.txt")); path = PathUtils.addExtension(new Path("file.txt"), ".abc"); Assert.assertEquals(path, new Path("file.txt.abc")); path = PathUtils.addExtension(new Path("file.txt.gpg"), ".txt", ".gpg"); Assert.assertEquals(path, new Path("file.txt.gpg.txt.gpg")); path = PathUtils.addExtension(new Path("file.txt.gpg"), ".tar.gz"); Assert.assertEquals(path, new Path("file.txt.gpg.tar.gz")); } }
@Override public final void writeImpl(FileAwareInputStream fileAwareInputStream) throws IOException { CopyableFile copyableFile = fileAwareInputStream.getFile(); if (encryptionConfig != null) { copyableFile.setDestination(PathUtils.addExtension(copyableFile.getDestination(), "." + EncryptionConfigParser.getEncryptionType(encryptionConfig))); } Path stagingFile = getStagingFilePath(copyableFile); if (this.actualProcessedCopyableFile.isPresent()) { throw new IOException(this.getClass().getCanonicalName() + " can only process one file."); } this.actualProcessedCopyableFile = Optional.of(copyableFile); this.fs.mkdirs(stagingFile.getParent()); writeImpl(fileAwareInputStream.getInputStream(), stagingFile, copyableFile, fileAwareInputStream); this.filesWritten.incrementAndGet(); }
private void deleteFilesOnSource(WorkUnitState state) throws IOException { CopyEntity copyEntity = CopySource.deserializeCopyEntity(state); if (copyEntity instanceof CopyableFile) { HadoopUtils.deletePath(this.sourceFs, ((CopyableFile) copyEntity).getOrigin().getPath(), true); HadoopUtils.deletePath(this.sourceFs, PathUtils.addExtension(((CopyableFile) copyEntity).getOrigin().getPath(), ReadyCopyableFileFilter.READY_EXTENSION), true); } } }
@Test public void testFilter() throws Exception { CopyableFileFilter readyFilter = new ReadyCopyableFileFilter(); List<CopyableFile> copyableFiles = Lists.newArrayList(); copyableFiles.add(CopyableFileUtils.getTestCopyableFile()); copyableFiles.add(CopyableFileUtils.getTestCopyableFile()); copyableFiles.add(CopyableFileUtils.getTestCopyableFile()); FileSystem sourceFs = Mockito.mock(FileSystem.class); Mockito.when(sourceFs.exists(PathUtils.addExtension(copyableFiles.get(0).getOrigin().getPath(), ".ready"))) .thenReturn(false); Mockito.when(sourceFs.exists(PathUtils.addExtension(copyableFiles.get(1).getOrigin().getPath(), ".ready"))) .thenReturn(true); Mockito.when(sourceFs.exists(PathUtils.addExtension(copyableFiles.get(2).getOrigin().getPath(), ".ready"))) .thenReturn(false); Collection<CopyableFile> filtered = readyFilter.filter(sourceFs, null, copyableFiles); Assert.assertEquals(filtered.size(), 1); }
newPath = PathUtils.addExtension(newPath, "conf"); newPath = new Path(this.outputPath, newPath.getName());
/** * * @param fsSpecStoreDirPath The directory path for specs. * @param uri Uri as the identifier of JobSpec * @return */ protected Path getPathForURI(Path fsSpecStoreDirPath, URI uri, String version) { return PathUtils.addExtension(PathUtils.mergePaths(fsSpecStoreDirPath, new Path(uri)), version); } }
Path readyFilePath = PathUtils.addExtension(cf.getOrigin().getPath(), READY_EXTENSION); try { if (sourceFs.exists(readyFilePath)) {
@Override public final void writeImpl(FileAwareInputStream fileAwareInputStream) throws IOException { CopyableFile copyableFile = fileAwareInputStream.getFile(); if (encryptionConfig != null) { copyableFile.setDestination(PathUtils.addExtension(copyableFile.getDestination(), "." + EncryptionConfigParser.getEncryptionType(encryptionConfig))); } Path stagingFile = getStagingFilePath(copyableFile); if (this.actualProcessedCopyableFile.isPresent()) { throw new IOException(this.getClass().getCanonicalName() + " can only process one file."); } this.actualProcessedCopyableFile = Optional.of(copyableFile); this.fs.mkdirs(stagingFile.getParent()); writeImpl(fileAwareInputStream.getInputStream(), stagingFile, copyableFile, fileAwareInputStream); this.filesWritten.incrementAndGet(); }
private void deleteFilesOnSource(WorkUnitState state) throws IOException { CopyEntity copyEntity = CopySource.deserializeCopyEntity(state); if (copyEntity instanceof CopyableFile) { HadoopUtils.deletePath(this.sourceFs, ((CopyableFile) copyEntity).getOrigin().getPath(), true); HadoopUtils.deletePath(this.sourceFs, PathUtils.addExtension(((CopyableFile) copyEntity).getOrigin().getPath(), ReadyCopyableFileFilter.READY_EXTENSION), true); } } }
newPath = PathUtils.addExtension(newPath, "conf"); newPath = new Path(this.outputPath, newPath.getName());