for (Config retainedConfig : retainedConfigs) { Path fullFilePath = new Path(testTempDirPath, PathUtils.withoutLeadingSeparator(new Path(retainedConfig .getString(TEST_DATA_PATH_LOCAL_KEY)))); Assert.assertTrue(this.fs.exists(fullFilePath), for (Config retainedConfig : deletedConfigs) { Path fullFilePath = new Path(testTempDirPath, PathUtils.withoutLeadingSeparator(new Path(retainedConfig .getString(TEST_DATA_PATH_LOCAL_KEY)))); Assert.assertFalse(this.fs.exists(fullFilePath), String.format("%s should be deleted", fullFilePath.toString())); for (Config permissionsConfig : permissionsConfigs) { Path fullFilePath = new Path(testTempDirPath, PathUtils.withoutLeadingSeparator(new Path(permissionsConfig .getString(TEST_DATA_PATH_LOCAL_KEY)))); Assert.assertEquals(this.fs.getFileStatus(fullFilePath).getPermission(),
public static Path getOutputFilePath(CopyableFile file, Path outputDir, CopyEntity.DatasetAndPartition datasetAndPartition) { Path destinationWithoutSchemeAndAuthority = PathUtils.getPathWithoutSchemeAndAuthority(file.getDestination()); return new Path(getPartitionOutputRoot(outputDir, datasetAndPartition), PathUtils.withoutLeadingSeparator(destinationWithoutSchemeAndAuthority)); }
new Path(testTempDirPath, PathUtils.withoutLeadingSeparator(new Path(fileToCreate .getString(TEST_DATA_PATH_LOCAL_KEY))));
private void createDatasetFiles() throws IOException { // Create writer output files Path datasetWriterOutputPath = new Path(writerOutputPath, copyEntity.getDatasetAndPartition(this.metadata).identifier()); Path outputPathWithCurrentDirectory = new Path(datasetWriterOutputPath, PathUtils.withoutLeadingSeparator(this.targetPath)); for (String path : relativeFilePaths) { Path pathToCreate = new Path(outputPathWithCurrentDirectory, path); fs.mkdirs(pathToCreate.getParent()); fs.create(pathToCreate); } }
@Test(dataProvider = "testFileDataProvider") public void testWrite(final String filePath, final String newFileName, final String expectedText) throws Exception { String expectedFileContents = "text"; String fileNameInArchive = "text.txt"; WorkUnitState state = TestUtils.createTestWorkUnitState(); state.setProp(ConfigurationKeys.WRITER_STAGING_DIR, new Path(testTempPath, "staging").toString()); state.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, new Path(testTempPath, "output").toString()); state.setProp(ConfigurationKeys.WRITER_FILE_PATH, "writer_file_path_" + RandomStringUtils.randomAlphabetic(5)); CopyableDatasetMetadata metadata = new CopyableDatasetMetadata(new TestCopyableDataset(new Path("/source"))); CopySource.serializeCopyableDataset(state, metadata); FileAwareInputStream fileAwareInputStream = getCompressedInputStream(filePath, newFileName); CopySource.serializeCopyEntity(state, fileAwareInputStream.getFile()); TarArchiveInputStreamDataWriter dataWriter = new TarArchiveInputStreamDataWriter(state, 1, 0); dataWriter.write(fileAwareInputStream); dataWriter.commit(); // the archive file contains file test.txt Path unArchivedFilePath = new Path(fileAwareInputStream.getFile().getDestination(), fileNameInArchive); // Path at which the writer writes text.txt Path taskOutputFilePath = new Path(new Path(state.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR), fileAwareInputStream.getFile().getDatasetAndPartition(metadata).identifier()), PathUtils.withoutLeadingSeparator(unArchivedFilePath)); Assert.assertEquals(IOUtils.toString(new FileInputStream(taskOutputFilePath.toString())).trim(), expectedFileContents); }
for (Config retainedConfig : retainedConfigs) { Path fullFilePath = new Path(testTempDirPath, PathUtils.withoutLeadingSeparator(new Path(retainedConfig .getString(TEST_DATA_PATH_LOCAL_KEY)))); Assert.assertTrue(this.fs.exists(fullFilePath), for (Config retainedConfig : deletedConfigs) { Path fullFilePath = new Path(testTempDirPath, PathUtils.withoutLeadingSeparator(new Path(retainedConfig .getString(TEST_DATA_PATH_LOCAL_KEY)))); Assert.assertFalse(this.fs.exists(fullFilePath), String.format("%s should be deleted", fullFilePath.toString())); for (Config permissionsConfig : permissionsConfigs) { Path fullFilePath = new Path(testTempDirPath, PathUtils.withoutLeadingSeparator(new Path(permissionsConfig .getString(TEST_DATA_PATH_LOCAL_KEY)))); Assert.assertEquals(this.fs.getFileStatus(fullFilePath).getPermission(),
public static Path getOutputFilePath(CopyableFile file, Path outputDir, CopyEntity.DatasetAndPartition datasetAndPartition) { Path destinationWithoutSchemeAndAuthority = PathUtils.getPathWithoutSchemeAndAuthority(file.getDestination()); return new Path(getPartitionOutputRoot(outputDir, datasetAndPartition), PathUtils.withoutLeadingSeparator(destinationWithoutSchemeAndAuthority)); }
new Path(testTempDirPath, PathUtils.withoutLeadingSeparator(new Path(fileToCreate .getString(TEST_DATA_PATH_LOCAL_KEY))));