Refine search
@Test(expected = UnsupportedOperationException.class) public void testMultiPathSetOnSinglePathIF2() { final DummyFileInputFormat format = new DummyFileInputFormat(); final String myPath = "/an/imaginary/path"; final String myPath2 = "/an/imaginary/path2"; format.setFilePaths(new Path(myPath), new Path(myPath2)); }
@Test public void testMkdirsCreatesParentDirectories() throws Exception { // this test applies to object stores as well, as rely on the fact that they // return true when things are not bad final Path directory = new Path(new Path(new Path(basePath, randomName()), randomName()), randomName()); assertTrue(fs.mkdirs(directory)); if (getFileSystemKind() != FileSystemKind.OBJECT_STORE) { assertTrue(fs.exists(directory)); } }
/** * Test if the AvroInputFormat is able to properly read data from an Avro * file as a GenericRecord. */ @Test public void testDeserializationGenericRecord() throws IOException { Configuration parameters = new Configuration(); AvroInputFormat<GenericRecord> format = new AvroInputFormat<>(new Path(testFile.getAbsolutePath()), GenericRecord.class); doTestDeserializationGenericRecord(format, parameters); }
@Test public void testExternalProgram() throws Exception { TestEnvironment.setAsContext( MINI_CLUSTER, PARALLELISM, Collections.singleton(new Path(JAR_FILE)), Collections.emptyList()); String testData = getClass().getResource(TEST_DATA_FILE).toString(); PackagedProgram program = new PackagedProgram(new File(JAR_FILE), new String[]{testData}); program.invokeInteractiveModeForExecution(); } }
@Test public void testFullUriMatching() throws Exception { EntropyInjectingFileSystem efs = new TestEntropyInjectingFs("s0mek3y", "12345678"); Path path = new Path("s3://hugo@myawesomehost:55522/path/s0mek3y/the/file"); assertEquals(new Path("s3://hugo@myawesomehost:55522/path/12345678/the/file"), EntropyInjector.resolveEntropy(path, efs, true)); assertEquals(new Path("s3://hugo@myawesomehost:55522/path/the/file"), EntropyInjector.resolveEntropy(path, efs, false)); }
@Test public void testPathOnlyMatching() throws Exception { EntropyInjectingFileSystem efs = new TestEntropyInjectingFs("_entropy_key_", "xyzz"); Path path = new Path("/path/_entropy_key_/file"); assertEquals(new Path("/path/xyzz/file"), EntropyInjector.resolveEntropy(path, efs, true)); assertEquals(new Path("/path/file"), EntropyInjector.resolveEntropy(path, efs, false)); }
@Test public void testEntropyNotFullSegment() throws Exception { EntropyInjectingFileSystem efs = new TestEntropyInjectingFs("_entropy_key_", "pqr"); Path path = new Path("s3://myhost:122/entropy-_entropy_key_-suffix/file"); assertEquals(new Path("s3://myhost:122/entropy-pqr-suffix/file"), EntropyInjector.resolveEntropy(path, efs, true)); assertEquals(new Path("s3://myhost:122/entropy--suffix/file"), EntropyInjector.resolveEntropy(path, efs, false)); }
@Test public void testRenameNonExistingFile() throws IOException { final FileSystem fs = FileSystem.getLocalFileSystem(); final File srcFile = new File(temporaryFolder.newFolder(), "someFile.txt"); final File destFile = new File(temporaryFolder.newFolder(), "target"); final Path srcFilePath = new Path(srcFile.toURI()); final Path destFilePath = new Path(destFile.toURI()); // this cannot succeed because the source file does not exist assertFalse(fs.rename(srcFilePath, destFilePath)); }
@Test public void testS3nKind() throws IOException { try { Class.forName("org.apache.hadoop.fs.s3native.NativeS3FileSystem"); } catch (ClassNotFoundException ignored) { // not in the classpath, cannot run this test log.info("Skipping test 'testS3nKind()' because the Native S3 file system is not in the class path"); return; } final FileSystem s3n = new Path("s3n://myId:mySecret@bucket/some/bucket/some/object").getFileSystem(); assertEquals(FileSystemKind.OBJECT_STORE, s3n.getKind()); }
@Test public void testS3Kind() throws IOException { try { Class.forName("org.apache.hadoop.fs.s3.S3FileSystem"); } catch (ClassNotFoundException ignored) { // not in the classpath, cannot run this test log.info("Skipping test 'testS3Kind()' because the S3 file system is not in the class path"); return; } final FileSystem s3 = new Path("s3://myId:mySecret@bucket/some/bucket/some/object").getFileSystem(); assertEquals(FileSystemKind.OBJECT_STORE, s3.getKind()); }
@Test public void testS3aKind() throws IOException { try { Class.forName("org.apache.hadoop.fs.s3a.S3AFileSystem"); } catch (ClassNotFoundException ignored) { // not in the classpath, cannot run this test log.info("Skipping test 'testS3aKind()' because the S3AFileSystem is not in the class path"); return; } final FileSystem s3a = new Path("s3a://myId:mySecret@bucket/some/bucket/some/object").getFileSystem(); assertEquals(FileSystemKind.OBJECT_STORE, s3a.getKind()); }
@Test public void testIncludeFileWithAnyCharacterMatcher() { GlobFilePathFilter matcher = new GlobFilePathFilter( Collections.singletonList("dir/?.txt"), Collections.<String>emptyList()); assertFalse(matcher.filterPath(new Path("dir/a.txt"))); assertTrue(matcher.filterPath(new Path("dir/aa.txt"))); }
@Test public void testStreamingCustomSplitJobWithCustomClassLoader() throws IOException, ProgramInvocationException { PackagedProgram streamingInputSplitTestProg = new PackagedProgram(new File(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)); TestStreamEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)), Collections.<URL>emptyList()); streamingInputSplitTestProg.invokeInteractiveModeForExecution(); }
@Test public void testUserCodeTypeJobWithCustomClassLoader() throws IOException, ProgramInvocationException { PackagedProgram userCodeTypeProg = new PackagedProgram(new File(USERCODETYPE_JAR_PATH)); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(USERCODETYPE_JAR_PATH)), Collections.<URL>emptyList()); userCodeTypeProg.invokeInteractiveModeForExecution(); }
@Test public void testDoubleStarPattern() { GlobFilePathFilter matcher = new GlobFilePathFilter( Collections.singletonList("**"), Collections.<String>emptyList()); assertFalse(matcher.filterPath(new Path("a"))); assertFalse(matcher.filterPath(new Path("a/b"))); assertFalse(matcher.filterPath(new Path("a/b/c"))); }
@Test public void testStreamingClassloaderJobWithCustomClassLoader() throws IOException, ProgramInvocationException { // regular streaming job PackagedProgram streamingProg = new PackagedProgram(new File(STREAMING_PROG_JAR_FILE)); TestStreamEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(STREAMING_PROG_JAR_FILE)), Collections.<URL>emptyList()); streamingProg.invokeInteractiveModeForExecution(); }
@Test public void testExcludeFilesNotInIncludePatterns() { GlobFilePathFilter matcher = new GlobFilePathFilter( Collections.singletonList("dir/*"), Collections.<String>emptyList()); assertFalse(matcher.filterPath(new Path("dir/file.txt"))); assertTrue(matcher.filterPath(new Path("dir1/file.txt"))); }