Refine search
@Test public void testToText() throws Exception { PipelineOptions options = PipelineOptionsFactory.create(); File file = tmpFolder.newFolder(); HDFSFileSink<String, NullWritable, Text> sink = HDFSFileSink.toText(file.toString()); doWrite(sink, options, Collections.singletonList(foobar)); List<String> strings = Files.readAllLines(new File(file.toString(), part0).toPath(), Charset.forName("UTF-8")); assertEquals(Collections.singletonList(foobar), strings); }
@Test public void testSourceSplitAtFraction() throws Exception { List<Integer> elements = new ArrayList<>(); Random random = new Random(); for (int i = 0; i < 25; i++) { elements.add(random.nextInt()); } CreateSource<Integer> source = CreateSource.fromIterable(elements, VarIntCoder.of()); SourceTestUtils.assertSplitAtFractionExhaustive(source, PipelineOptionsFactory.create()); } }
@Test public void testServiceLoader() { for (FileSystemRegistrar registrar : Lists.newArrayList(ServiceLoader.load(FileSystemRegistrar.class).iterator())) { if (registrar instanceof LocalFileSystemRegistrar) { Iterable<FileSystem> fileSystems = registrar.fromOptions(PipelineOptionsFactory.create()); assertThat(fileSystems, contains(instanceOf(LocalFileSystem.class))); return; } } fail("Expected to find " + LocalFileSystemRegistrar.class); } }
@Test public void testServiceLoader() { for (FileSystemRegistrar registrar : Lists.newArrayList(ServiceLoader.load(FileSystemRegistrar.class).iterator())) { if (registrar instanceof GcsFileSystemRegistrar) { Iterable<FileSystem> fileSystems = registrar.fromOptions(PipelineOptionsFactory.create()); assertThat(fileSystems, contains(instanceOf(GcsFileSystem.class))); return; } } fail("Expected to find " + GcsFileSystemRegistrar.class); } }
@Test public void testEmptyFilepatternTreatmentDefaultDisallow() throws IOException { PipelineOptions options = PipelineOptionsFactory.create(); TestFileBasedSource source = new TestFileBasedSource(new File(tempFolder.getRoot(), "doesNotExist").getPath(), 64, null); thrown.expect(FileNotFoundException.class); readFromSource(source, options); }
@Test public void testEmptyFilepatternTreatmentAllowIfWildcard() throws IOException { PipelineOptions options = PipelineOptionsFactory.create(); TestFileBasedSource source = new TestFileBasedSource( new File(tempFolder.getRoot(), "doesNotExist").getPath(), EmptyMatchTreatment.ALLOW_IF_WILDCARD, 64, null); thrown.expect(FileNotFoundException.class); readFromSource(source, options); }
@Test public void fromOptionsCreatesInstance() { PipelineOptions opts = PipelineOptionsFactory.create(); opts.setRunner(CrashingRunner.class); PipelineRunner<? extends PipelineResult> runner = PipelineRunner.fromOptions(opts); assertTrue("Should have created a CrashingRunner", runner instanceof CrashingRunner); }
@Test public void testUnableToGetDefaultProject() throws Exception { System.setProperty("user.home", tmpFolder.getRoot().getAbsolutePath()); DefaultProjectFactory projectFactory = spy(new DefaultProjectFactory()); when(projectFactory.getEnvironment()).thenReturn(ImmutableMap.of()); assertNull(projectFactory.create(PipelineOptionsFactory.create())); }
@Test public void testWhenRequiredOptionIsNeverSetOnSuperInterface() { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage( "Missing required value for " + "[public abstract java.lang.String org.apache.beam." + "sdk.options.PipelineOptionsValidatorTest$Required.getObject(), \"Fake Description\"]."); PipelineOptions options = PipelineOptionsFactory.create(); PipelineOptionsValidator.validate(Required.class, options); }
@Test public void testSplitAtFractionExhaustive() throws Exception { PipelineOptions options = PipelineOptionsFactory.create(); // Smaller file for exhaustive testing. File file = createFileWithData("file", createStringDataset(3, 20)); Metadata metadata = FileSystems.matchSingleFileSpec(file.getPath()); TestFileBasedSource source = new TestFileBasedSource(metadata, 1, 0, file.length(), null); assertSplitAtFractionExhaustive(source, options); }
@Test public void testOptionsIdIsSet() throws Exception { ObjectMapper mapper = new ObjectMapper() .registerModules(ObjectMapper.findModules(ReflectHelpers.findClassLoader())); PipelineOptions options = PipelineOptionsFactory.create(); // We purposely serialize/deserialize to get another instance. This allows to test if the // default has been set or not. PipelineOptions clone = mapper.readValue(mapper.writeValueAsString(options), PipelineOptions.class); // It is important that we don't call getOptionsId() before we have created the clone. assertEquals(options.getOptionsId(), clone.getOptionsId()); }
@Test public void testFullyReadSingleFile() throws Exception { PipelineOptions options = PipelineOptionsFactory.create(); List<KV<IntWritable, Text>> expectedResults = createRandomRecords(3, 10, 0); File file = createFileWithData("tmp.seq", expectedResults); HDFSFileSource<KV<IntWritable, Text>, IntWritable, Text> source = HDFSFileSource.from( file.toString(), SequenceFileInputFormat.class, IntWritable.class, Text.class); assertEquals(file.length(), source.getEstimatedSizeBytes(null)); assertThat(expectedResults, containsInAnyOrder(readFromSource(source, options).toArray())); }