public JavaPairDStream<Long, Integer> readStats(JavaStreamingContext jssc, String inputDirectory) { // Note: This example doesn't work until Spark 1.2 JavaPairDStream<LongWritable, Text> input = jssc.fileStream(inputDirectory, LongWritable.class, Text.class, TextInputFormat.class); // convert the input from Writables to native types JavaPairDStream<Long, Integer> usefulInput = input.mapToPair( new PairFunction<Tuple2<LongWritable, Text>, Long, Integer>() { public Tuple2<Long, Integer> call(Tuple2<LongWritable, Text> input) { return new Tuple2(input._1().get(), Integer.parseInt(input._2().toString())); } }); return usefulInput; }
@SuppressWarnings("unchecked") @Test public void testFileStream() throws IOException { File testDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark"); List<List<String>> expected = fileTestPrepare(testDir); JavaPairInputDStream<LongWritable, Text> inputStream = ssc.fileStream( testDir.toString(), LongWritable.class, Text.class, TextInputFormat.class, v1 -> Boolean.TRUE, true); JavaDStream<String> test = inputStream.map(v1 -> v1._2().toString()); JavaTestUtils.attachTestOutputStream(test); List<List<String>> result = JavaTestUtils.runStreams(ssc, 1, 1); assertOrderInvariantEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testFileStream() throws IOException { File testDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark"); List<List<String>> expected = fileTestPrepare(testDir); JavaPairInputDStream<LongWritable, Text> inputStream = ssc.fileStream( testDir.toString(), LongWritable.class, Text.class, TextInputFormat.class, v1 -> Boolean.TRUE, true); JavaDStream<String> test = inputStream.map(v1 -> v1._2().toString()); JavaTestUtils.attachTestOutputStream(test); List<List<String>> result = JavaTestUtils.runStreams(ssc, 1, 1); assertOrderInvariantEquals(expected, result); }