Refine search
@Override protected void testProgram() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); TestNonRichOutputFormat output = new TestNonRichOutputFormat(); env.createInput(new TestNonRichInputFormat()).output(output); try { env.execute(); } catch (Exception e){ // we didn't break anything by making everything rich. e.printStackTrace(); fail(e.getMessage()); } } }
public static void main(String[] args) throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.fromElements(1, 2).output(new DiscardingOutputFormat<Integer>()); env.execute().getNetRuntime(); } }
public static void main(String[] args) throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.fromElements(1, 2).output(new DiscardingOutputFormat<Integer>()); env.execute().getJobID(); } }
public static void main(String[] args) throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.fromElements(1, 2).output(new DiscardingOutputFormat<Integer>()); env.execute().getAccumulatorResult(ACCUMULATOR_NAME); } }
public static void main(String[] args) throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.fromElements(1, 2).output(new DiscardingOutputFormat<Integer>()); env.execute().getAllAccumulatorResults(); } }
@Override protected void testProgram() throws Exception { // test verifying the number of records read and written vs the accumulator counts readCalls = new ConcurrentLinkedQueue<Integer>(); writeCalls = new ConcurrentLinkedQueue<Integer>(); final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.createInput(new TestInputFormat(new Path(inputPath))).output(new TestOutputFormat()); JobExecutionResult result = env.execute(); Object a = result.getAllAccumulatorResults().get("DATA_SOURCE_ACCUMULATOR"); Object b = result.getAllAccumulatorResults().get("DATA_SINK_ACCUMULATOR"); long recordsRead = (Long) a; long recordsWritten = (Long) b; assertEquals(recordsRead, readCalls.size()); assertEquals(recordsWritten, writeCalls.size()); }
@Override protected void testProgram() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple2<String, String>> data = env.fromCollection(input); data.mapPartition(new TestMapPartition()).output(new LocalCollectionOutputFormat<Tuple2<String, Integer>>(result)); env.execute(); }
@Override protected void testProgram() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Integer> data = env.fromElements(1, 2, 3, 4, 5, 6, 7, 8); IterativeDataSet<Integer> iteration = data.iterate(10); DataSet<Integer> result = data.reduceGroup(new PickOneAllReduce()).withBroadcastSet(iteration, "bc"); final List<Integer> resultList = new ArrayList<Integer>(); iteration.closeWith(result).output(new LocalCollectionOutputFormat<Integer>(resultList)); env.execute(); Assert.assertEquals(8, resultList.get(0).intValue()); }
@Override protected void testProgram() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); IterativeDataSet<Long> iteration = env.generateSequence(1, 10).iterate(100); iteration.closeWith(iteration) .output(new LocalCollectionOutputFormat<Long>(result)); env.execute(); }
public static void main(String[] args) throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.fromElements(1, 2).output(new DiscardingOutputFormat<Integer>()); env.execute(); env.fromElements(1, 2).collect(); } }
@SuppressWarnings("serial") public static void main(String[] args) throws Exception { if (args.length < 2) { System.err.println("Usage: TestOptimizerPlan <input-file-path> <output-file-path>"); return; } ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple2<Long, Long>> input = env.readCsvFile(args[0]) .fieldDelimiter("\t").types(Long.class, Long.class); DataSet<Tuple2<Long, Long>> result = input.map( new MapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>() { public Tuple2<Long, Long> map(Tuple2<Long, Long> value){ return new Tuple2<Long, Long>(value.f0, value.f1 + 1); } }); result.writeAsCsv(args[1], "\n", "\t"); env.execute(); }
@SuppressWarnings("serial") public static void main(String[] args) throws Exception { if (args.length < 2) { System.err.println("Usage: TestOptimizerPlan <input-file-path> <output-file-path>"); return; } ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple2<Long, Long>> input = env.readCsvFile(args[0]) .fieldDelimiter("\t").types(Long.class, Long.class); DataSet<Tuple2<Long, Long>> result = input.map( new MapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>() { public Tuple2<Long, Long> map(Tuple2<Long, Long> value){ return new Tuple2<Long, Long>(value.f0, value.f1 + 1); } }); result.writeAsCsv(args[1], "\n", "\t"); env.execute(); }
@Test public void testConfigurableMapper() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); JobConf conf = new JobConf(); conf.set("my.filterPrefix", "Hello"); DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env); DataSet<Tuple2<IntWritable, Text>> hellos = ds. flatMap(new HadoopMapFunction<IntWritable, Text, IntWritable, Text>(new ConfigurableMapper(), conf)); String resultPath = tempFolder.newFile().toURI().toString(); hellos.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE); env.execute(); String expected = "(2,Hello)\n" + "(3,Hello world)\n" + "(4,Hello world, how are you?)\n"; compareResultsByLinesInMemory(expected, resultPath); }
@Override protected void testProgram() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(4); DataSet<Integer> data = env.fromElements(1, 2, 3, 4, 5, 6, 7, 8); IterativeDataSet<Integer> iteration = data.iterate(10); DataSet<Integer> result = data.reduceGroup(new PickOneAllReduce()).withBroadcastSet(iteration, "bc"); final List<Integer> resultList = new ArrayList<Integer>(); iteration.closeWith(result).output(new LocalCollectionOutputFormat<Integer>(resultList)); env.execute(); Assert.assertEquals(8, resultList.get(0).intValue()); }
@Test public void testUngroupedHadoopReducer() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env); DataSet<Tuple2<IntWritable, IntWritable>> commentCnts = ds. reduceGroup(new HadoopReduceFunction<IntWritable, Text, IntWritable, IntWritable>(new AllCommentCntReducer())); String resultPath = tempFolder.newFile().toURI().toString(); commentCnts.writeAsText(resultPath); env.execute(); String expected = "(42,15)\n"; compareResultsByLinesInMemory(expected, resultPath); }
@Test public void testNonPassingMapper() throws Exception{ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env); DataSet<Tuple2<IntWritable, Text>> nonPassingFlatMapDs = ds. flatMap(new HadoopMapFunction<IntWritable, Text, IntWritable, Text>(new NonPassingMapper())); String resultPath = tempFolder.newFile().toURI().toString(); nonPassingFlatMapDs.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE); env.execute(); compareResultsByLinesInMemory("\n", resultPath); }
@Test public void testIntSortingParallelism1() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Integer> ds = CollectionDataSets.getIntegerDataSet(env); ds.writeAsText(resultPath).sortLocalOutput("*", Order.DESCENDING).setParallelism(1); env.execute(); String expected = "5\n5\n5\n5\n5\n4\n4\n4\n4\n3\n3\n3\n2\n2\n1\n"; compareResultsByLinesInMemoryWithStrictOrder(expected, resultPath); }
@Override protected void testProgram() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<String> text = env.fromElements(WordCountData.TEXT); DataSet<Tuple2<String, Integer>> words = text.flatMap(new WordCount.Tokenizer()); DataSet<Tuple2<String, Integer>> result = words.groupBy(0).aggregate(Aggregations.SUM, 1); result.output(new LocalCollectionOutputFormat<Tuple2<String, Integer>>(resultsCollected)); env.execute("Word Count Collection"); } }
@Override protected void testProgram() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<String> input = env.readTextFile(dataPath); input.flatMap(new TokenizeLine()) .groupBy(0) .reduceGroup(new CountWords()) .writeAsCsv(resultPath, "\n", " "); this.result = env.execute(); }
@Test public void testStringSortingParallelism1() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<String> ds = CollectionDataSets.getStringDataSet(env); ds.writeAsText(resultPath).sortLocalOutput("*", Order.ASCENDING).setParallelism(1); env.execute(); String expected = "Hello\n" + "Hello world\n" + "Hello world, how are you?\n" + "Hi\n" + "I am fine.\n" + "LOL\n" + "Luke Skywalker\n" + "Random comment\n"; compareResultsByLinesInMemoryWithStrictOrder(expected, resultPath); }