public static void main(String[] args) throws Exception { final ParameterTool params = ParameterTool.fromArgs(args); final Path inputFile = Paths.get(params.getRequired("inputFile")); final Path inputDir = Paths.get(params.getRequired("inputDir")); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); env.registerCachedFile(inputFile.toString(), "test_data", false); env.registerCachedFile(inputDir.toString(), "test_dir", false); final Path containedFile; try (Stream<Path> files = Files.list(inputDir)) { containedFile = files.findAny().orElseThrow(() -> new RuntimeException("Input directory must not be empty.")); } env.fromElements(1) .map(new TestMapFunction( inputFile.toAbsolutePath().toString(), Files.size(inputFile), inputDir.toAbsolutePath().toString(), containedFile.getFileName().toString())) .writeAsText(params.getRequired("output"), FileSystem.WriteMode.OVERWRITE); env.execute("Distributed Cache Via Blob Test Program"); }
@Test(expected = IllegalArgumentException.class) public void fromElementsWithBaseTypeTest2() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.fromElements(SubClass.class, new SubClass(1, "Java"), new ParentClass(1, "hello")); }
@Test(expected = UnsupportedOperationException.class) public void testForwardFailsLowToHighParallelism() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Integer> src = env.fromElements(1, 2, 3); // this doesn't work because it goes from 1 to 3 src.forward().map(new NoOpIntMap()); env.execute(); }
@Test(expected = CompositeType.InvalidFieldReferenceException.class) public void testFailOnNestedPojoFieldAccessor() throws Exception { StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Data> dataStream = see.fromCollection(elements); dataStream.keyBy("aaa", "stats.count").sum("stats.nonExistingField"); }
@Test public void testStreamingDistributedCache() throws Exception { String textPath = createTempFile("count.txt", DATA); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.registerCachedFile(textPath, "cache_test"); env.readTextFile(textPath).flatMap(new WordChecker()); env.execute(); }
public static void main(String[] args) throws Exception { final ParameterTool params = ParameterTool.fromArgs(args); double errorRate = params.getDouble("error-rate", 0.0); int sleep = params.getInt("sleep", 1); String kafkaTopic = params.get("kafka-topic"); String brokers = params.get("brokers", "localhost:9092"); System.out.printf("Generating events to Kafka with standalone source with error rate %f and sleep delay %s millis\n", errorRate, sleep); System.out.println(); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env .addSource(new EventsGeneratorSource(errorRate, sleep)) .addSink(new FlinkKafkaProducer010<>(brokers, kafkaTopic, new EventDeSerializer())); // trigger program execution env.execute("State machine example Kafka events generator job"); }
public static void main(String[] args) throws Exception { // parse the parameters final ParameterTool params = ParameterTool.fromArgs(args); final long windowSize = params.getLong("windowSize", 2000); final long rate = params.getLong("rate", 3L); System.out.println("Using windowSize=" + windowSize + ", data rate=" + rate); System.out.println("To customize example, use: WindowJoin [--windowSize <window-size-in-millis>] [--rate <elements-per-second>]"); // obtain execution environment, run this example in "ingestion time" StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); // make parameters available in the web interface env.getConfig().setGlobalJobParameters(params); // create the data sources for both grades and salaries DataStream<Tuple2<String, Integer>> grades = GradeSource.getSource(env, rate); DataStream<Tuple2<String, Integer>> salaries = SalarySource.getSource(env, rate); // run the actual window join program // for testability, this functionality is in a separate method. DataStream<Tuple3<String, Integer, Integer>> joinedStream = runWindowJoin(grades, salaries, windowSize); // print the results with a single thread, rather than in parallel joinedStream.print().setParallelism(1); // execute program env.execute("Windowed Join Example"); }
@Before public void setUp() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); dataStream1 = env.fromElements("a1", "a2", "a3"); dataStream2 = env.fromElements("a1", "a2"); keySelector = element -> element; tsAssigner = TumblingEventTimeWindows.of(Time.milliseconds(1)); joinFunction = (first, second) -> first + second; }
@Test public void testOperatorChainWithObjectReuseAndNoOutputOperators() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().enableObjectReuse(); DataStream<Integer> input = env.fromElements(1, 2, 3); input.flatMap(new FlatMapFunction<Integer, Integer>() { @Override public void flatMap(Integer value, Collector<Integer> out) throws Exception { out.collect(value << 1); } }); env.execute(); } }
@Test public void fromElementsWithBaseTypeTest1() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.fromElements(ParentClass.class, new SubClass(1, "Java"), new ParentClass(1, "hello")); }
@Test public void testCheckpointConfigDefault() throws Exception { StreamExecutionEnvironment streamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment(); Assert.assertTrue(streamExecutionEnvironment.getCheckpointConfig().isFailOnCheckpointingErrors()); }
@Test public void testFilter() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env); StreamITCase.clear(); DataStream<Tuple5<Integer, Long, Integer, String, Long>> ds = JavaStreamTestData.get5TupleDataStream(env); tableEnv.registerDataStream("MyTable", ds, "a, b, c, d, e"); String sqlQuery = "SELECT a, b, e FROM MyTable WHERE c < 4"; Table result = tableEnv.sqlQuery(sqlQuery); DataStream<Row> resultSet = tableEnv.toAppendStream(result, Row.class); resultSet.addSink(new StreamITCase.StringSink<Row>()); env.execute(); List<String> expected = new ArrayList<>(); expected.add("1,1,1"); expected.add("2,2,2"); expected.add("2,3,1"); expected.add("3,4,2"); StreamITCase.compareWithList(expected); }
@Override public void go() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.addSource(new SourceFunction<String>() { @Override public void run(SourceContext<String> ctx) throws Exception { sync.block(); } @Override public void cancel() { sync.releaseBlocker(); } }).addSink(new PrintSinkFunction()); env.execute(); }
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); String outputPath = params.getRequired("outputPath"); int recordsPerSecond = params.getInt("recordsPerSecond", 10); int duration = params.getInt("durationInSecond", 60); int offset = params.getInt("offsetInSecond", 0); StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment(); sEnv.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); sEnv.enableCheckpointing(4000); sEnv.getConfig().setAutoWatermarkInterval(1000); // execute a simple pass through program. PeriodicSourceGenerator generator = new PeriodicSourceGenerator( recordsPerSecond, duration, offset); DataStream<Tuple> rows = sEnv.addSource(generator); DataStream<Tuple> result = rows .keyBy(1) .timeWindow(Time.seconds(5)) .sum(0); result.writeAsText(outputPath + "/result.txt", FileSystem.WriteMode.OVERWRITE) .setParallelism(1); sEnv.execute(); }
private static void runJob() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.fromElements(1, 2, 3) .print(); env.execute(); }
@Before public void setUp() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); dataStream1 = env.fromElements("a1", "a2", "a3"); dataStream2 = env.fromElements("a1", "a2"); keySelector = element -> element; tsAssigner = TumblingEventTimeWindows.of(Time.milliseconds(1L)); coGroupFunction = (CoGroupFunction<String, String, String>) (first, second, out) -> out.collect(""); }
@Test(expected = IllegalStateException.class) public void testExecutionWithEmptyIteration() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap); IterativeStream<Integer> iter1 = source.iterate(); iter1.map(noOpIntMap).print(); env.execute(); }
@Test(expected = UnsupportedOperationException.class) public void testIncorrectParallelism() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Integer> source = env.fromElements(1, 10); IterativeStream<Integer> iter1 = source.iterate(); SingleOutputStreamOperator<Integer> map1 = iter1.map(noOpIntMap); iter1.closeWith(map1).print(); }
@Test public void testMultiChainingWithoutObjectReuse() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableObjectReuse(); testMultiChaining(env); }
@Test public void testSelect() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env); StreamITCase.clear(); DataStream<Tuple3<Integer, Long, String>> ds = JavaStreamTestData.getSmall3TupleDataSet(env); Table in = tableEnv.fromDataStream(ds, "a,b,c"); tableEnv.registerTable("MyTable", in); String sqlQuery = "SELECT * FROM MyTable"; Table result = tableEnv.sqlQuery(sqlQuery); DataStream<Row> resultSet = tableEnv.toAppendStream(result, Row.class); resultSet.addSink(new StreamITCase.StringSink<Row>()); env.execute(); List<String> expected = new ArrayList<>(); expected.add("1,1,Hi"); expected.add("2,2,Hello"); expected.add("3,2,Hello world"); StreamITCase.compareWithList(expected); }