return message + ":" + messageFromPropsFile + ":" + orderedProperties; }) .writeAsText(params.getRequired("output"), FileSystem.WriteMode.OVERWRITE);
public static void main(String[] args) throws Exception { final ParameterTool params = ParameterTool.fromArgs(args); final Path inputFile = Paths.get(params.getRequired("inputFile")); final Path inputDir = Paths.get(params.getRequired("inputDir")); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); env.registerCachedFile(inputFile.toString(), "test_data", false); env.registerCachedFile(inputDir.toString(), "test_dir", false); final Path containedFile; try (Stream<Path> files = Files.list(inputDir)) { containedFile = files.findAny().orElseThrow(() -> new RuntimeException("Input directory must not be empty.")); } env.fromElements(1) .map(new TestMapFunction( inputFile.toAbsolutePath().toString(), Files.size(inputFile), inputDir.toAbsolutePath().toString(), containedFile.getFileName().toString())) .writeAsText(params.getRequired("output"), FileSystem.WriteMode.OVERWRITE); env.execute("Distributed Cache Via Blob Test Program"); }
public static void main(String[] args) throws Exception { if (!parseParameters(args)) { return; } StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Tuple3<Integer, Integer, EventType>> edges = getGraphStream(env); // 1. emit (vertexID, 1) or (vertexID, -1) for addition or deletion edges.flatMap(new EmitVerticesWithChange()) // group by vertex ID and maintain degree per vertex .keyBy(0).flatMap(new VertexDegreeCounts()) // group by degree and emit current count .keyBy(0).map(new DegreeDistributionMap()) .writeAsText(resultPath); env.execute("Streaming Degree Distribution"); }
@Test public void testSimpleWriteAndRead() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Event> input = env.fromElements( Event.of(1, "start", 1.0), Event.of(2, "middle", 2.0), Event.of(3, "end", 3.0), Event.of(4, "start", 4.0), Event.of(5, "middle", 5.0), Event.of(6, "end", 6.0) ); String path = tempFolder.newFile().toURI().toString(); input.transform("transformer", TypeInformation.of(Event.class), new StreamMap<>(new MapFunction<Event, Event>() { @Override public Event map(Event event) throws Exception { return event; } })).writeAsText(path); env.execute(); Assert.assertEquals(6, getLineCount(path)); }