/** * Creates a new {@link Stream} of tuples from the given {@link IRichSpout} * * @param spout the spout * @return the new stream */ public Stream<Tuple> newStream(IRichSpout spout) { return newStream(spout, 1); }
@Override public StormTopology build() { return streamBuilder.build(); }
private void doProcessCurGroup(TopologyBuilder topologyBuilder, List<ProcessorNode> group) { String boltId = UniqueIdGen.getInstance().getUniqueBoltId(); for (ProcessorNode processorNode : group) { processorNode.setComponentId(boltId); processorNode.setWindowedParentStreams(getWindowedParentStreams(processorNode)); } final Set<ProcessorNode> initialProcessors = initialProcessors(group); Set<Window<?, ?>> windowParams = getWindowParams(initialProcessors); if (windowParams.isEmpty()) { if (hasStatefulProcessor(group)) { addStatefulBolt(topologyBuilder, boltId, initialProcessors, group); } else { addBolt(topologyBuilder, boltId, initialProcessors, group); } } else if (windowParams.size() == 1) { addWindowedBolt(topologyBuilder, boltId, initialProcessors, windowParams.iterator().next(), group); } else { throw new IllegalStateException("More than one window config for current group " + group); } }
windowInfo.clear(); curGroup.clear(); TopologicalOrderIterator<Node, Edge> iterator = new TopologicalOrderIterator<>(graph, queue()); TopologyBuilder topologyBuilder = new TopologyBuilder(); while (iterator.hasNext()) { Node node = iterator.next(); if (node instanceof SpoutNode) { addSpout(topologyBuilder, (SpoutNode) node); } else if (node instanceof ProcessorNode) { handleProcessorNode((ProcessorNode) node, topologyBuilder); } else if (node instanceof PartitionNode) { updateNodeGroupingInfo((PartitionNode) node); processCurGroup(topologyBuilder); } else if (node instanceof WindowNode) { updateWindowInfo((WindowNode) node); processCurGroup(topologyBuilder); } else if (node instanceof SinkNode) { processCurGroup(topologyBuilder); addSink(topologyBuilder, (SinkNode) node); processCurGroup(topologyBuilder); mayBeAddTsField(); return topologyBuilder.createTopology();
@SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); Stream<Integer>[] evenAndOdd = builder /* * Create a stream of random numbers from a spout that * emits random integers by extracting the tuple value at index 0. */ .newStream(new RandomIntegerSpout(), new ValueMapper<Integer>(0)) /* * Split the stream of numbers into streams of * even and odd numbers. The first stream contains even * and the second contains odd numbers. */ .branch(x -> (x % 2) == 0, x -> (x % 2) == 1); evenAndOdd[0].forEach(x -> LOG.info("EVEN> " + x)); evenAndOdd[1].forEach(x -> LOG.info("ODD > " + x)); Config config = new Config(); String topoName = "branchExample"; if (args.length > 0) { topoName = args[0]; } config.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build()); }
public StreamsPlanCreator(Map<String, ISqlStreamsDataSource> sources, RexBuilder rexBuilder) { this.sources = sources; this.rexCompiler = new RexNodeToJavaCodeCompiler(rexBuilder); this.typeFactory = (JavaTypeFactory) rexBuilder.getTypeFactory(); this.streamBuilder = new StreamBuilder(); this.dataContext = new StormDataContext(); }
Node addNode(Node parent, Node child, int parallelism) { return streamBuilder.addNode(parent, child, parallelism); }
private void splitStatefulProcessor(ProcessorNode processorNode, TopologyBuilder topologyBuilder) { for (Node parent : StreamUtil.<Node>getParents(graph, processorNode)) { ProcessorNode identity = new ProcessorNode(new MapProcessor<>(new IdentityFunction<>()), UniqueIdGen.getInstance().getUniqueStreamId(), parent.getOutputFields()); addNode(parent, identity); graph.removeEdge(parent, processorNode); processorNode.removeParentStreams(parent); addNode(identity, processorNode); curGroup.add(identity); } processCurGroup(topologyBuilder); }
public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); // a stream of (number, square) pairs PairStream<Integer, Integer> squares = builder .newStream(new NumberSpout(x -> x * x), new PairValueMapper<>(0, 1)); // a stream of (number, cube) pairs PairStream<Integer, Integer> cubes = builder .newStream(new NumberSpout(x -> x * x * x), new PairValueMapper<>(0, 1)); // create a windowed stream of five seconds duration squares.window(TumblingWindows.of(Duration.seconds(5))) /* * Join the squares and the cubes stream within the window. * The values in the squares stream having the same key as that * of the cubes stream within the window will be joined together. */ .join(cubes) /** * The results should be of the form (number, (square, cube)) */ .print(); Config config = new Config(); String topoName = JoinExample.class.getName(); if (args.length > 0) { topoName = args[0]; } config.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build()); }
@Test public void testSpoutToBolt() throws Exception { Stream<Tuple> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID)); stream.to(newBolt()); StormTopology topology = streamBuilder.build(); assertEquals(1, topology.get_spouts_size()); assertEquals(1, topology.get_bolts_size()); String spoutId = topology.get_spouts().keySet().iterator().next(); Map<GlobalStreamId, Grouping> expected = new HashMap<>(); expected.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); assertEquals(expected, topology.get_bolts().values().iterator().next().get_common().get_inputs()); }
@Before public void setUp() throws Exception { streamBuilder = new StreamBuilder(); UniqueIdGen.getInstance().reset(); }
private Node addNode(Node parent, Node child, String parentStreamId) { return streamBuilder.addNode(parent, child, parentStreamId); }
public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); /** * The spout emits sequences of (Integer, Long, Long). TupleValueMapper can be used to extract fields * from the values and produce a stream of typed tuple (Tuple3<Integer, Long, Long> in this case. */ Stream<Tuple3<Integer, Long, Long>> stream = builder.newStream(new RandomIntegerSpout(), TupleValueMappers.of(0, 1, 2)); PairStream<Long, Integer> pairs = stream.mapToPair(t -> Pair.of(t._2 / 10000, t._1)); pairs.window(TumblingWindows.of(Count.of(10))).groupByKey().print(); String topoName = "test"; if (args.length > 0) { topoName = args[0]; } Config config = new Config(); config.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build()); } }
@Test public void testPartitionByKeySinglePartition() { TopologyContext mockContext = Mockito.mock(TopologyContext.class); OutputCollector mockCollector = Mockito.mock(OutputCollector.class); Stream<Integer> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); stream.mapToPair(x -> Pair.of(x, x)) .reduceByKey((x, y) -> x + y) .print(); StormTopology topology = streamBuilder.build(); assertEquals(1, topology.get_bolts_size()); } }
/** * Creates a new {@link Stream} of values from the given {@link IRichSpout} by extracting field(s) from tuples via the supplied {@link * TupleValueMapper}. * * @param spout the spout * @param valueMapper the value mapper * @param <T> the type of values in the resultant stream * @return the new stream */ public <T> Stream<T> newStream(IRichSpout spout, TupleValueMapper<T> valueMapper) { return newStream(spout).map(valueMapper); }
StreamFactory factory = StreamFactory.newInstance(); StreamBuilder builder = new StreamBuilder("") // Your file .format("delimited") .parser(new DelimitedParserBuilder(',')) // Sign to use as a delimiter .addRecord(Yourclass.class); // class to be mapped factory.define(builder);
private Node addNode(Node parent, Node child, String parentStreamId, int parallelism) { return streamBuilder.addNode(parent, child, parentStreamId, parallelism); }
@SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); /** * Computes average of the stream of numbers emitted by the spout. Internally the per-partition * sum and counts are accumulated and emitted to a downstream task where the partially accumulated * results are merged and the final result is emitted. */ builder.newStream(new RandomIntegerSpout(), new ValueMapper<Integer>(0), 2) .window(TumblingWindows.of(BaseWindowedBolt.Duration.seconds(5))) .filter(x -> x > 0 && x < 500) .aggregate(new Avg()) .print(); Config config = new Config(); String topoName = "AGG_EXAMPLE"; if (args.length > 0) { topoName = args[0]; } config.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build()); }
@Test public void testGroupBy() throws Exception { PairStream<String, String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new PairValueMapper<>(0, 1), 2); stream.window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))).aggregateByKey(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); assertEquals(Grouping.shuffle(new NullStruct()), bolt1.get_common().get_inputs().values().iterator().next()); assertEquals(Grouping.fields(Collections.singletonList("key")), bolt2.get_common().get_inputs().values().iterator().next()); }