windowLengthCount = new Count(((Number) topoConf.get(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT)).intValue()); } else if (topoConf.containsKey(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS)) { windowLengthDuration = new Duration( slidingIntervalCount = new Count(((Number) topoConf.get(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT)).intValue()); } else if (topoConf.containsKey(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS)) { slidingIntervalDuration = } else { slidingIntervalCount = new Count(1);
@Test public void testMultiPartitionByKeyWithRepartition() { TopologyContext mockContext = Mockito.mock(TopologyContext.class); OutputCollector mockCollector = Mockito.mock(OutputCollector.class); Map<GlobalStreamId, Grouping> expected = new HashMap<>(); expected.put(new GlobalStreamId("bolt2", "s3"), Grouping.fields(Collections.singletonList("key"))); expected.put(new GlobalStreamId("bolt2", "s3__punctuation"), Grouping.all(new NullStruct())); Stream<Integer> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); stream.mapToPair(x -> Pair.of(x, x)) .window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))) .reduceByKey((x, y) -> x + y) .repartition(10) .reduceByKey((x, y) -> 0) .print(); StormTopology topology = streamBuilder.build(); assertEquals(3, topology.get_bolts_size()); assertEquals(expected, topology.get_bolts().get("bolt3").get_common().get_inputs()); }
windowLengthCount = new Count(((Number) stormConf.get(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT)).intValue()); } else if (stormConf.containsKey(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS)) { windowLengthDuration = new Duration( slidingIntervalCount = new Count(((Number) stormConf.get(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT)).intValue()); } else if (stormConf.containsKey(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS)) { slidingIntervalDuration = new Duration(((Number) stormConf.get(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS)).intValue(), TimeUnit.MILLISECONDS); } else { slidingIntervalCount = new Count(1);
withWindow(windowLength, slidingInterval); } else if (windowConfig.getSlidingInterval() instanceof Window.Count) { BaseWindowedBolt.Count slidingInterval = new BaseWindowedBolt.Count(((Window.Count) windowConfig.getSlidingInterval()).getCount()); withWindow(windowLength, slidingInterval); } else { BaseWindowedBolt.Count windowLength = new BaseWindowedBolt.Count(((Window.Count) windowConfig.getWindowLength()).getCount()); if (windowConfig.getSlidingInterval() instanceof Window.Duration) { BaseWindowedBolt.Duration slidingInterval = new BaseWindowedBolt.Duration(((Window.Duration) windowConfig.getWindowLength()).getDurationMs(), TimeUnit.MILLISECONDS); withWindow(windowLength, slidingInterval); } else if (windowConfig.getSlidingInterval() instanceof Window.Count) { BaseWindowedBolt.Count slidingInterval = new BaseWindowedBolt.Count(((Window.Count) windowConfig.getWindowLength()).getCount()); withWindow(windowLength, slidingInterval); } else {
withWindow(windowLength, slidingInterval); } else if (windowConfig.getSlidingInterval() instanceof Window.Count) { Count slidingInterval = new Count(((Window.Count) windowConfig.getSlidingInterval()).getCount()); withWindow(windowLength, slidingInterval); } else { Count windowLength = new Count(((Window.Count) windowConfig.getWindowLength()).getCount()); if (windowConfig.getSlidingInterval() instanceof Window.Duration) { Duration slidingInterval = new Duration(((Window.Duration) windowConfig.getWindowLength()).getDurationMs(), TimeUnit.MILLISECONDS); withWindow(windowLength, slidingInterval); } else if (windowConfig.getSlidingInterval() instanceof Window.Count) { Count slidingInterval = new Count(((Window.Count) windowConfig.getWindowLength()).getCount()); withWindow(windowLength, slidingInterval); } else {
/** * A tuple count based window that slides with every incoming tuple. * * @param windowLength the number of tuples in the window */ public BaseWindowedBolt withWindow(Count windowLength) { return withWindowLength(windowLength).withSlidingInterval(new Count(1)); }
/** * A time duration based window that slides with every incoming tuple. * * @param windowLength the time duration of the window */ public BaseWindowedBolt withWindow(Duration windowLength) { return withWindowLength(windowLength).withSlidingInterval(new Count(1)); }
/** * Returns a {@link Count} of given value. * * @param value the count value * @return the Count */ public static Count of(int value) { return new Count(value); }
/** * Returns a {@link Count} of given value. * * @param value the count value * @return the Count */ public static Count of(int value) { return new Count(value); }
/** * A tuple count based window that slides with every incoming tuple. * * @param windowLength the number of tuples in the window */ public BaseWindowedBolt withWindow(Count windowLength) { return withWindowLength(windowLength).withSlidingInterval(new Count(1)); }
/** * A time duration based window that slides with every incoming tuple. * * @param windowLength the time duration of the window */ public BaseWindowedBolt withWindow(Duration windowLength) { return withWindowLength(windowLength).withSlidingInterval(new Count(1)); }
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("integer", new RandomIntegerSpout(), 1); builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(Count.of(30), Count.of(10)), 1) .shuffleGrouping("integer"); builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(Count.of(3)), 1) .shuffleGrouping("slidingsum"); builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("tumblingavg"); Config conf = new Config(); conf.setDebug(true); String topoName = "test"; if (args != null && args.length > 0) { topoName = args[0]; } conf.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology()); }
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new RandomIntegerSpout()); builder.setBolt("sumbolt", new WindowSumBolt().withWindow(new Count(5), new Count(3)) .withMessageIdField("msgid"), 1).shuffleGrouping("spout"); builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("sumbolt"); Config conf = new Config(); conf.setDebug(false); //conf.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider"); String topoName = "test"; if (args != null && args.length > 0) { topoName = args[0]; } conf.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology()); }
public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); /** * The spout emits sequences of (Integer, Long, Long). TupleValueMapper can be used to extract fields * from the values and produce a stream of typed tuple (Tuple3<Integer, Long, Long> in this case. */ Stream<Tuple3<Integer, Long, Long>> stream = builder.newStream(new RandomIntegerSpout(), TupleValueMappers.of(0, 1, 2)); PairStream<Long, Integer> pairs = stream.mapToPair(t -> Pair.of(t._2 / 10000, t._1)); pairs.window(TumblingWindows.of(Count.of(10))).groupByKey().print(); String topoName = "test"; if (args.length > 0) { topoName = args[0]; } Config config = new Config(); config.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build()); } }
@Test public void testEmit() throws Exception { Window<?, ?> window = TumblingWindows.of(BaseWindowedBolt.Count.of(2)); setUpWindowedProcessorBolt(new AggregateProcessor<>(new Count<>()), window); bolt.execute(getMockTupleWindow(mockTuple1, mockTuple2, mockTuple3)); ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class); ArgumentCaptor<String> os = ArgumentCaptor.forClass(String.class); Mockito.verify(mockOutputCollector, Mockito.times(2)).emit(os.capture(), values.capture()); assertEquals("outputstream", os.getAllValues().get(0)); assertEquals(new Values(3L), values.getAllValues().get(0)); assertEquals("outputstream__punctuation", os.getAllValues().get(1)); assertEquals(new Values(WindowNode.PUNCTUATION), values.getAllValues().get(1)); }
/** * A tuple count based window that slides with every incoming tuple. * * @param windowLength the number of tuples in the window */ public BaseWindowedBolt withWindow(Count windowLength) { return withWindowLength(windowLength).withSlidingInterval(new Count(1)); }
@Test public void testGroupBy() throws Exception { PairStream<String, String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new PairValueMapper<>(0, 1), 2); stream.window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))).aggregateByKey(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); assertEquals(Grouping.shuffle(new NullStruct()), bolt1.get_common().get_inputs().values().iterator().next()); assertEquals(Grouping.fields(Collections.singletonList("key")), bolt2.get_common().get_inputs().values().iterator().next()); }
@Test public void testMultiPartitionByKey() { TopologyContext mockContext = Mockito.mock(TopologyContext.class); OutputCollector mockCollector = Mockito.mock(OutputCollector.class); Stream<Integer> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); stream.mapToPair(x -> Pair.of(x, x)) .window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))) .reduceByKey((x, y) -> x + y) .reduceByKey((x, y) -> 0) .print(); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); }
/** * Returns a {@link Count} of given value. * * @param value the count value * @return the Count */ public static Count of(int value) { return new Count(value); }