windowLengthCount = new Count(((Number) topoConf.get(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT)).intValue()); } else if (topoConf.containsKey(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS)) { windowLengthDuration = new Duration( ((Number) topoConf.get(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS)).intValue(), TimeUnit.MILLISECONDS); } else if (topoConf.containsKey(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS)) { slidingIntervalDuration = new Duration(((Number) topoConf.get(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS)).intValue(), TimeUnit.MILLISECONDS); } else {
@Test public void testTimeBasedWindowExpiry() throws Exception { EvictionPolicy<Integer, ?> evictionPolicy = new TimeEvictionPolicy<Integer>(new Duration(100, TimeUnit.MILLISECONDS).value); windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new TimeTriggerPolicy<Integer>(new Duration(1, TimeUnit.DAYS).value, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy);
@Test public void testTimeBasedWindow() throws Exception { EvictionPolicy<Integer, ?> evictionPolicy = new TimeEvictionPolicy<Integer>(new Duration(1, TimeUnit.SECONDS).value); windowManager.setEvictionPolicy(evictionPolicy); new TimeTriggerPolicy<Integer>(new Duration(1, TimeUnit.DAYS).value, windowManager, evictionPolicy); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy);
/** * Returns a {@link Duration} corresponding to the the given value in days. * * @param days the number of days * @return the Duration */ public static Duration days(int days) { return new Duration(days, TimeUnit.DAYS); }
/** * Returns a {@link Duration} corresponding to the the given value in hours. * * @param hours the number of hours * @return the Duration */ public static Duration hours(int hours) { return new Duration(hours, TimeUnit.HOURS); }
@Override public TridentTopology createTopology() { ITridentDataSource source = KafkaSpoutFactory.getTridentSpout(config, true); TridentTopology topology = new TridentTopology(); topology.newStream("kafka", source) .each(new Fields("str"), new Parser(), new Fields("ip", "time")) .project(new Fields("ip", "time")) .parallelismHint(config.spoutThreads) .groupBy(new Fields("ip")).toStream() .slidingWindow(new BaseWindowedBolt.Duration((int) config.windowDuration, TimeUnit.MILLISECONDS), new BaseWindowedBolt.Duration((int) config.windowSlideStep, TimeUnit.MILLISECONDS), new InMemoryWindowsStoreFactory(), new Fields("ip", "time"), new Count(config), new Fields("ip", "count")) .parallelismHint(config.boltThreads); return topology; }
@Override public void setBolts(TopologyBuilder builder) { Duration windowDuration = new Duration((int) config.windowDuration, TimeUnit.MILLISECONDS); Duration windowSlide = new Duration((int) config.windowSlideStep, TimeUnit.MILLISECONDS); BoltDeclarer boltDeclarer = builder.setBolt("parser", new ParserBolt(), config.boltThreads); if (config.localShuffle) { boltDeclarer.localOrShuffleGrouping("spout"); } else { boltDeclarer.shuffleGrouping("spout"); } builder.setBolt("window", new SlidingWindowBolt(config) .withWindow(windowDuration, windowSlide), config.boltThreads).fieldsGrouping("parser", new Fields("ip")); }
public static void main(String[] args) throws Exception { if (!NimbusClient.isLocalOverride()) { throw new IllegalStateException("This example only works in local mode. " + "Run with storm local not storm jar"); } FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender")); FeederSpout ageSpout = new FeederSpout(new Fields("id", "age")); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("genderSpout", genderSpout); builder.setSpout("ageSpout", ageSpout); // inner join of 'age' and 'gender' records on 'id' field JoinBolt joiner = new JoinBolt("genderSpout", "id") .join("ageSpout", "id", "genderSpout") .select("genderSpout:id,ageSpout:id,gender,age") .withTumblingWindow(new BaseWindowedBolt.Duration(10, TimeUnit.SECONDS)); builder.setBolt("joiner", joiner) .fieldsGrouping("genderSpout", new Fields("id")) .fieldsGrouping("ageSpout", new Fields("id")); builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("joiner"); Config conf = new Config(); StormSubmitter.submitTopologyWithProgressBar("join-example", conf, builder.createTopology()); generateGenderData(genderSpout); generateAgeData(ageSpout); }
@Test public void testWindowConfig() { int windowLength = 9; TumblingCountWindow tumblingCountWindow = TumblingCountWindow.of(windowLength); Assert.assertTrue(tumblingCountWindow.getWindowLength() == windowLength); Assert.assertTrue(tumblingCountWindow.getSlidingLength() == windowLength); windowLength = 10; int slidingLength = 2; SlidingCountWindow slidingCountWindow = SlidingCountWindow.of(10, 2); Assert.assertTrue(slidingCountWindow.getWindowLength() == windowLength); Assert.assertTrue(slidingCountWindow.getSlidingLength() == slidingLength); windowLength = 20; TumblingDurationWindow tumblingDurationWindow = TumblingDurationWindow.of(new BaseWindowedBolt.Duration(windowLength, TimeUnit.SECONDS)); Assert.assertTrue(tumblingDurationWindow.getWindowLength() == windowLength * 1000); Assert.assertTrue(tumblingDurationWindow.getSlidingLength() == windowLength * 1000); windowLength = 50; slidingLength = 10; SlidingDurationWindow slidingDurationWindow = SlidingDurationWindow.of(new BaseWindowedBolt.Duration(windowLength, TimeUnit.SECONDS), new BaseWindowedBolt.Duration(slidingLength, TimeUnit.SECONDS)); Assert.assertTrue(slidingDurationWindow.getWindowLength() == windowLength * 1000); Assert.assertTrue(slidingDurationWindow.getSlidingLength() == slidingLength * 1000); }
@Test public void testExpireThreshold() throws Exception { int threshold = WindowManager.EXPIRE_EVENTS_THRESHOLD; int windowLength = 5; windowManager.setEvictionPolicy(new CountEvictionPolicy<Integer>(5)); TriggerPolicy<Integer, ?> triggerPolicy = new TimeTriggerPolicy<Integer>(new Duration(1, TimeUnit.HOURS).value, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); for (int i : seq(1, 5)) { windowManager.add(i); } // nothing expired yet assertTrue(listener.onExpiryEvents.isEmpty()); for (int i : seq(6, 10)) { windowManager.add(i); } for (int i : seq(11, threshold)) { windowManager.add(i); } // window should be compacted and events should be expired. assertEquals(seq(1, threshold - windowLength), listener.onExpiryEvents); }
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); BaseWindowedBolt bolt = new SlidingWindowSumBolt() .withWindow(new Duration(5, TimeUnit.SECONDS), new Duration(3, TimeUnit.SECONDS)) .withTimestampField("ts") .withLag(new Duration(5, TimeUnit.SECONDS)); builder.setSpout("integer", new RandomIntegerSpout(), 1); builder.setBolt("slidingsum", bolt, 1).shuffleGrouping("integer"); builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("slidingsum"); Config conf = new Config(); conf.setDebug(true); String topoName = "test"; if (args != null && args.length > 0) { topoName = args[0]; } conf.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology()); } }
@Test public void testWindowStrategyInstances() throws Exception { WindowStrategy<Object> tumblingCountStrategy = TumblingCountWindow.of(10).getWindowStrategy(); Assert.assertTrue(tumblingCountStrategy instanceof TumblingCountWindowStrategy); WindowStrategy<Object> slidingCountStrategy = SlidingCountWindow.of(100, 10).getWindowStrategy(); Assert.assertTrue(slidingCountStrategy instanceof SlidingCountWindowStrategy); WindowStrategy<Object> tumblingDurationStrategy = TumblingDurationWindow.of( new BaseWindowedBolt.Duration(10, TimeUnit.SECONDS)) .getWindowStrategy(); Assert.assertTrue(tumblingDurationStrategy instanceof TumblingDurationWindowStrategy); WindowStrategy<Object> slidingDurationStrategy = SlidingDurationWindow.of( new BaseWindowedBolt.Duration(10, TimeUnit.SECONDS), new BaseWindowedBolt.Duration(2, TimeUnit.SECONDS)) .getWindowStrategy(); Assert.assertTrue(slidingDurationStrategy instanceof SlidingDurationWindowStrategy); }
@Override public StormTopology newTopology() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(getSpoutName(), new TimeDataIncrementingSpout(), spoutExecutors); builder.setBolt(getBoltName(), new TimeDataVerificationBolt() .withWindow(new BaseWindowedBolt.Duration(windowSec, TimeUnit.SECONDS), new BaseWindowedBolt.Duration(slideSec, TimeUnit.SECONDS)) .withTimestampField(TimeData.getTimestampFieldName()) .withLag(new BaseWindowedBolt.Duration(10, TimeUnit.SECONDS)), boltExecutors) .globalGrouping(getSpoutName()); return builder.createTopology(); } }
@Override public StormTopology newTopology() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(getSpoutName(), new TimeDataIncrementingSpout(), spoutExecutors); builder.setBolt(getBoltName(), new TimeDataVerificationBolt() .withTumblingWindow(new BaseWindowedBolt.Duration(tumbleSec, TimeUnit.SECONDS)) .withLag(new BaseWindowedBolt.Duration(10, TimeUnit.SECONDS)) .withTimestampField(TimeData.getTimestampFieldName()), boltExecutors) .globalGrouping(getSpoutName()); return builder.createTopology(); } }
/** * Returns a {@link Duration} corresponding to the the given value in seconds. * * @param seconds the number of seconds * @return the Duration */ public static Duration seconds(int seconds) { return new Duration(seconds, TimeUnit.SECONDS); }
/** * Returns a {@link Duration} corresponding to the the given value in hours. * * @param hours the number of hours * @return the Duration */ public static Duration hours(int hours) { return new Duration(hours, TimeUnit.HOURS); }
/** * Returns a {@link Duration} corresponding to the the given value in minutes. * * @param minutes the number of minutes * @return the Duration */ public static Duration minutes(int minutes) { return new Duration(minutes, TimeUnit.MINUTES); }
/** * Returns a {@link Duration} corresponding to the the given value in days. * * @param days the number of days * @return the Duration */ public static Duration days(int days) { return new Duration(days, TimeUnit.DAYS); }
/** * Returns a {@link Duration} corresponding to the the given value in milli seconds. * * @param milliseconds the duration in milliseconds * @return the Duration */ public static Duration of(int milliseconds) { return new Duration(milliseconds, TimeUnit.MILLISECONDS); }