public StormTopology buildTopology() { return buildTopologyBuilder().createTopology(); }
public StormTopology buildTopology() { return buildTopologyBuilder().createTopology(); }
public static void test() { String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); topologyName = className[className.length - 1]; try { JStormHelper.runTopology(setBuilder().createTopology(), topologyName, conf, 100, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { e.printStackTrace(); Assert.fail("Failed"); } }
private static void submitTopology(TopologyBuilder builder) { try { if (local_mode(conf)) { LocalCluster cluster = new LocalCluster(); cluster.submitTopology(String.valueOf(conf.get("topology.name")), conf, builder.createTopology()); Thread.sleep(200000); cluster.shutdown(); } else { StormSubmitter.submitTopology(String.valueOf(conf.get("topology.name")), conf, builder.createTopology()); } } catch (Exception e) { e.printStackTrace(); } }
public static void SetRemoteTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException { String streamName = (String) conf.get(Config.TOPOLOGY_NAME); if (streamName == null) { streamName = "SequenceTest"; } TopologyBuilder builder = new TopologyBuilder(); SetBuilder(builder, conf); conf.put(Config.STORM_CLUSTER_MODE, "distributed"); StormSubmitter.submitTopology(streamName, conf, builder.createTopology()); }
public static void SetLocalTopology() throws Exception { TopologyBuilder builder = new TopologyBuilder(); conf.put(TOPOLOGY_BOLT_PARALLELISM_HINT, 1); SetBuilder(builder, conf); LOG.debug("test"); LOG.info("Submit log"); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("SplitMerge", conf, builder.createTopology()); Thread.sleep(60000); cluster.killTopology("SplitMerge"); cluster.shutdown(); }
@Override public StormTopology createTopology() { TopologyBuilder.putStormConf(ConfigExtension.TASK_BATCH_TUPLE, "true"); TopologyBuilder.putStormConf(Config.TOPOLOGY_ACKER_EXECUTORS, "0"); TopologyBuilder.putStormConf(ConfigExtension.TRANSACTION_TOPOLOGY, true); return super.createTopology(); }
public static void test() { String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { SkewedRollingTopWords rtw = new SkewedRollingTopWords(topologyName); JStormHelper.runTopology(rtw.getBuilder().createTopology(), topologyName, rtw.getTopologyConfig(), rtw.getRuntimeInSeconds(), new JStormHelper.CheckAckedFail(rtw.getTopologyConfig()), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { RollingTopWords rtw = new RollingTopWords(topologyName); JStormHelper.runTopology(rtw.getBuilder().createTopology(), topologyName, rtw.getTopologyConfig(), rtw.getRuntimeInSeconds(), new JStormHelper.CheckAckedFail(rtw.getTopologyConfig()), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void SetRemoteTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException { String streamName = (String) conf.get(Config.TOPOLOGY_NAME); if (streamName == null) { String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); streamName = className[className.length - 1]; } TopologyBuilder builder = new TopologyBuilder(); int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2); builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint); BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint); // localFirstGrouping is only for jstorm // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME); boltDeclarer.shuffleGrouping("spout"); // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60); conf.put(Config.STORM_CLUSTER_MODE, "distributed"); StormSubmitter.submitTopology(streamName, conf, builder.createTopology()); }
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException { Map config = new Config(); config.put(ConfigExtension.TOPOLOGY_MASTER_USER_DEFINED_STREAM_CLASS, "com.alipay.dw.jstorm.example.tm.TMUdfHandler"); config.put(Config.TOPOLOGY_WORKERS, 2); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("TMUdfSpout", new TMUdfSpout(), 2); builder.setBolt("TMUdfBolt", new TMUdfBolt(), 4); StormTopology topology = builder.createTopology(); StormSubmitter.submitTopology("TMUdfTopology", config, topology); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); int spoutNum = JStormUtils.parseInt(conf.get("spout.num"), 8); int countNum = JStormUtils.parseInt(conf.get("count.num"), 8); builder.setSpout("spout", new InOrderSpout(), spoutNum); builder.setBolt("count", new Check(), countNum).fieldsGrouping("spout", new Fields("c1")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new RandomSentenceSpout(), 5); builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word")); Config conf = new Config(); conf.setDebug(true); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); WordCount wordCountBolt = new WordCount(); builder.setBolt("count", wordCountBolt.sessionTimeWindow(Time.seconds(1L)) .withWindowStateMerger(wordCountBolt), count_Parallelism_hint) .fieldsGrouping("spout", new Fields("word")); //.allGrouping("spout", Common.WATERMARK_STREAM_ID); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); } catch (Exception e) { e.printStackTrace(); } }
public static void test() throws Exception { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); WordCount wordCountBolt = new WordCount(); builder.setBolt("count", wordCountBolt.sessionEventTimeWindow(Time.milliseconds(3L)) .withTimestampExtractor(wordCountBolt) .withWindowStateMerger(wordCountBolt), count_Parallelism_hint) .fieldsGrouping("spout", new Fields("word", "ts")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); }
public static void test() { String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { TopologyBuilder builder = new TopologyBuilder(); BaseWindowedBolt bolt = new SlidingWindowSumBolt() .withWindow(new Duration(5, TimeUnit.SECONDS), new Duration(3, TimeUnit.SECONDS)) .withTimestampField("ts").withLag(new Duration(5, TimeUnit.SECONDS)); builder.setSpout("integer", new RandomIntegerSpout(), 1); builder.setBolt("slidingsum", bolt, 1).shuffleGrouping("integer"); builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("slidingsum"); conf.setDebug(true); JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int split_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 2); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).localOrShuffleGrouping("spout"); builder.setBolt("count", new WordCount(), count_Parallelism_hint).fieldsGrouping("split", new Fields("word")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; isLocal = JStormHelper.localMode(conf); try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() throws Exception { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); WordCount wordCountBolt = new WordCount(); builder.setBolt("count", wordCountBolt.eventTimeWindow(Time.milliseconds(3L)) .withTimestampExtractor(wordCountBolt) .withWatermarkGenerator(new PeriodicWatermarkGenerator(Time.milliseconds(1L), Time.milliseconds(10L))) , count_Parallelism_hint) .fieldsGrouping("spout", new Fields("word", "ts")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); }
public static void test() { String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("integer", new RandomIntegerSpout(), 1); builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(new Count(30), new Count(10)), 1) .shuffleGrouping("integer"); builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(new Count(3)), 1) .shuffleGrouping("slidingsum"); builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("tumblingavg"); conf.setDebug(true); JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.fillInStackTrace(); Assert.fail("Failed to submit topology"); } }
public static void test() throws Exception { int spout_parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int split_parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1); int count_parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_parallelism_hint); builder.setBolt("split", new SplitSentence(), split_parallelism_hint).shuffleGrouping("spout"); builder.setBolt("count", new WordCount() .ingestionTimeWindow(Time.seconds(1L), Time.milliseconds(500L)) .withStateSize(Time.minutes(10)), count_parallelism_hint).fieldsGrouping("split", new Fields("word")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); }