public static void main(String[] args) throws Exception { Map conf = JStormHelper.getConfig(args); int spoutParallelism = JStormUtils.parseInt(conf.get(SPOUT_PARALLELISM_HINT), 1); int splitParallelism = JStormUtils.parseInt(conf.get(SPLIT_PARALLELISM_HINT), 2); int countParallelism = JStormUtils.parseInt(conf.get(COUNT_PARALLELISM_HINT), 2); TransactionTopologyBuilder builder = new TransactionTopologyBuilder(); builder.setSpout("spout", new ScheduleTxSpout(), spoutParallelism); builder.setBolt("split", new TxSplitSentence(), splitParallelism).localOrShuffleGrouping("spout"); builder.setBolt("count", new RocksDbCount(), countParallelism).fieldsGrouping("split", new Fields("word")); builder.enableHdfs(); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; StormSubmitter.submitTopology(topologyName, conf, builder.createTopology()); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new InOrderSpout(), 8); builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1")); conf.setMaxSpoutPending(20); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; if (isLocal) { drpc = new LocalDRPC(); } try { JStormHelper.runTopology(buildTopology(drpc), topologyName, conf, 60, new DrpcValidator(), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH); TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("top-n-words", "spout", spout, 2); builder.setBolt("count", new KeyedCountUpdater(), 5).fieldsGrouping("spout", new Fields("word")); builder.setBolt("bucketize", new Bucketize()).noneGrouping("count"); builder.setBolt("buckets", new BucketCountUpdater(), 5).fieldsGrouping("bucketize", new Fields("bucket")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.buildTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new InOrderSpout(), 8); builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1")); conf.setMaxSpoutPending(20); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; if (isLocal) { drpc = new LocalDRPC(); } try { JStormHelper.runTopology(buildDevicesTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new InOrderSpout(), 8); builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1")); conf.setMaxSpoutPending(20); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; if (isLocal) { drpc = new LocalDRPC(); } try { JStormHelper.runTopology(buildVehiclesTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
private void wireTopology() throws InterruptedException { String spoutId = "wordGenerator"; String counterId = "counter"; String intermediateRankerId = "intermediateRanker"; String totalRankerId = "finalRanker"; builder.setSpout(spoutId, new TestWordSpout(), 5); builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word")); builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId, new Fields("obj")); builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId); }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new InOrderSpout(), 8); builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1")); Config conf = new Config(); conf.setMaxSpoutPending(20); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; if (isLocal) { drpc = new LocalDRPC(); } try { JStormHelper.runTopology(buildTopology(drpc), topologyName, conf, 60, new DrpcValidator(), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); int spoutNum = JStormUtils.parseInt(conf.get("spout.num"), 8); int countNum = JStormUtils.parseInt(conf.get("count.num"), 8); builder.setSpout("spout", new InOrderSpout(), spoutNum); builder.setBolt("count", new Check(), countNum).fieldsGrouping("spout", new Fields("c1")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new RandomSentenceSpout(), 5); builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word")); Config conf = new Config(); conf.setDebug(true); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); WordCount wordCountBolt = new WordCount(); builder.setBolt("count", wordCountBolt.sessionTimeWindow(Time.seconds(1L)) .withWindowStateMerger(wordCountBolt), count_Parallelism_hint) .fieldsGrouping("spout", new Fields("word")); //.allGrouping("spout", Common.WATERMARK_STREAM_ID); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); } catch (Exception e) { e.printStackTrace(); } }
public static void test() throws Exception { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); WordCount wordCountBolt = new WordCount(); builder.setBolt("count", wordCountBolt.sessionEventTimeWindow(Time.milliseconds(3L)) .withTimestampExtractor(wordCountBolt) .withWindowStateMerger(wordCountBolt), count_Parallelism_hint) .fieldsGrouping("spout", new Fields("word", "ts")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); }
private void wireTopology() throws InterruptedException { String spoutId = "wordGenerator"; String counterId = "counter"; String aggId = "aggregator"; String intermediateRankerId = "intermediateRanker"; String totalRankerId = "finalRanker"; builder.setSpout(spoutId, new TestWordSpout(), 5); builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId, new Fields("word")); builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId, new Fields("obj")); builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(aggId, new Fields("obj")); builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId); }
public static void test() { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int split_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 2); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).localOrShuffleGrouping("spout"); builder.setBolt("count", new WordCount(), count_Parallelism_hint).fieldsGrouping("split", new Fields("word")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; isLocal = JStormHelper.localMode(conf); try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() throws Exception { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); WordCount wordCountBolt = new WordCount(); builder.setBolt("count", wordCountBolt.eventTimeWindow(Time.milliseconds(3L)) .withTimestampExtractor(wordCountBolt) .withWatermarkGenerator(new PeriodicWatermarkGenerator(Time.milliseconds(1L), Time.milliseconds(10L))) , count_Parallelism_hint) .fieldsGrouping("spout", new Fields("word", "ts")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); }
public static void test() throws Exception { int spout_parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int split_parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1); int count_parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_parallelism_hint); builder.setBolt("split", new SplitSentence(), split_parallelism_hint).shuffleGrouping("spout"); builder.setBolt("count", new WordCount() .ingestionTimeWindow(Time.seconds(1L), Time.milliseconds(500L)) .withStateSize(Time.minutes(10)), count_parallelism_hint).fieldsGrouping("split", new Fields("word")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); }
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException { Config conf = JStormHelper.getConfig(args); int spoutParallelism = JStormUtils.parseInt(conf.get(SPOUT_PARALLELISM_HINT), 1); int splitParallelism = JStormUtils.parseInt(conf.get(SPLIT_PARALLELISM_HINT), 2); int countParallelism = JStormUtils.parseInt(conf.get(COUNT_PARALLELISM_HINT), 2); boolean isValueSpout = JStormUtils.parseBoolean(conf.get("is.value.spout"), false); TransactionTopologyBuilder builder = new TransactionTopologyBuilder(); if (isValueSpout) builder.setSpoutWithAck("spout", new BatchAckerValueSpout(), spoutParallelism); else builder.setSpoutWithAck("spout", new BatchAckerSpout(), spoutParallelism); builder.setBoltWithAck("split", new BatchAckerSplit(), splitParallelism).localOrShuffleGrouping("spout");; builder.setBoltWithAck("count", new BatchAckerCount(), countParallelism).fieldsGrouping("split", new Fields("word")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; StormSubmitter.submitTopology(topologyName, conf, builder.createTopology()); } }
count_Parallelism_hint).fieldsGrouping("split", new Fields("word"));
public static void test() throws Exception { TransactionTopologyBuilder builder = new TransactionTopologyBuilder(); if (isLocal) { conf.put("tuple.num.per.batch", 100); conf.put("transaction.scheduler.spout", false); conf.put("transaction.exactly.cache.type", "default"); conf.put("transaction.topology", true); } int spoutParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int splitParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1); int countParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); builder.setSpout("spout", new TxFastRandomSentenceSpout(), spoutParallelismHint); builder.setBolt("split", new TxSplitSentence(), splitParallelismHint).localOrShuffleGrouping("spout"); WordCount wordCount = new WordCount(); builder.setBolt("count", wordCount .timeWindow(Time.seconds(60L)) .withTransactionStateOperator(wordCount), countParallelismHint).fieldsGrouping("split", new Fields("word")); builder.enableHdfs(); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; StormSubmitter.submitTopology(topologyName, conf, builder.createTopology()); }
public static void test() throws Exception { TransactionTopologyBuilder builder = new TransactionTopologyBuilder(); if (isLocal) { conf.put("tuple.num.per.batch", 100); conf.put("transaction.scheduler.spout", false); conf.put("transaction.exactly.cache.type", "default"); conf.put("transaction.topology", true); } int spoutParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int splitParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1); int countParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); builder.setSpout("spout", new TxFastRandomSentenceSpout(), spoutParallelismHint); builder.setBolt("split", new SplitSentence(), splitParallelismHint).localOrShuffleGrouping("spout"); builder.setBolt("count", new WordCount() .timeWindow(Time.seconds(1L)) .withStateSize(Time.hours(2)), countParallelismHint).fieldsGrouping("split", new Fields("word")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); }
public static void test(Config conf) throws Exception { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int split_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).shuffleGrouping("spout"); long windowSize = JStormUtils.parseLong(conf.get("window.size.sec"), 10); long stateWindowSize = JStormUtils.parseLong(conf.get("state.window.size.sec"), 60); builder.setBolt("count", new WordCount() .timeWindow(Time.seconds(windowSize)) .withStateSize(Time.seconds(stateWindowSize)), count_Parallelism_hint).fieldsGrouping("split", new Fields("word")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; boolean isLocal = true; if (conf.containsKey("storm.cluster.mode")) { isLocal = StormConfig.local_mode(conf); } // RUN_LONG_TIME = true JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); }