public BatchTopologyBuilder(String topologyName) { topologyBuilder = new TopologyBuilder(); System.setProperty(ConfigExtension.TASK_BATCH_TUPLE, "false"); spoutDeclarer = topologyBuilder.setSpout(BatchDef.SPOUT_TRIGGER, new BatchSpoutTrigger(), 1); }
public RollingTopWords(String topologyName) throws InterruptedException { builder = new TopologyBuilder(); this.topologyName = topologyName; topologyConfig = createTopologyConfiguration(); runtimeInSeconds = DEFAULT_RUNTIME_IN_SECONDS; wireTopology(); }
public SkewedRollingTopWords(String topologyName) throws InterruptedException { builder = new TopologyBuilder(); this.topologyName = topologyName; topologyConfig = createTopologyConfiguration(); runtimeInSeconds = DEFAULT_RUNTIME_IN_SECONDS; wireTopology(); }
public static void SetRemoteTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException { String streamName = (String) conf.get(Config.TOPOLOGY_NAME); if (streamName == null) { streamName = "SequenceTest"; } TopologyBuilder builder = new TopologyBuilder(); SetBuilder(builder, conf); conf.put(Config.STORM_CLUSTER_MODE, "distributed"); StormSubmitter.submitTopology(streamName, conf, builder.createTopology()); }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; if (isLocal) { drpc = new LocalDRPC(); } try { JStormHelper.runTopology(buildTopology(drpc), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void SetLocalTopology() throws Exception { TopologyBuilder builder = new TopologyBuilder(); conf.put(TOPOLOGY_BOLT_PARALLELISM_HINT, 1); SetBuilder(builder, conf); LOG.debug("test"); LOG.info("Submit log"); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("SplitMerge", conf, builder.createTopology()); Thread.sleep(60000); cluster.killTopology("SplitMerge"); cluster.shutdown(); }
public static void SetRemoteTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException { String streamName = (String) conf.get(Config.TOPOLOGY_NAME); if (streamName == null) { String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); streamName = className[className.length - 1]; } TopologyBuilder builder = new TopologyBuilder(); int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2); builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint); BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint); // localFirstGrouping is only for jstorm // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME); boltDeclarer.shuffleGrouping("spout"); // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60); conf.put(Config.STORM_CLUSTER_MODE, "distributed"); StormSubmitter.submitTopology(streamName, conf, builder.createTopology()); }
private static TopologyBuilder setupBuilder() throws Exception { TopologyBuilder builder = new TopologyBuilder(); int writerParallel = JStormUtils.parseInt(conf.get("topology.producer.parallel"), 1); int spoutParallel = JStormUtils.parseInt(conf.get("topology.consumer.parallel"), 1); builder.setSpout("OnsConsumer", new ConsumerSpout(), spoutParallel); builder.setBolt("OnsProducer", new ProducerBolt(), writerParallel).localFirstGrouping("OnsConsumer"); return builder; }
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException { Map config = new Config(); config.put(ConfigExtension.TOPOLOGY_MASTER_USER_DEFINED_STREAM_CLASS, "com.alipay.dw.jstorm.example.tm.TMUdfHandler"); config.put(Config.TOPOLOGY_WORKERS, 2); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("TMUdfSpout", new TMUdfSpout(), 2); builder.setBolt("TMUdfBolt", new TMUdfBolt(), 4); StormTopology topology = builder.createTopology(); StormSubmitter.submitTopology("TMUdfTopology", config, topology); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new InOrderSpout(), 8); builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1")); conf.setMaxSpoutPending(20); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; if (isLocal) { drpc = new LocalDRPC(); } try { JStormHelper.runTopology(buildTopology(drpc), topologyName, conf, 60, new DrpcValidator(), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new InOrderSpout(), 8); builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1")); conf.setMaxSpoutPending(20); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; if (isLocal) { drpc = new LocalDRPC(); } try { JStormHelper.runTopology(buildDevicesTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new InOrderSpout(), 8); builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1")); conf.setMaxSpoutPending(20); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; if (isLocal) { drpc = new LocalDRPC(); } try { JStormHelper.runTopology(buildVehiclesTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new InOrderSpout(), 8); builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1")); Config conf = new Config(); conf.setMaxSpoutPending(20); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; if (isLocal) { drpc = new LocalDRPC(); } try { JStormHelper.runTopology(buildTopology(drpc), topologyName, conf, 60, new DrpcValidator(), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); int spoutNum = JStormUtils.parseInt(conf.get("spout.num"), 8); int countNum = JStormUtils.parseInt(conf.get("count.num"), 8); builder.setSpout("spout", new InOrderSpout(), spoutNum); builder.setBolt("count", new Check(), countNum).fieldsGrouping("spout", new Fields("c1")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new RandomSentenceSpout(), 5); builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word")); Config conf = new Config(); conf.setDebug(true); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); WordCount wordCountBolt = new WordCount(); builder.setBolt("count", wordCountBolt.sessionTimeWindow(Time.seconds(1L)) .withWindowStateMerger(wordCountBolt), count_Parallelism_hint) .fieldsGrouping("spout", new Fields("word")); //.allGrouping("spout", Common.WATERMARK_STREAM_ID); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); } catch (Exception e) { e.printStackTrace(); } }
public static void test() throws Exception { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); WordCount wordCountBolt = new WordCount(); builder.setBolt("count", wordCountBolt.sessionEventTimeWindow(Time.milliseconds(3L)) .withTimestampExtractor(wordCountBolt) .withWindowStateMerger(wordCountBolt), count_Parallelism_hint) .fieldsGrouping("spout", new Fields("word", "ts")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); }
public static void test() { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int split_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 2); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).localOrShuffleGrouping("spout"); builder.setBolt("count", new WordCount(), count_Parallelism_hint).fieldsGrouping("split", new Fields("word")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; isLocal = JStormHelper.localMode(conf); try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() throws Exception { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); WordCount wordCountBolt = new WordCount(); builder.setBolt("count", wordCountBolt.eventTimeWindow(Time.milliseconds(3L)) .withTimestampExtractor(wordCountBolt) .withWatermarkGenerator(new PeriodicWatermarkGenerator(Time.milliseconds(1L), Time.milliseconds(10L))) , count_Parallelism_hint) .fieldsGrouping("spout", new Fields("word", "ts")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); }
public static void test() throws Exception { int spout_parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int split_parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1); int count_parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_parallelism_hint); builder.setBolt("split", new SplitSentence(), split_parallelism_hint).shuffleGrouping("spout"); builder.setBolt("count", new WordCount() .ingestionTimeWindow(Time.seconds(1L), Time.milliseconds(500L)) .withStateSize(Time.minutes(10)), count_parallelism_hint).fieldsGrouping("split", new Fields("word")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); }