/** * Define a new bolt in this topology with parallelism of just one thread. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. * @param bolt the bolt * @return use the returned object to declare the inputs to this component * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public BoltDeclarer setBolt(String id, IRichBolt bolt) throws IllegalArgumentException { return setBolt(id, bolt, null); }
/** * Define a new bolt in this topology. This defines a basic bolt, which is a * simpler to use but more restricted kind of bolt. Basic bolts are intended * for non-aggregation processing and automate the anchoring/acking process to * achieve proper reliability in the topology. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. * @param bolt the basic bolt * @return use the returned object to declare the inputs to this component * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public BoltDeclarer setBolt(String id, IBasicBolt bolt) throws IllegalArgumentException { return setBolt(id, bolt, null); }
public BoltDeclarer setBolt(String id, IControlBolt bolt) { return setBolt(id, bolt, null); }
/** * Define a new bolt in this topology with parallelism of just one thread. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. * @param bolt the bolt * @return use the returned object to declare the inputs to this component */ public BoltDeclarer setBolt(String id, IRichBolt bolt) { return setBolt(id, bolt, null); }
/** * Define a new bolt in this topology. This defines a control bolt, which is a simpler to use but more restricted kind of bolt. Control bolts are intended for * making sending control message more simply * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. * @param bolt the control bolt * @param parallelism_hint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process somwehere around * the cluster. * @return use the returned object to declare the inputs to this component */ public BoltDeclarer setBolt(String id, IControlBolt bolt, Number parallelism_hint) { return setBolt(id, new ControlBoltExecutor(bolt), parallelism_hint); } public BoltDeclarer setBolt(String id, IControlBolt bolt) {
public BoltDeclarer setBolt(String id, IWindowedBolt bolt) throws IllegalArgumentException { return setBolt(id, new backtype.storm.topology.WindowedBoltExecutor(bolt), null); }
/** * Define a new bolt in this topology. This defines a windowed bolt, intended * for windowing operations. The {@link IWindowedBolt#execute(TupleWindow)} method * is triggered for each window interval with the list of current events in the window. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. * @param bolt the windowed bolt * @param parallelism_hint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process somwehere around the cluster. * @return use the returned object to declare the inputs to this component * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public BoltDeclarer setBolt(String id, IWindowedBolt bolt, Number parallelism_hint) throws IllegalArgumentException { return setBolt(id, new backtype.storm.topology.WindowedBoltExecutor(bolt), parallelism_hint); }
/** * Define a new bolt in this topology. This defines a basic bolt, which is a * simpler to use but more restricted kind of bolt. Basic bolts are intended * for non-aggregation processing and automate the anchoring/acking process to * achieve proper reliability in the topology. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. * @param bolt the basic bolt * @param parallelism_hint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process somewhere around the cluster. * @return use the returned object to declare the inputs to this component * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public BoltDeclarer setBolt(String id, IBasicBolt bolt, Number parallelism_hint) throws IllegalArgumentException { return setBolt(id, new BasicBoltExecutor(bolt), parallelism_hint); }
private void wireTopology() throws InterruptedException { String spoutId = "wordGenerator"; String counterId = "counter"; String intermediateRankerId = "intermediateRanker"; String totalRankerId = "finalRanker"; builder.setSpout(spoutId, new TestWordSpout(), 5); builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word")); builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId, new Fields("obj")); builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId); }
private void wireTopology() throws InterruptedException { String spoutId = "wordGenerator"; String counterId = "counter"; String aggId = "aggregator"; String intermediateRankerId = "intermediateRanker"; String totalRankerId = "finalRanker"; builder.setSpout(spoutId, new TestWordSpout(), 5); builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId, new Fields("word")); builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId, new Fields("obj")); builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(aggId, new Fields("obj")); builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId); }
/** * Define a new bolt in this topology. This defines a stateful windowed bolt, intended for stateful * windowing operations. The {@link IStatefulWindowedBolt#execute(TupleWindow)} method is triggered * for each window interval with the list of current events in the window. During initialization of * this bolt {@link IStatefulWindowedBolt#initState(State)} is invoked with its previously saved state. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. * @param bolt the stateful windowed bolt * @param parallelism_hint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process somwehere around the cluster. * @param <T> the type of the state (e.g. {@link backtype.storm.state.KeyValueState}) * @return use the returned object to declare the inputs to this component * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public <T extends State> BoltDeclarer setBolt(String id, IStatefulWindowedBolt<T> bolt, Number parallelism_hint) throws IllegalArgumentException { hasStatefulBolt = true; return setBolt(id, new StatefulBoltExecutor<T>(new StatefulWindowedBoltExecutor<T>(bolt)), parallelism_hint); }
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException { Map config = new Config(); config.put(ConfigExtension.TOPOLOGY_MASTER_USER_DEFINED_STREAM_CLASS, "com.alipay.dw.jstorm.example.tm.TMUdfHandler"); config.put(Config.TOPOLOGY_WORKERS, 2); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("TMUdfSpout", new TMUdfSpout(), 2); builder.setBolt("TMUdfBolt", new TMUdfBolt(), 4); StormTopology topology = builder.createTopology(); StormSubmitter.submitTopology("TMUdfTopology", config, topology); } }
private static TopologyBuilder setupBuilder() throws Exception { TopologyBuilder builder = new TopologyBuilder(); int writerParallel = JStormUtils.parseInt(conf.get("topology.producer.parallel"), 1); int spoutParallel = JStormUtils.parseInt(conf.get("topology.consumer.parallel"), 1); builder.setSpout("OnsConsumer", new ConsumerSpout(), spoutParallel); builder.setBolt("OnsProducer", new ProducerBolt(), writerParallel).localFirstGrouping("OnsConsumer"); return builder; }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new RandomSentenceSpout(), 5); builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word")); Config conf = new Config(); conf.setDebug(true); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new InOrderSpout(), 8); builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1")); conf.setMaxSpoutPending(20); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; if (isLocal) { drpc = new LocalDRPC(); } try { JStormHelper.runTopology(buildTopology(drpc), topologyName, conf, 60, new DrpcValidator(), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); int spoutNum = JStormUtils.parseInt(conf.get("spout.num"), 8); int countNum = JStormUtils.parseInt(conf.get("count.num"), 8); builder.setSpout("spout", new InOrderSpout(), spoutNum); builder.setBolt("count", new Check(), countNum).fieldsGrouping("spout", new Fields("c1")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public BoltDeclarer setBolt(String id, IBasicBolt bolt, int parallel) { CoordinatedBolt coordinatedBolt = new CoordinatedBolt(bolt); BoltDeclarer boltDeclarer = topologyBuilder.setBolt(id, coordinatedBolt, parallel); if (bolt instanceof IPrepareCommit) { boltDeclarer.allGrouping(BatchDef.SPOUT_TRIGGER, BatchDef.PREPARE_STREAM_ID); } if (bolt instanceof ICommitter) { boltDeclarer.allGrouping(BatchDef.SPOUT_TRIGGER, BatchDef.COMMIT_STREAM_ID); boltDeclarer.allGrouping(BatchDef.SPOUT_TRIGGER, BatchDef.REVERT_STREAM_ID); } if (bolt instanceof IPostCommit) { boltDeclarer.allGrouping(BatchDef.SPOUT_TRIGGER, BatchDef.POST_STREAM_ID); } return boltDeclarer; }
public static void test() throws Exception { int spout_parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int split_parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1); int count_parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_parallelism_hint); builder.setBolt("split", new SplitSentence(), split_parallelism_hint).shuffleGrouping("spout"); builder.setBolt("count", new WordCount() .ingestionTimeWindow(Time.seconds(1L), Time.milliseconds(500L)) .withStateSize(Time.minutes(10)), count_parallelism_hint).fieldsGrouping("split", new Fields("word")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); }
public static void test() throws Exception { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); WordCount wordCountBolt = new WordCount(); builder.setBolt("count", wordCountBolt.sessionEventTimeWindow(Time.milliseconds(3L)) .withTimestampExtractor(wordCountBolt) .withWindowStateMerger(wordCountBolt), count_Parallelism_hint) .fieldsGrouping("spout", new Fields("word", "ts")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); }
public static void test() throws Exception { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); WordCount wordCountBolt = new WordCount(); builder.setBolt("count", wordCountBolt.eventTimeWindow(Time.milliseconds(3L)) .withTimestampExtractor(wordCountBolt) .withWatermarkGenerator(new PeriodicWatermarkGenerator(Time.milliseconds(1L), Time.milliseconds(10L))) , count_Parallelism_hint) .fieldsGrouping("spout", new Fields("word", "ts")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); }