@Override protected StormTopology getTopologyKafkaSpout(KafkaSpoutConfig<String, String> spoutConfig) { final TopologyBuilder tp = new TopologyBuilder(); tp.setSpout("kafka_spout", new KafkaSpout<>(spoutConfig), 1); tp.setBolt("kafka_bolt", new KafkaSpoutTestBolt()).shuffleGrouping("kafka_spout", STREAM); return tp.createTopology(); }
static StormTopology getTopology() { // 1 - Setup Const Spout -------- ConstSpout spout = new ConstSpout("some data").withOutputFields("str"); // 2 - Setup Topology -------- TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, spout, 1); return builder.createTopology(); }
protected int run(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new RandomSentenceSpout(), 4); builder.setBolt("split", new SplitSentence(), 4).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), 4).fieldsGrouping("split", new Fields("word")); Config conf = new Config(); conf.setMaxTaskParallelism(3); String topologyName = "word-count"; conf.setNumWorkers(3); if (args != null && args.length > 0) { topologyName = args[0]; } return submit(topologyName, conf, builder); }
builder.setSpout(QUERY_COMPONENT, new QuerySpout(config), querySpoutParallelism) .setCPULoad(querySpoutCPULoad).setMemoryLoad(querySpoutMemoryOnHeapLoad, querySpoutMemoryOffHeapLoad); builder.setSpout(TICK_COMPONENT, new TickSpout(config), tickSpoutParallelism) .setCPULoad(tickSpoutCPULoad).setMemoryLoad(tickSpoutMemoryOnheapLoad, tickSpoutMemoryOffHeapLoad); builder.setBolt(FILTER_COMPONENT, new FilterBolt(recordComponent, config), filterBoltParallelism) .shuffleGrouping(recordComponent) .allGrouping(QUERY_COMPONENT, QUERY_STREAM) builder.setBolt(JOIN_COMPONENT, new JoinBolt(config), joinBoltParallelism) .fieldsGrouping(QUERY_COMPONENT, QUERY_STREAM, new Fields(ID_FIELD)) .fieldsGrouping(QUERY_COMPONENT, METADATA_STREAM, new Fields(ID_FIELD)) .fieldsGrouping(FILTER_COMPONENT, DATA_STREAM, new Fields(ID_FIELD)) .allGrouping(TICK_COMPONENT, TICK_STREAM) .setCPULoad(joinBoltCPULoad).setMemoryLoad(joinBoltMemoryOnHeapLoad, joinBoltMemoryOffHeapLoad); builder.setBolt(TopologyConstants.RESULT_COMPONENT, new ResultBolt(config), resultBoltParallelism) .shuffleGrouping(JOIN_COMPONENT, RESULT_STREAM) .setCPULoad(resultBoltCPULoad).setMemoryLoad(resultBoltMemoryOnHeapLoad, resultBoltMemoryOffHeapLoad); log.info("Windowing is disabled. Skipping hooking in the Loop Bolt..."); } else { builder.setBolt(LOOP_COMPONENT, new LoopBolt(config), loopBoltParallelism) .shuffleGrouping(JOIN_COMPONENT, FEEDBACK_STREAM) .setCPULoad(loopBoltCPULoad).setMemoryLoad(loopBoltMemoryOnHeapLoad, loopBoltMemoryOffHeapLoad); StormSubmitter.submitTopology(name, stormConfig, builder.createTopology());
private GeneralTopologyContext getContext(final Fields fields) { TopologyBuilder builder = new TopologyBuilder(); return new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap(), new HashMap(), new HashMap(), "") { @Override public Fields getComponentOutputFields(String componentId, String streamId) { return fields; } }; }
if (boltObj instanceof IRichBolt) { if (declarer == null) { declarer = builder.setBolt(stream.getTo(), (IRichBolt) boltObj, topologyDef.parallelismForBolt(stream.getTo())); declarer = builder.setBolt( stream.getTo(), (IBasicBolt) boltObj, declarer = builder.setBolt( stream.getTo(), (IWindowedBolt) boltObj, declarer = builder.setBolt( stream.getTo(), (IStatefulBolt) boltObj, case FIELDS: declarer.fieldsGrouping(stream.getFrom(), streamId, new Fields(grouping.getArgs())); break; case ALL:
/** * Define a new bolt in this topology with parallelism of just one thread. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. * @param bolt the bolt * @return use the returned object to declare the inputs to this component * * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public BoltDeclarer setBolt(String id, IRichBolt bolt) throws IllegalArgumentException { return setBolt(id, bolt, null); }
/** * Define a new spout in this topology. * * @param id the id of this component. This id is referenced by other components that want to consume this spout's outputs. * @param spout the spout * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public SpoutDeclarer setSpout(String id, IRichSpout spout) throws IllegalArgumentException { return setSpout(id, spout, null); }
/** * Submits the topology under a specific name **/ protected int submit(String name, Config conf, TopologyBuilder builder) { try { StormSubmitter.submitTopology(name, conf, builder.createTopology()); } catch (Exception e) { e.printStackTrace(); return -1; } return 0; }
builder.setSpout(QUERY_COMPONENT, new QuerySpout(config), querySpoutParallelism) .setCPULoad(querySpoutCPULoad).setMemoryLoad(querySpoutMemoryOnHeapLoad, querySpoutMemoryOffHeapLoad); builder.setSpout(TICK_COMPONENT, new TickSpout(config), tickSpoutParallelism) .setCPULoad(tickSpoutCPULoad).setMemoryLoad(tickSpoutMemoryOnheapLoad, tickSpoutMemoryOffHeapLoad); builder.setBolt(FILTER_COMPONENT, new FilterBolt(recordComponent, config), filterBoltParallelism) .shuffleGrouping(recordComponent) .allGrouping(QUERY_COMPONENT, QUERY_STREAM) builder.setBolt(JOIN_COMPONENT, new JoinBolt(config), joinBoltParallelism) .fieldsGrouping(QUERY_COMPONENT, QUERY_STREAM, new Fields(ID_FIELD)) .fieldsGrouping(QUERY_COMPONENT, METADATA_STREAM, new Fields(ID_FIELD)) .fieldsGrouping(FILTER_COMPONENT, DATA_STREAM, new Fields(ID_FIELD)) .allGrouping(TICK_COMPONENT, TICK_STREAM) .setCPULoad(joinBoltCPULoad).setMemoryLoad(joinBoltMemoryOnHeapLoad, joinBoltMemoryOffHeapLoad); builder.setBolt(TopologyConstants.RESULT_COMPONENT, new ResultBolt(config), resultBoltParallelism) .shuffleGrouping(JOIN_COMPONENT, RESULT_STREAM) .setCPULoad(resultBoltCPULoad).setMemoryLoad(resultBoltMemoryOnHeapLoad, resultBoltMemoryOffHeapLoad); log.info("Windowing is disabled. Skipping hooking in the Loop Bolt..."); } else { builder.setBolt(LOOP_COMPONENT, new LoopBolt(config), loopBoltParallelism) .shuffleGrouping(JOIN_COMPONENT, FEEDBACK_STREAM) .setCPULoad(loopBoltCPULoad).setMemoryLoad(loopBoltMemoryOnHeapLoad, loopBoltMemoryOffHeapLoad); StormSubmitter.submitTopology(name, stormConfig, builder.createTopology());
private Tuple generateTestTuple(Object id, Object msg, Object city, Object state) { TopologyBuilder builder = new TopologyBuilder(); GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap(), new HashMap(), new HashMap(), "") { @Override public Fields getComponentOutputFields(String componentId, String streamId) { return new Fields("id", "msg", "city", "state"); } }; return new TupleImpl(topologyContext, new Values(id, msg, city, state), "", 1, ""); }
@Override public void setBolts(TopologyBuilder builder) { BoltDeclarer boltDeclarer = builder.setBolt("split", new SplitStreamBolt(), config.boltThreads); if (config.localShuffle) { boltDeclarer.localOrShuffleGrouping("spout"); } else { boltDeclarer.shuffleGrouping("spout"); } builder.setBolt("count", new WordCountBolt(config), config.boltThreads).fieldsGrouping("split", new Fields("ip")); }
/** * Define a new bolt in this topology. This defines a windowed bolt, intended for windowing operations. The {@link * IWindowedBolt#execute(TupleWindow)} method is triggered for each window interval with the list of current events in the window. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. * @param bolt the windowed bolt * @return use the returned object to declare the inputs to this component * * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public BoltDeclarer setBolt(String id, IWindowedBolt bolt) throws IllegalArgumentException { return setBolt(id, bolt, null); }
/** * Define a new spout in this topology. * * @param id the id of this component. This id is referenced by other components that want to consume this spout's outputs. * @param supplier lambda expression that implements tuple generating for this spout * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public SpoutDeclarer setSpout(String id, SerializableSupplier<?> supplier) throws IllegalArgumentException { return setSpout(id, supplier, null); }
public static StormTopology buildTopology(int numSpout, int numBolt, int spoutParallelism, int boltParallelism) { return topologyBuilder(numSpout, numBolt, spoutParallelism, boltParallelism).createTopology(); }
public static StormTopology getStormTopology() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(WORD, new FixedOrderWordSpout(), SPOUT_EXECUTORS); builder.setBolt(EXCLAIM_1, new ExclamationTopology.ExclamationBolt(), 3).shuffleGrouping(WORD); builder.setBolt(EXCLAIM_2, new ExclamationTopology.ExclamationBolt(), EXCLAIM_2_EXECUTORS).shuffleGrouping(EXCLAIM_1); return builder.createTopology(); } }
private Tuple generateTestTuple(Object id, Object msg) { TopologyBuilder builder = new TopologyBuilder(); GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap(), new HashMap(), new HashMap(), "") { @Override public Fields getComponentOutputFields(String componentId, String streamId) { return new Fields("id", "msg"); } }; return new TupleImpl(topologyContext, new Values(id, msg), "", 1, ""); }
@Override public void setBolts(TopologyBuilder builder) { Duration windowDuration = new Duration((int) config.windowDuration, TimeUnit.MILLISECONDS); Duration windowSlide = new Duration((int) config.windowSlideStep, TimeUnit.MILLISECONDS); BoltDeclarer boltDeclarer = builder.setBolt("parser", new ParserBolt(), config.boltThreads); if (config.localShuffle) { boltDeclarer.localOrShuffleGrouping("spout"); } else { boltDeclarer.shuffleGrouping("spout"); } builder.setBolt("window", new SlidingWindowBolt(config) .withWindow(windowDuration, windowSlide), config.boltThreads).fieldsGrouping("parser", new Fields("ip")); }
/** * Define a new bolt in this topology. This defines a basic bolt, which is a simpler to use but more restricted kind of bolt. Basic * bolts are intended for non-aggregation processing and automate the anchoring/acking process to achieve proper reliability in the * topology. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. * @param bolt the basic bolt * @return use the returned object to declare the inputs to this component * * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public BoltDeclarer setBolt(String id, IBasicBolt bolt) throws IllegalArgumentException { return setBolt(id, bolt, null); }
/** * If the topology has at least one stateful bolt add a {@link CheckpointSpout} component to the topology. */ private void maybeAddCheckpointSpout() { if (hasStatefulBolt) { setSpout(CHECKPOINT_COMPONENT_ID, new CheckpointSpout(), 1); } }