public static void test() { MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH); TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("global-count", "spout", spout, 1); builder.setBolt("partial-count", new BatchCount(), 2).noneGrouping("spout"); builder.setBolt("sum", new UpdateGlobalCount(), 1).globalGrouping("partial-count"); conf.setDebug(true); conf.setMaxSpoutPending(3); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.buildTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
.globalGrouping(masterCoordinator(c.batchGroupId), MasterBatchCoordinator.BATCH_STREAM_ID) .globalGrouping(masterCoordinator(c.batchGroupId), MasterBatchCoordinator.SUCCESS_STREAM_ID);
private void wireTopology() throws InterruptedException { String spoutId = "wordGenerator"; String counterId = "counter"; String intermediateRankerId = "intermediateRanker"; String totalRankerId = "finalRanker"; builder.setSpout(spoutId, new TestWordSpout(), 5); builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word")); builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId, new Fields("obj")); builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId); }
private void wireTopology() throws InterruptedException { String spoutId = "wordGenerator"; String counterId = "counter"; String aggId = "aggregator"; String intermediateRankerId = "intermediateRanker"; String totalRankerId = "finalRanker"; builder.setSpout(spoutId, new TestWordSpout(), 5); builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId, new Fields("word")); builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId, new Fields("obj")); builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(aggId, new Fields("obj")); builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId); }
private static StormTopology buildTopology(AppArgs appArgs) { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(KAFKA_SPOUT_ID, KafkaSpoutFactory.getKafkaSpout(appArgs), 2); builder.setBolt(FILTER_BOLT_ID, new TwitterRawJsonConvertorBolt()) .shuffleGrouping(KAFKA_SPOUT_ID); builder.setBolt(HDFS_BOLT_ID, HdfsBoltBuilder.build(appArgs)) .shuffleGrouping(FILTER_BOLT_ID); builder.setBolt(HBASE_BOLT_ID, HBaseBoltBuilder.build(appArgs, "hbaseConfig")) .shuffleGrouping(FILTER_BOLT_ID); builder.setBolt("USERS_MAX_FOLLOWERS", new UsersWithMaxFollowers()).shuffleGrouping(FILTER_BOLT_ID); builder.setBolt("TOPN_USERS_MAX_FOLLOWERS", new TopNUsersWithMaxFollowers()) .globalGrouping("USERS_MAX_FOLLOWERS"); builder.setBolt("LOCATION_BY_TWEETS", new LocationByTweets()).shuffleGrouping(FILTER_BOLT_ID); builder.setBolt("TOPN_LOCATION_BY_TWEETS", new TopNLocationByTweets()).globalGrouping("LOCATION_BY_TWEETS"); builder.setBolt("MYSQL_WRITER", new MySQLDataWriterBolt(),2) .fieldsGrouping("TOPN_USERS_MAX_FOLLOWERS", new Fields("tableName")) .fieldsGrouping("TOPN_LOCATION_BY_TWEETS", new Fields("tableName")); return builder.createTopology(); }
.globalGrouping("RaceTBTradeCount") .globalGrouping("RaceTmallTradeCount");
builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("count-bolt");
builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("count-bolt");
builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("count-bolt");
break; case GLOBAL: declarer.globalGrouping(stream.getFrom(), streamId); break; case LOCAL_OR_SHUFFLE:
builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("count-bolt");
builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("count-bolt");
@Override public StormTopology buildTopology() { spout.setFields(new Fields(Field.TWEET)); builder.setSpout(Component.SPOUT, spout, spoutThreads); builder.setBolt(Component.TOPIC_EXTRACTOR, new TopicExtractorBolt(), topicExtractorThreads) .shuffleGrouping(Component.SPOUT); builder.setBolt(Component.COUNTER, new RollingCountBolt(counterFrequency), counterThreads) .fieldsGrouping(Component.TOPIC_EXTRACTOR, new Fields(Field.WORD)); builder.setBolt(Component.INTERMEDIATE_RANKER, new IntermediateRankingsBolt(topk, iRankerFrequency), iRankerThreads) .fieldsGrouping(Component.COUNTER, new Fields(Field.OBJ)); builder.setBolt(Component.TOTAL_RANKER, new TotalRankingsBolt(topk, tRankerFrequnecy), tRankerThreads) .globalGrouping(Component.INTERMEDIATE_RANKER); builder.setBolt(Component.SINK, sink, sinkThreads) .shuffleGrouping(Component.TOTAL_RANKER); return builder.createTopology(); }
.globalGrouping(Component.SPOUT); .globalGrouping(Component.SLIDING_WINDOW);
builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("count-bolt");
builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("exclaim2");
@Override public StormTopology buildTopology() { spout.setFields(new Fields(Field.IP, Field.URL, Field.CLIENT_KEY)); builder.setSpout(Component.SPOUT, spout, spoutThreads); // First layer of bolts builder.setBolt(Component.REPEATS, new RepeatVisitBolt(), repeatsThreads) .fieldsGrouping(Component.SPOUT, new Fields(Field.URL, Field.CLIENT_KEY)); builder.setBolt(Component.GEOGRAPHY, new GeographyBolt(), geographyThreads) .shuffleGrouping(Component.SPOUT); // second layer of bolts, commutative in nature builder.setBolt(Component.TOTAL_STATS, new VisitStatsBolt(), totalStatsThreads) .globalGrouping(Component.REPEATS); builder.setBolt(Component.GEO_STATS, new GeoStatsBolt(), geoStatsThreads) .fieldsGrouping(Component.GEOGRAPHY, new Fields(Field.COUNTRY)); // sinks builder.setBolt(Component.SINK_VISIT, visitSink, visitSinkThreads) .shuffleGrouping(Component.TOTAL_STATS); builder.setBolt(Component.SINK_LOCATION, locationSink, locationSinkThreads) .fieldsGrouping(Component.GEO_STATS, new Fields(Field.COUNTRY)); return builder.createTopology(); }
builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("rolling-count-bolt");
builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("rolling-count-bolt");
builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("count-bolt");