private StormTopology mkValidateTopology() { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestWordSpout(true), 3)); Map<String, BoltDetails> boltMap = Collections.singletonMap("2", Thrift.prepareBoltDetails( Collections.singletonMap( Utils.getGlobalStreamId("1", null), Thrift.prepareFieldsGrouping(Collections.singletonList("word"))), new TestWordCounter(), 4)); return Thrift.buildTopology(spoutMap, boltMap); }
private StormTopology mkInvalidateTopology2() { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestWordSpout(true), 3)); Map<String, BoltDetails> boltMap = Collections.singletonMap("2", Thrift.prepareBoltDetails( Collections.singletonMap( Utils.getGlobalStreamId("1", null), Thrift.prepareFieldsGrouping(Collections.singletonList("non-exists-field"))), new TestWordCounter(), 4)); return Thrift.buildTopology(spoutMap, boltMap); }
private StormTopology mkInvalidateTopology1() { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestWordSpout(true), 3)); Map<String, BoltDetails> boltMap = Collections.singletonMap("2", Thrift.prepareBoltDetails( Collections.singletonMap( Utils.getGlobalStreamId("3", null), Thrift.prepareFieldsGrouping(Collections.singletonList("word"))), new TestWordCounter(), 4)); return Thrift.buildTopology(spoutMap, boltMap); }
private StormTopology mkInvalidateTopology3() { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestWordSpout(true), 3)); Map<String, BoltDetails> boltMap = Collections.singletonMap("2", Thrift.prepareBoltDetails( Collections.singletonMap( Utils.getGlobalStreamId("1", "non-exists-stream"), Thrift.prepareFieldsGrouping(Collections.singletonList("word"))), new TestWordCounter(), 4)); return Thrift.buildTopology(spoutMap, boltMap); }
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("word", new TestWordSpout(), 10); builder.setBolt("exclaim1", new ExclamationLoggingBolt(), 3).shuffleGrouping("word"); builder.setBolt("exclaim2", new ExclamationLoggingBolt(), 2).shuffleGrouping("exclaim1"); Config conf = new Config(); conf.setDebug(true); String topoName = MultipleLoggerTopology.class.getName(); if (args != null && args.length > 0) { topoName = args[0]; } conf.setNumWorkers(2); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology()); }
protected int run(String[] args) { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("word", new TestWordSpout(), 10); builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word"); builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1"); conf.setDebug(true); String topologyName = "test"; conf.setNumWorkers(3); if (args != null && args.length > 0) { topologyName = args[0]; } return submit(topologyName, conf, builder); }
.withSimulatedTime() .build()) { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestWordSpout(true), 3));
String intermediateRankerId = "intermediateRanker"; String totalRankerId = "finalRanker"; builder.setSpout(spoutId, new TestWordSpout(), 5); builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word")); builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId,
@Test public void testLocalTransport() throws Exception { Config topoConf = new Config(); topoConf.put(Config.TOPOLOGY_WORKERS, 2); topoConf.put(Config.STORM_MESSAGING_TRANSPORT, "org.apache.storm.messaging.netty.Context"); try (ILocalCluster cluster = new LocalCluster.Builder().withSimulatedTime() .withSupervisors(1).withPortsPerSupervisor(2) .withDaemonConf(topoConf).build()) { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("1", new TestWordSpout(true), 2); builder.setBolt("2", new TestGlobalCount(), 6).shuffleGrouping("1"); StormTopology stormTopology = builder.createTopology(); List<FixedTuple> fixedTuples = new ArrayList<>(); for (int i = 0; i < 12; i++) { fixedTuples.add(new FixedTuple(Collections.singletonList("a"))); fixedTuples.add(new FixedTuple(Collections.singletonList("b"))); } Map<String, List<FixedTuple>> data = new HashMap<>(); data.put("1", fixedTuples); MockedSources mockedSources = new MockedSources(data); CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam(); completeTopologyParam.setMockedSources(mockedSources); Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, stormTopology, completeTopologyParam); Assert.assertEquals(6 * 4, Testing.readTuples(results, "2").size()); } }
builder.setSpout("1", new TestWordSpout(true), 1); builder.setBolt("2", new TestGlobalCount(), 1) .setNumTasks(10)
@Test public void testMultiTasksPerCluster() throws Exception { try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .withSupervisors(4) .build()) { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestWordSpout(true))); Map<String, BoltDetails> boltMap = new HashMap<>(); boltMap.put("2", Thrift.prepareBoltDetails( Collections.singletonMap( Utils.getGlobalStreamId("1", null), Thrift.prepareAllGrouping()), new EmitTaskIdBolt(), 3, Collections.singletonMap(Config.TOPOLOGY_TASKS, 6))); StormTopology topology = Thrift.buildTopology(spoutMap, boltMap); MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", Collections.singletonList(new FixedTuple(new Values("a"))))); CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam(); completeTopologyParams.setMockedSources(mockedSources); Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topology, completeTopologyParams); assertThat(Testing.readTuples(results, "2"), containsInAnyOrder( new Values(0), new Values(1), new Values(2), new Values(3), new Values(4), new Values(5) )); } }
String intermediateRankerId = "intermediateRanker"; String totalRankerId = "finalRanker"; builder.setSpout(spoutId, new TestWordSpout(), 5); builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId, new Fields("word")); builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId, new Fields("obj"));
public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); StreamState<String, Long> ss = builder.newStream(new TestWordSpout(), new ValueMapper<String>(0), 2)
@Test public void testMemoryLoadLargerThanMaxHeapSize() throws Exception { // Topology will not be able to be successfully scheduled: Config TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB=128.0 < 129.0, // Largest memory requirement of a component in the topology). TopologyBuilder builder1 = new TopologyBuilder(); builder1.setSpout("wordSpout1", new TestWordSpout(), 4); StormTopology stormTopology1 = builder1.createTopology(); Config config1 = new Config(); config1.put(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN, "org.apache.storm.networktopography.DefaultRackDNSToSwitchMapping"); config1.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, DefaultSchedulingPriorityStrategy.class.getName()); config1.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, DefaultResourceAwareStrategy.class.getName()); config1.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 10.0); config1.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 0.0); config1.put(Config.TOPOLOGY_PRIORITY, 0); config1.put(Config.TOPOLOGY_SUBMITTER_USER, "zhuo"); config1.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 128.0); config1.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 129.0); try { Nimbus.validateTopologyWorkerMaxHeapSizeConfigs(config1, stormTopology1, 768.0); fail("Expected exception not thrown"); } catch (IllegalArgumentException e) { //Expected... } }
builder1.setSpout("wordSpout1", new TestWordSpout(), 1); builder1.setSpout("wordSpout2", new TestWordSpout(), 1); builder1.setBolt("wordCountBolt1", new TestWordCounter(), 1).shuffleGrouping("wordSpout1").shuffleGrouping("wordSpout2"); builder1.setBolt("wordCountBolt2", new TestWordCounter(), 1).shuffleGrouping("wordCountBolt1"); builder2.setSpout("wordSpoutX", new TestWordSpout(), 1); builder2.setSpout("wordSpoutY", new TestWordSpout(), 1); StormTopology stormTopology2 = builder2.createTopology(); Map<ExecutorDetails, String> executorMap2 = genExecsAndComps(stormTopology2);
public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); builder.newStream(new TestWordSpout(), new ValueMapper<String>(0), 2) .window(TumblingWindows.of(BaseWindowedBolt.Duration.seconds(2)))
builder1.setSpout("wordSpout", new TestWordSpout(), 1).setCPULoad(20.0).setMemoryLoad(200.0); builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(20.0).setMemoryLoad(200.0); StormTopology stormTopology1 = builder1.createTopology();
public static StormTopology createTestTopology() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("words", new TestWordSpout(), generateParallelismHint()); builder.setBolt("count", new TestWordCounter(), generateParallelismHint()).shuffleGrouping("words"); builder.setBolt("globalCount", new TestGlobalCount(), generateParallelismHint()).shuffleGrouping("count"); return builder.createTopology(); }
builder.newStream(new TestWordSpout(), new ValueMapper<String>(0))
builder1.setSpout("wordSpout1", new TestWordSpout(), 4); StormTopology stormTopology1 = builder1.createTopology(); Config config1 = new Config(); builder2.setSpout("wordSpout2", new TestWordSpout(), 5); StormTopology stormTopology2 = builder2.createTopology(); Config config2 = new Config();