private StormTopology mkValidateTopology() { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestWordSpout(true), 3)); Map<String, BoltDetails> boltMap = Collections.singletonMap("2", Thrift.prepareBoltDetails( Collections.singletonMap( Utils.getGlobalStreamId("1", null), Thrift.prepareFieldsGrouping(Collections.singletonList("word"))), new TestWordCounter(), 4)); return Thrift.buildTopology(spoutMap, boltMap); }
private StormTopology mkInvalidateTopology2() { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestWordSpout(true), 3)); Map<String, BoltDetails> boltMap = Collections.singletonMap("2", Thrift.prepareBoltDetails( Collections.singletonMap( Utils.getGlobalStreamId("1", null), Thrift.prepareFieldsGrouping(Collections.singletonList("non-exists-field"))), new TestWordCounter(), 4)); return Thrift.buildTopology(spoutMap, boltMap); }
private StormTopology mkInvalidateTopology1() { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestWordSpout(true), 3)); Map<String, BoltDetails> boltMap = Collections.singletonMap("2", Thrift.prepareBoltDetails( Collections.singletonMap( Utils.getGlobalStreamId("3", null), Thrift.prepareFieldsGrouping(Collections.singletonList("word"))), new TestWordCounter(), 4)); return Thrift.buildTopology(spoutMap, boltMap); }
private StormTopology mkInvalidateTopology3() { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestWordSpout(true), 3)); Map<String, BoltDetails> boltMap = Collections.singletonMap("2", Thrift.prepareBoltDetails( Collections.singletonMap( Utils.getGlobalStreamId("1", "non-exists-stream"), Thrift.prepareFieldsGrouping(Collections.singletonList("word"))), new TestWordCounter(), 4)); return Thrift.buildTopology(spoutMap, boltMap); }
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("word", new TestWordSpout(), 10); builder.setBolt("exclaim1", new ExclamationLoggingBolt(), 3).shuffleGrouping("word"); builder.setBolt("exclaim2", new ExclamationLoggingBolt(), 2).shuffleGrouping("exclaim1"); Config conf = new Config(); conf.setDebug(true); String topoName = MultipleLoggerTopology.class.getName(); if (args != null && args.length > 0) { topoName = args[0]; } conf.setNumWorkers(2); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology()); }
protected int run(String[] args) { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("word", new TestWordSpout(), 10); builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word"); builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1"); conf.setDebug(true); String topologyName = "test"; conf.setNumWorkers(3); if (args != null && args.length > 0) { topologyName = args[0]; } return submit(topologyName, conf, builder); }
.withSimulatedTime() .build()) { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestWordSpout(true), 3));
builder1.setSpout("wordSpout", new TestWordSpout(), 2).setCPULoad(250.0).setMemoryLoad(1000.0, 200.0); builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(100.0) .setMemoryLoad(500.0, 100.0);
String intermediateRankerId = "intermediateRanker"; String totalRankerId = "finalRanker"; builder.setSpout(spoutId, new TestWordSpout(), 5); builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word")); builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId,
@Test public void testLocalTransport() throws Exception { Config topoConf = new Config(); topoConf.put(Config.TOPOLOGY_WORKERS, 2); topoConf.put(Config.STORM_MESSAGING_TRANSPORT, "org.apache.storm.messaging.netty.Context"); try (ILocalCluster cluster = new LocalCluster.Builder().withSimulatedTime() .withSupervisors(1).withPortsPerSupervisor(2) .withDaemonConf(topoConf).build()) { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("1", new TestWordSpout(true), 2); builder.setBolt("2", new TestGlobalCount(), 6).shuffleGrouping("1"); StormTopology stormTopology = builder.createTopology(); List<FixedTuple> fixedTuples = new ArrayList<>(); for (int i = 0; i < 12; i++) { fixedTuples.add(new FixedTuple(Collections.singletonList("a"))); fixedTuples.add(new FixedTuple(Collections.singletonList("b"))); } Map<String, List<FixedTuple>> data = new HashMap<>(); data.put("1", fixedTuples); MockedSources mockedSources = new MockedSources(data); CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam(); completeTopologyParam.setMockedSources(mockedSources); Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, stormTopology, completeTopologyParam); Assert.assertEquals(6 * 4, Testing.readTuples(results, "2").size()); } }
builder.setSpout("1", new TestWordSpout(true), 1); builder.setBolt("2", new TestGlobalCount(), 1) .setNumTasks(10)
@Test public void testMultiTasksPerCluster() throws Exception { try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .withSupervisors(4) .build()) { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestWordSpout(true))); Map<String, BoltDetails> boltMap = new HashMap<>(); boltMap.put("2", Thrift.prepareBoltDetails( Collections.singletonMap( Utils.getGlobalStreamId("1", null), Thrift.prepareAllGrouping()), new EmitTaskIdBolt(), 3, Collections.singletonMap(Config.TOPOLOGY_TASKS, 6))); StormTopology topology = Thrift.buildTopology(spoutMap, boltMap); MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", Collections.singletonList(new FixedTuple(new Values("a"))))); CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam(); completeTopologyParams.setMockedSources(mockedSources); Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topology, completeTopologyParams); assertThat(Testing.readTuples(results, "2"), containsInAnyOrder( new Values(0), new Values(1), new Values(2), new Values(3), new Values(4), new Values(5) )); } }
String intermediateRankerId = "intermediateRanker"; String totalRankerId = "finalRanker"; builder.setSpout(spoutId, new TestWordSpout(), 5); builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId, new Fields("word")); builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId, new Fields("obj"));
.withDaemonConf(Collections.singletonMap(Config.STORM_LOCAL_MODE_ZMQ, !useLocalMessaging)) .build()) { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestWordSpout(true), 3)); Map<String, BoltDetails> boltMap = new HashMap<>(); boltMap.put("2",
public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); StreamState<String, Long> ss = builder.newStream(new TestWordSpout(), new ValueMapper<String>(0), 2)
builder1.setSpout("wordSpout1", new TestWordSpout(), 1).setCPULoad(300.0).setMemoryLoad(2000.0, 48.0); StormTopology stormTopology1 = builder1.createTopology(); Config config1 = new Config(); builder2.setSpout("wordSpout2", new TestWordSpout(), 4).setCPULoad(100.0).setMemoryLoad(500.0, 12.0); StormTopology stormTopology2 = builder2.createTopology(); Config config2 = new Config(); builder3.setSpout("wordSpout3", new TestWordSpout(), 4).setCPULoad(20.0).setMemoryLoad(200.0, 56.0); StormTopology stormTopology3 = builder3.createTopology(); Config config3 = new Config(); builder4.setSpout("wordSpout4", new TestWordSpout(), 12).setCPULoad(30.0).setMemoryLoad(100.0, 0.0); StormTopology stormTopology4 = builder4.createTopology(); Config config4 = new Config(); builder5.setSpout("wordSpout5", new TestWordSpout(), 40).setCPULoad(25.0).setMemoryLoad(100.0, 28.0); StormTopology stormTopology5 = builder5.createTopology(); Config config5 = new Config();
@Test public void testMemoryLoadLargerThanMaxHeapSize() throws Exception { // Topology will not be able to be successfully scheduled: Config TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB=128.0 < 129.0, // Largest memory requirement of a component in the topology). TopologyBuilder builder1 = new TopologyBuilder(); builder1.setSpout("wordSpout1", new TestWordSpout(), 4); StormTopology stormTopology1 = builder1.createTopology(); Config config1 = new Config(); config1.put(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN, "org.apache.storm.networktopography.DefaultRackDNSToSwitchMapping"); config1.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, DefaultSchedulingPriorityStrategy.class.getName()); config1.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, DefaultResourceAwareStrategy.class.getName()); config1.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 10.0); config1.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 0.0); config1.put(Config.TOPOLOGY_PRIORITY, 0); config1.put(Config.TOPOLOGY_SUBMITTER_USER, "zhuo"); config1.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 128.0); config1.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 129.0); try { Nimbus.validateTopologyWorkerMaxHeapSizeConfigs(config1, stormTopology1, 768.0); fail("Expected exception not thrown"); } catch (IllegalArgumentException e) { //Expected... } }
builder1.setSpout("wordSpout1", new TestWordSpout(), 1); builder1.setSpout("wordSpout2", new TestWordSpout(), 1); builder1.setBolt("wordCountBolt1", new TestWordCounter(), 1).shuffleGrouping("wordSpout1").shuffleGrouping("wordSpout2"); builder1.setBolt("wordCountBolt2", new TestWordCounter(), 1).shuffleGrouping("wordCountBolt1"); builder2.setSpout("wordSpoutX", new TestWordSpout(), 1); builder2.setSpout("wordSpoutY", new TestWordSpout(), 1); StormTopology stormTopology2 = builder2.createTopology(); Map<ExecutorDetails, String> executorMap2 = genExecsAndComps(stormTopology2);
builder1.setSpout("wordSpout1", new TestWordSpout(), 3); StormTopology stormTopology1 = builder1.createTopology(); Config config1 = new Config(); builder2.setSpout("wordSpout2", new TestWordSpout(), 2); StormTopology stormTopology2 = builder2.createTopology(); Config config2 = new Config();
public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); builder.newStream(new TestWordSpout(), new ValueMapper<String>(0), 2) .window(TumblingWindows.of(BaseWindowedBolt.Duration.seconds(2)))