/** * add mock data for the spout. * * @param spoutId the spout to be mocked * @param streamId the stream of the spout to be mocked * @param valueses the mocked data */ public void addMockData(String spoutId, String streamId, Values... valueses) { if (!data.containsKey(spoutId)) { data.put(spoutId, new ArrayList<FixedTuple>()); } List<FixedTuple> tuples = data.get(spoutId); for (int i = 0; i < valueses.length; i++) { FixedTuple tuple = new FixedTuple(streamId, valueses[i]); tuples.add(tuple); } }
public FixedTupleSpout(List tuples, Fields fields) { _id = UUID.randomUUID().toString(); synchronized (acked) { acked.put(_id, 0); } synchronized (failed) { failed.put(_id, 0); } _tuples = new ArrayList<FixedTuple>(); for (Object o : tuples) { FixedTuple ft; if (o instanceof FixedTuple) { ft = (FixedTuple) o; } else { ft = new FixedTuple((List) o); } _tuples.add(ft); } _fields = fields; }
public void execute(Tuple input) { String component = input.getSourceComponent(); Map<String, List<FixedTuple>> captured = emitted_tuples.get(_name); if (!captured.containsKey(component)) { captured.put(component, new ArrayList<FixedTuple>()); } captured.get(component).add(new FixedTuple(input.getSourceStreamId(), input.getValues())); _collector.ack(input); }
private boolean tryCompleteWordCountTopology(LocalCluster cluster, StormTopology topology) throws Exception { try { List<FixedTuple> testTuples = Arrays.asList("nathan", "bob", "joey", "nathan").stream() .map(value -> new FixedTuple(new Values(value))) .collect(Collectors.toList()); MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples)); CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam(); completeTopologyParam.setMockedSources(mockedSources); completeTopologyParam.setStormConf(Collections.singletonMap(Config.TOPOLOGY_WORKERS, 2)); Testing.completeTopology(cluster, topology, completeTopologyParam); return false; } catch (InvalidTopologyException e) { return true; } }
@Test public void testKryoDecoratorsConfig() throws Exception { Map<String, Object> daemonConf = new HashMap<>(); daemonConf.put(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS, true); daemonConf.put(Config.TOPOLOGY_KRYO_DECORATORS, "this-is-overridden"); try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .withDaemonConf(daemonConf) .build()) { TopologyBuilder topologyBuilder = new TopologyBuilder(); topologyBuilder.setSpout("1", new TestPlannerSpout(new Fields("conf"))); topologyBuilder.setBolt("2", new TestConfBolt(Collections.singletonMap(Config.TOPOLOGY_KRYO_DECORATORS, Arrays.asList("one", "two")))) .shuffleGrouping("1"); List<FixedTuple> testTuples = Arrays.asList(new Values(Config.TOPOLOGY_KRYO_DECORATORS)).stream() .map(value -> new FixedTuple(value)) .collect(Collectors.toList()); MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples)); CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam(); completeTopologyParams.setMockedSources(mockedSources); completeTopologyParams.setStormConf(Collections.singletonMap(Config.TOPOLOGY_KRYO_DECORATORS, Arrays.asList("one", "three"))); Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topologyBuilder.createTopology(), completeTopologyParams); List<Object> concatValues = Testing.readTuples(results, "2").stream() .flatMap(values -> values.stream()) .collect(Collectors.toList()); assertThat(concatValues.get(0), is(Config.TOPOLOGY_KRYO_DECORATORS)); assertThat(concatValues.get(1), is(Arrays.asList("one", "two", "three"))); } }
.map(value -> new FixedTuple(new Values(value))) .collect(Collectors.toList()); Map<String, String> kryoRegister = new HashMap<>();
@Test public void testHooks() throws Exception { try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .build()) { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(new Fields("conf")))); Map<String, BoltDetails> boltMap = Collections.singletonMap("2", Thrift.prepareBoltDetails( Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new HooksBolt())); StormTopology topology = Thrift.buildTopology(spoutMap, boltMap); List<FixedTuple> testTuples = Arrays.asList(1, 1, 1, 1).stream() .map(value -> new FixedTuple(new Values(value))) .collect(Collectors.toList()); MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples)); CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam(); completeTopologyParams.setMockedSources(mockedSources); Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topology, completeTopologyParams); List<List<Object>> expectedTuples = Arrays.asList( Arrays.asList(0, 0, 0, 0), Arrays.asList(2, 1, 0, 1), Arrays.asList(4, 1, 1, 2), Arrays.asList(6, 2, 1, 3)); assertThat(Testing.readTuples(results, "2"), is(expectedTuples)); } }
.map(value -> new FixedTuple(new Values(value))) .collect(Collectors.toList());
@Test public void testLocalTransport() throws Exception { Config topoConf = new Config(); topoConf.put(Config.TOPOLOGY_WORKERS, 2); topoConf.put(Config.STORM_MESSAGING_TRANSPORT, "org.apache.storm.messaging.netty.Context"); try (ILocalCluster cluster = new LocalCluster.Builder().withSimulatedTime() .withSupervisors(1).withPortsPerSupervisor(2) .withDaemonConf(topoConf).build()) { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("1", new TestWordSpout(true), 2); builder.setBolt("2", new TestGlobalCount(), 6).shuffleGrouping("1"); StormTopology stormTopology = builder.createTopology(); List<FixedTuple> fixedTuples = new ArrayList<>(); for (int i = 0; i < 12; i++) { fixedTuples.add(new FixedTuple(Collections.singletonList("a"))); fixedTuples.add(new FixedTuple(Collections.singletonList("b"))); } Map<String, List<FixedTuple>> data = new HashMap<>(); data.put("1", fixedTuples); MockedSources mockedSources = new MockedSources(data); CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam(); completeTopologyParam.setMockedSources(mockedSources); Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, stormTopology, completeTopologyParam); Assert.assertEquals(6 * 4, Testing.readTuples(results, "2").size()); } }
fixedTuples.add(new FixedTuple(Collections.singletonList("a"))); fixedTuples.add(new FixedTuple(Collections.singletonList("b")));
@Test public void testMultiTasksPerCluster() throws Exception { try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .withSupervisors(4) .build()) { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestWordSpout(true))); Map<String, BoltDetails> boltMap = new HashMap<>(); boltMap.put("2", Thrift.prepareBoltDetails( Collections.singletonMap( Utils.getGlobalStreamId("1", null), Thrift.prepareAllGrouping()), new EmitTaskIdBolt(), 3, Collections.singletonMap(Config.TOPOLOGY_TASKS, 6))); StormTopology topology = Thrift.buildTopology(spoutMap, boltMap); MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", Collections.singletonList(new FixedTuple(new Values("a"))))); CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam(); completeTopologyParams.setMockedSources(mockedSources); Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topology, completeTopologyParams); assertThat(Testing.readTuples(results, "2"), containsInAnyOrder( new Values(0), new Values(1), new Values(2), new Values(3), new Values(4), new Values(5) )); } }
.map(value -> new FixedTuple(new Values(value))) .collect(Collectors.toList());
/** * add mock data for the spout. * * @param spoutId the spout to be mocked * @param streamId the stream of the spout to be mocked * @param valueses the mocked data */ public void addMockData(String spoutId, String streamId, Values... valueses) { if (!data.containsKey(spoutId)) { data.put(spoutId, new ArrayList<FixedTuple>()); } List<FixedTuple> tuples = data.get(spoutId); for (int i = 0; i < valueses.length; i++) { FixedTuple tuple = new FixedTuple(streamId, valueses[i]); tuples.add(tuple); } }
public FixedTupleSpout(List tuples, Fields fields) { _id = UUID.randomUUID().toString(); synchronized(acked) { acked.put(_id, 0); } synchronized(failed) { failed.put(_id, 0); } _tuples = new ArrayList<FixedTuple>(); for(Object o: tuples) { FixedTuple ft; if(o instanceof FixedTuple) { ft = (FixedTuple) o; } else { ft = new FixedTuple((List) o); } _tuples.add(ft); } _fields = fields; }
public void execute(Tuple input) { String component = input.getSourceComponent(); Map<String, List<FixedTuple>> captured = emitted_tuples.get(_name); if(!captured.containsKey(component)) { captured.put(component, new ArrayList<FixedTuple>()); } captured.get(component).add(new FixedTuple(input.getSourceStreamId(), input.getValues())); _collector.ack(input); }