private static boolean isEmptyInputs(ComponentCommon common) { if (common.get_inputs() == null) { return true; } else { return common.get_inputs().isEmpty(); } }
private Set<String> getInputsTo(ComponentCommon comp) { Set<String> ret = new HashSet<>(); for (GlobalStreamId globalId : comp.get_inputs().keySet()) { ret.add(globalId.get_componentId()); } return ret; }
/** * Gets the declared inputs to the specified component. * * @return A map from subscribed component/stream to the grouping subscribed with. */ public Map<GlobalStreamId, Grouping> getSources(String componentId) { return getComponentCommon(componentId).get_inputs(); }
public java.lang.Object getFieldValue(_Fields field) { switch (field) { case INPUTS: return get_inputs(); case STREAMS: return get_streams(); case PARALLELISM_HINT: return get_parallelism_hint(); case JSON_CONF: return get_json_conf(); } throw new java.lang.IllegalStateException(); }
/** * Gets information about who is consuming the outputs of the specified component, and how. * * @return Map from stream id to component id to the Grouping used. */ public Map<String, Map<String, Grouping>> getTargets(String componentId) { Map<String, Map<String, Grouping>> ret = new HashMap<>(); for (String otherComponentId : getComponentIds()) { Map<GlobalStreamId, Grouping> inputs = getComponentCommon(otherComponentId).get_inputs(); for (Map.Entry<GlobalStreamId, Grouping> entry : inputs.entrySet()) { GlobalStreamId id = entry.getKey(); if (id.get_componentId().equals(componentId)) { Map<String, Grouping> curr = ret.get(id.get_streamId()); if (curr == null) { curr = new HashMap<>(); } curr.put(otherComponentId, entry.getValue()); ret.put(id.get_streamId(), curr); } } } return ret; }
/** * For bolts that has incoming streams from spouts (the root bolts), add checkpoint stream from checkpoint spout to its input. For other * bolts, add checkpoint stream from the previous bolt to its input. */ private void addCheckPointInputs(ComponentCommon component) { Set<GlobalStreamId> checkPointInputs = new HashSet<>(); for (GlobalStreamId inputStream : component.get_inputs().keySet()) { String sourceId = inputStream.get_componentId(); if (_spouts.containsKey(sourceId)) { checkPointInputs.add(new GlobalStreamId(CHECKPOINT_COMPONENT_ID, CHECKPOINT_STREAM_ID)); } else { checkPointInputs.add(new GlobalStreamId(sourceId, CHECKPOINT_STREAM_ID)); } } for (GlobalStreamId streamId : checkPointInputs) { component.put_to_inputs(streamId, Grouping.all(new NullStruct())); } }
spoutSpecMapEntry.getValue().get_common().get_inputs().entrySet().stream().map( UIHelpers::getInputMap).collect(Collectors.toList()) ); boltEntry.getValue().get_common().get_inputs().entrySet().stream().map( UIHelpers::getInputMap).collect(Collectors.toList()) );
String componentId = entry.getKey(); ComponentCommon common = getComponentCommon(entry.getValue()); Map<GlobalStreamId, Grouping> inputs = common.get_inputs(); for (Map.Entry<GlobalStreamId, Grouping> input : inputs.entrySet()) { String sourceStreamId = input.getKey().get_streamId();
Bolt bolt = boltSpec.getValue(); ComponentCommon common = bolt.get_common(); Map<GlobalStreamId, Grouping> inputs = common.get_inputs(); if (inputs != null) { for (Map.Entry<GlobalStreamId, Grouping> input : inputs.entrySet()) {
@Test public void testGroupBy() throws Exception { PairStream<String, String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new PairValueMapper<>(0, 1), 2); stream.window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))).aggregateByKey(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); assertEquals(Grouping.shuffle(new NullStruct()), bolt1.get_common().get_inputs().values().iterator().next()); assertEquals(Grouping.fields(Collections.singletonList("key")), bolt2.get_common().get_inputs().values().iterator().next()); }
@Test public void testSpoutToBolt() throws Exception { Stream<Tuple> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID)); stream.to(newBolt()); StormTopology topology = streamBuilder.build(); assertEquals(1, topology.get_spouts_size()); assertEquals(1, topology.get_bolts_size()); String spoutId = topology.get_spouts().keySet().iterator().next(); Map<GlobalStreamId, Grouping> expected = new HashMap<>(); expected.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); assertEquals(expected, topology.get_bolts().values().iterator().next().get_common().get_inputs()); }
@Test public void testGlobalAggregate() throws Exception { Stream<String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0), 2); stream.aggregate(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); String spoutId = topology.get_spouts().keySet().iterator().next(); Map<GlobalStreamId, Grouping> expected1 = new HashMap<>(); expected1.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); Map<GlobalStreamId, Grouping> expected2 = new HashMap<>(); expected2.put(new GlobalStreamId("bolt1", "s1"), Grouping.fields(Collections.emptyList())); expected2.put(new GlobalStreamId("bolt1", "s1__punctuation"), Grouping.all(new NullStruct())); assertEquals(expected1, bolt1.get_common().get_inputs()); assertEquals(expected2, bolt2.get_common().get_inputs()); }
@Test public void testBranch() throws Exception { Stream<Tuple> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID)); Stream<Tuple>[] streams = stream.branch(x -> true); StormTopology topology = streamBuilder.build(); assertEquals(1, topology.get_spouts_size()); assertEquals(1, topology.get_bolts_size()); Map<GlobalStreamId, Grouping> expected = new HashMap<>(); String spoutId = topology.get_spouts().keySet().iterator().next(); expected.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); assertEquals(expected, topology.get_bolts().values().iterator().next().get_common().get_inputs()); assertEquals(1, streams.length); assertEquals(1, streams[0].node.getOutputStreams().size()); String parentStream = streams[0].node.getOutputStreams().iterator().next() + "-branch"; assertEquals(1, streams[0].node.getParents(parentStream).size()); Node processorNdoe = streams[0].node.getParents(parentStream).iterator().next(); assertTrue(processorNdoe instanceof ProcessorNode); assertTrue(((ProcessorNode) processorNdoe).getProcessor() instanceof BranchProcessor); assertTrue(processorNdoe.getParents("default").iterator().next() instanceof SpoutNode); }
@Test public void testMultiPartitionByKeyWithRepartition() { TopologyContext mockContext = Mockito.mock(TopologyContext.class); OutputCollector mockCollector = Mockito.mock(OutputCollector.class); Map<GlobalStreamId, Grouping> expected = new HashMap<>(); expected.put(new GlobalStreamId("bolt2", "s3"), Grouping.fields(Collections.singletonList("key"))); expected.put(new GlobalStreamId("bolt2", "s3__punctuation"), Grouping.all(new NullStruct())); Stream<Integer> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); stream.mapToPair(x -> Pair.of(x, x)) .window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))) .reduceByKey((x, y) -> x + y) .repartition(10) .reduceByKey((x, y) -> 0) .print(); StormTopology topology = streamBuilder.build(); assertEquals(3, topology.get_bolts_size()); assertEquals(expected, topology.get_bolts().get("bolt3").get_common().get_inputs()); }
@Test public void testStatefulTopology() { builder.setSpout("spout1", makeDummySpout()); builder.setSpout("spout2", makeDummySpout()); builder.setBolt("bolt1", makeDummyStatefulBolt(), 1) .shuffleGrouping("spout1").shuffleGrouping("spout2"); builder.setBolt("bolt2", makeDummyStatefulBolt(), 1).shuffleGrouping("spout1"); builder.setBolt("bolt3", makeDummyStatefulBolt(), 1) .shuffleGrouping("bolt1").shuffleGrouping("bolt2"); StormTopology topology = builder.createTopology(); Assert.assertNotNull(topology); Set<String> spouts = topology.get_spouts().keySet(); // checkpoint spout should 've been added Assert.assertEquals(ImmutableSet.of("spout1", "spout2", "$checkpointspout"), spouts); // bolt1, bolt2 should also receive from checkpoint spout Assert.assertEquals(ImmutableSet.of(new GlobalStreamId("spout1", "default"), new GlobalStreamId("spout2", "default"), new GlobalStreamId("$checkpointspout", "$checkpoint")), topology.get_bolts().get("bolt1").get_common().get_inputs().keySet()); Assert.assertEquals(ImmutableSet.of(new GlobalStreamId("spout1", "default"), new GlobalStreamId("$checkpointspout", "$checkpoint")), topology.get_bolts().get("bolt2").get_common().get_inputs().keySet()); // bolt3 should also receive from checkpoint streams of bolt1, bolt2 Assert.assertEquals(ImmutableSet.of(new GlobalStreamId("bolt1", "default"), new GlobalStreamId("bolt1", "$checkpoint"), new GlobalStreamId("bolt2", "default"), new GlobalStreamId("bolt2", "$checkpoint")), topology.get_bolts().get("bolt3").get_common().get_inputs().keySet()); }
/** * Gets the declared inputs to the specified component. * * @return A map from subscribed component/stream to the grouping subscribed with. */ public Map<GlobalStreamId, Grouping> getSources(String componentId) { return getComponentCommon(componentId).get_inputs(); }
public static Map<String, Set<String>> getAdjacencyMap(StormTopology topology, boolean removeSystemComponent) { Map<String, Set<String>> adjacencyMap = new HashMap<>(); for (Map.Entry<String, Bolt> entry : topology.get_bolts().entrySet()) { String boltName = entry.getKey(); Map<GlobalStreamId, Grouping> inputs = entry.getValue().get_common().get_inputs(); for (Map.Entry<GlobalStreamId, Grouping> input : inputs.entrySet()) { String inputComponentId = input.getKey().get_componentId(); Set<String> components = adjacencyMap.containsKey(inputComponentId) ? adjacencyMap.get(inputComponentId) : new HashSet<String>(); components.add(boltName); components = removeSystemComponent ? removeSystemComponents(components) : components; if (!removeSystemComponent || !isSystemComponent(inputComponentId)) { adjacencyMap.put(inputComponentId, components); } } } return adjacencyMap; }
public Object getFieldValue(_Fields field) { switch (field) { case INPUTS: return get_inputs(); case STREAMS: return get_streams(); case PARALLELISM_HINT: return get_parallelism_hint(); case JSON_CONF: return get_json_conf(); } throw new IllegalStateException(); }
/** * Gets information about who is consuming the outputs of the specified component, * and how. * * @return Map from stream id to component id to the Grouping used. */ public Map<String, Map<String, Grouping>> getTargets(String componentId) { Map<String, Map<String, Grouping>> ret = new HashMap<>(); for(String otherComponentId: getComponentIds()) { Map<GlobalStreamId, Grouping> inputs = getComponentCommon(otherComponentId).get_inputs(); for(Map.Entry<GlobalStreamId, Grouping> entry: inputs.entrySet()) { GlobalStreamId id = entry.getKey(); if(id.get_componentId().equals(componentId)) { Map<String, Grouping> curr = ret.get(id.get_streamId()); if(curr==null) curr = new HashMap<>(); curr.put(otherComponentId, entry.getValue()); ret.put(id.get_streamId(), curr); } } } return ret; }
/** * For bolts that has incoming streams from spouts (the root bolts), * add checkpoint stream from checkpoint spout to its input. For other bolts, * add checkpoint stream from the previous bolt to its input. */ private void addCheckPointInputs(ComponentCommon component) { Set<GlobalStreamId> checkPointInputs = new HashSet<>(); for (GlobalStreamId inputStream : component.get_inputs().keySet()) { String sourceId = inputStream.get_componentId(); if (_spouts.containsKey(sourceId)) { checkPointInputs.add(new GlobalStreamId(CHECKPOINT_COMPONENT_ID, CHECKPOINT_STREAM_ID)); } else { checkPointInputs.add(new GlobalStreamId(sourceId, CHECKPOINT_STREAM_ID)); } } for (GlobalStreamId streamId : checkPointInputs) { component.put_to_inputs(streamId, Grouping.all(new NullStruct())); } }