public java.lang.Object getFieldValue(_Fields field) { switch (field) { case SPOUT_OBJECT: return get_spout_object(); case COMMON: return get_common(); } throw new java.lang.IllegalStateException(); }
public static Map<String, NormalizedResourceRequest> getSpoutsResources(StormTopology topology, Map<String, Object> topologyConf) { Map<String, NormalizedResourceRequest> spoutResources = new HashMap<>(); if (topology.get_spouts() != null) { for (Map.Entry<String, SpoutSpec> spout : topology.get_spouts().entrySet()) { NormalizedResourceRequest topologyResources = new NormalizedResourceRequest(spout.getValue().get_common(), topologyConf, spout.getKey()); if (LOG.isTraceEnabled()) { LOG.trace("Turned {} into {}", spout.getValue().get_common().get_json_conf(), topologyResources); } spoutResources.put(spout.getKey(), topologyResources); } } return spoutResources; }
public static ComponentCommon getComponentCommon(Object component) { ComponentCommon common = null; if (component instanceof StateSpoutSpec) { common = ((StateSpoutSpec) component).get_common(); } else if (component instanceof SpoutSpec) { common = ((SpoutSpec) component).get_common(); } else if (component instanceof Bolt) { common = ((Bolt) component).get_common(); } return common; }
private static void addLagResultForKafkaSpout(Map<String, Map<String, Object>> finalResult, String spoutId, SpoutSpec spoutSpec) throws IOException { ComponentCommon componentCommon = spoutSpec.get_common(); String json = componentCommon.get_json_conf(); if (json != null && !json.isEmpty()) { Map<String, Object> jsonMap = null; try { jsonMap = (Map<String, Object>) JSONValue.parseWithException(json); } catch (ParseException e) { throw new IOException(e); } if (jsonMap.containsKey(TOPICS_CONFIG) && jsonMap.containsKey(GROUPID_CONFIG) && jsonMap.containsKey(BOOTSTRAP_CONFIG)) { finalResult.put(spoutId, getLagResultForNewKafkaSpout(spoutId, spoutSpec)); } } }
static Map<String, Map<String, Double>> getSpoutsResources(StormTopology topology, Map<String, Object> topologyConf) { Map<String, Map<String, Double>> spoutResources = new HashMap<>(); if (topology.get_spouts() != null) { for (Map.Entry<String, SpoutSpec> spout : topology.get_spouts().entrySet()) { Map<String, Double> topologyResources = parseResources(spout.getValue().get_common().get_json_conf()); checkInitialization(topologyResources, spout.getValue().toString(), topologyConf); spoutResources.put(spout.getKey(), topologyResources); } } return spoutResources; }
public static NormalizedResourceRequest getSpoutResources(StormTopology topology, Map<String, Object> topologyConf, String componentId) { if (topology.get_spouts() != null) { SpoutSpec spout = topology.get_spouts().get(componentId); return new NormalizedResourceRequest(spout.get_common(), topologyConf, componentId); } return null; }
public static ComponentCommon getComponentCommon(StormTopology topology, String id) { if (topology.get_spouts().containsKey(id)) { return topology.get_spouts().get(id).get_common(); } if (topology.get_bolts().containsKey(id)) { return topology.get_bolts().get(id).get_common(); } if (topology.get_state_spouts().containsKey(id)) { return topology.get_state_spouts().get(id).get_common(); } throw new IllegalArgumentException("Could not find component with id " + id); }
for (String streamName : spoutSpec.get_common().get_streams().keySet()) { if (streamName.contains(".")) { LOG.warn("Metrics for stream name '{}' will be reported as '{}'.", streamName, streamName.replace('.', '_'));
public static ComponentCommon getComponentCommon(StormTopology topology, String componentId) { Bolt b = topology.get_bolts().get(componentId); if (b != null) { return b.get_common(); } SpoutSpec s = topology.get_spouts().get(componentId); if (s != null) { return s.get_common(); } StateSpoutSpec ss = topology.get_state_spouts().get(componentId); if (ss != null) { return ss.get_common(); } throw new IllegalArgumentException("Could not find component common for " + componentId); } }
private static Map<String, Object> getLagResultForKafka(String spoutId, SpoutSpec spoutSpec) throws IOException { ComponentCommon componentCommon = spoutSpec.get_common(); String json = componentCommon.get_json_conf(); Map<String, Object> result = null;
NormalizedResourceRequest topologyResources = new NormalizedResourceRequest(spout.getValue().get_common(), topologyConf, spout.getKey()); for (Map.Entry<ExecutorDetails, String> anExecutorToComponent :
for (String streamName : spoutSpec.get_common().get_streams().keySet()) { if (streamName.contains(".")) { throw new WrappedInvalidTopologyException(String.format("Stream name '%s' contains illegal character '.'", streamName));
ComponentCommon spoutCommon = spoutSpec.get_common(); Map<String, Double> resourcesUpdate = NormalizedResources .RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resourceUpdatesMap.get(spoutName));
spoutSpecMapEntry.getValue().get_common().get_inputs().entrySet().stream().map( UIHelpers::getInputMap).collect(Collectors.toList()) );
for (Map.Entry<String, SpoutSpec> entry : spouts.entrySet()) { Component spout = ret.get(entry.getKey()); for (String parentId : getInputsTo(entry.getValue().get_common())) { ret.get(parentId).addChild(spout);
for (Map.Entry<String, SpoutSpec> spoutEntry : topology.get_spouts().entrySet()) { String id = spoutEntry.getKey(); for (Entry<String, StreamInfo> streamEntry : spoutEntry.getValue().get_common().get_streams().entrySet()) { String stream = streamEntry.getKey(); StreamInfo info = streamEntry.getValue();
ComponentCommon common = spout.get_common(); Map<String, Object> spoutConf = componentConf(spout); spoutConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS,
public static Map<ExecutorDetails, String> genExecsAndComps(StormTopology topology) { Map<ExecutorDetails, String> retMap = new HashMap<>(); int startTask = 0; int endTask = 0; for (Map.Entry<String, SpoutSpec> entry : topology.get_spouts().entrySet()) { SpoutSpec spout = entry.getValue(); String spoutId = entry.getKey(); int spoutParallelism = spout.get_common().get_parallelism_hint(); for (int i = 0; i < spoutParallelism; i++) { retMap.put(new ExecutorDetails(startTask, endTask), spoutId); startTask++; endTask++; } } for (Map.Entry<String, Bolt> entry : topology.get_bolts().entrySet()) { String boltId = entry.getKey(); Bolt bolt = entry.getValue(); int boltParallelism = bolt.get_common().get_parallelism_hint(); for (int i = 0; i < boltParallelism; i++) { retMap.put(new ExecutorDetails(startTask, endTask), boltId); startTask++; endTask++; } } return retMap; }
String componentConfRaw = readStormTopology.get_spouts().get("spout-1").get_common().get_json_conf();
@Test public void testRepartition() throws Exception { Stream<String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); stream.repartition(3).filter(x -> true).repartition(2).filter(x -> true).aggregate(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(1, topology.get_spouts_size()); SpoutSpec spout = topology.get_spouts().get("spout1"); assertEquals(4, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); Bolt bolt3 = topology.get_bolts().get("bolt3"); Bolt bolt4 = topology.get_bolts().get("bolt4"); assertEquals(1, spout.get_common().get_parallelism_hint()); assertEquals(1, bolt1.get_common().get_parallelism_hint()); assertEquals(3, bolt2.get_common().get_parallelism_hint()); assertEquals(2, bolt3.get_common().get_parallelism_hint()); assertEquals(2, bolt4.get_common().get_parallelism_hint()); }