@Override public String getComponent() { return stream.get_componentId(); } });
@Override public String getComponent() { return stream.get_componentId(); } });
@Override public String getComponent() { return stream.get_componentId(); }
@Override public String getComponent() { return stream.get_componentId(); } });
Map<String, Set<String>> getBoltBatchToComponentSubscriptions(String id) { Map<String, Set<String>> ret = new HashMap(); for(GlobalStreamId s: getBoltSubscriptionStreams(id)) { String b = _batchIds.get(s); if(!ret.containsKey(b)) ret.put(b, new HashSet()); ret.get(b).add(s.get_componentId()); } return ret; }
public Object getFieldValue(_Fields field) { switch (field) { case COMPONENT_ID: return get_componentId(); case STREAM_ID: return get_streamId(); } throw new IllegalStateException(); }
@Override public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List<Integer> tasks) { List<Integer> sourceTasks = new ArrayList<Integer>(context.getComponentTasks(stream.get_componentId())); Collections.sort(sourceTasks); if (sourceTasks.size() != tasks.size()) { throw new RuntimeException("Can only do an identity grouping when source and target have same number of tasks"); } tasks = new ArrayList<Integer>(tasks); Collections.sort(tasks); for (int i = 0; i < sourceTasks.size(); i++) { int s = sourceTasks.get(i); int t = tasks.get(i); _precomputed.put(s, Arrays.asList(t)); } }
public Object getFieldValue(_Fields field) { switch (field) { case COMPONENT_ID: return get_componentId(); case STREAM_ID: return get_streamId(); } throw new IllegalStateException(); }
public Map<String, List<Integer>> getThisSourceComponentTasks() { Map<String, List<Integer>> ret = new HashMap<>(); Map<GlobalStreamId, Grouping> sources = getThisSources(); Set<String> sourceComponents = new HashSet<>(); if (sources != null) { for (GlobalStreamId globalStreamId : sources.keySet()) { sourceComponents.add(globalStreamId.get_componentId()); } } for (String component : sourceComponents) { ret.put(component, getComponentTasks(component)); } return ret; }
public static Set<String> getUpstreamComponents(String componentId, TopologyContext context) { Set<String> upstreamComponents = new HashSet<>(); ComponentCommon componentCommon = Utils.getComponentCommon(context.getRawTopology(), componentId); Set<GlobalStreamId> input = componentCommon.get_inputs().keySet(); for (GlobalStreamId stream : input) { upstreamComponents.add(stream.get_componentId()); } return upstreamComponents; }
/** * returns the total number of input checkpoint streams across * all input tasks to this component. */ private int getCheckpointInputTaskCount(TopologyContext context) { int count = 0; for (GlobalStreamId inputStream : context.getThisSources().keySet()) { if (CheckpointSpout.CHECKPOINT_STREAM_ID.equals(inputStream.get_streamId())) { count += context.getComponentTasks(inputStream.get_componentId()).size(); } } return count; }
public static Set<String> getDownstreamComponents(String componentId, StormTopology topology) { Set<String> components = new HashSet<>(); Map<String, Bolt> bolts = topology.get_bolts(); for (Entry<String, Bolt> entry : bolts.entrySet()) { String downstreamComponentId = entry.getKey(); Bolt bolt = entry.getValue(); Set<GlobalStreamId> input = bolt.get_common().get_inputs().keySet(); for (GlobalStreamId stream : input) { String upstreamComponentId = stream.get_componentId(); if (upstreamComponentId.equals(componentId)) { components.add(downstreamComponentId); break; } } } return components; }
/** * Gets the declared output fields for the specified global stream id. */ public Fields getComponentOutputFields(GlobalStreamId id) { return getComponentOutputFields(id.get_componentId(), id.get_streamId()); }
@Override public BoltDeclarer grouping(GlobalStreamId id, Grouping grouping) { return grouping(id.get_componentId(), id.get_streamId(), grouping); } }
private static boolean hasDownstreamComponent(StormTopology topology, String outComponentId, String outStreamId) { boolean ret = false; for (String componentId : ThriftTopologyUtils.getComponentIds(topology)) { ComponentCommon componentCommon = Utils.getComponentCommon(topology, componentId); Set<GlobalStreamId> inputs = componentCommon.get_inputs().keySet(); for (GlobalStreamId input : inputs) { if (input.get_componentId().equals(outComponentId) && input.get_streamId().equals(outStreamId)) return true; } } return ret; } }
@Override public BoltDeclarer grouping(GlobalStreamId id, Grouping grouping) { return grouping(id.get_componentId(), id.get_streamId(), grouping); } }
/** * Gets the declared output fields for the specified global stream id. */ public Fields getComponentOutputFields(GlobalStreamId id) { return getComponentOutputFields(id.get_componentId(), id.get_streamId()); }
public static Set<Integer> getDownstreamTasks(String componentId, TopologyContext context) { Set<Integer> tasks = new HashSet<>(); StormTopology topology = context.getRawTopology(); Map<String, Bolt> bolts = topology.get_bolts(); for (Entry<String, Bolt> entry : bolts.entrySet()) { String downstreamComponentId = entry.getKey(); Bolt bolt = entry.getValue(); Set<GlobalStreamId> input = bolt.get_common().get_inputs().keySet(); for (GlobalStreamId stream : input) { String upstreamComponentId = stream.get_componentId(); if (upstreamComponentId.equals(componentId)) { tasks.addAll(context.getComponentTasks(downstreamComponentId)); break; } } } return tasks; }
/** * Add watermark stream to source components of window bolts */ private void maybeAddWatermarkInputs(ComponentCommon common, IRichBolt bolt) { if (bolt instanceof WindowedBoltExecutor) { Set<String> comps = new HashSet<>(); for (GlobalStreamId globalStreamId : common.get_inputs().keySet()) { comps.add(globalStreamId.get_componentId()); } for (String comp : comps) { common.put_to_inputs( new GlobalStreamId(comp, Common.WATERMARK_STREAM_ID), Grouping.all(new NullStruct())); } } }
@Override public void initState(T state) { if (stateInitialized) { LOG.warn("State is already initialized. Ignoring initState"); return; } statefulWindowedBolt.initState((T) state); // query the streamState for each input task stream and compute recoveryStates for (GlobalStreamId streamId : topologyContext.getThisSources().keySet()) { for (int taskId : topologyContext.getComponentTasks(streamId.get_componentId())) { WindowState windowState = streamState.get(new TaskStream(taskId, streamId)); if (windowState != null) { recoveryStates.put(new TaskStream(taskId, streamId), windowState); } } } LOG.debug("recoveryStates {}", recoveryStates); stateInitialized = true; start(); }