/** * Define a new bolt in this topology. This defines a stateful windowed bolt, intended for stateful * windowing operations. The {@link IStatefulWindowedBolt#execute(TupleWindow)} method is triggered * for each window interval with the list of current events in the window. During initialization of * this bolt {@link IStatefulWindowedBolt#initState(State)} is invoked with its previously saved state. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. * @param bolt the stateful windowed bolt * @param parallelism_hint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process somwehere around the cluster. * @param <T> the type of the state (e.g. {@link backtype.storm.state.KeyValueState}) * @return use the returned object to declare the inputs to this component * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public <T extends State> BoltDeclarer setBolt(String id, IStatefulWindowedBolt<T> bolt, Number parallelism_hint) throws IllegalArgumentException { hasStatefulBolt = true; return setBolt(id, new StatefulBoltExecutor<T>(new StatefulWindowedBoltExecutor<T>(bolt)), parallelism_hint); }
private void handleRecovery(Tuple input) { long msgId = getMsgId(input); TaskStream taskStream = TaskStream.fromTuple(input); WindowState state = recoveryStates.get(taskStream); LOG.debug("handleRecovery, recoveryStates {}", recoveryStates); if (state != null) { LOG.debug("Tuple msgid {}, saved state {}", msgId, state); if (msgId <= state.lastExpired) { LOG.debug("Ignoring tuple since msg id {} <= lastExpired id {}", msgId, state.lastExpired); outputCollector.ack(input); } else if (msgId <= state.lastEvaluated) { super.execute(input); } else { LOG.debug("Tuple msg id {} > lastEvaluated id {}, adding to pendingTuples and clearing recovery state " + "for taskStream {}", msgId, state.lastEvaluated, taskStream); pendingTuples.add(input); clearRecoveryState(taskStream); } } else { pendingTuples.add(input); } }
private void updateState(Map<TaskStream, WindowState> state, List<Tuple> tuples, boolean newEvents) { for (Tuple tuple : tuples) { TaskStream taskStream = TaskStream.fromTuple(tuple); WindowState curState = state.get(taskStream); WindowState newState; if ((newState = getUpdatedState(curState, getMsgId(tuple), newEvents)) != null) { state.put(taskStream, newState); } } }
@Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { prepare(stormConf, context, collector, getWindowState(stormConf, context)); }
@Override public void execute(Tuple input) { if (!isStateInitialized()) { throw new IllegalStateException("execute invoked before initState with input tuple " + input); } else if (isRecovering()) { handleRecovery(input); } else { super.execute(input); } }
@Override protected void start() { if (!isStateInitialized() || isRecovering()) { LOG.debug("Will invoke start after recovery is complete."); } else { super.start(); } }
private void clearRecoveryState(TaskStream stream) { recoveryStates.remove(stream); if (!isRecovering()) { super.start(); LOG.debug("Recovery complete, processing {} pending tuples", pendingTuples.size()); for (Tuple tuple : pendingTuples) { super.execute(tuple); } } }
void prepare(Map stormConf, TopologyContext context, OutputCollector collector, KeyValueState<TaskStream, WindowState> windowState) { init(stormConf, context, collector, windowState); super.prepare(stormConf, context, collector); }
@Override public void preCommit(long txid) { if (!isStateInitialized() || (!isRecovering() && prePrepared)) { LOG.debug("Commit streamState, txid {}", txid); streamState.commit(txid); } else { LOG.debug("Still recovering, ignoring preCommit and not committing streamState."); } }
@Override public void execute(Tuple input) { if (!isStateInitialized()) { throw new IllegalStateException("execute invoked before initState with input tuple " + input); } else if (isRecovering()) { handleRecovery(input); } else { super.execute(input); } }
@Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { prepare(stormConf, context, collector, getWindowState(stormConf, context)); }
private void clearRecoveryState(TaskStream stream) { recoveryStates.remove(stream); if (!isRecovering()) { super.start(); LOG.debug("Recovery complete, processing {} pending tuples", pendingTuples.size()); for (Tuple tuple : pendingTuples) { super.execute(tuple); } } }
void prepare(Map stormConf, TopologyContext context, OutputCollector collector, KeyValueState<TaskStream, WindowState> windowState) { init(stormConf, context, collector, windowState); super.prepare(stormConf, context, collector); }
@Override public void prePrepare(long txid) { if (!isStateInitialized()) { LOG.warn("Cannot prepare before initState"); } else if (!isRecovering()) { LOG.debug("Prepare streamState, txid {}", txid); streamState.prepareCommit(txid); prePrepared = true; } else { LOG.debug("Still recovering, ignoring prePrepare and not preparing streamState."); } }
private void updateState(Map<TaskStream, WindowState> state, List<Tuple> tuples, boolean newEvents) { for (Tuple tuple : tuples) { TaskStream taskStream = TaskStream.fromTuple(tuple); WindowState curState = state.get(taskStream); WindowState newState; if ((newState = getUpdatedState(curState, getMsgId(tuple), newEvents)) != null) { state.put(taskStream, newState); } } }
private void handleRecovery(Tuple input) { long msgId = getMsgId(input); TaskStream taskStream = TaskStream.fromTuple(input); WindowState state = recoveryStates.get(taskStream); LOG.debug("handleRecovery, recoveryStates {}", recoveryStates); if (state != null) { LOG.debug("Tuple msgid {}, saved state {}", msgId, state); if (msgId <= state.lastExpired) { LOG.debug("Ignoring tuple since msg id {} <= lastExpired id {}", msgId, state.lastExpired); outputCollector.ack(input); } else if (msgId <= state.lastEvaluated) { super.execute(input); } else { LOG.debug("Tuple msg id {} > lastEvaluated id {}, adding to pendingTuples and clearing recovery state " + "for taskStream {}", msgId, state.lastEvaluated, taskStream); pendingTuples.add(input); clearRecoveryState(taskStream); } } else { pendingTuples.add(input); } }
/** * Define a new bolt in this topology. This defines a stateful windowed bolt, intended for stateful * windowing operations. The {@link IStatefulWindowedBolt#execute(TupleWindow)} method is triggered * for each window interval with the list of current events in the window. During initialization of * this bolt {@link IStatefulWindowedBolt#initState(State)} is invoked with its previously saved state. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. * @param bolt the stateful windowed bolt * @param parallelism_hint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process somwehere around the cluster. * @param <T> the type of the state (e.g. {@link org.apache.storm.state.KeyValueState}) * @return use the returned object to declare the inputs to this component * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public <T extends State> BoltDeclarer setBolt(String id, IStatefulWindowedBolt<T> bolt, Number parallelism_hint) throws IllegalArgumentException { hasStatefulBolt = true; return setBolt(id, new StatefulBoltExecutor<T>(new StatefulWindowedBoltExecutor<T>(bolt)), parallelism_hint); }
@Override protected void start() { if (!isStateInitialized() || isRecovering()) { LOG.debug("Will invoke start after recovery is complete."); } else { super.start(); } }
@Override public void preCommit(long txid) { if (!isStateInitialized() || (!isRecovering() && prePrepared)) { LOG.debug("Commit streamState, txid {}", txid); streamState.commit(txid); } else { LOG.debug("Still recovering, ignoring preCommit and not committing streamState."); } }
@Override public void prePrepare(long txid) { if (!isStateInitialized()) { LOG.warn("Cannot prepare before initState"); } else if (!isRecovering()) { LOG.debug("Prepare streamState, txid {}", txid); streamState.prepareCommit(txid); prePrepared = true; } else { LOG.debug("Still recovering, ignoring prePrepare and not preparing streamState."); } }