@Override public void prepare(Map<String, Object> topoConf, TopologyContext context, OutputCollector collector) { prepare(topoConf, context, collector, getWindowState(topoConf, context)); }
private void handleRecovery(Tuple input) { long msgId = getMsgId(input); TaskStream taskStream = TaskStream.fromTuple(input); WindowState state = recoveryStates.get(taskStream); LOG.debug("handleRecovery, recoveryStates {}", recoveryStates); if (state != null) { LOG.debug("Tuple msgid {}, saved state {}", msgId, state); if (msgId <= state.lastExpired) { LOG.debug("Ignoring tuple since msg id {} <= lastExpired id {}", msgId, state.lastExpired); outputCollector.ack(input); } else if (msgId <= state.lastEvaluated) { super.execute(input); } else { LOG.debug("Tuple msg id {} > lastEvaluated id {}, adding to pendingTuples and clearing recovery state " + "for taskStream {}", msgId, state.lastEvaluated, taskStream); pendingTuples.add(input); clearRecoveryState(taskStream); } } else { pendingTuples.add(input); } }
private void updateState(Map<TaskStream, WindowState> state, List<Tuple> tuples, boolean newEvents) { for (Tuple tuple : tuples) { TaskStream taskStream = TaskStream.fromTuple(tuple); WindowState curState = state.get(taskStream); WindowState newState; if ((newState = getUpdatedState(curState, getMsgId(tuple), newEvents)) != null) { state.put(taskStream, newState); } } }
@Override public void execute(Tuple input) { if (!isStateInitialized()) { throw new IllegalStateException("execute invoked before initState with input tuple " + input); } else if (isRecovering()) { handleRecovery(input); } else { super.execute(input); } }
@Override protected void start() { if (!isStateInitialized() || isRecovering()) { LOG.debug("Will invoke start after recovery is complete."); } else { super.start(); } }
@Test public void testExecute() throws Exception { mockStormConf.put(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME, "msgid"); mockStormConf.put(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT, 5); mockStormConf.put(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT, 5); KeyValueState<TaskStream, WindowState> mockState; mockState = Mockito.mock(KeyValueState.class); executor.prepare(mockStormConf, mockTopologyContext, mockOutputCollector, mockState); executor.initState(null); List<Tuple> tuples = getMockTuples(5); for (Tuple tuple : tuples) { executor.execute(tuple); } Mockito.verify(mockBolt, Mockito.times(1)).execute(getTupleWindow(tuples)); WindowState expectedState = new WindowState(Long.MIN_VALUE, 4); Mockito.verify(mockState, Mockito.times(1)).put(Mockito.any(TaskStream.class), Mockito.eq(expectedState)); }
@Before public void setUp() throws Exception { mockBolt = Mockito.mock(IStatefulWindowedBolt.class); mockTopologyContext = Mockito.mock(TopologyContext.class); mockOutputCollector = Mockito.mock(OutputCollector.class); executor = new StatefulWindowedBoltExecutor<>(mockBolt); }
@Test public void testPrepareWithMsgid() throws Exception { mockStormConf.put(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME, "msgid"); mockStormConf.put(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT, 5); mockStormConf.put(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT, 5); executor.prepare(mockStormConf, mockTopologyContext, mockOutputCollector); }
void prepare(Map<String, Object> topoConf, TopologyContext context, OutputCollector collector, KeyValueState<TaskStream, WindowState> windowState) { init(topoConf, context, collector, windowState); super.prepare(topoConf, context, collector); }
private void clearRecoveryState(TaskStream stream) { recoveryStates.remove(stream); if (!isRecovering()) { super.start(); LOG.debug("Recovery complete, processing {} pending tuples", pendingTuples.size()); for (Tuple tuple : pendingTuples) { super.execute(tuple); } } }
@Override public void preCommit(long txid) { if (!isStateInitialized() || (!isRecovering() && prePrepared)) { LOG.debug("Commit streamState, txid {}", txid); streamState.commit(txid); } else { LOG.debug("Still recovering, ignoring preCommit and not committing streamState."); } }
@Override public void execute(Tuple input) { if (!isStateInitialized()) { throw new IllegalStateException("execute invoked before initState with input tuple " + input); } else if (isRecovering()) { handleRecovery(input); } else { super.execute(input); } }
@Test public void testRecovery() throws Exception { mockStormConf.put(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME, "msgid"); mockStormConf.put(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT, 5); mockStormConf.put(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT, 5); KeyValueState<TaskStream, WindowState> mockState; mockState = Mockito.mock(KeyValueState.class); Map<GlobalStreamId, Grouping> mockMap = Mockito.mock(Map.class); Mockito.when(mockTopologyContext.getThisSources()).thenReturn(mockMap); Mockito.when(mockTopologyContext.getComponentTasks(Mockito.anyString())).thenReturn(Collections.singletonList(1)); Mockito.when(mockMap.keySet()).thenReturn(Collections.singleton(new GlobalStreamId("a", "s"))); WindowState mockWindowState = new WindowState(4, 4); Mockito.when(mockState.get(Mockito.any(TaskStream.class))).thenReturn(mockWindowState); executor.prepare(mockStormConf, mockTopologyContext, mockOutputCollector, mockState); executor.initState(null); List<Tuple> tuples = getMockTuples(10); for (Tuple tuple : tuples) { executor.execute(tuple); } WindowState expectedState = new WindowState(4, 9); Mockito.verify(mockState, Mockito.times(1)).put(Mockito.any(TaskStream.class), Mockito.eq(expectedState)); }
/** * Define a new bolt in this topology. This defines a stateful windowed bolt, intended for stateful windowing operations. The {@link * IStatefulWindowedBolt#execute(TupleWindow)} method is triggered for each window interval with the list of current events in the * window. During initialization of this bolt {@link IStatefulWindowedBolt#initState(State)} is invoked with its previously saved * state. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's * outputs. * @param bolt the stateful windowed bolt * @param parallelism_hint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process * somwehere around the cluster. * @param <T> the type of the state (e.g. {@link org.apache.storm.state.KeyValueState}) * @return use the returned object to declare the inputs to this component * * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public <T extends State> BoltDeclarer setBolt(String id, IStatefulWindowedBolt<T> bolt, Number parallelism_hint) throws IllegalArgumentException { hasStatefulBolt = true; IStatefulBolt<T> executor; if (bolt.isPersistent()) { executor = new PersistentWindowedBoltExecutor<>(bolt); } else { executor = new StatefulWindowedBoltExecutor<T>(bolt); } return setBolt(id, new StatefulBoltExecutor<T>(executor), parallelism_hint); }
@Test(expected = IllegalArgumentException.class) public void testPrepare() throws Exception { executor.prepare(mockStormConf, mockTopologyContext, mockOutputCollector); }
void prepare(Map stormConf, TopologyContext context, OutputCollector collector, KeyValueState<TaskStream, WindowState> windowState) { init(stormConf, context, collector, windowState); super.prepare(stormConf, context, collector); }
private void clearRecoveryState(TaskStream stream) { recoveryStates.remove(stream); if (!isRecovering()) { super.start(); LOG.debug("Recovery complete, processing {} pending tuples", pendingTuples.size()); for (Tuple tuple : pendingTuples) { super.execute(tuple); } } }
@Override public void prePrepare(long txid) { if (!isStateInitialized()) { LOG.warn("Cannot prepare before initState"); } else if (!isRecovering()) { LOG.debug("Prepare streamState, txid {}", txid); streamState.prepareCommit(txid); prePrepared = true; } else { LOG.debug("Still recovering, ignoring prePrepare and not preparing streamState."); } }
@Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { prepare(stormConf, context, collector, getWindowState(stormConf, context)); }
private void updateState(Map<TaskStream, WindowState> state, List<Tuple> tuples, boolean newEvents) { for (Tuple tuple : tuples) { TaskStream taskStream = TaskStream.fromTuple(tuple); WindowState curState = state.get(taskStream); WindowState newState; if ((newState = getUpdatedState(curState, getMsgId(tuple), newEvents)) != null) { state.put(taskStream, newState); } } }