@Override public void initState(T state) { if (stateInitialized) { String msg = "initState invoked when the state is already initialized"; LOG.warn(msg); throw new IllegalStateException(msg); } else { statefulWindowedBolt.initState(state); stateInitialized = true; start(); } }
@Override public void preRollback() { LOG.debug("Rollback streamState, stateInitialized {}", stateInitialized); statefulWindowedBolt.preRollback(); state.rollback(stateInitialized); if (stateInitialized) { restoreWindowSystemState(); } }
@Override public long maxEventsInMemory() { return maxEventsInMemory > 0 ? maxEventsInMemory : IStatefulWindowedBolt.super.maxEventsInMemory(); }
@Test public void testExecute() throws Exception { mockStormConf.put(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME, "msgid"); mockStormConf.put(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT, 5); mockStormConf.put(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT, 5); KeyValueState<TaskStream, WindowState> mockState; mockState = Mockito.mock(KeyValueState.class); executor.prepare(mockStormConf, mockTopologyContext, mockOutputCollector, mockState); executor.initState(null); List<Tuple> tuples = getMockTuples(5); for (Tuple tuple : tuples) { executor.execute(tuple); } Mockito.verify(mockBolt, Mockito.times(1)).execute(getTupleWindow(tuples)); WindowState expectedState = new WindowState(Long.MIN_VALUE, 4); Mockito.verify(mockState, Mockito.times(1)).put(Mockito.any(TaskStream.class), Mockito.eq(expectedState)); }
@Override public void prePrepare(long txid) { if (stateInitialized) { LOG.debug("Prepare streamState, txid {}", txid); statefulWindowedBolt.prePrepare(txid); state.prepareCommit(txid); prePrepared = true; } else { String msg = "Cannot prepare before initState"; LOG.warn(msg); throw new IllegalStateException(msg); } }
@Override public void preCommit(long txid) { // preCommit can be invoked during recovery before the state is initialized if (prePrepared || !stateInitialized) { LOG.debug("Commit streamState, txid {}", txid); statefulWindowedBolt.preCommit(txid); state.commit(txid); } else { String msg = "preCommit before prePrepare in initialized state"; LOG.warn(msg); throw new IllegalStateException(msg); } }
/** * Define a new bolt in this topology. This defines a stateful windowed bolt, intended for stateful windowing operations. The {@link * IStatefulWindowedBolt#execute(TupleWindow)} method is triggered for each window interval with the list of current events in the * window. During initialization of this bolt {@link IStatefulWindowedBolt#initState(State)} is invoked with its previously saved * state. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's * outputs. * @param bolt the stateful windowed bolt * @param parallelism_hint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process * somwehere around the cluster. * @param <T> the type of the state (e.g. {@link org.apache.storm.state.KeyValueState}) * @return use the returned object to declare the inputs to this component * * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public <T extends State> BoltDeclarer setBolt(String id, IStatefulWindowedBolt<T> bolt, Number parallelism_hint) throws IllegalArgumentException { hasStatefulBolt = true; IStatefulBolt<T> executor; if (bolt.isPersistent()) { executor = new PersistentWindowedBoltExecutor<>(bolt); } else { executor = new StatefulWindowedBoltExecutor<T>(bolt); } return setBolt(id, new StatefulBoltExecutor<T>(executor), parallelism_hint); }
executor.getWindowManager().add(new WaterMarkEvent<>(activationTs)); Mockito.verify(mockBolt, Mockito.times(tupleCount / WINDOW_EVENT_COUNT)).execute(Mockito.any());
@Test public void testRollbackBeforeInit() throws Exception { executor.preRollback(); Mockito.verify(mockBolt, Mockito.times(1)).preRollback(); // partition ids ArgumentCaptor<String> pkCatptor = ArgumentCaptor.forClass(String.class); Mockito.verify(mockPartitionState, Mockito.times(1)).rollback(); Mockito.verify(mockWindowState, Mockito.times(1)).rollback(); Mockito.verify(mockSystemState, Mockito.times(1)).rollback(); }
@Override public void initState(T state) { if (stateInitialized) { LOG.warn("State is already initialized. Ignoring initState"); return; } statefulWindowedBolt.initState((T) state); // query the streamState for each input task stream and compute recoveryStates for (GlobalStreamId streamId : topologyContext.getThisSources().keySet()) { for (int taskId : topologyContext.getComponentTasks(streamId.get_componentId())) { WindowState windowState = streamState.get(new TaskStream(taskId, streamId)); if (windowState != null) { recoveryStates.put(new TaskStream(taskId, streamId), windowState); } } } LOG.debug("recoveryStates {}", recoveryStates); stateInitialized = true; start(); }
return null; }).when(mockBolt).execute(Mockito.any());
void prepare(Map<String, Object> topoConf, TopologyContext context, OutputCollector collector, KeyValueState<Long, WindowPartition<Tuple>> windowState, KeyValueState<String, Deque<Long>> partitionState, KeyValueState<String, Optional<?>> windowSystemState) { outputCollector = collector; this.windowSystemState = windowSystemState; state = new WindowState<>(windowState, partitionState, windowSystemState, this::getState, statefulWindowedBolt.maxEventsInMemory()); doPrepare(topoConf, context, new NoAckOutputCollector(collector), state, true); restoreWindowSystemState(); }
@Test public void testRollbackAfterInit() throws Exception { executor.initState(null); executor.prePrepare(0); executor.preRollback(); Mockito.verify(mockBolt, Mockito.times(1)).preRollback(); Mockito.verify(mockPartitionState, Mockito.times(1)).rollback(); ArgumentCaptor<String> stringArgumentCaptor = ArgumentCaptor.forClass(String.class); Mockito.verify(mockPartitionState, Mockito.times(2)).put(stringArgumentCaptor.capture(), partitionValuesCaptor.capture()); Mockito.verify(mockWindowState, Mockito.times(1)).rollback(); Mockito.verify(mockSystemState, Mockito.times(1)).rollback(); Mockito.verify(mockSystemState, Mockito.times(2)).iterator(); }
@Override public void initState(T state) { if (stateInitialized) { LOG.warn("State is already initialized. Ignoring initState"); return; } statefulWindowedBolt.initState((T) state); // query the streamState for each input task stream and compute recoveryStates for (GlobalStreamId streamId : topologyContext.getThisSources().keySet()) { for (int taskId : topologyContext.getComponentTasks(streamId.get_componentId())) { WindowState windowState = streamState.get(new TaskStream(taskId, streamId)); if (windowState != null) { recoveryStates.put(new TaskStream(taskId, streamId), windowState); } } } LOG.debug("recoveryStates {}", recoveryStates); stateInitialized = true; start(); }