@Override public void execute(Tuple input) { if (!stateInitialized) { throw new IllegalStateException("execute invoked before initState with input tuple " + input); } super.execute(input); // StatefulBoltExecutor does the actual ack when the state is saved. outputCollector.ack(input); }
private void clearRecoveryState(TaskStream stream) { recoveryStates.remove(stream); if (!isRecovering()) { super.start(); LOG.debug("Recovery complete, processing {} pending tuples", pendingTuples.size()); for (Tuple tuple : pendingTuples) { super.execute(tuple); } } }
@Override public void execute(Tuple input) { if (!isStateInitialized()) { throw new IllegalStateException("execute invoked before initState with input tuple " + input); } else if (isRecovering()) { handleRecovery(input); } else { super.execute(input); } }
private void handleRecovery(Tuple input) { long msgId = getMsgId(input); TaskStream taskStream = TaskStream.fromTuple(input); WindowState state = recoveryStates.get(taskStream); LOG.debug("handleRecovery, recoveryStates {}", recoveryStates); if (state != null) { LOG.debug("Tuple msgid {}, saved state {}", msgId, state); if (msgId <= state.lastExpired) { LOG.debug("Ignoring tuple since msg id {} <= lastExpired id {}", msgId, state.lastExpired); outputCollector.ack(input); } else if (msgId <= state.lastEvaluated) { super.execute(input); } else { LOG.debug("Tuple msg id {} > lastEvaluated id {}, adding to pendingTuples and clearing recovery state " + "for taskStream {}", msgId, state.lastEvaluated, taskStream); pendingTuples.add(input); clearRecoveryState(taskStream); } } else { pendingTuples.add(input); } }
@Test(expected = IllegalArgumentException.class) public void testExecuteWithoutTs() throws Exception { executor.execute(getTuple("s1", new Fields("a"), new Values(1), "s1Src")); }
@Test public void testExecuteWithTs() throws Exception { long[] timestamps = { 603, 605, 607, 618, 626, 636 }; for (long ts : timestamps) { executor.execute(getTuple("s1", new Fields("ts"), new Values(ts), "s1Src")); } //Thread.sleep(120); executor.waterMarkEventGenerator.run(); //System.out.println(testWindowedBolt.tupleWindows); assertEquals(3, testWindowedBolt.tupleWindows.size()); TupleWindow first = testWindowedBolt.tupleWindows.get(0); assertArrayEquals(new long[]{ 603, 605, 607 }, new long[]{ (long) first.get().get(0).getValue(0), (long) first.get().get(1).getValue(0), (long) first.get().get(2).getValue(0) }); TupleWindow second = testWindowedBolt.tupleWindows.get(1); assertArrayEquals(new long[]{ 603, 605, 607, 618 }, new long[]{ (long) second.get().get(0).getValue(0), (long) second.get().get(1).getValue(0), (long) second.get().get(2).getValue(0), (long) second.get().get(3).getValue(0) }); TupleWindow third = testWindowedBolt.tupleWindows.get(2); assertArrayEquals(new long[]{ 618, 626 }, new long[]{ (long) third.get().get(0).getValue(0), (long) third.get().get(1).getValue(0) }); }
private void clearRecoveryState(TaskStream stream) { recoveryStates.remove(stream); if (!isRecovering()) { super.start(); LOG.debug("Recovery complete, processing {} pending tuples", pendingTuples.size()); for (Tuple tuple : pendingTuples) { super.execute(tuple); } } }
@Override public void execute(Tuple input) { if (!isStateInitialized()) { throw new IllegalStateException("execute invoked before initState with input tuple " + input); } else if (isRecovering()) { handleRecovery(input); } else { super.execute(input); } }
private void handleRecovery(Tuple input) { long msgId = getMsgId(input); TaskStream taskStream = TaskStream.fromTuple(input); WindowState state = recoveryStates.get(taskStream); LOG.debug("handleRecovery, recoveryStates {}", recoveryStates); if (state != null) { LOG.debug("Tuple msgid {}, saved state {}", msgId, state); if (msgId <= state.lastExpired) { LOG.debug("Ignoring tuple since msg id {} <= lastExpired id {}", msgId, state.lastExpired); outputCollector.ack(input); } else if (msgId <= state.lastEvaluated) { super.execute(input); } else { LOG.debug("Tuple msg id {} > lastEvaluated id {}, adding to pendingTuples and clearing recovery state " + "for taskStream {}", msgId, state.lastEvaluated, taskStream); pendingTuples.add(input); clearRecoveryState(taskStream); } } else { pendingTuples.add(input); } }