/** * If the topology has at least one stateful bolt * add a {@link CheckpointSpout} component to the topology. */ private void maybeAddCheckpointSpout() { if (hasStatefulBolt) { setSpout(CheckpointSpout.CHECKPOINT_COMPONENT_ID, new CheckpointSpout(), 1); } }
private boolean shouldCheckpoint() { return !recovering && !checkpointStepInProgress && (curTxState.getState() != CheckPointState.State.COMMITTED || checkpointIntervalElapsed()); }
@Override public void nextTuple() { if (shouldRecover()) { handleRecovery(); startProgress(); } else if (shouldCheckpoint()) { doCheckpoint(); startProgress(); } else { Utils.sleep(sleepInterval); } }
@Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { open(context, collector, loadCheckpointInterval(conf), loadCheckpointState(conf, context)); }
@Override public void ack(Object msgId) { LOG.debug("Got ack with txid {}, current txState {}", msgId, curTxState); if (curTxState.getTxid() == ((Number) msgId).longValue()) { if (recovering) { handleRecoveryAck(); } else { handleCheckpointAck(); } } else { LOG.warn("Ack msgid {}, txState.txid {} mismatch", msgId, curTxState.getTxid()); } resetProgress(); }
private void doCheckpoint() { LOG.debug("In checkpoint"); if (curTxState.getState() == CheckPointState.State.COMMITTED) { saveTxState(curTxState.nextState(false)); lastCheckpointTs = System.currentTimeMillis(); } Action action = curTxState.nextAction(false); emit(curTxState.getTxid(), action); }
@Override public void execute(Tuple input) { if (CheckpointSpout.isCheckpoint(input)) { processCheckpoint(input); } else { handleTuple(input); } }
private void handleRecovery() { LOG.debug("In recovery"); Action action = curTxState.nextAction(true); emit(curTxState.getTxid(), action); }
@Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { open(context, collector, loadCheckpointInterval(conf), loadCheckpointState(conf, context)); }
@Override public void ack(Object msgId) { LOG.debug("Got ack with txid {}, current txState {}", msgId, curTxState); if (curTxState.getTxid() == ((Number) msgId).longValue()) { if (recovering) { handleRecoveryAck(); } else { handleCheckpointAck(); } } else { LOG.warn("Ack msgid {}, txState.txid {} mismatch", msgId, curTxState.getTxid()); } resetProgress(); }
private void doCheckpoint() { LOG.debug("In checkpoint"); if (curTxState.getState() == CheckPointState.State.COMMITTED) { saveTxState(curTxState.nextState(false)); lastCheckpointTs = System.currentTimeMillis(); } Action action = curTxState.nextAction(false); emit(curTxState.getTxid(), action); }
@Override public void execute(Tuple input) { if (CheckpointSpout.isCheckpoint(input)) { processCheckpoint(input); } else { handleTuple(input); } }
private void handleRecovery() { LOG.debug("In recovery"); Action action = curTxState.nextAction(true); emit(curTxState.getTxid(), action); }
@Override public void nextTuple() { if (shouldRecover()) { handleRecovery(); startProgress(); } else if (shouldCheckpoint()) { doCheckpoint(); startProgress(); } else { Utils.sleep(sleepInterval); } }
private boolean shouldCheckpoint() { return !recovering && !checkpointStepInProgress && (curTxState.getState() != CheckPointState.State.COMMITTED || checkpointIntervalElapsed()); }
/** * If the topology has at least one stateful bolt * add a {@link CheckpointSpout} component to the topology. */ private void maybeAddCheckpointSpout() { if (hasStatefulBolt) { setSpout(CheckpointSpout.CHECKPOINT_COMPONENT_ID, new CheckpointSpout(), 1); } }