/** * Define a new bolt in this topology. This defines a windowed bolt, intended for windowing operations. The {@link * IWindowedBolt#execute(TupleWindow)} method is triggered for each window interval with the list of current events in the window. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's * outputs. * @param bolt the windowed bolt * @param parallelism_hint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process * somwehere around the cluster. * @return use the returned object to declare the inputs to this component * * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public BoltDeclarer setBolt(String id, IWindowedBolt bolt, Number parallelism_hint) throws IllegalArgumentException { return setBolt(id, new WindowedBoltExecutor(bolt), parallelism_hint); }
@Override public void execute(Tuple input) { if (!stateInitialized) { throw new IllegalStateException("execute invoked before initState with input tuple " + input); } super.execute(input); // StatefulBoltExecutor does the actual ack when the state is saved. outputCollector.ack(input); }
void prepare(Map<String, Object> topoConf, TopologyContext context, OutputCollector collector, KeyValueState<TaskStream, WindowState> windowState) { init(topoConf, context, collector, windowState); super.prepare(topoConf, context, collector); }
protected void validate(Map<String, Object> topoConf, Count windowLengthCount, Duration windowLengthDuration, Count slidingIntervalCount, Duration slidingIntervalDuration) { int topologyTimeout = getTopologyTimeoutMillis(topoConf); int maxSpoutPending = getMaxSpoutPending(topoConf); if (windowLengthCount == null && windowLengthDuration == null) { throw new IllegalArgumentException("Window length is not specified"); } if (windowLengthDuration != null && slidingIntervalDuration != null) { ensureDurationLessThanTimeout(windowLengthDuration.value + slidingIntervalDuration.value, topologyTimeout); } else if (windowLengthDuration != null) { ensureDurationLessThanTimeout(windowLengthDuration.value, topologyTimeout); } else if (slidingIntervalDuration != null) { ensureDurationLessThanTimeout(slidingIntervalDuration.value, topologyTimeout); } if (windowLengthCount != null && slidingIntervalCount != null) { ensureCountLessThanMaxPending(windowLengthCount.value + slidingIntervalCount.value, maxSpoutPending); } else if (windowLengthCount != null) { ensureCountLessThanMaxPending(windowLengthCount.value, maxSpoutPending); } else if (slidingIntervalCount != null) { ensureCountLessThanMaxPending(slidingIntervalCount.value, maxSpoutPending); } }
maxLagMs, getComponentStreams(context)); } else { if (topoConf.containsKey(Config.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM)) { validate(topoConf, windowLengthCount, windowLengthDuration, slidingIntervalCount, slidingIntervalDuration); evictionPolicy = getEvictionPolicy(windowLengthCount, windowLengthDuration); triggerPolicy = getTriggerPolicy(slidingIntervalCount, slidingIntervalDuration, manager, evictionPolicy); manager.setEvictionPolicy(evictionPolicy);
@Before public void setUp() { testWindowedBolt = new TestWindowedBolt(); testWindowedBolt.withTimestampField("ts"); executor = new WindowedBoltExecutor(testWindowedBolt); Map<String, Object> conf = new HashMap<>(); conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 100000); conf.put(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS, 20); conf.put(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS, 10); conf.put(Config.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS, 5); // trigger manually to avoid timing issues conf.put(Config.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS, 100000); executor.prepare(conf, getTopologyContext(), getOutputCollector()); }
@Override public void declareOutputFields(OutputFieldsDeclarer declarer) { String lateTupleStream = (String) getComponentConfiguration().get(Config.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM); if (lateTupleStream != null) { declarer.declareStream(lateTupleStream, new Fields(LATE_TUPLE_FIELD)); } bolt.declareOutputFields(declarer); }
private void validate(Map stormConf, Count windowLengthCount, Duration windowLengthDuration, Count slidingIntervalCount, Duration slidingIntervalDuration) { int topologyTimeout = getTopologyTimeoutMillis(stormConf); int maxSpoutPending = getMaxSpoutPending(stormConf); if (windowLengthCount == null && windowLengthDuration == null) { throw new IllegalArgumentException("Window length is not specified"); } if (windowLengthDuration != null && slidingIntervalDuration != null) { ensureDurationLessThanTimeout(windowLengthDuration.value + slidingIntervalDuration.value, topologyTimeout); } else if (windowLengthDuration != null) { ensureDurationLessThanTimeout(windowLengthDuration.value, topologyTimeout); } else if (slidingIntervalDuration != null) { ensureDurationLessThanTimeout(slidingIntervalDuration.value, topologyTimeout); } if (windowLengthCount != null && slidingIntervalCount != null) { ensureCountLessThanMaxPending(windowLengthCount.value + slidingIntervalCount.value, maxSpoutPending); } else if (windowLengthCount != null) { ensureCountLessThanMaxPending(windowLengthCount.value, maxSpoutPending); } else if (slidingIntervalCount != null) { ensureCountLessThanMaxPending(slidingIntervalCount.value, maxSpoutPending); } }
maxLagMs, getComponentStreams(context)); } else { if (stormConf.containsKey(Config.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM)) { validate(stormConf, windowLengthCount, windowLengthDuration, slidingIntervalCount, slidingIntervalDuration); evictionPolicy = getEvictionPolicy(windowLengthCount, windowLengthDuration); triggerPolicy = getTriggerPolicy(slidingIntervalCount, slidingIntervalDuration, manager, evictionPolicy); manager.setEvictionPolicy(evictionPolicy);
@Test public void testPrepareLateTupleStreamWithoutTs() throws Exception { Map<String, Object> conf = new HashMap<>(); conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 100000); conf.put(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS, 20); conf.put(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS, 10); conf.put(Config.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM, "$late"); conf.put(Config.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS, 5); conf.put(Config.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS, 10); testWindowedBolt = new TestWindowedBolt(); executor = new WindowedBoltExecutor(testWindowedBolt); TopologyContext context = getTopologyContext(); // emulate the call of withLateTupleStream method Mockito.when(context.getThisStreams()).thenReturn(new HashSet<>(Arrays.asList("default", "$late"))); try { executor.prepare(conf, context, getOutputCollector()); fail(); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), is("Late tuple stream can be defined only when specifying a timestamp field")); } }
@Override public void declareOutputFields(OutputFieldsDeclarer declarer) { String lateTupleStream = (String) getComponentConfiguration().get(Config.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM); if (lateTupleStream != null) { declarer.declareStream(lateTupleStream, new Fields(LATE_TUPLE_FIELD)); } bolt.declareOutputFields(declarer); }
@Test public void testPrepareLateTupleStreamWithoutBuilder() throws Exception { Map<String, Object> conf = new HashMap<>(); conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 100000); conf.put(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS, 20); conf.put(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS, 10); conf.put(Config.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM, "$late"); conf.put(Config.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS, 5); conf.put(Config.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS, 10); testWindowedBolt = new TestWindowedBolt(); testWindowedBolt.withTimestampField("ts"); executor = new WindowedBoltExecutor(testWindowedBolt); TopologyContext context = getTopologyContext(); try { executor.prepare(conf, context, getOutputCollector()); fail(); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), is("Stream for late tuples must be defined with the builder method withLateTupleStream")); } }
@Override public void execute(Tuple input) { if (!isStateInitialized()) { throw new IllegalStateException("execute invoked before initState with input tuple " + input); } else if (isRecovering()) { handleRecovery(input); } else { super.execute(input); } }
void prepare(Map stormConf, TopologyContext context, OutputCollector collector, KeyValueState<TaskStream, WindowState> windowState) { init(stormConf, context, collector, windowState); super.prepare(stormConf, context, collector); }
/** * Define a new bolt in this topology. This defines a windowed bolt, intended * for windowing operations. The {@link IWindowedBolt#execute(TupleWindow)} method * is triggered for each window interval with the list of current events in the window. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. * @param bolt the windowed bolt * @param parallelism_hint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process somwehere around the cluster. * @return use the returned object to declare the inputs to this component * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public BoltDeclarer setBolt(String id, IWindowedBolt bolt, Number parallelism_hint) throws IllegalArgumentException { return setBolt(id, new WindowedBoltExecutor(bolt), parallelism_hint); }
private void handleRecovery(Tuple input) { long msgId = getMsgId(input); TaskStream taskStream = TaskStream.fromTuple(input); WindowState state = recoveryStates.get(taskStream); LOG.debug("handleRecovery, recoveryStates {}", recoveryStates); if (state != null) { LOG.debug("Tuple msgid {}, saved state {}", msgId, state); if (msgId <= state.lastExpired) { LOG.debug("Ignoring tuple since msg id {} <= lastExpired id {}", msgId, state.lastExpired); outputCollector.ack(input); } else if (msgId <= state.lastEvaluated) { super.execute(input); } else { LOG.debug("Tuple msg id {} > lastEvaluated id {}, adding to pendingTuples and clearing recovery state " + "for taskStream {}", msgId, state.lastEvaluated, taskStream); pendingTuples.add(input); clearRecoveryState(taskStream); } } else { pendingTuples.add(input); } }
@Test(expected = IllegalArgumentException.class) public void testExecuteWithoutTs() throws Exception { executor.execute(getTuple("s1", new Fields("a"), new Values(1), "s1Src")); }
@Test public void testExecuteWithTs() throws Exception { long[] timestamps = { 603, 605, 607, 618, 626, 636 }; for (long ts : timestamps) { executor.execute(getTuple("s1", new Fields("ts"), new Values(ts), "s1Src")); } //Thread.sleep(120); executor.waterMarkEventGenerator.run(); //System.out.println(testWindowedBolt.tupleWindows); assertEquals(3, testWindowedBolt.tupleWindows.size()); TupleWindow first = testWindowedBolt.tupleWindows.get(0); assertArrayEquals(new long[]{ 603, 605, 607 }, new long[]{ (long) first.get().get(0).getValue(0), (long) first.get().get(1).getValue(0), (long) first.get().get(2).getValue(0) }); TupleWindow second = testWindowedBolt.tupleWindows.get(1); assertArrayEquals(new long[]{ 603, 605, 607, 618 }, new long[]{ (long) second.get().get(0).getValue(0), (long) second.get().get(1).getValue(0), (long) second.get().get(2).getValue(0), (long) second.get().get(3).getValue(0) }); TupleWindow third = testWindowedBolt.tupleWindows.get(2); assertArrayEquals(new long[]{ 618, 626 }, new long[]{ (long) third.get().get(0).getValue(0), (long) third.get().get(1).getValue(0) }); }