@Override public void execute(TupleWindow inputWindow) { List<Tuple> tuplesInWindow = inputWindow.get(); List<Tuple> newTuples = inputWindow.getNew(); List<Tuple> expiredTuples = inputWindow.getExpired(); LOG.info("tuplesInWindow.size() = " + tuplesInWindow.size()); LOG.info("newTuples.size() = " + newTuples.size()); LOG.info("expiredTuples.size() = " + expiredTuples.size()); LOG.info(StringDecorator.decorate(componentId, "tuplesInWindow = " + tuplesInWindow.stream() .map(t -> t.getValues()) .collect(Collectors.toList()))); collector.emit(new Values("dummyValue")); }
@Override public void execute(TupleWindow inputWindow) { collector.emit(new Values(inputWindow.get().size())); }
@Override public void execute(TupleWindow inputWindow) { LOG.trace("Window triggered at {}, inputWindow {}", new Date(), inputWindow); if (delegate.isEventTimestamp()) { delegate.setEventTimestamp(inputWindow.getEndTimestamp()); } for (Tuple tuple : inputWindow.get()) { Pair<Object, String> valueAndStream = delegate.getValueAndStream(tuple); if (!StreamUtil.isPunctuation(valueAndStream.getFirst())) { delegate.process(valueAndStream.getFirst(), valueAndStream.getSecond()); } } for (String stream : delegate.getInitialStreams()) { delegate.process(PUNCTUATION, stream); } }
/** * Logs information about the {@link TupleWindow}. * * @param window The tuple window. */ private void log(TupleWindow window) { // summarize the newly received tuples LongSummaryStatistics received = window.get() .stream() .map(tuple -> getField(TIMESTAMP_TUPLE_FIELD, tuple, Long.class)) .collect(Collectors.summarizingLong(Long::longValue)); LOG.debug("Tuple(s) received; count={}, min={}, max={}, range={} ms", received.getCount(), received.getMin(), received.getMax(), received.getMax() - received.getMin()); if (window.getExpired().size() > 0) { // summarize the expired tuples LongSummaryStatistics expired = window.getExpired() .stream() .map(tuple -> getField(TIMESTAMP_TUPLE_FIELD, tuple, Long.class)) .collect(Collectors.summarizingLong(Long::longValue)); LOG.debug("Tuple(s) expired; count={}, min={}, max={}, range={} ms, lag={} ms", expired.getCount(), expired.getMin(), expired.getMax(), expired.getMax() - expired.getMin(), received.getMin() - expired.getMin()); } }
@Override public void execute(TupleWindow inputWindow) { int sum = 0; List<Tuple> tuplesInWindow = inputWindow.get(); LOG.debug("Events in current window: " + tuplesInWindow.size()); if (tuplesInWindow.size() > 0) { /* * Since this is a tumbling window calculation, * we use all the tuples in the window to compute the avg. */ for (Tuple tuple : tuplesInWindow) { sum += (int) tuple.getValue(0); } collector.emit(new Values(sum / tuplesInWindow.size())); } }
@Override public void execute(TupleWindow inputWindow) { List<Tuple> tuplesInWindow = inputWindow.get(); List<Tuple> newTuples = inputWindow.getNew(); List<Tuple> expiredTuples = inputWindow.getExpired(); LOG.info("tuplesInWindow.size() = " + tuplesInWindow.size()); LOG.info("newTuples.size() = " + newTuples.size()); LOG.info("expiredTuples.size() = " + expiredTuples.size()); List<TimeData> dataInWindow = tuplesInWindow.stream() .map(TimeData::fromTuple) .collect(Collectors.toList()); final String jsonData = TimeData.toString(dataInWindow); LOG.info(StringDecorator.decorate(componentId, jsonData)); collector.emit(new Values("dummyValue")); }
private TupleWindow getMockTupleWindow(Tuple... tuples) { TupleWindow tupleWindow = Mockito.mock(TupleWindow.class); Mockito.when(tupleWindow.get()).thenReturn(Arrays.asList(tuples)); return tupleWindow; } }
@Override public void execute(TupleWindow inputWindow) { /* * The inputWindow gives a view of * (a) all the events in the window * (b) events that expired since last activation of the window * (c) events that newly arrived since last activation of the window */ List<Tuple> tuplesInWindow = inputWindow.get(); List<Tuple> newTuples = inputWindow.getNew(); List<Tuple> expiredTuples = inputWindow.getExpired(); LOG.debug("Events in current window: " + tuplesInWindow.size()); /* * Instead of iterating over all the tuples in the window to compute * the sum, the values for the new events are added and old events are * subtracted. Similar optimizations might be possible in other * windowing computations. */ for (Tuple tuple : newTuples) { sum += (int) tuple.getValue(0); } for (Tuple tuple : expiredTuples) { sum -= (int) tuple.getValue(0); } collector.emit(new Values(sum)); }
@Override public Void answer(InvocationOnMock invocation) throws Throwable { TupleWindow window = (TupleWindow) invocation.getArguments()[0]; // iterate the tuples Assert.assertEquals(WINDOW_EVENT_COUNT, window.get().size()); // iterating multiple times should produce same events Assert.assertEquals(WINDOW_EVENT_COUNT, window.get().size()); Assert.assertEquals(WINDOW_EVENT_COUNT, window.get().size()); return null; } }).when(mockBolt).execute(Mockito.any());
@Override public void execute(TupleWindow inputWindow) { /* * The inputWindow gives a view of * (a) all the events in the window * (b) events that expired since last activation of the window * (c) events that newly arrived since last activation of the window */ List<Tuple> tuplesInWindow = inputWindow.get(); List<Tuple> newTuples = inputWindow.getNew(); List<Tuple> expiredTuples = inputWindow.getExpired(); LOG.debug("Events in current window: " + tuplesInWindow.size()); /* * Instead of iterating over all the tuples in the window to compute * the sum, the values for the new events are added and old events are * subtracted. Similar optimizations might be possible in other * windowing computations. */ for (Tuple tuple : newTuples) { sum += (int) tuple.getValue(0); } for (Tuple tuple : expiredTuples) { sum -= (int) tuple.getValue(0); } collector.emit(new Values(sum)); }
@Override public void execute(TupleWindow inputWindow) { for (Tuple tuple : inputWindow.get()) { sum += tuple.getIntegerByField("value"); } state.put("sum", sum); collector.emit(new Values(sum)); }
@Test public void testExecuteWithTs() throws Exception { long[] timestamps = { 603, 605, 607, 618, 626, 636 }; for (long ts : timestamps) { executor.execute(getTuple("s1", new Fields("ts"), new Values(ts), "s1Src")); } //Thread.sleep(120); executor.waterMarkEventGenerator.run(); //System.out.println(testWindowedBolt.tupleWindows); assertEquals(3, testWindowedBolt.tupleWindows.size()); TupleWindow first = testWindowedBolt.tupleWindows.get(0); assertArrayEquals(new long[]{ 603, 605, 607 }, new long[]{ (long) first.get().get(0).getValue(0), (long) first.get().get(1).getValue(0), (long) first.get().get(2).getValue(0) }); TupleWindow second = testWindowedBolt.tupleWindows.get(1); assertArrayEquals(new long[]{ 603, 605, 607, 618 }, new long[]{ (long) second.get().get(0).getValue(0), (long) second.get().get(1).getValue(0), (long) second.get().get(2).getValue(0), (long) second.get().get(3).getValue(0) }); TupleWindow third = testWindowedBolt.tupleWindows.get(2); assertArrayEquals(new long[]{ 618, 626 }, new long[]{ (long) third.get().get(0).getValue(0), (long) third.get().get(1).getValue(0) }); }
@Override public void execute(TupleWindow inputWindow) { // 1) Perform Join List<Tuple> currentWindow = inputWindow.get(); JoinAccumulator joinResult = hashJoin(currentWindow); // 2) Emit results for (ResultRecord resultRecord : joinResult.getRecords()) { ArrayList<Object> outputTuple = resultRecord.getOutputFields(); if (outputStreamName == null) { // explicit anchoring emits to corresponding input tuples only, as default window anchoring will anchor them to all // tuples in window collector.emit(resultRecord.tupleList, outputTuple); } else { // explicitly anchor emits to corresponding input tuples only, as default window anchoring will anchor them to all tuples // in window collector.emit(outputStreamName, resultRecord.tupleList, outputTuple); } } }
@Override public void execute(TupleWindow inputWindow) { Map<String, Long[]> counts = new HashMap<String, Long[]>(); for (Tuple tuple : inputWindow.get()) { Long time = tuple.getLong(1); String ip = tuple.getString(0); Long[] timeAndCount = counts.get(ip); if (null == timeAndCount) { timeAndCount = new Long[2]; timeAndCount[0] = time; timeAndCount[1] = 0L; } timeAndCount[0] = Math.min(timeAndCount[0], time); timeAndCount[1]++; counts.put(ip, timeAndCount); } LatencyReporter latencyReporter = new KafkaReporter(config.reporterTopic, config.brokerList); for (Long[] timeAndCount : counts.values()) { for (int i = 0; i < timeAndCount[1]; i++) { latencyReporter.report(timeAndCount[0], System.currentTimeMillis()); } } }
@Override public void execute(TupleWindow window) { if(LOG.isDebugEnabled()) { log(window); } try { // handle each tuple in the window for(Tuple tuple : window.get()) { handleMessage(tuple); } // time to flush active profiles? if(activeFlushSignal.isTimeToFlush()) { flushActive(); } } catch (Throwable e) { LOG.error("Unexpected error", e); collector.reportError(e); } }
/** * Process the tuple window and optionally emit new tuples based on the tuples in the input window. * * @param inputWindow */ @Override public void execute(TupleWindow inputWindow) { ++windowId; LOG.debug("Window activated, window id {}, number of tuples in window {}", windowId, inputWindow.get().size()); Map<String, Tuple> eventIdToTupleMap = new HashMap<>(); try { StreamlineEvent event; for (Tuple input : inputWindow.get()) { if ((event = getStreamlineEventFromTuple(input)) != null) { LOG.debug("++++++++ Executing tuple [{}] which contains StreamlineEvent [{}]", input, event); eventIdToTupleMap.put(event.getId(), input); processAndEmit(event, eventIdToTupleMap); } } // force evaluation of the last group by processAndEmit(GROUP_BY_TRIGGER_EVENT, eventIdToTupleMap); // current group is processed and result emitted eventIdToTupleMap.clear(); } catch (Exception e) { collector.reportError(e); LOG.error("", e); } }
@Override public void execute(TupleWindow inputWindow) { // 1) Perform Join List<Tuple> currentWindow = inputWindow.get(); JoinAccumulator joinResult = hashJoin(currentWindow); // 2) Emit results for (ResultRecord resultRecord : joinResult.getRecords()) { ArrayList<Object> outputTuple = resultRecord.getOutputFields(); if ( outputStreamName==null ) // explicit anchoring emits to corresponding input tuples only, as default window anchoring will anchor them to all tuples in window collector.emit( resultRecord.tupleList, outputTuple ); else // explicitly anchor emits to corresponding input tuples only, as default window anchoring will anchor them to all tuples in window collector.emit( outputStreamName, resultRecord.tupleList, outputTuple ); } }