Refine search
public static StormTopology buildTopology(WindowsStoreFactory windowStore, WindowConfig windowConfig) throws Exception { FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"), new Values("the man went to the store and bought some candy"), new Values("four score and seven years ago"), new Values("how many apples can you eat"), new Values("to be or not to be the person")); spout.setCycle(true); TridentTopology topology = new TridentTopology(); Stream stream = topology.newStream("spout1", spout).parallelismHint(16).each(new Fields("sentence"), new Split(), new Fields("word")) .window(windowConfig, windowStore, new Fields("word"), new CountAsAggregator(), new Fields("count")) .peek(new Consumer() { @Override public void accept(TridentTuple input) { LOG.info("Received tuple: [{}]", input); } }); return topology.build(); }
@Override public Values toValues(TransactionalValue<ITuple> tuple) { Values values = new Values(); values.add(tuple.getTxid()); for (String valueField : tupleFields) { if (tuple.getVal() != null) { values.add(tuple.getVal().getValueByField(valueField)); } else { values.add(null); } } return values; }
@Override public T fromValues(List<Values> values) { if (values.size() == 0) { return null; } else if (values.size() == 1) { ByteBuffer bytes = (ByteBuffer) values.get(0).get(0); return serializer.deserialize(bytes.array()); } else { throw new IllegalArgumentException("Can only convert single values, " + values.size() + " encountered"); } }
@Override public Values apply(Tuple input) { Values values = new Values(); for (String field : fieldNames) { values.add(input.getValueByField(field)); } return values; } }
/** * Forwards the checkpoint tuple downstream. * * @param checkpointTuple the checkpoint tuple * @param action the action (prepare, commit, rollback or initstate) * @param txid the transaction id. */ protected void handleCheckpoint(Tuple checkpointTuple, Action action, long txid) { collector.emit(CHECKPOINT_STREAM_ID, checkpointTuple, new Values(txid, action)); collector.ack(checkpointTuple); }
@Override public void execute(Tuple input) { sum += ((Number) input.getValueByField("value")).longValue(); LOG.debug("{} sum = {}", name, sum); kvState.put("sum", sum); collector.emit(input, new Values(sum)); collector.ack(input); }
public void execute(Tuple tuple) { Object requestId = tuple.getValue(0); if (tuple.getSourceComponent().equals(returnComponent)) { returns.put(requestId, tuple); } else { results.put(requestId, tuple); } if (returns.containsKey(requestId) && results.containsKey(requestId)) { Tuple result = results.remove(requestId); Tuple returner = returns.remove(requestId); LOG.debug(result.getValue(1).toString()); List<Tuple> anchors = new ArrayList<>(); anchors.add(result); anchors.add(returner); _collector.emit(anchors, new Values("" + result.getValue(1), returner.getValue(1))); _collector.ack(result); _collector.ack(returner); } }
public void execute(Tuple input) { Integer sourceId = input.getInteger(0); Long eventId = input.getLong(1); Long recentEvent = recentEventId.get(sourceId); if (null != recentEvent && eventId <= recentEvent) { String error = "Error: event id is not in strict order! event source Id: " + sourceId + ", last event Id: " + recentEvent + ", current event Id: " + eventId; _collector.emit(input, new Values(error)); } recentEventId.put(sourceId, eventId); _collector.ack(input); }
private boolean checkFinishId(Tuple tup, TupleType type) { Object id = tup.getValue(0); boolean failed = false; failed = true; for (Tuple t : track.ackTuples) { _collector.fail(t); while (outTasks.hasNext()) { int task = outTasks.next(); int numTuples = Utils.get(track.taskEmittedTuples, task, 0); _collector.emitDirect(task, Constants.COORDINATED_STREAM_ID, tup, new Values(id, numTuples)); _collector.ack(t);
@Test public void testTestTuple() throws Exception { Tuple tuple = Testing.testTuple(new Values("james", "bond")); assertThat(tuple.getValues(), is(new Values("james", "bond"))); assertThat(tuple.getSourceStreamId(), is(Utils.DEFAULT_STREAM_ID)); assertThat(tuple.getFields().toList(), is(Arrays.asList("field1", "field2"))); assertThat(tuple.getSourceComponent(), is("component")); }
@Test public void testHooks() throws Exception { try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .build()) { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(new Fields("conf")))); Map<String, BoltDetails> boltMap = Collections.singletonMap("2", Thrift.prepareBoltDetails( Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new HooksBolt())); StormTopology topology = Thrift.buildTopology(spoutMap, boltMap); List<FixedTuple> testTuples = Arrays.asList(1, 1, 1, 1).stream() .map(value -> new FixedTuple(new Values(value))) .collect(Collectors.toList()); MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples)); CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam(); completeTopologyParams.setMockedSources(mockedSources); Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topology, completeTopologyParams); List<List<Object>> expectedTuples = Arrays.asList( Arrays.asList(0, 0, 0, 0), Arrays.asList(2, 1, 0, 1), Arrays.asList(4, 1, 1, 2), Arrays.asList(6, 2, 1, 3)); assertThat(Testing.readTuples(results, "2"), is(expectedTuples)); } }
@Test public void testEmitTs() throws Exception { Tuple tupleWithTs = Mockito.mock(Tuple.class); setUpMockTuples(tupleWithTs); Mockito.when(tupleWithTs.getLongByField("ts")).thenReturn(12345L); setUpProcessorBolt(new FilterProcessor(x -> true), "ts"); bolt.execute(tupleWithTs); ArgumentCaptor<Collection> anchor = ArgumentCaptor.forClass(Collection.class); ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class); ArgumentCaptor<String> os = ArgumentCaptor.forClass(String.class); Mockito.verify(mockOutputCollector).emit(os.capture(), anchor.capture(), values.capture()); assertEquals("outputstream", os.getValue()); assertArrayEquals(new Object[]{ tupleWithTs }, anchor.getValue().toArray()); assertEquals(new Values(100, 12345L), values.getValue()); Mockito.verify(mockOutputCollector, Mockito.times(1)).ack(tupleWithTs); }
@Override public void execute(TupleWindow inputWindow) { int sum = 0; List<Tuple> tuplesInWindow = inputWindow.get(); LOG.debug("Events in current window: " + tuplesInWindow.size()); if (tuplesInWindow.size() > 0) { /* * Since this is a tumbling window calculation, * we use all the tuples in the window to compute the avg. */ for (Tuple tuple : tuplesInWindow) { sum += (int) tuple.getValue(0); } collector.emit(new Values(sum / tuplesInWindow.size())); } }