@Override public long extractTimestamp(Tuple tuple) { return tuple.getLongByField(fieldName); }
private long getMsgId(Tuple input) { return input.getLongByField(msgIdFieldName); }
@Override public Writable key(Tuple tuple) { if (this.key == null) { this.key = new LongWritable(); } this.key.set(tuple.getLongByField(this.keyField)); return this.key; }
public static TimeData fromTuple(Tuple tuple) { return new TimeData(tuple.getIntegerByField(NUMBER_FIELD_NAME), new Date(tuple.getLongByField(TIMESTAMP_FIELD_NAME))); }
private List<Tuple> getMockTuples(int count) { List<Tuple> mockTuples = new ArrayList<>(); for (long i = 0; i < count; i++) { Tuple mockTuple = Mockito.mock(Tuple.class); Mockito.when(mockTuple.getLongByField("msgid")).thenReturn(i); Mockito.when(mockTuple.getSourceTask()).thenReturn(1); Mockito.when(mockTuple.getSourceGlobalStreamId()).thenReturn(new GlobalStreamId("a", "s")); mockTuples.add(mockTuple); } return mockTuples; } }
@Test public void testCommitBeforeInitstate() throws Exception { Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default"); Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(COMMIT); Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(100)); executor.execute(mockCheckpointTuple); Mockito.verify(mockOutputCollector, Mockito.times(1)).ack(mockCheckpointTuple); Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(ROLLBACK); Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(100)); executor.execute(mockCheckpointTuple); Mockito.verify(mockState, Mockito.times(1)).rollback(); }
long txid = input.getLongByField(CHECKPOINT_FIELD_TXID); if (shouldProcessTransaction(action, txid)) { LOG.debug("Processing action {}, txid {}", action, txid);
@Test public void testRollback() throws Exception { Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default"); executor.execute(mockTuple); Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(ROLLBACK); Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(0)); Mockito.doNothing().when(mockOutputCollector).ack(mockCheckpointTuple); executor.execute(mockCheckpointTuple); Mockito.verify(mockState, Mockito.times(1)).rollback(); }
@Test public void testCommit() throws Exception { Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default"); executor.execute(mockTuple); Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(COMMIT); Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(0)); Mockito.doNothing().when(mockOutputCollector).ack(mockCheckpointTuple); executor.execute(mockCheckpointTuple); Mockito.verify(mockBolt, Mockito.times(1)).preCommit(new Long(0)); Mockito.verify(mockState, Mockito.times(1)).commit(new Long(0)); }
@Test public void testHandleTuple() throws Exception { Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default"); executor.execute(mockTuple); Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(INITSTATE); Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(0)); Mockito.doNothing().when(mockOutputCollector).ack(mockCheckpointTuple); executor.execute(mockCheckpointTuple); Mockito.verify(mockBolt, Mockito.times(1)).execute(mockTuple); Mockito.verify(mockBolt, Mockito.times(1)).initState(Mockito.any(KeyValueState.class)); }
@Test public void testPrepareAndRollbackBeforeInitstate() throws Exception { Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default"); executor.execute(mockTuple); Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(PREPARE); Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(100)); executor.execute(mockCheckpointTuple); Mockito.verify(mockOutputCollector, Mockito.times(1)).fail(mockCheckpointTuple); Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(ROLLBACK); Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(100)); Mockito.doNothing().when(mockOutputCollector).ack(mockCheckpointTuple); executor.execute(mockCheckpointTuple); Mockito.verify(mockState, Mockito.times(1)).rollback(); }
@Override public void execute(Tuple input) { LOG.debug("** EventLoggerBolt got tuple from sourceComponent {}, with values {}", input.getSourceComponent(), input.getValues()); Object msgId = input.getValueByField(FIELD_MESSAGE_ID); EventInfo eventInfo = new EventInfo(input.getLongByField(FIELD_TS), input.getSourceComponent(), input.getSourceTask(), msgId, (List<Object>) input.getValueByField(FIELD_VALUES)); for (IEventLogger eventLogger : eventLoggers) { eventLogger.log(eventInfo); } }
@Test public void testEmitTs() throws Exception { Tuple tupleWithTs = Mockito.mock(Tuple.class); setUpMockTuples(tupleWithTs); Mockito.when(tupleWithTs.getLongByField("ts")).thenReturn(12345L); setUpProcessorBolt(new FilterProcessor(x -> true), "ts"); bolt.execute(tupleWithTs); ArgumentCaptor<Collection> anchor = ArgumentCaptor.forClass(Collection.class); ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class); ArgumentCaptor<String> os = ArgumentCaptor.forClass(String.class); Mockito.verify(mockOutputCollector).emit(os.capture(), anchor.capture(), values.capture()); assertEquals("outputstream", os.getValue()); assertArrayEquals(new Object[]{ tupleWithTs }, anchor.getValue().toArray()); assertEquals(new Values(100, 12345L), values.getValue()); Mockito.verify(mockOutputCollector, Mockito.times(1)).ack(tupleWithTs); }
@Test public void testPrepareAndCommit() throws Exception { Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default"); Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(INITSTATE); Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(0)); executor.execute(mockCheckpointTuple); executor.execute(mockTuple); Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(PREPARE); Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(100)); executor.execute(mockCheckpointTuple); executor.execute(mockTuple); Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(COMMIT); Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(100)); executor.execute(mockCheckpointTuple); mockOutputCollector.ack(mockTuple); Mockito.verify(mockState, Mockito.times(1)).commit(new Long(100)); Mockito.verify(mockBolt, Mockito.times(2)).execute(mockTuple); Mockito.verify(mockOutputCollector, Mockito.times(1)).ack(mockTuple); } }
void processAndAck(Tuple input) { RefCountedTuple refCountedTuple = new RefCountedTuple(input); setAnchor(refCountedTuple); if (isEventTimestamp()) { setEventTimestamp(input.getLongByField(getTimestampField())); } Pair<Object, String> valueAndStream = getValueAndStream(input); process(valueAndStream.getFirst(), valueAndStream.getSecond()); ack(refCountedTuple); }
ts = tuple.getLongByField(tupleTimestampField);
/** * This bolt consumes telemetry messages and determines if the message is needed * by any of the profiles. The message is then routed to one or more downstream * bolts that are responsible for building each profile * * <p>The outgoing tuples are timestamped so that Storm's window and event-time * processing functionality can recognize the time of each message. * * <p>The timestamp that is attached to each outgoing tuple is what decides if * the Profiler is operating on processing time or event time. * * @param input The tuple. */ @Override public void execute(Tuple input) { try { LOG.debug("Received message; topic={}, partition={}, offset={}, kafkaTimestamp={}", input.contains(TOPIC.getFieldName()) ? input.getStringByField(TOPIC.getFieldName()): "unknown", input.contains(PARTITION.getFieldName()) ? input.getIntegerByField(PARTITION.getFieldName()): "unknown", input.contains(OFFSET.getFieldName()) ? input.getLongByField(OFFSET.getFieldName()): "unknown", input.contains(TIMESTAMP.getFieldName()) ? input.getLongByField(TIMESTAMP.getFieldName()): "unknown"); doExecute(input); } catch (Throwable t) { LOG.error("Unexpected error", t); collector.reportError(t); } finally { collector.ack(input); } }
@Override public long extractTimestamp(Tuple tuple) { return tuple.getLongByField(fieldName); }
@Override public void execute(Tuple input, BasicOutputCollector basicOutputCollector) { hashOperations.put(WORD_COUNT_REAL_TIME_KEY, input.getStringByField("obj"), input.getLongByField("count")); }
@Override public void execute(Tuple input) { LOG.debug("** EventLoggerBolt got tuple from sourceComponent {}, with values {}", input.getSourceComponent(), input.getValues()); Object msgId = input.getValueByField(FIELD_MESSAGE_ID); EventInfo eventInfo = new EventInfo(input.getLongByField(FIELD_TS), input.getSourceComponent(), input.getSourceTask(), msgId, (List<Object>) input.getValueByField(FIELD_VALUES)); for (IEventLogger eventLogger : eventLoggers) { eventLogger.log(eventInfo); } }