Refine search
@Override protected void process(Tuple input) { MqttMessage message = this.mapper.toMessage(input); try { this.publisher.publish(message); this.collector.ack(input); } catch (Exception e) { LOG.warn("Error publishing MQTT message. Failing tuple.", e); // should we fail the tuple or kill the worker? collector.reportError(e); collector.fail(input); } }
private void failBatch(TrackedBatch tracked, FailedException e) { if (e != null && e instanceof ReportedFailedException) { _collector.reportError(e); } tracked.failed = true; if (tracked.delayedAck != null) { _collector.fail(tracked.delayedAck); tracked.delayedAck = null; } }
/** * Forwards the checkpoint tuple downstream. * * @param checkpointTuple the checkpoint tuple * @param action the action (prepare, commit, rollback or initstate) * @param txid the transaction id. */ protected void handleCheckpoint(Tuple checkpointTuple, Action action, long txid) { collector.emit(CHECKPOINT_STREAM_ID, checkpointTuple, new Values(txid, action)); collector.ack(checkpointTuple); }
@Test public void testNonBatchErrorPath() throws Exception { ParserConfigurations configurations = getConfigurations(1); String sensorType = "test"; Tuple t = mock(Tuple.class); when(t.getValueByField(eq("message"))).thenThrow(new IllegalStateException()); WriterBolt bolt = new WriterBolt(new WriterHandler(writer), configurations, sensorType); bolt.prepare(new HashMap(), topologyContext, outputCollector); verify(writer, times(1)).init(); bolt.execute(t); verify(outputCollector, times(1)).ack(t); verify(writer, times(0)).write(eq(sensorType), any(), any(), any()); verify(outputCollector, times(1)).reportError(any()); verify(outputCollector, times(0)).fail(any()); } @Test
@Test public void testCommitBeforeInitstate() throws Exception { Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default"); Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(COMMIT); Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(100)); executor.execute(mockCheckpointTuple); Mockito.verify(mockOutputCollector, Mockito.times(1)).ack(mockCheckpointTuple); Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(ROLLBACK); Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(100)); executor.execute(mockCheckpointTuple); Mockito.verify(mockState, Mockito.times(1)).rollback(); }
@Test public void testSourceTypeMissing() throws Exception { // setup the bolt BulkMessageWriterBolt<IndexingConfigurations> bulkMessageWriterBolt = new BulkMessageWriterBolt<IndexingConfigurations>( "zookeeperUrl", "INDEXING") .withBulkMessageWriter(bulkMessageWriter) .withMessageGetter(MessageGetters.JSON_FROM_FIELD.name()) .withMessageGetterField("message"); bulkMessageWriterBolt.setCuratorFramework(client); bulkMessageWriterBolt.setZKCache(cache); bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(sensorType, new FileInputStream(sampleSensorIndexingConfigPath)); // initialize the bolt bulkMessageWriterBolt.declareOutputFields(declarer); Map stormConf = new HashMap(); bulkMessageWriterBolt.prepare(stormConf, topologyContext, outputCollector); // create a message with no source type JSONObject message = (JSONObject) new JSONParser().parse(sampleMessageString); message.remove("source.type"); when(tuple.getValueByField("message")).thenReturn(message); // the tuple should be handled as an error and ack'd bulkMessageWriterBolt.execute(tuple); verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), any()); verify(outputCollector, times(1)).ack(tuple); }
@Test public void testExecuteTuple() throws Exception { Mockito.when(mockWaterMarkEventGenerator.track(Mockito.any(), Mockito.anyLong())).thenReturn(true); Tuple mockTuple = Mockito.mock(Tuple.class); executor.initState(null); executor.waterMarkEventGenerator = mockWaterMarkEventGenerator; executor.execute(mockTuple); // should be ack-ed once Mockito.verify(mockOutputCollector, Mockito.times(1)).ack(mockTuple); }
@Test public void shouldThrowExceptionOnFailedExecute() { when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8)); when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic"); doThrow(new IllegalStateException("parserRunner.execute failed")).when(parserRunner).execute(eq("yaf"), any(), eq(parserConfigurations)); verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); verify(outputCollector, times(1)).reportError(any(IllegalStateException.class)); verify(outputCollector, times(1)).ack(t1);
@SuppressWarnings("unchecked") @Test public void testExecuteShouldReportError() throws ExecutionException { joinBolt.withMaxCacheSize(100); joinBolt.withMaxTimeRetain(10000); joinBolt.prepare(new HashMap(), topologyContext, outputCollector); when(tuple.getValueByField("key")).thenReturn(key); when(tuple.getValueByField("message")).thenReturn(new JSONObject()); joinBolt.cache = mock(LoadingCache.class); when(joinBolt.cache.get(any())).thenThrow(new RuntimeException(new Exception("join exception"))); joinBolt.execute(tuple); RuntimeException expectedExecutionException = new RuntimeException(new Exception("join exception")); MetronError error = new MetronError() .withErrorType(Constants.ErrorType.ENRICHMENT_ERROR) .withMessage("Joining problem: {}") .withThrowable(expectedExecutionException) .addRawMessage(new JSONObject()); verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); verify(outputCollector, times(1)).reportError(any(ExecutionException.class)); verify(outputCollector, times(1)).ack(eq(tuple)); verifyNoMoreInteractions(outputCollector); } }
joinBolt.prepare(new HashMap(), topologyContext, outputCollector); Tuple geoTuple = mock(Tuple.class); when(geoTuple.getValueByField("key")).thenReturn(key); when(geoTuple.getSourceStreamId()).thenReturn("geo"); when(geoTuple.getValueByField("message")).thenReturn(geoMessage); joinBolt.execute(geoTuple); joinBolt.execute(stellarTuple); verify(outputCollector, times(1)).emit(eq("message"), any(tuple.getClass()), eq(new Values(key, joinedMessage))); verify(outputCollector, times(1)).ack(messageTuple); verifyNoMoreInteractions(outputCollector);
@Test public void testEmitTs() throws Exception { Tuple tupleWithTs = Mockito.mock(Tuple.class); setUpMockTuples(tupleWithTs); Mockito.when(tupleWithTs.getLongByField("ts")).thenReturn(12345L); setUpProcessorBolt(new FilterProcessor(x -> true), "ts"); bolt.execute(tupleWithTs); ArgumentCaptor<Collection> anchor = ArgumentCaptor.forClass(Collection.class); ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class); ArgumentCaptor<String> os = ArgumentCaptor.forClass(String.class); Mockito.verify(mockOutputCollector).emit(os.capture(), anchor.capture(), values.capture()); assertEquals("outputstream", os.getValue()); assertArrayEquals(new Object[]{ tupleWithTs }, anchor.getValue().toArray()); assertEquals(new Values(100, 12345L), values.getValue()); Mockito.verify(mockOutputCollector, Mockito.times(1)).ack(tupleWithTs); }
when(bulkMessageWriter.getName()).thenReturn("hdfs"); messageList = new ArrayList<>(); for(int i = 0; i < 4; i++) { when(tuple.getValueByField("message")).thenReturn(fullMessageList.get(i)); tupleList.add(tuple); messageList.add(fullMessageList.get(i)); when(tuple.getValueByField("message")).thenReturn(fullMessageList.get(4)); tupleList.add(tuple); messageList.add(fullMessageList.get(4)); verify(outputCollector, times(5)).ack(tuple); when(tuple.getValueByField("message")).thenReturn(fullMessageList.get(0)); UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.FATAL); for(int i = 0; i < 5; i++) { verify(outputCollector, times(5)).ack(tuple); verify(outputCollector, times(5)).emit(eq(Constants.ERROR_STREAM), any(Values.class)); verify(outputCollector, times(1)).reportError(any(Throwable.class));
@Test public void executeShouldHandleTickTuple() throws Exception { when(t1.getSourceComponent()).thenReturn("__system"); when(t1.getSourceStreamId()).thenReturn("__tick"); ParserConfigurations parserConfigurations = mock(ParserConfigurations.class); ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserRunner, new HashMap<String, WriterHandler>() {{ put("yaf", writerHandler); }}) { @Override public ParserConfigurations getConfigurations() { return parserConfigurations; } }; parserBolt.setMessageGetStrategy(messageGetStrategy); parserBolt.setOutputCollector(outputCollector); parserBolt.execute(t1); verify(writerHandler, times(1)).flush(parserConfigurations, messageGetStrategy); verify(outputCollector, times(1)).ack(t1); }
@Test public void testActivation() throws Exception { Mockito.when(mockWaterMarkEventGenerator.track(Mockito.any(), Mockito.anyLong())).thenReturn(true); executor.initState(null); executor.waterMarkEventGenerator = mockWaterMarkEventGenerator; mockTuples.forEach(t -> executor.execute(t)); Mockito.verify(mockOutputCollector, Mockito.times(WINDOW_EVENT_COUNT)).ack(tupleCaptor.capture()); Assert.assertArrayEquals(mockTuples.toArray(), tupleCaptor.getAllValues().toArray());
@Test public void shouldExecuteOnError() throws Exception { when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8)); when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic"); MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {{ add("yaf"); verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); verify(outputCollector, times(1)).ack(t1);
@Test public void handleErrorShouldEmitAndReportError() throws Exception { Throwable e = new Exception("error"); MetronError error = new MetronError().withMessage("error message").withThrowable(e); OutputCollector collector = mock(OutputCollector.class); ErrorUtils.handleError(collector, error); verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); verify(collector, times(1)).reportError(any()); } }
@Test public void test() { StandAloneSplitBolt splitBolt = spy(new StandAloneSplitBolt("zookeeperUrl")); splitBolt.setCuratorFramework(client); splitBolt.setZKCache(cache); doCallRealMethod().when(splitBolt).reloadCallback(anyString(), any(ConfigurationType.class)); splitBolt.prepare(new HashMap(), topologyContext, outputCollector); splitBolt.declareOutputFields(declarer); verify(declarer, times(1)).declareStream(eq("message"), argThat(new FieldsMatcher("key", "message", "subgroup"))); for(String streamId: streamIds) { verify(declarer, times(1)).declareStream(eq(streamId), argThat(new FieldsMatcher("key", "message"))); doReturn(streamMessageMap).when(splitBolt).splitMessage(sampleMessage); splitBolt.execute(tuple); verify(outputCollector, times(1)).emit(eq("message"), any(tuple.getClass()), eq(new Values(key, sampleMessage, ""))); verify(outputCollector, times(1)).emit(eq("geo"), eq(new Values(key, geoMessage))); verify(outputCollector, times(1)).emit(eq("host"), eq(new Values(key, hostMessage))); verify(outputCollector, times(1)).emit(eq("hbaseEnrichment"), eq(new Values(key, hbaseEnrichmentMessage))); verify(outputCollector, times(1)).ack(tuple); streamMessageMap = new HashMap<>(); streamMessageMap.put("host", null);