/** * Constructor for a message having content, a {@link Signal} and a sequence number. * * @param id The ID associated with the message. * @param content The content of the message. * @param signal The Signal to be sent with the message. * @param sequence The sequence number of the message. */ public PubSubMessage(String id, String content, Signal signal, int sequence) { this(id, content, new Metadata(signal, null), sequence); }
private void emitMetaSignal(String id, Metadata.Signal signal) { log.error("Emitting {} signal to the feedback stream for {}", signal, id); collector.emit(FEEDBACK_STREAM, new Values(id, new Metadata(signal, null))); }
private void emitMetaSignal(String id, Metadata.Signal signal) { log.error("Emitting {} signal to the feedback stream for {}", signal, id); collector.emit(FEEDBACK_STREAM, new Values(id, new Metadata(signal, null))); }
private Metadata withSignal(Metadata metadata, Metadata.Signal signal) { // Don't change the non-readonly bits of metadata in place since that might affect tuples emitted but pending. Metadata copy = new Metadata(signal, null); if (metadata != null) { copy.setContent(metadata.getContent()); } return copy; }
private Metadata withSignal(Metadata metadata, Metadata.Signal signal) { // Don't change the non-readonly bits of metadata in place since that might affect tuples emitted but pending. Metadata copy = new Metadata(signal, null); if (metadata != null) { copy.setContent(metadata.getContent()); } return copy; }
@Override public void send(PubSubMessage message) { // Put responseURL in the metadata so the ResponsePublisher knows to which host to send the response Metadata metadata = message.getMetadata(); metadata = metadata == null ? new Metadata() : metadata; metadata.setContent(resultURL); message.setMetadata(metadata); sendToURL(queryURL, message); } }
@Test(expectedExceptions = PubSubException.class) public void testFailing() throws Exception { injectedMockBolt.setFailNumber(1); Assert.assertEquals(injectedMockBolt.getCount(), 0); PubSubMessage message = new PubSubMessage("foo", "{}", new Metadata(null, makeReturnInfo("a", "testHost", 80))); publisher.send(message); }
@Test public void testExecuteStillAcksWhenPublisherThrows() throws Exception { // Execute a few tuples // Closing the publisher will cause CustomPublisher to throw publisher.close(); bolt.execute(makeTuple("42", "This is a PubSubMessage", new Metadata())); bolt.execute(makeTuple("43", "This is also a PubSubMessage", new Metadata())); bolt.execute(makeTuple("44", "This is still a PubSubMessage", new Metadata())); // Assert that no tuples were sent, committed, or acked Assert.assertTrue(publisher.getSent().isEmpty()); Assert.assertEquals(collector.getAckedCount(), 3); }
@Test public void testExecuteMessagesAreSent() { List<PubSubMessage> expected = asList(new PubSubMessage("42", "This is a PubSubMessage", new Metadata()), new PubSubMessage("43", "This is also a PubSubMessage", new Metadata()), new PubSubMessage("44", "This is still a PubSubMessage", new Metadata())); List<Tuple> tuples = new ArrayList<>(); expected.forEach(m -> tuples.add(makeTuple(m.getId(), m.getContent(), m.getMetadata()))); for (int i = 0; i < tuples.size(); i++) { bolt.execute(tuples.get(i)); Assert.assertEquals(publisher.getSent().get(i).getId(), expected.get(i).getId()); Assert.assertEquals(publisher.getSent().get(i).getContent(), expected.get(i).getContent()); Assert.assertEquals(publisher.getSent().get(i).getMetadata(), expected.get(i).getMetadata()); Assert.assertTrue(collector.wasNthAcked(tuples.get(i), i + 1)); Assert.assertEquals(collector.getAckedCount(), i + 1); } }
@Test public void testAcksEvenOnException() throws Exception { publisher.close(); bolt.execute(makeTuple("42", new Metadata(Metadata.Signal.KILL, null))); bolt.execute(makeTuple("43", new Metadata(Metadata.Signal.FAIL, null))); Assert.assertTrue(publisher.getSent().isEmpty()); Assert.assertEquals(collector.getAckedCount(), 2); }
@Test public void testMetadataIsNotReplaced() { Metadata actualMetadata = new Metadata(null, "foo"); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", actualMetadata); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE); Metadata expectedMetadata = new Metadata(Metadata.Signal.COMPLETE, "foo"); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), expectedMetadata); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, "foo")); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }
@Test public void testMetaTupleRemovingQueries() { CustomCollector collector = new CustomCollector(); TestQueryBolt bolt = new TestQueryBolt(new BulletStormConfig()); ComponentUtils.prepare(bolt, collector); Map<String, Querier> queries = bolt.getQueries(); queries.put("foo", null); Tuple complete = makeIDTuple(Type.METADATA_TUPLE, "foo", new Metadata(Signal.COMPLETE, null)); bolt.execute(complete); Assert.assertFalse(queries.containsKey("foo")); queries.put("foo", null); Tuple fail = makeIDTuple(Type.METADATA_TUPLE, "foo", new Metadata(Signal.KILL, null)); bolt.execute(fail); Assert.assertFalse(queries.containsKey("foo")); }
@Test public void testJoining() { Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42"); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testRegularMetaTupleIgnored() { CustomCollector collector = new CustomCollector(); TestQueryBolt bolt = new TestQueryBolt(new BulletStormConfig()); ComponentUtils.prepare(bolt, collector); Map<String, Querier> queries = bolt.getQueries(); queries.put("foo", null); Tuple meta = makeIDTuple(Type.METADATA_TUPLE, "foo", new Metadata(Signal.ACKNOWLEDGE, null)); bolt.execute(meta); Assert.assertTrue(queries.containsKey("foo")); }
@Test public void testRawQueryDoneButNotTimedOutWithExcessRecords() { Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 5), EMPTY); bolt.execute(query); // This will send 2 batches of 3 records each (total of 6 records). List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 5, 3); List<BulletRecord> actualSent = sent.subList(0, 5); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(actualSent).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testMissingMetadataIsEmitted() { Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}"); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }
@Test public void testCompleteSignal() { config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); config.validate(); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1)); Tuple complete = TupleUtils.makeIDTuple(TupleClassifier.Type.METADATA_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); bolt.execute(complete); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0)); }
@Test public void testKillSignal() { config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); config.validate(); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1)); Tuple kill = TupleUtils.makeIDTuple(TupleClassifier.Type.METADATA_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null)); bolt.execute(kill); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0)); }
@Test public void testDataWithoutQuery() { sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 2); Assert.assertEquals(collector.getEmittedCount(), 0); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}"); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }
@Test public void testRateLimitingOnCombine() { RateLimitError rateLimitError = new RateLimitError(42.0, 5.0); bolt = new RateLimitedJoinBolt(2, rateLimitError, config); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 10)); bolt.execute(query); // After consuming the 3rd one, it is rate limited and the fourth is not consumed List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 4); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent.subList(0, 3)).add(rateLimitError.makeMeta()).asJSON(), new Metadata(Metadata.Signal.FAIL, null)); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }