@Test public void testBadAggregation() { Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(TOP_K, 5), EMPTY); bolt.execute(query); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 0); }
@Test public void testRateLimitErrorFromUpstreamWithoutQuery() { RateLimitError rateLimitError = new RateLimitError(2000.0, 1000.0); Tuple error = TupleUtils.makeIDTuple(TupleClassifier.Type.ERROR_TUPLE, "42", rateLimitError); bolt.execute(error); Assert.assertEquals(collector.getEmittedCount(), 0); }
@Test public void testImproperQueryCountingMetrics() { config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); setup(new JoinBolt(config)); Assert.assertEquals(context.getLongMetric(TopologyConstants.CREATED_QUERIES_METRIC), Long.valueOf(0)); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0)); Assert.assertEquals(context.getLongMetric(TopologyConstants.IMPROPER_QUERIES_METRIC), Long.valueOf(0)); Tuple badQuery = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{", EMPTY); bolt.execute(badQuery); bolt.execute(badQuery); Assert.assertEquals(context.getLongMetric(TopologyConstants.CREATED_QUERIES_METRIC), Long.valueOf(0)); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0)); Assert.assertEquals(context.getLongMetric(TopologyConstants.IMPROPER_QUERIES_METRIC), Long.valueOf(2)); }
@Test public void testUnknownTuple() { Tuple query = TupleUtils.makeRawTuple(TopologyConstants.RECORD_COMPONENT, TopologyConstants.RECORD_STREAM, RecordBox.get().add("a", "b").getRecord()); bolt.execute(query); Assert.assertFalse(collector.wasAcked(query)); }
@Test public void testKillSignal() { config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); config.validate(); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1)); Tuple kill = TupleUtils.makeIDTuple(TupleClassifier.Type.METADATA_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null)); bolt.execute(kill); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0)); }
@Test public void testCompleteSignal() { config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); config.validate(); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1)); Tuple complete = TupleUtils.makeIDTuple(TupleClassifier.Type.METADATA_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); bolt.execute(complete); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0)); }
@Test public void testJoining() { Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42"); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testErrorInQueryWithoutMetadata() { Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{\"aggregation\": {\"type\": \"garbage\"}}"); bolt.execute(query); Assert.assertEquals(collector.getEmittedCount(), 1); BulletError expectedError = Aggregation.TYPE_NOT_SUPPORTED_ERROR; Meta expectedMetadata = Meta.of(expectedError); List<Object> expected = TupleUtils.makeTuple("42", Clip.of(expectedMetadata).asJSON(), FAILED).getValues(); List<Object> actual = collector.getNthTupleEmittedTo(TopologyConstants.RESULT_STREAM, 1).get(); Assert.assertTrue(isSameResult(actual, expected)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getEmittedCount(), 1); }
@Test public void testRawQueryDoneButNotTimedOutWithExcessRecords() { Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 5), EMPTY); bolt.execute(query); // This will send 2 batches of 3 records each (total of 6 records). List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 5, 3); List<BulletRecord> actualSent = sent.subList(0, 5); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(actualSent).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testQueryNotDoneButIsDurationBased() { // This bolt will be done after combining RAW_MAX_SIZE - 1 times and is a shouldBuffer, so it is buffered. bolt = new DonableJoinBolt(config, RAW_MAX_SIZE - 1, true); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1); Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE); // Should make isDone true but query is a a shouldBuffer so it gets buffered. bolt.execute(tick); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED); // We need to tick the default query tickout to make the query emit for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_POST_FINISH_BUFFER_TICKS - 1; ++i) { bolt.execute(tick); Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); } // Should emit on the last tick bolt.execute(tick); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testMissingMetadataIsEmitted() { Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}"); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }
@Test public void testErrorEmittedProperly() { Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "garbage", EMPTY); bolt.execute(query); Assert.assertEquals(collector.getEmittedCount(), 1); String error = ParsingError.GENERIC_JSON_ERROR + ":\ngarbage\n" + "IllegalStateException: Expected BEGIN_OBJECT but was STRING at line 1 column 1 path $"; BulletError expectedError = ParsingError.makeError(error, ParsingError.GENERIC_JSON_RESOLUTION); Meta expectedMetadata = Meta.of(expectedError); List<Object> expected = TupleUtils.makeTuple("42", Clip.of(expectedMetadata).asJSON(), FAILED).getValues(); List<Object> actual = collector.getNthTupleEmittedTo(TopologyConstants.RESULT_STREAM, 1).get(); Assert.assertTrue(isSameResult(actual, expected)); }
@Test public void testUnknownAggregation() { // Lowercase "top" is not valid and will not be parsed since there is no enum for it // In this case aggregation type should be set to null and an error should be emitted Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{\"aggregation\": {\"type\": \"garbage\"}}", EMPTY); bolt.execute(query); Assert.assertEquals(collector.getEmittedCount(), 1); BulletError expectedError = Aggregation.TYPE_NOT_SUPPORTED_ERROR; Meta expectedMetadata = Meta.of(expectedError); List<Object> expected = TupleUtils.makeTuple("42", Clip.of(expectedMetadata).asJSON(), FAILED).getValues(); List<Object> actual = collector.getNthTupleEmittedTo(TopologyConstants.RESULT_STREAM, 1).get(); Assert.assertTrue(isSameResult(actual, expected)); }
@Test public void testDataWithoutQuery() { sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 2); Assert.assertEquals(collector.getEmittedCount(), 0); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}"); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }
@Test public void testMetadataIsNotReplaced() { Metadata actualMetadata = new Metadata(null, "foo"); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", actualMetadata); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE); Metadata expectedMetadata = new Metadata(Metadata.Signal.COMPLETE, "foo"); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), expectedMetadata); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, "foo")); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }
@Test public void testUnhandledExceptionErrorEmitted() { // An empty query should throw an null-pointer exception which should be caught in JoinBolt // and an error should be emitted Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "", EMPTY); bolt.execute(query); sendRawRecordTuplesTo(bolt, "42"); Assert.assertEquals(collector.getEmittedCount(), 1); String error = ParsingError.GENERIC_JSON_ERROR + ":\n\nNullPointerException: "; BulletError expectedError = ParsingError.makeError(error, ParsingError.GENERIC_JSON_RESOLUTION); Meta expectedMetadata = Meta.of(expectedError); List<Object> expected = TupleUtils.makeTuple("42", Clip.of(expectedMetadata).asJSON(), FAILED).getValues(); List<Object> actual = collector.getNthTupleEmittedTo(TopologyConstants.RESULT_STREAM, 1).get(); Assert.assertTrue(isSameResult(actual, expected)); }
@Test public void testRateLimitingWithTicks() { RateLimitError rateLimitError = new RateLimitError(42.0, 5.0); bolt = new RateLimitedJoinBolt(2, rateLimitError, config); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 10)); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 2); Assert.assertEquals(collector.getEmittedCount(), 0); Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE); bolt.execute(tick); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(rateLimitError.makeMeta()).asJSON(), new Metadata(Metadata.Signal.FAIL, null)); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }
@Test public void testQueryIdentifierMetadata() { config = configWithRawMaxAndEmptyMeta(); enableMetadataInConfig(config, Concept.QUERY_METADATA.getName(), "meta"); enableMetadataInConfig(config, Concept.QUERY_ID.getName(), "id"); setup(new JoinBolt(config)); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42"); Meta meta = new Meta(); meta.add("meta", singletonMap("id", "42")); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(meta).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testRateLimitingOnCombine() { RateLimitError rateLimitError = new RateLimitError(42.0, 5.0); bolt = new RateLimitedJoinBolt(2, rateLimitError, config); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 10)); bolt.execute(query); // After consuming the 3rd one, it is rate limited and the fourth is not consumed List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 4); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent.subList(0, 3)).add(rateLimitError.makeMeta()).asJSON(), new Metadata(Metadata.Signal.FAIL, null)); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }
@Test public void testUnknownConceptMetadata() { config = configWithRawMaxAndEmptyMeta(); enableMetadataInConfig(config, Concept.QUERY_METADATA.getName(), "meta"); enableMetadataInConfig(config, Concept.QUERY_ID.getName(), "id"); enableMetadataInConfig(config, "foo", "bar"); setup(new JoinBolt(config)); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42"); Meta meta = new Meta(); meta.add("meta", singletonMap("id", "42")); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(meta).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }