/** * Write this error as a JSON Bullet error response in the {@link Meta} of a {@link Clip}. * * @return A String JSON version of this error. */ public String asJSONClip() { return Clip.of(Meta.of(this)).asJSON(); } }
/** * Write this error as a JSON Bullet error response in the {@link Meta} of a {@link Clip}. * * @return A String JSON version of this error. */ public String asJSONClip() { return Clip.of(Meta.of(this)).asJSON(); } }
private void emitResult(String id, Metadata metadata, Clip result) { // Metadata should not be checked. It could be null. collector.emit(RESULT_STREAM, new Values(id, result.asJSON(), metadata)); }
private void emitResult(String id, Metadata metadata, Clip result) { // Metadata should not be checked. It could be null. collector.emit(RESULT_STREAM, new Values(id, result.asJSON(), metadata)); }
@Test public void testFailJoiningForNoQuery() { List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42"); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED); Assert.assertFalse(wasResultEmitted(expected)); Assert.assertEquals(collector.getEmittedCount(), 0); }
@Test public void testErrorInQueryWithoutMetadata() { Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{\"aggregation\": {\"type\": \"garbage\"}}"); bolt.execute(query); Assert.assertEquals(collector.getEmittedCount(), 1); BulletError expectedError = Aggregation.TYPE_NOT_SUPPORTED_ERROR; Meta expectedMetadata = Meta.of(expectedError); List<Object> expected = TupleUtils.makeTuple("42", Clip.of(expectedMetadata).asJSON(), FAILED).getValues(); List<Object> actual = collector.getNthTupleEmittedTo(TopologyConstants.RESULT_STREAM, 1).get(); Assert.assertTrue(isSameResult(actual, expected)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getEmittedCount(), 1); }
@Test public void testJoining() { Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42"); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testRawQueryDoneButNotTimedOutWithExcessRecords() { Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 5), EMPTY); bolt.execute(query); // This will send 2 batches of 3 records each (total of 6 records). List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 5, 3); List<BulletRecord> actualSent = sent.subList(0, 5); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(actualSent).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testMissingMetadataIsEmitted() { Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}"); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }
@Test public void testErrorEmittedProperly() { Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "garbage", EMPTY); bolt.execute(query); Assert.assertEquals(collector.getEmittedCount(), 1); String error = ParsingError.GENERIC_JSON_ERROR + ":\ngarbage\n" + "IllegalStateException: Expected BEGIN_OBJECT but was STRING at line 1 column 1 path $"; BulletError expectedError = ParsingError.makeError(error, ParsingError.GENERIC_JSON_RESOLUTION); Meta expectedMetadata = Meta.of(expectedError); List<Object> expected = TupleUtils.makeTuple("42", Clip.of(expectedMetadata).asJSON(), FAILED).getValues(); List<Object> actual = collector.getNthTupleEmittedTo(TopologyConstants.RESULT_STREAM, 1).get(); Assert.assertTrue(isSameResult(actual, expected)); }
@Test public void testUnknownAggregation() { // Lowercase "top" is not valid and will not be parsed since there is no enum for it // In this case aggregation type should be set to null and an error should be emitted Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{\"aggregation\": {\"type\": \"garbage\"}}", EMPTY); bolt.execute(query); Assert.assertEquals(collector.getEmittedCount(), 1); BulletError expectedError = Aggregation.TYPE_NOT_SUPPORTED_ERROR; Meta expectedMetadata = Meta.of(expectedError); List<Object> expected = TupleUtils.makeTuple("42", Clip.of(expectedMetadata).asJSON(), FAILED).getValues(); List<Object> actual = collector.getNthTupleEmittedTo(TopologyConstants.RESULT_STREAM, 1).get(); Assert.assertTrue(isSameResult(actual, expected)); }
@Test public void testDataWithoutQuery() { sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 2); Assert.assertEquals(collector.getEmittedCount(), 0); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}"); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }
@Test public void testMetadataIsNotReplaced() { Metadata actualMetadata = new Metadata(null, "foo"); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", actualMetadata); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE); Metadata expectedMetadata = new Metadata(Metadata.Signal.COMPLETE, "foo"); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), expectedMetadata); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, "foo")); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }
@Test public void testUnhandledExceptionErrorEmitted() { // An empty query should throw an null-pointer exception which should be caught in JoinBolt // and an error should be emitted Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "", EMPTY); bolt.execute(query); sendRawRecordTuplesTo(bolt, "42"); Assert.assertEquals(collector.getEmittedCount(), 1); String error = ParsingError.GENERIC_JSON_ERROR + ":\n\nNullPointerException: "; BulletError expectedError = ParsingError.makeError(error, ParsingError.GENERIC_JSON_RESOLUTION); Meta expectedMetadata = Meta.of(expectedError); List<Object> expected = TupleUtils.makeTuple("42", Clip.of(expectedMetadata).asJSON(), FAILED).getValues(); List<Object> actual = collector.getNthTupleEmittedTo(TopologyConstants.RESULT_STREAM, 1).get(); Assert.assertTrue(isSameResult(actual, expected)); }
@Test public void testJoiningAfterLateArrivalMakingQueryFinishBeforeTickout() { bolt = new DonableJoinBolt(config, 2, true); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 3), EMPTY); bolt.execute(query); // This calls isDone twice. So the query is done on the next tick List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED); // Tick once to get the query done rotated into buffer. Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE); bolt.execute(tick); Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); // Now we satisfy the query and see if it causes an emission List<BulletRecord> sentLate = sendRawRecordTuplesTo(bolt, "42", 1); sent.addAll(sentLate); expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testQueryNotDoneButIsDurationBased() { // This bolt will be done after combining RAW_MAX_SIZE - 1 times and is a shouldBuffer, so it is buffered. bolt = new DonableJoinBolt(config, RAW_MAX_SIZE - 1, true); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1); Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE); // Should make isDone true but query is a a shouldBuffer so it gets buffered. bolt.execute(tick); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED); // We need to tick the default query tickout to make the query emit for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_POST_FINISH_BUFFER_TICKS - 1; ++i) { bolt.execute(tick); Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); } // Should emit on the last tick bolt.execute(tick); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testQueryIdentifierMetadata() { config = configWithRawMaxAndEmptyMeta(); enableMetadataInConfig(config, Concept.QUERY_METADATA.getName(), "meta"); enableMetadataInConfig(config, Concept.QUERY_ID.getName(), "id"); setup(new JoinBolt(config)); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42"); Meta meta = new Meta(); meta.add("meta", singletonMap("id", "42")); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(meta).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testRateLimitingOnCombine() { RateLimitError rateLimitError = new RateLimitError(42.0, 5.0); bolt = new RateLimitedJoinBolt(2, rateLimitError, config); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 10)); bolt.execute(query); // After consuming the 3rd one, it is rate limited and the fourth is not consumed List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 4); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent.subList(0, 3)).add(rateLimitError.makeMeta()).asJSON(), new Metadata(Metadata.Signal.FAIL, null)); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }
@Test public void testUnknownConceptMetadata() { config = configWithRawMaxAndEmptyMeta(); enableMetadataInConfig(config, Concept.QUERY_METADATA.getName(), "meta"); enableMetadataInConfig(config, Concept.QUERY_ID.getName(), "id"); enableMetadataInConfig(config, "foo", "bar"); setup(new JoinBolt(config)); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42"); Meta meta = new Meta(); meta.add("meta", singletonMap("id", "42")); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(meta).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testRateLimitingWithTicks() { RateLimitError rateLimitError = new RateLimitError(42.0, 5.0); bolt = new RateLimitedJoinBolt(2, rateLimitError, config); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 10)); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 2); Assert.assertEquals(collector.getEmittedCount(), 0); Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE); bolt.execute(tick); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(rateLimitError.makeMeta()).asJSON(), new Metadata(Metadata.Signal.FAIL, null)); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }