@Test public void testUnknownTuple() { Tuple query = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "", ""); bolt.execute(query); Assert.assertFalse(collector.wasAcked(query)); }
@Test public void testQueryErrorsAreSilentlyIgnored() { Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{'aggregation': { 'type': null }}"); bolt.execute(query); BulletRecord record = RecordBox.get().add("field", "b235gf23b").getRecord(); Tuple someTuple = makeRecordTuple(record); bolt.execute(someTuple); bolt.execute(someTuple); Assert.assertEquals(collector.getEmittedCount(), 0); }
@Test public void testFilteringSlidingWindow() { Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeSimpleAggregationFilterQuery("field", singletonList("b235gf23b"), EQUALS, RAW, 5, Window.Unit.RECORD, 1, Window.Unit.RECORD, 1), METADATA); bolt.execute(query); BulletRecord record = RecordBox.get().add("field", "b235gf23b").getRecord(); Tuple matching = makeRecordTuple(record); bolt.execute(matching); bolt.execute(matching); bolt.execute(matching); bolt.execute(matching); Tuple expected = makeSlidingTuple(TupleClassifier.Type.DATA_TUPLE, "42", record); Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 4, expected)); }
@Test public void testKillSignal() { Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeSimpleAggregationFilterQuery("field", singletonList("b235gf23b"), EQUALS, RAW, 5, Window.Unit.RECORD, 1, Window.Unit.RECORD, 1), METADATA); bolt.execute(query); BulletRecord record = RecordBox.get().add("field", "b235gf23b").getRecord(); Tuple matching = makeRecordTuple(record); bolt.execute(matching); bolt.execute(matching); Tuple expected = makeSlidingTuple(TupleClassifier.Type.DATA_TUPLE, "42", record); Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 2, expected)); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple kill = makeIDTuple(TupleClassifier.Type.METADATA_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null)); bolt.execute(kill); bolt.execute(matching); bolt.execute(matching); Assert.assertEquals(collector.getEmittedCount(), 2); }
@Test public void testDuplicateQueryIds() { Tuple queryA = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeFieldFilterQuery("b235gf23b"), METADATA); Tuple queryB = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "43", makeFilterQuery("timestamp", asList("1", "2", "3", "45"), NOT_EQUALS), METADATA); Assert.assertEquals(bolt.getManager().size(), 0); bolt.execute(queryA); bolt.execute(queryB); Assert.assertEquals(bolt.getManager().size(), 2); bolt.execute(queryA); bolt.execute(queryB); Assert.assertEquals(bolt.getManager().size(), 2); }
@Test public void testBadJson() { Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "'filters' : [], ", METADATA); bolt.execute(query); BulletRecord record = RecordBox.get().add("field", "b235gf23b").getRecord(); Tuple matching = makeRecordTuple(record); bolt.execute(matching); Tuple expected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", record); Assert.assertTrue(collector.wasAcked(query)); Assert.assertTrue(collector.wasAcked(matching)); Assert.assertFalse(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, expected)); }
@Test public void testQueryNotDone() { bolt = ComponentUtils.prepare(new DonableFilterBolt(), collector); Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeFieldFilterQuery("b235gf23b"), METADATA); bolt.execute(query); BulletRecord record = RecordBox.get().add("field", "b235gf23b").getRecord(); Tuple matching = makeRecordTuple(record); bolt.execute(matching); Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE); bolt.execute(tick); bolt.execute(tick); Tuple expected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", record); Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 1, expected)); }
@Test public void testStatisticsReporting() { config.set(BulletStormConfig.FILTER_BOLT_STATS_REPORT_TICKS, 10); config.validate(); bolt = ComponentUtils.prepare(new HashMap<>(), new FilterBolt(TopologyConstants.RECORD_COMPONENT, config), collector); Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE); for (int i = 0; i < 10; ++i) { Assert.assertEquals(bolt.getStatsTickCount(), i); bolt.execute(tick); } Assert.assertEquals(bolt.getStatsTickCount(), 0); } }
@Test public void testQueryDone() { bolt = ComponentUtils.prepare(new DonableFilterBolt(), collector); Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeFieldFilterQuery("b235gf23b"), METADATA); bolt.execute(query); BulletRecord nonMatching = RecordBox.get().add("field", "foo").getRecord(); Tuple notMatching = makeRecordTuple(nonMatching); bolt.execute(notMatching); Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE); bolt.execute(tick); bolt.execute(tick); BulletRecord record = RecordBox.get().add("field", "b235gf23b").getRecord(); Tuple matching = makeRecordTuple(record); bolt.execute(matching); Tuple expected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", record); Assert.assertFalse(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, expected)); }
@Test public void testDifferentQueryMatchingSameTuple() { Tuple queryA = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeFieldFilterQuery("b235gf23b"), METADATA); Tuple queryB = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "43", makeFilterQuery("timestamp", asList("1", "2", "3", "45"), EQUALS), METADATA); bolt.execute(queryA); bolt.execute(queryB); BulletRecord record = RecordBox.get().add("field", "b235gf23b").add("timestamp", 45L).getRecord(); Tuple matching = makeRecordTuple(record); bolt.execute(matching); Tuple expectedA = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", record); Tuple expectedB = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "43", record); Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 1, expectedA)); Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 1, expectedB)); }
@Test public void testProjectionAndFiltering() { Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeProjectionFilterQuery("map_field.id", singletonList("123"), EQUALS, Pair.of("field", "id"), Pair.of("map_field.id", "mid")), METADATA); bolt.execute(query); BulletRecord record = RecordBox.get().add("field", "b235gf23b").add("timestamp", 92L) .addMap("map_field", Pair.of("id", "123"), Pair.of("bar", "foo")) .getRecord(); Tuple matching = makeRecordTuple(record); bolt.execute(matching); BulletRecord expectedRecord = RecordBox.get().add("id", "b235gf23b").add("mid", "123").getRecord(); Tuple expected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", expectedRecord); Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 1, expected)); }
@Test public void testProjection() { Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeProjectionQuery(Pair.of("field", "id"), Pair.of("map_field.id", "mid")), METADATA); bolt.execute(query); BulletRecord record = RecordBox.get().add("field", "b235gf23b").add("timestamp", 92L) .addMap("map_field", Pair.of("id", "123"), Pair.of("bar", "foo")) .getRecord(); Tuple matching = makeRecordTuple(record); bolt.execute(matching); BulletRecord expectedRecord = RecordBox.get().add("id", "b235gf23b").add("mid", "123").getRecord(); Tuple expected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", expectedRecord); Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 1, expected)); }
@Test public void testFailQueryInitialization() { bolt = ComponentUtils.prepare(new NoQueryFilterBolt(), collector); Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeFieldFilterQuery("b235gf23b"), METADATA); bolt.execute(query); BulletRecord record = RecordBox.get().add("field", "b235gf23b").getRecord(); Tuple matching = makeRecordTuple(record); bolt.execute(matching); Tuple expected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", record); Assert.assertFalse(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, expected)); }
@Test public void testFilteringUsingProjectedName() { Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeProjectionFilterQuery("mid", singletonList("123"), EQUALS, Pair.of("field", "id"), Pair.of("map_field.id", "mid")), METADATA); bolt.execute(query); BulletRecord record = RecordBox.get().add("field", "b235gf23b").add("timestamp", 92L) .addMap("map_field", Pair.of("id", "123"), Pair.of("bar", "foo")) .getRecord(); Tuple matching = makeRecordTuple(record); bolt.execute(matching); BulletRecord expectedRecord = RecordBox.get().add("id", "b235gf23b").add("mid", "123").getRecord(); Tuple expected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", expectedRecord); Assert.assertFalse(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, expected)); }
@Test public void testProjectionNotLosingFilterColumn() { Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeProjectionFilterQuery("timestamp", singletonList("92"), EQUALS, Pair.of("field", "id"), Pair.of("map_field.id", "mid")), METADATA); bolt.execute(query); BulletRecord record = RecordBox.get().add("field", "b235gf23b").add("timestamp", 92L) .addMap("map_field", Pair.of("id", "123"), Pair.of("bar", "foo")) .getRecord(); Tuple matching = makeRecordTuple(record); bolt.execute(matching); BulletRecord expectedRecord = RecordBox.get().add("id", "b235gf23b").add("mid", "123").getRecord(); Tuple expected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", expectedRecord); Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, expected)); }
@Test public void testFilteringLatency() { config = new BulletStormConfig(); // Don't use the overridden aggregation default size but turn on built in metrics config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); collector = new CustomCollector(); CustomTopologyContext context = new CustomTopologyContext(); bolt = new FilterBolt(TopologyConstants.RECORD_COMPONENT, config); ComponentUtils.prepare(new HashMap<>(), bolt, context, collector); Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeFieldFilterQuery("bar"), METADATA); bolt.execute(query); BulletRecord record = RecordBox.get().add("field", "foo").getRecord(); long start = System.currentTimeMillis(); IntStream.range(0, 10).mapToObj(i -> makeRecordTuple(record, System.currentTimeMillis())) .forEach(bolt::execute); long end = System.currentTimeMillis(); double actualLatecy = context.getDoubleMetric(TopologyConstants.LATENCY_METRIC); Assert.assertTrue(actualLatecy <= end - start); }
@Test public void testFiltering() { Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeFieldFilterQuery("b235gf23b"), METADATA); bolt.execute(query); BulletRecord record = RecordBox.get().add("field", "b235gf23b").getRecord(); Tuple matching = makeRecordTuple(record); bolt.execute(matching); BulletRecord anotherRecord = RecordBox.get().add("field", "wontmatch").getRecord(); Tuple nonMatching = makeRecordTuple(anotherRecord); bolt.execute(nonMatching); Tuple expected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", record); Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 1, expected)); Tuple anotherExpected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", anotherRecord); Assert.assertFalse(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, anotherExpected)); }
@Test public void testQueryNotDoneAndThenDone() { bolt = ComponentUtils.prepare(new DonableFilterBolt(), collector); Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeFieldFilterQuery("b235gf23b"), METADATA); bolt.execute(query); BulletRecord record = RecordBox.get().add("field", "b235gf23b").getRecord(); Tuple matching = makeRecordTuple(record); bolt.execute(matching); Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE); bolt.execute(tick); bolt.execute(tick); Tuple expected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", record); Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 1, expected)); BulletRecord anotherRecord = RecordBox.get().add("field", "b235gf23b").add("mid", "2342").getRecord(); Tuple anotherExpected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", anotherRecord); Assert.assertFalse(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, anotherExpected)); }
@Test public void testDifferentQueryMatchingDifferentTuple() { Tuple queryA = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeFieldFilterQuery("b235gf23b"), METADATA); Tuple queryB = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "43", makeFilterQuery("timestamp", asList("1", "2", "3", "45"), NOT_EQUALS), METADATA); bolt.execute(queryA); bolt.execute(queryB); BulletRecord recordA = RecordBox.get().add("field", "b235gf23b").add("timestamp", 45L).getRecord(); BulletRecord recordB = RecordBox.get().add("field", "b235gf23b").add("timestamp", 42L).getRecord(); Tuple matchingA = makeRecordTuple(recordA); Tuple matchingB = makeRecordTuple(recordB); bolt.execute(matchingA); bolt.execute(matchingB); Tuple expectedAA = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", recordA); Tuple expectedAB = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", recordB); Tuple expectedB = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "43", recordB); Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 1, expectedAA)); Assert.assertFalse(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, expectedAB)); Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 1, expectedB)); }
@Test public void testTuplesCustomSource() { bolt = ComponentUtils.prepare(new FilterBolt("CustomSource", oneRecordConfig()), collector); Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeFieldFilterQuery("b235gf23b"), METADATA); bolt.execute(query); BulletRecord record = RecordBox.get().add("field", "b235gf23b").getRecord(); Tuple matching = TupleUtils.makeRawTuple("CustomSource", TopologyConstants.RECORD_STREAM, record); bolt.execute(matching); Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE); bolt.execute(tick); BulletRecord anotherRecord = RecordBox.get().add("field", "wontmatch").getRecord(); Tuple nonMatching = TupleUtils.makeRawTuple("CustomSource", TopologyConstants.RECORD_STREAM, anotherRecord); bolt.execute(nonMatching); Tuple expected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", record); Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 1, expected)); Tuple notExpected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", anotherRecord); Assert.assertFalse(wasRawRecordEmitted(notExpected)); }