@Test public void testTickIntervalIsHighEnough() { BulletStormConfig config = new BulletStormConfig((Config) null); config.set(BulletStormConfig.TICK_SPOUT_INTERVAL, 1); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TICK_SPOUT_INTERVAL), BulletStormConfig.DEFAULT_TICK_SPOUT_INTERVAL); config.set(BulletStormConfig.TICK_SPOUT_INTERVAL, BulletStormConfig.TICK_INTERVAL_MINIMUM); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TICK_SPOUT_INTERVAL), BulletStormConfig.TICK_INTERVAL_MINIMUM); }
private static BulletStormConfig oneRecordConfig() { BulletStormConfig config = new BulletStormConfig(); // Set aggregation default size to 1 since most queries here are RAW with filtering and projections. This // makes them isClosedForPartition even if they are not done. immediately. config.set(BulletStormConfig.AGGREGATION_DEFAULT_SIZE, 1); config.validate(); return config; }
@Test public void testIntervalMappingNotPresent() { BulletStormConfig config = new BulletStormConfig((Config) null); config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_EMIT_INTERVAL_MAPPING, new HashMap<>()); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_EMIT_INTERVAL_MAPPING), BulletStormConfig.DEFAULT_TOPOLOGY_METRICS_BUILT_IN_EMIT_INTERVAL_MAPPING); config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, false); config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_EMIT_INTERVAL_MAPPING, new HashMap<>()); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_EMIT_INTERVAL_MAPPING), emptyMap()); }
@Test public void testLoopBoltOverridesIsAMapWithStringKeys() { BulletStormConfig config = new BulletStormConfig((Config) null); config.set(BulletStormConfig.LOOP_BOLT_PUBSUB_OVERRIDES, null); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.LOOP_BOLT_PUBSUB_OVERRIDES), BulletStormConfig.DEFAULT_LOOP_BOLT_PUBSUB_OVERRIDES); config.set(BulletStormConfig.LOOP_BOLT_PUBSUB_OVERRIDES, new HashMap<>()); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.LOOP_BOLT_PUBSUB_OVERRIDES), BulletStormConfig.DEFAULT_LOOP_BOLT_PUBSUB_OVERRIDES); config.set(BulletStormConfig.LOOP_BOLT_PUBSUB_OVERRIDES, singletonMap(1, "foo")); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.LOOP_BOLT_PUBSUB_OVERRIDES), BulletStormConfig.DEFAULT_LOOP_BOLT_PUBSUB_OVERRIDES); config.set(BulletStormConfig.LOOP_BOLT_PUBSUB_OVERRIDES, singletonMap("foo", singletonList("bar"))); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.LOOP_BOLT_PUBSUB_OVERRIDES), singletonMap("foo", singletonList("bar"))); Map<String, Object> overrides = new HashMap<>(); overrides.put("foo", 1L); overrides.put("bar", new ArrayList<>()); config.set(BulletStormConfig.LOOP_BOLT_PUBSUB_OVERRIDES, overrides); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.LOOP_BOLT_PUBSUB_OVERRIDES), overrides); }
@Test public void testProperMetricsConsumers() { BulletStormConfig config = new BulletStormConfig((Config) null); Assert.assertEquals(config.get(BulletStormConfig.TOPOLOGY_METRICS_CLASSES), BulletStormConfig.DEFAULT_TOPOLOGY_METRICS_CLASSES); // Test removing all metrics config.set(BulletStormConfig.TOPOLOGY_METRICS_ENABLE, true); config.set(BulletStormConfig.TOPOLOGY_METRICS_CLASSES, new ArrayList<>()); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TOPOLOGY_METRICS_CLASSES), new ArrayList<>()); config.set(BulletStormConfig.TOPOLOGY_METRICS_CLASSES, singletonList(1)); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TOPOLOGY_METRICS_CLASSES), BulletStormConfig.DEFAULT_TOPOLOGY_METRICS_CLASSES); config.set(BulletStormConfig.TOPOLOGY_METRICS_CLASSES, singletonList("foo")); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TOPOLOGY_METRICS_CLASSES), BulletStormConfig.DEFAULT_TOPOLOGY_METRICS_CLASSES); config.set(BulletStormConfig.TOPOLOGY_METRICS_CLASSES, singletonList(CustomIMetricsConsumer.class.getName())); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TOPOLOGY_METRICS_CLASSES), singletonList(CustomIMetricsConsumer.class.getName())); }
config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_EMIT_INTERVAL_MAPPING), BulletStormConfig.DEFAULT_TOPOLOGY_METRICS_BUILT_IN_EMIT_INTERVAL_MAPPING); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_EMIT_INTERVAL_MAPPING), BulletStormConfig.DEFAULT_TOPOLOGY_METRICS_BUILT_IN_EMIT_INTERVAL_MAPPING); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_EMIT_INTERVAL_MAPPING), BulletStormConfig.DEFAULT_TOPOLOGY_METRICS_BUILT_IN_EMIT_INTERVAL_MAPPING); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_EMIT_INTERVAL_MAPPING), BulletStormConfig.DEFAULT_TOPOLOGY_METRICS_BUILT_IN_EMIT_INTERVAL_MAPPING); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_EMIT_INTERVAL_MAPPING), singletonMap(TopologyConstants.LATENCY_METRIC, 60));
@Test(expectedExceptions = IllegalStateException.class) public void testPreStartDelayIsTooSmall() { BulletStormConfig config = new BulletStormConfig((Config) null); config.set(BulletStormConfig.TICK_SPOUT_INTERVAL, 100); config.set(BulletStormConfig.JOIN_BOLT_WINDOW_PRE_START_DELAY_TICKS, 3); config.set(BulletStormConfig.WINDOW_MIN_EMIT_EVERY, 400); config.validate(); }
@Test(expectedExceptions = IllegalStateException.class) public void testWindowMinEmitIsTooSmall() { BulletStormConfig config = new BulletStormConfig((Config) null); config.set(BulletStormConfig.TICK_SPOUT_INTERVAL, 100); config.set(BulletStormConfig.JOIN_BOLT_WINDOW_PRE_START_DELAY_TICKS, 3); config.set(BulletStormConfig.WINDOW_MIN_EMIT_EVERY, 100); config.validate(); } }
private static BulletStormConfig configWithRawMaxAndNoMeta() { BulletStormConfig config = new BulletStormConfig(); config.set(BulletStormConfig.RAW_AGGREGATION_MAX_SIZE, RAW_MAX_SIZE); config.set(BulletStormConfig.RESULT_METADATA_ENABLE, false); config.validate(); return config; }
private static BulletStormConfig configWithRawMaxAndEmptyMeta() { BulletStormConfig config = new BulletStormConfig(); config.set(BulletStormConfig.RAW_AGGREGATION_MAX_SIZE, RAW_MAX_SIZE); config.set(BulletStormConfig.RESULT_METADATA_ENABLE, true); config.validate(); config.set(BulletStormConfig.RESULT_METADATA_METRICS, new HashMap<>()); return config; }
@Test public void testStatisticsReporting() { config.set(BulletStormConfig.FILTER_BOLT_STATS_REPORT_TICKS, 10); config.validate(); bolt = ComponentUtils.prepare(new HashMap<>(), new FilterBolt(TopologyConstants.RECORD_COMPONENT, config), collector); Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE); for (int i = 0; i < 10; ++i) { Assert.assertEquals(bolt.getStatsTickCount(), i); bolt.execute(tick); } Assert.assertEquals(bolt.getStatsTickCount(), 0); } }
@Test public void testKillSignal() { config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); config.validate(); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1)); Tuple kill = TupleUtils.makeIDTuple(TupleClassifier.Type.METADATA_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null)); bolt.execute(kill); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0)); }
@Test public void testCompleteSignal() { config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); config.validate(); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1)); Tuple complete = TupleUtils.makeIDTuple(TupleClassifier.Type.METADATA_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); bolt.execute(complete); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0)); }
@Test public void testPreStartDelayIsEnoughForWindowMinEmit() { BulletStormConfig config = new BulletStormConfig((Config) null); config.set(BulletStormConfig.TICK_SPOUT_INTERVAL, 100); config.set(BulletStormConfig.JOIN_BOLT_WINDOW_PRE_START_DELAY_TICKS, 2); config.set(BulletStormConfig.WINDOW_MIN_EMIT_EVERY, 400); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TICK_SPOUT_INTERVAL), 100); Assert.assertEquals(config.get(BulletStormConfig.JOIN_BOLT_WINDOW_PRE_START_DELAY_TICKS), 2); Assert.assertEquals(config.get(BulletStormConfig.WINDOW_MIN_EMIT_EVERY), 400); config.set(BulletStormConfig.TICK_SPOUT_INTERVAL, 100); config.set(BulletStormConfig.JOIN_BOLT_WINDOW_PRE_START_DELAY_TICKS, 5); config.set(BulletStormConfig.WINDOW_MIN_EMIT_EVERY, 700); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TICK_SPOUT_INTERVAL), 100); Assert.assertEquals(config.get(BulletStormConfig.JOIN_BOLT_WINDOW_PRE_START_DELAY_TICKS), 5); Assert.assertEquals(config.get(BulletStormConfig.WINDOW_MIN_EMIT_EVERY), 700); config.set(BulletStormConfig.TICK_SPOUT_INTERVAL, 20); config.set(BulletStormConfig.JOIN_BOLT_WINDOW_PRE_START_DELAY_TICKS, 3); config.set(BulletStormConfig.WINDOW_MIN_EMIT_EVERY, 100); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TICK_SPOUT_INTERVAL), 20); Assert.assertEquals(config.get(BulletStormConfig.JOIN_BOLT_WINDOW_PRE_START_DELAY_TICKS), 3); Assert.assertEquals(config.get(BulletStormConfig.WINDOW_MIN_EMIT_EVERY), 100); }
@Test public void testMetricsUpdateOnMetricsEnabled() { CustomTopologyContext context = new CustomTopologyContext(); CustomCollector collector = new CustomCollector(); BulletStormConfig config = new BulletStormConfig(); config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); config.validate(); TestQueryBolt bolt = new TestQueryBolt(config); ComponentUtils.prepare(new HashMap<>(), bolt, context, collector); Assert.assertTrue(bolt.isMetricsEnabled()); bolt.execute(null); IMetric averager = context.getRegisteredMetricByName("foo"); IMetric counter = context.getRegisteredMetricByName("bar"); Assert.assertEquals(averager.getValueAndReset(), 1.0); Assert.assertEquals(counter.getValueAndReset(), 1L); }
@Test public void testQueryFinishedWhileBeingDelayed() { config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); config.validate(); bolt = new DonableJoinBolt(config, 2, false); setup(bolt);
@Test public void testRateLimitErrorFromUpstream() { config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); config.validate(); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1)); RateLimitError rateLimitError = new RateLimitError(2000.0, 1000.0); Tuple error = TupleUtils.makeIDTuple(TupleClassifier.Type.ERROR_TUPLE, "42", rateLimitError); bolt.execute(error); Assert.assertEquals(collector.getEmittedCount(), 2); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0)); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(rateLimitError.makeMeta()).asJSON(), new Metadata(Metadata.Signal.FAIL, null)); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
config.set(BulletStormConfig.JOIN_BOLT_WINDOW_PRE_START_DELAY_TICKS, 3); config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); config.validate(); bolt = new DonableJoinBolt(config, 2, false); setup(bolt);
@Test public void testQueryClosedWhileFinishedTerminatesTheQuery() { config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); config.validate(); bolt = new DonableJoinBolt(config, 2, true); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 5, Window.Unit.RECORD, 1, Window.Unit.RECORD, 1), EMPTY); bolt.execute(query); Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE); // Third tick marks it done and start buffering bolt.execute(tick); bolt.execute(tick); // Begins buffering in query tickout buffer after here bolt.execute(tick); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1)); List<BulletRecord> sent = sendSlidingWindowWithRawRecordTuplesTo(bolt, "42", 2); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0)); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getEmittedCount(), 2); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testWindowClosedOnTickIsImmediatelyEmitted() { config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); config.validate(); bolt = new ClosableJoinBolt(config, 3, false); setup(bolt);