@Test public void testTickIntervalIsHighEnough() { BulletStormConfig config = new BulletStormConfig((Config) null); config.set(BulletStormConfig.TICK_SPOUT_INTERVAL, 1); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TICK_SPOUT_INTERVAL), BulletStormConfig.DEFAULT_TICK_SPOUT_INTERVAL); config.set(BulletStormConfig.TICK_SPOUT_INTERVAL, BulletStormConfig.TICK_INTERVAL_MINIMUM); config.validate(); Assert.assertEquals(config.get(BulletStormConfig.TICK_SPOUT_INTERVAL), BulletStormConfig.TICK_INTERVAL_MINIMUM); }
@Test public void testGettingNonStormSettingsOnly() { BulletStormConfig config = new BulletStormConfig((Config) null); config.set(BulletStormConfig.CUSTOM_STORM_SETTING_PREFIX + "storm.foo", "bar"); config.set(BulletStormConfig.CUSTOM_STORM_SETTING_PREFIX + "bar", "baz"); Map<String, Object> settings = config.getCustomStormSettings(); Assert.assertEquals(settings.size(), 2); Assert.assertEquals(settings.get("storm.foo"), "bar"); Assert.assertEquals(settings.get("bar"), "baz"); }
/** * Gets all the custom settings defined with {@link #CUSTOM_STORM_SETTING_PREFIX}. The prefix is removed. * * @return A {@link Map} of these custom settings. */ public Map<String, Object> getCustomStormSettings() { return getAllWithPrefix(Optional.empty(), CUSTOM_STORM_SETTING_PREFIX, true); }
private static BulletStormConfig oneRecordConfig() { BulletStormConfig config = new BulletStormConfig(); // Set aggregation default size to 1 since most queries here are RAW with filtering and projections. This // makes them isClosedForPartition even if they are not done. immediately. config.set(BulletStormConfig.AGGREGATION_DEFAULT_SIZE, 1); config.validate(); return config; }
@Override protected Publisher createPublisher() throws PubSubException { PubSub pubSub = PubSub.from(config); // Map is always not null and is validated to be a proper BulletStormConfig Map<String, Object> overrides = (Map<String, Object>) config.getAs(BulletStormConfig.LOOP_BOLT_PUBSUB_OVERRIDES, Map.class); log.info("Loaded pubsub overrides: {}", overrides); BulletStormConfig modified = new BulletStormConfig(config); overrides.forEach(modified::set); pubSub.switchContext(PubSub.Context.QUERY_SUBMISSION, modified); log.info("Switched the PubSub into query submission mode"); Publisher publisher = pubSub.getPublisher(); log.info("Setup PubSub: {} with Publisher: {}", pubSub, publisher); return publisher; }
/** * Create a new DRPCConfig by reading in a file. * * @param file The file containing DRPC settings. */ public DRPCConfig(String file) { // Load and merge with default bullet-storm settings. Storm defaults also contain the DRPC settings. this(new BulletStormConfig(file)); VALIDATOR.validate(this); }
@Test public void testStatisticsReporting() { config.set(BulletStormConfig.FILTER_BOLT_STATS_REPORT_TICKS, 10); config.validate(); bolt = ComponentUtils.prepare(new HashMap<>(), new FilterBolt(TopologyConstants.RECORD_COMPONENT, config), collector); Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE); for (int i = 0; i < 10; ++i) { Assert.assertEquals(bolt.getStatsTickCount(), i); bolt.execute(tick); } Assert.assertEquals(bolt.getStatsTickCount(), 0); } }
private static DRPCConfig makeEmpty() { BulletStormConfig config = new BulletStormConfig(); config.set(BulletStormConfig.WINDOW_DISABLE, true); return new DRPCConfig(config); }
@Test public void testNoFiles() { BulletStormConfig config = new BulletStormConfig((String) null); Assert.assertEquals(config.get(BulletStormConfig.TOPOLOGY_NAME), BulletStormConfig.DEFAULT_TOPOLOGY_NAME); Assert.assertEquals(config.get(BulletStormConfig.QUERY_MAX_DURATION), BulletStormConfig.DEFAULT_QUERY_MAX_DURATION); Assert.assertEquals(config.get(BulletStormConfig.AGGREGATION_MAX_SIZE), BulletStormConfig.DEFAULT_AGGREGATION_MAX_SIZE); config = new BulletStormConfig(""); Assert.assertEquals(config.get(BulletStormConfig.TOPOLOGY_NAME), BulletStormConfig.DEFAULT_TOPOLOGY_NAME); }
String name = config.getAs(BulletStormConfig.TOPOLOGY_NAME, String.class); Number querySpoutParallelism = config.getAs(BulletStormConfig.QUERY_SPOUT_PARALLELISM, Number.class); Number querySpoutCPULoad = config.getAs(BulletStormConfig.QUERY_SPOUT_CPU_LOAD, Number.class); Number querySpoutMemoryOnHeapLoad = config.getAs(BulletStormConfig.QUERY_SPOUT_MEMORY_ON_HEAP_LOAD, Number.class); Number querySpoutMemoryOffHeapLoad = config.getAs(BulletStormConfig.QUERY_SPOUT_MEMORY_OFF_HEAP_LOAD, Number.class); Number tickSpoutCPULoad = config.getAs(BulletStormConfig.TICK_SPOUT_CPU_LOAD, Number.class); Number tickSpoutMemoryOnheapLoad = config.getAs(BulletStormConfig.TICK_SPOUT_MEMORY_ON_HEAP_LOAD, Number.class); Number tickSpoutMemoryOffHeapLoad = config.getAs(BulletStormConfig.TICK_SPOUT_MEMORY_OFF_HEAP_LOAD, Number.class); Number filterBoltParallelism = config.getAs(BulletStormConfig.FILTER_BOLT_PARALLELISM, Number.class); Number filterBoltCPULoad = config.getAs(BulletStormConfig.FILTER_BOLT_CPU_LOAD, Number.class); Number filterBoltMemoryOnheapLoad = config.getAs(BulletStormConfig.FILTER_BOLT_MEMORY_ON_HEAP_LOAD, Number.class); Number filterBoltMemoryOffHeapLoad = config.getAs(BulletStormConfig.FILTER_BOLT_MEMORY_OFF_HEAP_LOAD, Number.class); Number joinBoltParallelism = config.getAs(BulletStormConfig.JOIN_BOLT_PARALLELISM, Number.class); Number joinBoltCPULoad = config.getAs(BulletStormConfig.JOIN_BOLT_CPU_LOAD, Number.class); Number joinBoltMemoryOnHeapLoad = config.getAs(BulletStormConfig.JOIN_BOLT_MEMORY_ON_HEAP_LOAD, Number.class); Number joinBoltMemoryOffHeapLoad = config.getAs(BulletStormConfig.JOIN_BOLT_MEMORY_OFF_HEAP_LOAD, Number.class); Number resultBoltParallelism = config.getAs(BulletStormConfig.RESULT_BOLT_PARALLELISM, Number.class); Number resultBoltCPULoad = config.getAs(BulletStormConfig.RESULT_BOLT_CPU_LOAD, Number.class); Number resultBoltMemoryOnHeapLoad = config.getAs(BulletStormConfig.RESULT_BOLT_MEMORY_ON_HEAP_LOAD, Number.class); Number resultBoltMemoryOffHeapLoad = config.getAs(BulletStormConfig.RESULT_BOLT_MEMORY_OFF_HEAP_LOAD, Number.class); Number loopBoltParallelism = config.getAs(BulletStormConfig.LOOP_BOLT_PARALLELISM, Number.class); Number loopBoltCPULoad = config.getAs(BulletStormConfig.LOOP_BOLT_CPU_LOAD, Number.class); Number loopBoltMemoryOnHeapLoad = config.getAs(BulletStormConfig.LOOP_BOLT_MEMORY_ON_HEAP_LOAD, Number.class);
@Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { // Add the Storm Config and the context as is, in case any PubSubs need it. config.set(BulletStormConfig.STORM_CONFIG, conf); config.set(BulletStormConfig.STORM_CONTEXT, context); this.collector = collector; }
@Test public void testHookingInCustomMetricsConsumer() { builder.setSpout("source", new CustomIRichSpout(), 10); config.set(BulletStormConfig.TOPOLOGY_METRICS_ENABLE, true); config.set(BulletStormConfig.TOPOLOGY_METRICS_CLASSES, singletonList(CustomIMetricsConsumer.class.getName())); Assert.assertNull(config.get(CustomIMetricsConsumer.CUSTOM_METRICS_REGISTERED)); Assert.assertFalse(builder.isTopologyCreated()); submitWithTopology("source"); Assert.assertTrue(builder.isTopologyCreated()); Assert.assertTrue((Boolean) config.get(CustomIMetricsConsumer.CUSTOM_METRICS_REGISTERED)); }
/** * Main. Launches a remote Storm topology. * @param args The input args. * @throws Exception if any. */ public static void main(String[] args) throws Exception { OptionSet options = PARSER.parse(args); if (options.has(HELP_ARG) || !options.has(CONFIGURATION_ARG)) { printHelp(); return; } String yamlPath = (String) options.valueOf(CONFIGURATION_ARG); BulletStormConfig config = new BulletStormConfig(yamlPath); log.info(config.toString()); StormUtils.submit(config, new TopologyBuilder()); }
private static void enableMetadataInConfig(BulletStormConfig config, String metaConcept, String key) { Map<String, String> metadataConfig = (Map<String, String>) config.getOrDefault(BulletStormConfig.RESULT_METADATA_METRICS, new HashMap<>()); metadataConfig.put(metaConcept, key); config.set(BulletStormConfig.RESULT_METADATA_ENABLE, true); config.set(BulletStormConfig.RESULT_METADATA_METRICS, metadataConfig); }
/** * Creates an instance of this class with the given non-null config. * * @param config The non-null {@link BulletStormConfig} which is the config for this component. */ public TickSpout(BulletStormConfig config) { super(config); tickInterval = config.getAs(BulletStormConfig.TICK_SPOUT_INTERVAL, Integer.class); }
@Test public void testSwitchingIntoQueryPublishing() { Assert.assertEquals(publisher.getContext(), PubSub.Context.QUERY_SUBMISSION); // Config is modified in place Assert.assertEquals(config.get("fake.setting"), "foo"); // But original setting is preserved Assert.assertEquals(config.get(BulletStormConfig.TOPOLOGY_NAME), "test"); }
/** * Constructor that loads the defaults and augments it with defaults. * * @param other The other config to wrap. */ public BulletStormConfig(Config other) { // Load Bullet and Storm defaults. Then merge the other. super(DEFAULT_STORM_CONFIGURATION); merge(other); VALIDATOR.validate(this); }
private static BulletStormConfig configWithRawMaxAndNoMeta() { BulletStormConfig config = new BulletStormConfig(); config.set(BulletStormConfig.RAW_AGGREGATION_MAX_SIZE, RAW_MAX_SIZE); config.set(BulletStormConfig.RESULT_METADATA_ENABLE, false); config.validate(); return config; }
/** * Create a new DRPCConfig by reading in a file. * * @param file The file containing DRPC settings. */ public DRPCConfig(String file) { // Load and merge with default bullet-storm settings. Storm defaults also contain the DRPC settings. this(new BulletStormConfig(file)); VALIDATOR.validate(this); }
@Test public void testKillSignal() { config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); config.validate(); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1)); Tuple kill = TupleUtils.makeIDTuple(TupleClassifier.Type.METADATA_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null)); bolt.execute(kill); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0)); }