sd.setCPULoad(spoutConf.cpuLoad);
masterCoord.setCPULoad(cpuLoad);
spoutDeclarer.setCPULoad(cpuLoad);
builder1.setSpout("wordSpout", new TestWordSpout(), 2).setCPULoad(250.0).setMemoryLoad(1000.0, 200.0); builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(100.0) .setMemoryLoad(500.0, 100.0);
builder1.setSpout("wordSpout1", new TestWordSpout(), 1).setCPULoad(300.0).setMemoryLoad(2000.0, 48.0); StormTopology stormTopology1 = builder1.createTopology(); Config config1 = new Config(); builder2.setSpout("wordSpout2", new TestWordSpout(), 4).setCPULoad(100.0).setMemoryLoad(500.0, 12.0); StormTopology stormTopology2 = builder2.createTopology(); Config config2 = new Config(); builder3.setSpout("wordSpout3", new TestWordSpout(), 4).setCPULoad(20.0).setMemoryLoad(200.0, 56.0); StormTopology stormTopology3 = builder3.createTopology(); Config config3 = new Config(); builder4.setSpout("wordSpout4", new TestWordSpout(), 12).setCPULoad(30.0).setMemoryLoad(100.0, 0.0); StormTopology stormTopology4 = builder4.createTopology(); Config config4 = new Config(); builder5.setSpout("wordSpout5", new TestWordSpout(), 40).setCPULoad(25.0).setMemoryLoad(100.0, 28.0); StormTopology stormTopology5 = builder5.createTopology(); Config config5 = new Config();
builder1.setSpout("wordSpout", new TestWordSpout(), 1).setCPULoad(20.0).setMemoryLoad(200.0); builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(20.0).setMemoryLoad(200.0); StormTopology stormTopology1 = builder1.createTopology();
private static void buildSpouts(ExecutionContext context, TopologyBuilder builder) throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, NoSuchFieldException { for (SpoutDef sd : context.getTopologyDef().getSpouts()) { IRichSpout spout = buildSpout(sd, context); SpoutDeclarer declarer = builder.setSpout(sd.getId(), spout, sd.getParallelism()); if (sd.getOnHeapMemoryLoad() > -1) { if (sd.getOffHeapMemoryLoad() > -1) { declarer.setMemoryLoad(sd.getOnHeapMemoryLoad(), sd.getOffHeapMemoryLoad()); } else { declarer.setMemoryLoad(sd.getOnHeapMemoryLoad()); } } if (sd.getCpuLoad() > -1) { declarer.setCPULoad(sd.getCpuLoad()); } if (sd.getNumTasks() > -1) { declarer.setNumTasks(sd.getNumTasks()); } context.addSpout(sd.getId(), spout); } }
SpoutDeclarer spout = builder.setSpout("word", new TestWordSpout(), 10).setCPULoad(20);
/** * This submits a topology after loading the given spout with the given configuration as the source of * {@link com.yahoo.bullet.record.BulletRecord} using the given {@link TopologyBuilder}. * * @param builder The {@link TopologyBuilder} to use to add the topology to. * @param spout The name of the instance of an {@link org.apache.storm.topology.IRichSpout} to load. * @param args The arguments to pass to the constructor of this spout (otherwise the default constructor is used). * @param config The Storm settings for this Bullet topology. * @param parallelism The parallelism of the spout component. * @param cpuLoad The CPU load for the Storm RAS scheduler. * @param onHeapMemoryLoad The on heap memory load for the Storm RAS scheduler. * @param offHeapMemoryLoad The off heap memory load for the Storm RAS scheduler. * @throws Exception if there were issues creating the topology. */ public static void submit(TopologyBuilder builder, String spout, List<String> args, BulletStormConfig config, Number parallelism, Number cpuLoad, Number onHeapMemoryLoad, Number offHeapMemoryLoad) throws Exception { builder.setSpout(TopologyConstants.RECORD_COMPONENT, ReflectionUtils.getSpout(spout, args), parallelism) .setCPULoad(cpuLoad).setMemoryLoad(onHeapMemoryLoad, offHeapMemoryLoad); log.info("Added spout {} with Parallelism {}, CPU load {}, On-heap memory {}, Off-heap memory {}", spout, parallelism, cpuLoad, onHeapMemoryLoad, offHeapMemoryLoad); submit(config, TopologyConstants.RECORD_COMPONENT, builder); }
private static void addBulletSpout(BulletStormConfig config, TopologyBuilder builder) throws Exception { String bulletSpoutClassName = config.getAs(BulletStormConfig.BULLET_SPOUT_CLASS_NAME, String.class); List<String> bulletSpoutArgs = config.getAs(BulletStormConfig.BULLET_SPOUT_ARGS, List.class); Number bulletSpoutParallelism = config.getAs(BulletStormConfig.BULLET_SPOUT_PARALLELISM, Number.class); Number bulletSpoutCPULoad = config.getAs(BulletStormConfig.BULLET_SPOUT_CPU_LOAD, Number.class); Number bulletSpoutMemoryOnHeapLoad = config.getAs(BulletStormConfig.BULLET_SPOUT_MEMORY_ON_HEAP_LOAD, Number.class); Number bulletSpoutMemoryOffHeapLoad = config.getAs(BulletStormConfig.BULLET_SPOUT_MEMORY_OFF_HEAP_LOAD, Number.class); builder.setSpout(TopologyConstants.RECORD_COMPONENT, ReflectionUtils.getSpout(bulletSpoutClassName, bulletSpoutArgs), bulletSpoutParallelism) .setCPULoad(bulletSpoutCPULoad) .setMemoryLoad(bulletSpoutMemoryOnHeapLoad, bulletSpoutMemoryOffHeapLoad); log.info("Added spout with Parallelism {}, CPU load {}, On-heap memory {}, Off-heap memory {}", bulletSpoutParallelism, bulletSpoutCPULoad, bulletSpoutMemoryOnHeapLoad, bulletSpoutMemoryOffHeapLoad); }
masterCoord.setCPULoad(cpuLoad);
spoutDeclarer.setCPULoad(cpuLoad);
private static void addDSLSpout(BulletStormConfig config, TopologyBuilder builder) { Number dslSpoutParallelism = config.getAs(BulletStormConfig.DSL_SPOUT_PARALLELISM, Number.class); Number dslSpoutCPULoad = config.getAs(BulletStormConfig.DSL_SPOUT_CPU_LOAD, Number.class); Number dslSpoutMemoryOnHeapLoad = config.getAs(BulletStormConfig.DSL_SPOUT_MEMORY_ON_HEAP_LOAD, Number.class); Number dslSpoutMemoryOffHeapLoad = config.getAs(BulletStormConfig.DSL_SPOUT_MEMORY_OFF_HEAP_LOAD, Number.class); Boolean dslBoltEnable = config.getAs(BulletStormConfig.DSL_BOLT_ENABLE, Boolean.class); builder.setSpout(dslBoltEnable ? TopologyConstants.DATA_COMPONENT : TopologyConstants.RECORD_COMPONENT, new DSLSpout(config), dslSpoutParallelism) .setCPULoad(dslSpoutCPULoad) .setMemoryLoad(dslSpoutMemoryOnHeapLoad, dslSpoutMemoryOffHeapLoad); log.info("Added DSLSpout with Parallelism {}, CPU load {}, On-heap memory {}, Off-heap memory {}", dslSpoutParallelism, dslSpoutCPULoad, dslSpoutMemoryOnHeapLoad, dslSpoutMemoryOffHeapLoad); if (dslBoltEnable) { Number dslBoltParallelism = config.getAs(BulletStormConfig.DSL_BOLT_PARALLELISM, Number.class); Number dslBoltCPULoad = config.getAs(BulletStormConfig.DSL_BOLT_CPU_LOAD, Number.class); Number dslBoltMemoryOnHeapLoad = config.getAs(BulletStormConfig.DSL_BOLT_MEMORY_ON_HEAP_LOAD, Number.class); Number dslBoltMemoryOffHeapLoad = config.getAs(BulletStormConfig.DSL_BOLT_MEMORY_OFF_HEAP_LOAD, Number.class); builder.setBolt(TopologyConstants.RECORD_COMPONENT, new DSLBolt(config), dslBoltParallelism) .shuffleGrouping(TopologyConstants.DATA_COMPONENT) .setCPULoad(dslBoltCPULoad) .setMemoryLoad(dslBoltMemoryOnHeapLoad, dslBoltMemoryOffHeapLoad); log.info("Added DSLBolt with Parallelism {}, CPU load {}, On-heap memory {}, Off-heap memory {}", dslBoltParallelism, dslBoltCPULoad, dslBoltMemoryOnHeapLoad, dslBoltMemoryOffHeapLoad); } }
private static void buildSpouts(ExecutionContext context, TopologyBuilder builder) throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, NoSuchFieldException { for (SpoutDef sd : context.getTopologyDef().getSpouts()) { IRichSpout spout = buildSpout(sd, context); SpoutDeclarer declarer = builder.setSpout(sd.getId(), spout, sd.getParallelism()); if (sd.getOnHeapMemoryLoad() > -1) { if (sd.getOffHeapMemoryLoad() > -1) { declarer.setMemoryLoad(sd.getOnHeapMemoryLoad(), sd.getOffHeapMemoryLoad()); } else { declarer.setMemoryLoad(sd.getOnHeapMemoryLoad()); } } if (sd.getCpuLoad() > -1) { declarer.setCPULoad(sd.getCpuLoad()); } if (sd.getNumTasks() > -1) { declarer.setNumTasks(sd.getNumTasks()); } context.addSpout(sd.getId(), spout); } }
.setCPULoad(querySpoutCPULoad).setMemoryLoad(querySpoutMemoryOnHeapLoad, querySpoutMemoryOffHeapLoad); .setCPULoad(tickSpoutCPULoad).setMemoryLoad(tickSpoutMemoryOnheapLoad, tickSpoutMemoryOffHeapLoad);
.setCPULoad(querySpoutCPULoad).setMemoryLoad(querySpoutMemoryOnHeapLoad, querySpoutMemoryOffHeapLoad); .setCPULoad(tickSpoutCPULoad).setMemoryLoad(tickSpoutMemoryOnheapLoad, tickSpoutMemoryOffHeapLoad);