public void setMaxSpoutPending(int max) { setMaxSpoutPending(this, max); }
public static void test() { conf.setMaxSpoutPending(20); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; if (isLocal) { drpc = new LocalDRPC(); } try { JStormHelper.runTopology(buildTopology(drpc), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new InOrderSpout(), 8); builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1")); conf.setMaxSpoutPending(20); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; if (isLocal) { drpc = new LocalDRPC(); } try { JStormHelper.runTopology(buildDevicesTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new InOrderSpout(), 8); builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1")); conf.setMaxSpoutPending(20); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; if (isLocal) { drpc = new LocalDRPC(); } try { JStormHelper.runTopology(buildVehiclesTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new InOrderSpout(), 8); builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1")); conf.setMaxSpoutPending(20); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; if (isLocal) { drpc = new LocalDRPC(); } try { JStormHelper.runTopology(buildTopology(drpc), topologyName, conf, 60, new DrpcValidator(), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new InOrderSpout(), 8); builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1")); Config conf = new Config(); conf.setMaxSpoutPending(20); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; if (isLocal) { drpc = new LocalDRPC(); } try { JStormHelper.runTopology(buildTopology(drpc), topologyName, conf, 60, new DrpcValidator(), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH); TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("global-count", "spout", spout, 1); builder.setBolt("partial-count", new BatchCount(), 2).noneGrouping("spout"); builder.setBolt("sum", new UpdateGlobalCount(), 1).globalGrouping("partial-count"); conf.setDebug(true); conf.setMaxSpoutPending(3); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.buildTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public void setMaxSpoutPending(int max) { setMaxSpoutPending(this, max); }
public void setMaxSpoutPending(int max) { setMaxSpoutPending(this, max); }
public void setMaxSpoutPending(int max) { setMaxSpoutPending(this, max); }
public static void main(String[] args) throws Exception { Config conf = new Config(); conf.setMaxSpoutPending(20); // This topology can only be run as local because it is a toy example LocalDRPC drpc = new LocalDRPC(); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("hashtagCounter", conf, buildTopology(drpc)); // Query 100 times for hashtag "california" for illustrating the effect of the lambda architecture for(int i = 0; i < 100; i++) { System.out.println("Result for hashtag 'california' -> " + drpc.execute("hashtags", "california")); Thread.sleep(1000); } } }
private static Config createTopologyConfiguration(Properties prop, boolean localMode) { Config conf = new Config(); List<String> dprcServers = new ArrayList<String>(); dprcServers.add("localhost"); conf.put(Config.DRPC_SERVERS, dprcServers); conf.put(Config.DRPC_PORT, 3772); if (!localMode) conf.put(Config.STORM_CLUSTER_MODE, new String("distributed")); conf.put("UNIQUE_WORDS_EXPECTED", prop.getProperty("UNIQUE_WORDS_EXPECTED")); conf.put("PATH_TO_OOV_FILE", prop.getProperty("PATH_TO_OOV_FILE")); conf.put("L", prop.getProperty("L")); conf.put("BucketsParallelism", prop.getProperty("BucketsParallelism")); conf.put("k", prop.getProperty("k")); conf.put("QUEUE_SIZE", prop.getProperty("QUEUE_SIZE")); List<String> countAggKeepFields = new ArrayList<String>(); countAggKeepFields.add("tweet_obj"); countAggKeepFields.add("coltweet_obj"); conf.put("countAggKeepFields", countAggKeepFields); conf.put("THRESHOLD", prop.getProperty("THRESHOLD")); conf.put("RECENT_TWEETS_TO_COMPARE_WITH", prop.getProperty("RECENT_TWEETS_TO_COMPARE_WITH")); conf.setDebug(false); conf.setNumWorkers(Integer.valueOf((String) prop .get("NUMBER_OF_WORKERS"))); conf.setMaxSpoutPending(50000000); return conf; }
@Override public Config prepareConfig() { if (preparedConfig == null) { preparedConfig = new Config(); preparedConfig.setMaxSpoutPending(500); preparedConfig.setNumWorkers(numberOfWorkers); preparedConfig.setFallBackOnJavaSerialization(false); preparedConfig.setSkipMissingKryoRegistrations(false); JenaStormUtils.registerSerializers(preparedConfig); } return preparedConfig; }
config.setNumWorkers(1); config.setMaxTaskParallelism(10); config.setMaxSpoutPending(10000); config.put(Config.NIMBUS_HOST, nimbusIp); config.put(Config.NIMBUS_THRIFT_PORT, 6627);
/** * @param topologyName * @param conf * @param builder * @throws AlreadyAliveException * @throws InvalidTopologyException */ public static void submitStormTopology(String topologyName, Config conf, TopologyBuilder builder) throws AlreadyAliveException, InvalidTopologyException { int numWorkers = ConfigUtility.getInt(conf, "num.workers", 1); int maxSpoutPending = ConfigUtility.getInt(conf, "max.spout.pending", 1000); int maxTaskParalleism = ConfigUtility.getInt(conf, "max.task.parallelism", 100); conf.setNumWorkers(numWorkers); conf.setMaxSpoutPending(maxSpoutPending); conf.setMaxTaskParallelism(maxTaskParalleism); StormSubmitter.submitTopology(topologyName, conf, builder.createTopology()); }
/** * @param args * @throws InterruptedException */ public static void main(String[] args) throws InterruptedException { // Add transactional spout MemoryTransactionalSpout spout = new MemoryTransactionalSpout(values, new Fields("shortid", "url", "user", "date"), 3); TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder( "shorturl-count", "spout", spout, 2); // Build TupleTableConifg TupleTableConfig ttConfig = new TupleTableConfig("shorturl", "shortid"); ttConfig.setBatch(false); ttConfig.addColumn("data", "clicks"); ttConfig.addColumn("daily", "date"); builder.setBolt("hbase-counters", new HBaseCountersBatchBolt(ttConfig), 2) .fieldsGrouping("spout", new Fields("shortid")); LocalCluster cluster = new LocalCluster(); Config stormConfig = new Config(); stormConfig.setDebug(true); stormConfig.setMaxSpoutPending(3); cluster.submitTopology("hbase-example", stormConfig, builder.buildTopology()); Thread.sleep(10000); cluster.shutdown(); }
public static void main(String[] args) { final Config conf = new Config(); conf.setDebug(false); conf.setNumWorkers(2); conf.setMaxSpoutPending(1); conf.setFallBackOnJavaSerialization(false); conf.setSkipMissingKryoRegistrations(false); final LocalCluster cluster = new LocalCluster(); final TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("randomSpout1", new RandomFieldSpout(2, 0, 0, 1)); // (nfields,seed,min,max) builder.setSpout("randomSpout2", new RandomFieldSpout(2, 10, 0, 1)); // (nfields,seed,min,max) JoinBolt.connectNewBolt(builder); final StormTopology topology = builder.createTopology(); cluster.submitTopology("playTopology", conf, topology); Utils.sleep(10000); cluster.killTopology("playTopology"); cluster.shutdown(); } }
public static void main(String[] args) { final Config conf = new Config(); conf.setDebug(false); conf.setNumWorkers(2); conf.setMaxSpoutPending(1); conf.setFallBackOnJavaSerialization(false); conf.setSkipMissingKryoRegistrations(false); final LocalCluster cluster = new LocalCluster(); final TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("randomSpout1", new RandomFieldSpout(2, 0, 0, 1)); // (nfields,seed,min,max) builder.setSpout("randomSpout2", new RandomFieldSpout(2, 10, 0, 1)); // (nfields,seed,min,max) JoinBolt.connectNewBolt(builder); final StormTopology topology = builder.createTopology(); cluster.submitTopology("playTopology", conf, topology); Utils.sleep(10000); cluster.killTopology("playTopology"); cluster.shutdown(); } }
public void run() { StormTopology topology = new FlowmixBuilder() .setFlowLoader(new SimpleFlowLoaderSpout(provider.getFlows(), 60000)) .setEventsLoader(new MockEventGeneratorSpout(getMockEvents(), 10)) .setOutputBolt(new PrinterBolt()) .setParallelismHint(6) .create() .createTopology(); Config conf = new Config(); conf.setNumWorkers(20); conf.setMaxSpoutPending(5000); conf.setDebug(false); conf.registerSerialization(BaseEvent.class, EventSerializer.class); conf.setSkipMissingKryoRegistrations(false); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("example-topology", conf, topology); }