public void setMessageTimeoutSecs(int secs) { put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, secs); }
public void setOptimize(boolean isOn) { put(Config.TOPOLOGY_OPTIMIZE, isOn); }
public void setNumAckers(int numTasks) { put(Config.TOPOLOGY_ACKERS, numTasks); }
public void setSkipMissingKryoRegistrations(boolean skip) { put(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS, skip); }
public void setMaxTaskParallelism(int max) { put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, max); }
public void setNumWorkers(int workers) { put(Config.TOPOLOGY_WORKERS, workers); }
public void setDebug(boolean isOn) { put(Config.TOPOLOGY_DEBUG, isOn); }
public void setMaxSpoutPending(int max) { put(Config.TOPOLOGY_MAX_SPOUT_PENDING, max); }
public void setStatsSampleRate(double rate) { put(Config.TOPOLOGY_STATS_SAMPLE_RATE, rate); }
public void setFallBackOnJavaSerialization(boolean fallback) { put(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION, fallback); }
@Override public void process(Config conf, CommandLine commandLine) throws ParseException { try { List<File> jarFiles = validateFiles(commandLine.getOptionValue("libjars")); Map<String, String> jars = new HashMap<>(jarFiles.size()); List<String> names = new ArrayList<>(jarFiles.size()); for (File f : jarFiles) { jars.put(f.getName(), f.getAbsolutePath()); names.add(f.getName()); } conf.put(TOPOLOGY_LIB_PATH, jars); conf.put(TOPOLOGY_LIB_NAME, names); } catch (IOException e) { throw new ParseException(e.getMessage()); } } }
public void SetRemoteTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException { Config conf = getConf(); StormTopology topology = buildTopology(); conf.put(Config.STORM_CLUSTER_MODE, "distributed"); String streamName = (String) conf.get(Config.TOPOLOGY_NAME); if (streamName == null) { streamName = "SequenceTest"; } if (streamName.contains("zeromq")) { conf.put(Config.STORM_MESSAGING_TRANSPORT, "com.alibaba.jstorm.message.zeroMq.MQContext"); } else { conf.put(Config.STORM_MESSAGING_TRANSPORT, "com.alibaba.jstorm.message.netty.NettyContext"); } StormSubmitter.submitTopology(streamName, conf, topology); }
private List getRegisteredSerializations() { if(!containsKey(Config.TOPOLOGY_KRYO_REGISTER)) { put(Config.TOPOLOGY_KRYO_REGISTER, new ArrayList()); } return (List) get(Config.TOPOLOGY_KRYO_REGISTER); } }
public static void test() throws Exception { TransactionTopologyBuilder builder = new TransactionTopologyBuilder(); if (isLocal) { conf.put("tuple.num.per.batch", 100); conf.put("transaction.scheduler.spout", false); conf.put("transaction.exactly.cache.type", "default"); conf.put("transaction.topology", true); } int spoutParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int splitParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1); int countParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); builder.setSpout("spout", new TxFastRandomSentenceSpout(), spoutParallelismHint); builder.setBolt("split", new TxSplitSentence(), splitParallelismHint).localOrShuffleGrouping("spout"); WordCount wordCount = new WordCount(); builder.setBolt("count", wordCount .timeWindow(Time.seconds(60L)) .withTransactionStateOperator(wordCount), countParallelismHint).fieldsGrouping("split", new Fields("word")); builder.enableHdfs(); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; StormSubmitter.submitTopology(topologyName, conf, builder.createTopology()); }
public static void test() throws Exception { TransactionTopologyBuilder builder = new TransactionTopologyBuilder(); if (isLocal) { conf.put("tuple.num.per.batch", 100); conf.put("transaction.scheduler.spout", false); conf.put("transaction.exactly.cache.type", "default"); conf.put("transaction.topology", true); } int spoutParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int splitParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1); int countParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); builder.setSpout("spout", new TxFastRandomSentenceSpout(), spoutParallelismHint); builder.setBolt("split", new SplitSentence(), splitParallelismHint).localOrShuffleGrouping("spout"); builder.setBolt("count", new WordCount() .timeWindow(Time.seconds(1L)) .withStateSize(Time.hours(2)), countParallelismHint).fieldsGrouping("split", new Fields("word")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); }
conf.put(Config.ISOLATION_SCHEDULER_MACHINES, hosts);
public static void test() throws Exception { TransactionTopologyBuilder builder = new TransactionTopologyBuilder(); if (isLocal) { conf.put("tuple.num.per.batch", 5); conf.put("transaction.scheduler.spout", false); conf.put("transaction.exactly.cache.type", "default"); } int spoutParallelism = JStormUtils.parseInt(conf.get(SPOUT_PARALLELISM_HINT), 1); int splitParallelism = JStormUtils.parseInt(conf.get(SPLIT_PARALLELISM_HINT), 2); int countParallelism = JStormUtils.parseInt(conf.get(COUNT_PARALLELISM_HINT), 2); boolean isScheduleSpout = JStormUtils.parseBoolean(conf.get("transaction.scheduler.spout"), true); if (isScheduleSpout) // Generate batch by configured time. "transaction.schedule.batch.delay.ms: 1000 # 1sec" builder.setSpout("spout", new ScheduleTxSpout(), spoutParallelism); else // Generate batch by user when calling emitBarrier builder.setSpout("spout", new BasicTxSpout(), spoutParallelism, false); builder.setBolt("split", new TxSplitSentence(), splitParallelism).localOrShuffleGrouping("spout"); builder.setBolt("count", new TxWordCount(), countParallelism).fieldsGrouping("split", new Fields("word")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; StormSubmitter.submitTopology(topologyName, conf, builder.createTopology()); }
conf.put(ConfigExtension.TOPOLOGY_BACKPRESSURE_ENABLE, true); conf.setNumWorkers(8);
conf.put(Config.TOPOLOGY_WORKERS, workerNum);
@Override public Map<String, Object> getComponentConfiguration() { Config conf = new Config(); // send tick tuples every second conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 1); return conf; }