public Emitter(Map conf) { Object c = conf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING); if (c == null) _maxSpoutPending = 1; else _maxSpoutPending = Utils.getInt(c); }
public Emitter(Map conf) { Object c = conf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING); if (c == null) _maxSpoutPending = 1; else _maxSpoutPending = Utils.getInt(c); }
public static Integer getInt(Object o) { Integer result = getInt(o, null); if (null == result) { throw new IllegalArgumentException("Don't know how to convert null to int"); } return result; }
public int getMaxBufferSize(Map conf) { return Utils.getInt(conf.get(_buffConf)); } }
public int getNumThreads(Map conf) { return Utils.getInt(conf.get(_threadsConf)); }
public int getPort(Map conf) { return Utils.getInt(conf.get(_portConf)); }
public Emitter(Map conf) { Object c = conf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING); if(c==null) _maxSpoutPending = 1; else _maxSpoutPending = Utils.getInt(c); }
protected static void setupBuilder(CuratorFrameworkFactory.Builder builder, String zkStr, Map conf, ZookeeperAuthInfo auth) { builder.connectString(zkStr) .connectionTimeoutMs(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT))) .sessionTimeoutMs(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT))) .retryPolicy( new StormBoundedExponentialBackoffRetry(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)), Utils.getInt(conf .get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING)), Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)))); if (auth != null && auth.scheme != null && auth.payload != null) { builder = builder.authorization(auth.scheme, auth.payload); } }
/** * Invoked once immediately after construction * * @param storm_conf Storm configuration */ @Override public void prepare(Map storm_conf) { int timeout = Utils.getInt(storm_conf.get(Config.STORM_GROUP_MAPPING_SERVICE_CACHE_DURATION_SECS)); cachedGroups = new TimeCacheMap<String, Set<String>>(timeout); }
private CuratorFramework newCurator(Map conf, KafkaSpoutConfig config) throws Exception { String serverPorts = ""; List<Host> zkServers = config.zkServers; for (Host server : zkServers) { serverPorts = serverPorts + server.getHost() + ":" + server.getPort() + ","; } return CuratorFrameworkFactory.newClient(serverPorts, Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), 15000, new RetryNTimes( Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)), Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)))); }
public static CuratorFramework newCurator(Map conf, List<String> servers, Object port, String root, ZookeeperAuthInfo auth) { List<String> serverPorts = new ArrayList<>(); for (String zkServer : servers) { serverPorts.add(zkServer + ":" + Utils.getInt(port)); } String zkStr = StringUtils.join(serverPorts, ",") + root; CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder(); setupBuilder(builder, zkStr, conf, auth); return builder.build(); }
public static NimbusClient getConfiguredClient(Map conf) { String nimbusHost = (String) conf.get(Config.NIMBUS_HOST); int nimbusPort = Utils.getInt(conf.get(Config.NIMBUS_THRIFT_PORT)); return new NimbusClient(nimbusHost, nimbusPort); }
public int maxTopologyMessageTimeout() { Integer max = Utils.getInt(_stormConf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)); for (String spout : getRawTopology().get_spouts().keySet()) { ComponentCommon common = getComponentCommon(spout); String jsonConf = common.get_json_conf(); if (jsonConf != null) { Map conf = (Map) Utils.from_json(jsonConf); Object comp = conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS); if (comp != null) { max = Math.max(Utils.getInt(comp), max); } } } return max; }
protected List<AsyncLoopThread> setDeserializeThreads() { WorkerTopologyContext workerTopologyContext = contextMaker.makeWorkerTopologyContext(sysTopology); int tasksNum = shutdownTasks.size(); double workerRatio = ConfigExtension.getWorkerDeserializeThreadRatio(stormConf); int workerDeserThreadNum = Utils.getInt(Math.ceil(workerRatio * tasksNum)); if (workerDeserThreadNum > 0 && tasksNum > 0) { double average = tasksNum / (double) workerDeserThreadNum; for (int i = 0; i < workerDeserThreadNum; i++) { int startRunTaskIndex = Utils.getInt(Math.rint(average * i)); deserializeThreads.add(new AsyncLoopThread(new WorkerDeserializeRunnable( shutdownTasks, stormConf, workerTopologyContext, startRunTaskIndex, i))); } } return deserializeThreads; }
protected List<AsyncLoopThread> setSerializeThreads() { WorkerTopologyContext workerTopologyContext = contextMaker.makeWorkerTopologyContext(sysTopology); int tasksNum = shutdownTasks.size(); double workerRatio = ConfigExtension.getWorkerSerializeThreadRatio(stormConf); int workerSerialThreadNum = Utils.getInt(Math.ceil(workerRatio * tasksNum)); if (workerSerialThreadNum > 0 && tasksNum > 0) { double average = tasksNum / (double) workerSerialThreadNum; for (int i = 0; i < workerSerialThreadNum; i++) { int startRunTaskIndex = Utils.getInt(Math.rint(average * i)); serializeThreads.add(new AsyncLoopThread(new WorkerSerializeRunnable( shutdownTasks, stormConf, workerTopologyContext, startRunTaskIndex, i))); } } return serializeThreads; }
public int maxTopologyMessageTimeout(Map<String, Object> topologyConfig) { Integer max = Utils.getInt(topologyConfig.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)); for(String spout: getRawTopology().get_spouts().keySet()) { ComponentCommon common = getComponentCommon(spout); String jsonConf = common.get_json_conf(); if(jsonConf!=null) { Map conf = (Map) JSONValue.parse(jsonConf); Object comp = conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS); if(comp!=null) { max = Math.max(Utils.getInt(comp), max); } } } return max; } }
@SuppressWarnings("rawtypes") NettyClientAsync(Map conf, ChannelFactory factory, String host, int port, ReconnectRunnable reconnector, final Set<Integer> sourceTasks, final Set<Integer> targetTasks) { super(conf, factory, host, port, reconnector); clientChannelFactory = factory; initFlowCtrl(conf, sourceTasks, targetTasks); flushCheckInterval = Utils.getInt(conf.get(Config.STORM_NETTY_FLUSH_CHECK_INTERVAL_MS), 5); flusher = new NettyClientFlush(flushCheckInterval); flusher.start(); start(); }
/** * initialization per Storm configuration */ @SuppressWarnings("rawtypes") public void prepare(Map stormConf) { this.stormConf = stormConf; int maxWorkers = Utils.getInt(stormConf.get(Config.STORM_MESSAGING_NETTY_CLIENT_WORKER_THREADS)); ThreadFactory bossFactory = new NettyRenameThreadFactory(MetricDef.NETTY_CLI + "boss"); ThreadFactory workerFactory = new NettyRenameThreadFactory(MetricDef.NETTY_CLI + "worker"); if (maxWorkers > 0) { clientChannelFactory = new NioClientSocketChannelFactory(Executors.newCachedThreadPool(bossFactory), Executors.newCachedThreadPool(workerFactory), maxWorkers); } else { clientChannelFactory = new NioClientSocketChannelFactory(Executors.newCachedThreadPool(bossFactory), Executors.newCachedThreadPool(workerFactory)); } reconnector = new ReconnectRunnable(); new AsyncLoopThread(reconnector, true, Thread.MIN_PRIORITY, true); }
@Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { _rand = new Random(Utils.secureRandomLong()); _state = TransactionalState.newCoordinatorState(conf, (String) conf.get(Config.TOPOLOGY_TRANSACTIONAL_ID), _spout.getComponentConfiguration()); _coordinatorState = new RotatingTransactionalState(_state, META_DIR, true); _collector = collector; _coordinator = _spout.getCoordinator(conf, context); _currTransaction = getStoredCurrTransaction(_state); Object active = conf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING); if (active == null) { _maxTransactionActive = 1; } else { _maxTransactionActive = Utils.getInt(active); } _initializer = new StateInitializer(); }
@Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { _state = TransactionalState.newCoordinatorState(conf, (String) conf.get(Config.TOPOLOGY_TRANSACTIONAL_ID), _spout.getComponentConfiguration()); _coordinatorState = new RotatingTransactionalState(_state, META_DIR, true); _collector = collector; _coordinator = _spout.getCoordinator(conf, context); _currTransaction = getStoredCurrTransaction(_state); Object active = conf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING); if(active==null) { _maxTransactionActive = 1; } else { _maxTransactionActive = Utils.getInt(active); } _initializer = new StateInitializer(); }