public static Double getDouble(Object o) { Double result = getDouble(o, null); if (null == result) { throw new IllegalArgumentException("Don't know how to convert null to double"); } return result; }
public static int samplingRate(Map conf) { double rate = Utils.getDouble(conf.get(Config.TOPOLOGY_STATS_SAMPLE_RATE)); if (rate != 0) { return (int) (1 / rate); } throw new IllegalArgumentException("Illegal topology.stats.sample.rate in conf: " + rate); }
private Map<String, Double> mkSupervisorCapacities(Map conf) { Map<String, Double> ret = new HashMap<String, Double>(); Double mem = Utils.getDouble(conf.get(Config.SUPERVISOR_MEMORY_CAPACITY_MB), 4096.0); ret.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, mem); Double cpu = Utils.getDouble(conf.get(Config.SUPERVISOR_CPU_CAPACITY), 400.0); ret.put(Config.SUPERVISOR_CPU_CAPACITY, cpu); return ret; }
private static void validateTopologyWorkerMaxHeapSizeMBConfigs(Map<String, Object> stormConf, StormTopology topology) { double largestMemReq = getMaxExecutorMemoryUsageForTopo(topology, stormConf); Double topologyWorkerMaxHeapSize = Utils.getDouble(stormConf.get(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB)); if(topologyWorkerMaxHeapSize < largestMemReq) { throw new IllegalArgumentException("Topology will not be able to be successfully scheduled: Config TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB=" +Utils.getDouble(stormConf.get(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB)) + " < " + largestMemReq + " (Largest memory requirement of a component in the topology). Perhaps set TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB to a larger amount"); } }
public static void checkInitMem(Map<String, Double> topology_resources, String Com, Map topologyConf) { if (!topology_resources.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB)) { topology_resources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, org.apache.storm.utils.Utils.getDouble(topologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB), null)); debugMessage("ONHEAP", Com, topologyConf); } if (!topology_resources.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB)) { topology_resources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, org.apache.storm.utils.Utils.getDouble(topologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB), null)); debugMessage("OFFHEAP", Com, topologyConf); } }
public static void checkInitCPU(Map<String, Double> topology_resources, String Com, Map topologyConf) { if (!topology_resources.containsKey(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT)) { topology_resources.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, org.apache.storm.utils.Utils.getDouble(topologyConf.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT), null)); debugMessage("CPU", Com, topologyConf); } }
/** * initializes member variables */ private void initConfigs() { this.topologyWorkerMaxHeapSize = Utils.getDouble(this.topologyConf.get(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB), null); this.topologyPriority = Utils.getInt(this.topologyConf.get(Config.TOPOLOGY_PRIORITY), null); assert this.topologyWorkerMaxHeapSize != null; assert this.topologyPriority != null; }
/** * Add default resource requirements for a executor */ private void addDefaultResforExec(ExecutorDetails exec) { Double topologyComponentCpuPcorePercent = Utils.getDouble(this.topologyConf.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT), null); Double topologyComponentResourcesOffheapMemoryMb = Utils.getDouble(this.topologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB), null); Double topologyComponentResourcesOnheapMemoryMb = Utils.getDouble(this.topologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB), null); assert topologyComponentCpuPcorePercent != null; assert topologyComponentResourcesOffheapMemoryMb != null; assert topologyComponentResourcesOnheapMemoryMb != null; Map<String, Double> defaultResourceList = new HashMap<>(); defaultResourceList.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, topologyComponentCpuPcorePercent); defaultResourceList.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, topologyComponentResourcesOffheapMemoryMb); defaultResourceList.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, topologyComponentResourcesOnheapMemoryMb); LOG.debug("Scheduling Executor: {} with memory requirement as onHeap: {} - offHeap: {} " + "and CPU requirement: {}", exec, this.topologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB), this.topologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB), this.topologyConf.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT)); addResourcesForExec(exec, defaultResourceList); }
public static Map<String, Double> parseResources(String input) { Map<String, Double> topology_resources = new HashMap<String, Double>(); JSONParser parser = new JSONParser(); LOG.debug("Input to parseResources {}", input); try { if (input != null) { Object obj = parser.parse(input); JSONObject jsonObject = (JSONObject) obj; if (jsonObject.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB)) { Double topoMemOnHeap = org.apache.storm.utils.Utils .getDouble(jsonObject.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB), null); topology_resources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, topoMemOnHeap); } if (jsonObject.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB)) { Double topoMemOffHeap = org.apache.storm.utils.Utils .getDouble(jsonObject.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB), null); topology_resources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, topoMemOffHeap); } if (jsonObject.containsKey(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT)) { Double topoCPU = org.apache.storm.utils.Utils.getDouble(jsonObject.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT), null); topology_resources.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, topoCPU); } LOG.debug("Topology Resources {}", topology_resources); } } catch (ParseException e) { LOG.error("Failed to parse component resources is:" + e.toString(), e); return null; } return topology_resources; }