public static Map<String, Integer> getComponentParallelism(Map<String, Object> topoConf, StormTopology topology) throws InvalidTopologyException { Map<String, Integer> ret = new HashMap<>(); Map<String, Object> components = StormCommon.allComponents(topology); for (Map.Entry<String, Object> entry : components.entrySet()) { ret.put(entry.getKey(), getComponentParallelism(topoConf, entry.getValue())); } return ret; }
public static double getEstimatedTotalHeapMemoryRequiredByTopo(Map<String, Object> topoConf, StormTopology topology) throws InvalidTopologyException { Map<String, Integer> componentParallelism = getComponentParallelism(topoConf, topology); double totalMemoryRequired = 0.0; for (Map.Entry<String, NormalizedResourceRequest> entry : ResourceUtils.getBoltsResources(topology, topoConf).entrySet()) { int parallelism = componentParallelism.getOrDefault(entry.getKey(), 1); double memoryRequirement = entry.getValue().getOnHeapMemoryMb(); totalMemoryRequired += memoryRequirement * parallelism; } for (Map.Entry<String, NormalizedResourceRequest> entry : ResourceUtils.getSpoutsResources(topology, topoConf).entrySet()) { int parallelism = componentParallelism.getOrDefault(entry.getKey(), 1); double memoryRequirement = entry.getValue().getOnHeapMemoryMb(); totalMemoryRequired += memoryRequirement * parallelism; } return totalMemoryRequired; }
private static StormTopology normalizeTopology(Map<String, Object> topoConf, StormTopology topology) throws InvalidTopologyException { StormTopology ret = topology.deepCopy(); for (Object comp : StormCommon.allComponents(ret).values()) { Map<String, Object> mergedConf = StormCommon.componentConf(comp); mergedConf.put(Config.TOPOLOGY_TASKS, ServerUtils.getComponentParallelism(topoConf, comp)); String jsonConf = JSONValue.toJSONString(mergedConf); StormCommon.getComponentCommon(comp).set_json_conf(jsonConf); } return ret; }