public ExecutorInfo deepCopy() { return new ExecutorInfo(this); }
private static ExecutorInfo toExecInfo(List<Long> exec) { return new ExecutorInfo(exec.get(0).intValue(), exec.get(1).intValue()); }
private static SupervisorWorkerHeartbeat nextMockedWorkerbeat() { List<ExecutorInfo> executorInfos = new ArrayList<>(); executorInfos.add(new ExecutorInfo(1, 1)); executorInfos.add(new ExecutorInfo(2, 2)); executorInfos.add(new ExecutorInfo(3, 3)); executorInfos.add(new ExecutorInfo(4, 4)); SupervisorWorkerHeartbeat heartbeat = new SupervisorWorkerHeartbeat(); heartbeat.set_executors(executorInfos); // generate a random storm id heartbeat.set_storm_id("storm_name_example_" + rand.nextInt(MOCKED_STORM_NUM)); heartbeat.set_time_secs(1221212121); return heartbeat; }
/** * Performs a deep copy on <i>other</i>. */ public SupervisorWorkerHeartbeat(SupervisorWorkerHeartbeat other) { __isset_bitfield = other.__isset_bitfield; if (other.is_set_storm_id()) { this.storm_id = other.storm_id; } if (other.is_set_executors()) { java.util.List<ExecutorInfo> __this__executors = new java.util.ArrayList<ExecutorInfo>(other.executors.size()); for (ExecutorInfo other_element : other.executors) { __this__executors.add(new ExecutorInfo(other_element)); } this.executors = __this__executors; } this.time_secs = other.time_secs; }
/** * Performs a deep copy on <i>other</i>. */ public LSWorkerHeartbeat(LSWorkerHeartbeat other) { __isset_bitfield = other.__isset_bitfield; this.time_secs = other.time_secs; if (other.is_set_topology_id()) { this.topology_id = other.topology_id; } if (other.is_set_executors()) { java.util.List<ExecutorInfo> __this__executors = new java.util.ArrayList<ExecutorInfo>(other.executors.size()); for (ExecutorInfo other_element : other.executors) { __this__executors.add(new ExecutorInfo(other_element)); } this.executors = __this__executors; } this.port = other.port; }
public void doHeartBeat() throws IOException { LocalState state = ConfigUtils.workerState(workerState.conf, workerState.workerId); LSWorkerHeartbeat lsWorkerHeartbeat = new LSWorkerHeartbeat(Time.currentTimeSecs(), workerState.topologyId, workerState.localExecutors.stream() .map(executor -> new ExecutorInfo( executor.get(0).intValue(), executor.get(1).intValue())) .collect(Collectors.toList()), workerState.port); state.setWorkerHeartBeat(lsWorkerHeartbeat); state.cleanup(60); // this is just in case supervisor is down so that disk doesn't fill up. // it shouldn't take supervisor 120 seconds between listing dir and reading it heartbeatToMasterIfLocalbeatFail(lsWorkerHeartbeat); }
/** * Used for local test. */ public static SupervisorWorkerHeartbeat thriftifyRpcWorkerHb(String stormId, List<Long> executorId) { SupervisorWorkerHeartbeat supervisorWorkerHeartbeat = new SupervisorWorkerHeartbeat(); supervisorWorkerHeartbeat.set_storm_id(stormId); supervisorWorkerHeartbeat .set_executors(Collections.singletonList(new ExecutorInfo(executorId.get(0).intValue(), executorId.get(1).intValue()))); supervisorWorkerHeartbeat.set_time_secs(Time.currentTimeSecs()); return supervisorWorkerHeartbeat; }
/** * Performs a deep copy on <i>other</i>. */ public ClusterWorkerHeartbeat(ClusterWorkerHeartbeat other) { __isset_bitfield = other.__isset_bitfield; if (other.is_set_storm_id()) { this.storm_id = other.storm_id; } if (other.is_set_executor_stats()) { java.util.Map<ExecutorInfo,ExecutorStats> __this__executor_stats = new java.util.HashMap<ExecutorInfo,ExecutorStats>(other.executor_stats.size()); for (java.util.Map.Entry<ExecutorInfo, ExecutorStats> other_element : other.executor_stats.entrySet()) { ExecutorInfo other_element_key = other_element.getKey(); ExecutorStats other_element_value = other_element.getValue(); ExecutorInfo __this__executor_stats_copy_key = new ExecutorInfo(other_element_key); ExecutorStats __this__executor_stats_copy_value = new ExecutorStats(other_element_value); __this__executor_stats.put(__this__executor_stats_copy_key, __this__executor_stats_copy_value); } this.executor_stats = __this__executor_stats; } this.time_secs = other.time_secs; this.uptime_secs = other.uptime_secs; }
/** * need to take executor->node+port in explicitly so that we don't run into a situation where a long dead worker with a skewed clock * overrides all the timestamps. By only checking heartbeats with an assigned node+port, and only reading executors from that heartbeat * that are actually assigned, we avoid situations like that. * * @param stormId topology id * @param executorNodePort executor id -> node + port * @return mapping of executorInfo -> executor beat */ @Override public Map<ExecutorInfo, ExecutorBeat> executorBeats(String stormId, Map<List<Long>, NodeInfo> executorNodePort) { Map<ExecutorInfo, ExecutorBeat> executorWhbs = new HashMap<>(); Map<NodeInfo, List<List<Long>>> nodePortExecutors = Utils.reverseMap(executorNodePort); for (Map.Entry<NodeInfo, List<List<Long>>> entry : nodePortExecutors.entrySet()) { String node = entry.getKey().get_node(); Long port = entry.getKey().get_port_iterator().next(); ClusterWorkerHeartbeat whb = getWorkerHeartbeat(stormId, node, port); List<ExecutorInfo> executorInfoList = new ArrayList<>(); for (List<Long> list : entry.getValue()) { executorInfoList.add(new ExecutorInfo(list.get(0).intValue(), list.get(list.size() - 1).intValue())); } if (whb != null) { executorWhbs.putAll(ClusterUtils.convertExecutorBeats(executorInfoList, whb)); } } return executorWhbs; }
public static ClusterWorkerHeartbeat thriftifyZkWorkerHb(Map<String, Object> heartbeat) { ClusterWorkerHeartbeat ret = new ClusterWorkerHeartbeat(); ret.set_uptime_secs(getByKeyOr0(heartbeat, UPTIME).intValue()); ret.set_storm_id((String) heartbeat.get("storm-id")); ret.set_time_secs(getByKeyOr0(heartbeat, TIME_SECS).intValue()); Map<ExecutorInfo, ExecutorStats> convertedStats = new HashMap<>(); Map<List<Integer>, ExecutorStats> executorStats = getMapByKey(heartbeat, EXECUTOR_STATS); if (executorStats != null) { for (Map.Entry<List<Integer>, ExecutorStats> entry : executorStats.entrySet()) { List<Integer> executor = entry.getKey(); ExecutorStats stats = entry.getValue(); if (null != stats) { convertedStats.put(new ExecutorInfo(executor.get(0), executor.get(1)), stats); } } } ret.set_executor_stats(convertedStats); return ret; }
/** * Performs a deep copy on <i>other</i>. */ public LocalAssignment(LocalAssignment other) { __isset_bitfield = other.__isset_bitfield; if (other.is_set_topology_id()) { this.topology_id = other.topology_id; } if (other.is_set_executors()) { java.util.List<ExecutorInfo> __this__executors = new java.util.ArrayList<ExecutorInfo>(other.executors.size()); for (ExecutorInfo other_element : other.executors) { __this__executors.add(new ExecutorInfo(other_element)); } this.executors = __this__executors; } if (other.is_set_resources()) { this.resources = new WorkerResources(other.resources); } this.total_node_shared = other.total_node_shared; if (other.is_set_owner()) { this.owner = other.owner; } }
static List<ExecutorInfo> mkExecutorInfoList(int... executors) { ArrayList<ExecutorInfo> ret = new ArrayList<>(executors.length); for (int exec : executors) { ExecutorInfo execInfo = new ExecutorInfo(); execInfo.set_task_start(exec); execInfo.set_task_end(exec); ret.add(execInfo); } return ret; }
private static ExecutorAggregateStats thriftifyExecAggStats(String compId, String compType, Map m) { ExecutorSummary executorSummary = new ExecutorSummary(); List executor = (List) m.get(EXECUTOR_ID); executorSummary.set_executor_info(new ExecutorInfo(((Number) executor.get(0)).intValue(), ((Number) executor.get(1)).intValue())); executorSummary.set_component_id(compId); executorSummary.set_host((String) m.get(HOST)); executorSummary.set_port(getByKeyOr0(m, PORT).intValue()); int uptime = getByKeyOr0(m, ClientStatsUtil.UPTIME).intValue(); executorSummary.set_uptime_secs(uptime); ExecutorAggregateStats stats = new ExecutorAggregateStats(); stats.set_exec_summary(executorSummary); if (compType.equals(ClientStatsUtil.SPOUT)) { stats.set_stats(thriftifySpoutAggStats(m)); } else { stats.set_stats(thriftifyBoltAggStats(m)); } return stats; }
@Override public void read(org.apache.storm.thrift.protocol.TProtocol prot, SupervisorWorkerHeartbeat struct) throws org.apache.storm.thrift.TException { org.apache.storm.thrift.protocol.TTupleProtocol iprot = (org.apache.storm.thrift.protocol.TTupleProtocol) prot; struct.storm_id = iprot.readString(); struct.set_storm_id_isSet(true); { org.apache.storm.thrift.protocol.TList _list875 = new org.apache.storm.thrift.protocol.TList(org.apache.storm.thrift.protocol.TType.STRUCT, iprot.readI32()); struct.executors = new java.util.ArrayList<ExecutorInfo>(_list875.size); ExecutorInfo _elem876; for (int _i877 = 0; _i877 < _list875.size; ++_i877) { _elem876 = new ExecutorInfo(); _elem876.read(iprot); struct.executors.add(_elem876); } } struct.set_executors_isSet(true); struct.time_secs = iprot.readI32(); struct.set_time_secs_isSet(true); } }
/** * Performs a deep copy on <i>other</i>. */ public ExecutorSummary(ExecutorSummary other) { __isset_bitfield = other.__isset_bitfield; if (other.is_set_executor_info()) { this.executor_info = new ExecutorInfo(other.executor_info); } if (other.is_set_component_id()) { this.component_id = other.component_id; } if (other.is_set_host()) { this.host = other.host; } this.port = other.port; this.uptime_secs = other.uptime_secs; if (other.is_set_stats()) { this.stats = new ExecutorStats(other.stats); } }
@Override public void read(org.apache.storm.thrift.protocol.TProtocol prot, LSWorkerHeartbeat struct) throws org.apache.storm.thrift.TException { org.apache.storm.thrift.protocol.TTupleProtocol iprot = (org.apache.storm.thrift.protocol.TTupleProtocol) prot; struct.time_secs = iprot.readI32(); struct.set_time_secs_isSet(true); struct.topology_id = iprot.readString(); struct.set_topology_id_isSet(true); { org.apache.storm.thrift.protocol.TList _list825 = new org.apache.storm.thrift.protocol.TList(org.apache.storm.thrift.protocol.TType.STRUCT, iprot.readI32()); struct.executors = new java.util.ArrayList<ExecutorInfo>(_list825.size); ExecutorInfo _elem826; for (int _i827 = 0; _i827 < _list825.size; ++_i827) { _elem826 = new ExecutorInfo(); _elem826.read(iprot); struct.executors.add(_elem826); } } struct.set_executors_isSet(true); struct.port = iprot.readI32(); struct.set_port_isSet(true); } }
for (int _i799 = 0; _i799 < _list797.size; ++_i799) _elem798 = new ExecutorInfo(); _elem798.read(iprot); struct.executors.add(_elem798);
@Override public void read(org.apache.storm.thrift.protocol.TProtocol prot, ClusterWorkerHeartbeat struct) throws org.apache.storm.thrift.TException { org.apache.storm.thrift.protocol.TTupleProtocol iprot = (org.apache.storm.thrift.protocol.TTupleProtocol) prot; struct.storm_id = iprot.readString(); struct.set_storm_id_isSet(true); { org.apache.storm.thrift.protocol.TMap _map778 = new org.apache.storm.thrift.protocol.TMap(org.apache.storm.thrift.protocol.TType.STRUCT, org.apache.storm.thrift.protocol.TType.STRUCT, iprot.readI32()); struct.executor_stats = new java.util.HashMap<ExecutorInfo,ExecutorStats>(2*_map778.size); ExecutorInfo _key779; ExecutorStats _val780; for (int _i781 = 0; _i781 < _map778.size; ++_i781) { _key779 = new ExecutorInfo(); _key779.read(iprot); _val780 = new ExecutorStats(); _val780.read(iprot); struct.executor_stats.put(_key779, _val780); } } struct.set_executor_stats_isSet(true); struct.time_secs = iprot.readI32(); struct.set_time_secs_isSet(true); struct.uptime_secs = iprot.readI32(); struct.set_uptime_secs_isSet(true); } }
@Override public void read(org.apache.storm.thrift.protocol.TProtocol prot, ExecutorSummary struct) throws org.apache.storm.thrift.TException { org.apache.storm.thrift.protocol.TTupleProtocol iprot = (org.apache.storm.thrift.protocol.TTupleProtocol) prot; struct.executor_info = new ExecutorInfo(); struct.executor_info.read(iprot); struct.set_executor_info_isSet(true); struct.component_id = iprot.readString(); struct.set_component_id_isSet(true); struct.host = iprot.readString(); struct.set_host_isSet(true); struct.port = iprot.readI32(); struct.set_port_isSet(true); struct.uptime_secs = iprot.readI32(); struct.set_uptime_secs_isSet(true); java.util.BitSet incoming = iprot.readBitSet(1); if (incoming.get(0)) { struct.stats = new ExecutorStats(); struct.stats.read(iprot); struct.set_stats_isSet(true); } } }
for (int _i872 = 0; _i872 < _list870.size; ++_i872) _elem871 = new ExecutorInfo(); _elem871.read(iprot); struct.executors.add(_elem871);