@Override public byte[] get_data(String path, boolean watch) { return stateStorage.get_data(path, watch); }
@Override public NimbusInfo getLeader(Runnable callback) { if (null != callback) { this.leaderInfoCallback.set(callback); } return Utils.javaDeserialize(this.stateStorage.get_data(ClusterUtils.LEADERINFO_SUBTREE, callback != null), NimbusInfo.class); }
@Override public PrivateWorkerKey getPrivateWorkerKey(WorkerTokenServiceType type, String topologyId, long keyVersion) { String path = ClusterUtils.secretKeysPath(type, topologyId, keyVersion); byte[] data = stateStorage.get_data(path, false); if (data == null) { LOG.debug("Could not find entry at {} will sync to see if that fixes it", path); //We didn't find it, but there are races, so we want to check again after a sync stateStorage.sync_path(path); data = stateStorage.get_data(path, false); } return ClusterUtils.maybeDeserialize(data, PrivateWorkerKey.class); }
@Override public LogConfig topologyLogConfig(String stormId, Runnable cb) { if (cb != null) { logConfigCallback.put(stormId, cb); } String path = ClusterUtils.logConfigPath(stormId); return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, cb != null), LogConfig.class); }
@Override public StormBase stormBase(String stormId, Runnable callback) { if (callback != null) { stormBaseCallback.put(stormId, callback); } return ClusterUtils.maybeDeserialize(stateStorage.get_data(ClusterUtils.stormPath(stormId), callback != null), StormBase.class); }
@Override public Assignment remoteAssignmentInfo(String stormId, Runnable callback) { if (callback != null) { assignmentInfoCallback.put(stormId, callback); } byte[] serialized = stateStorage.get_data(ClusterUtils.assignmentPath(stormId), callback != null); return ClusterUtils.maybeDeserialize(serialized, Assignment.class); }
@Override public Credentials credentials(String stormId, Runnable callback) { if (callback != null) { credentialsCallback.put(stormId, callback); } String path = ClusterUtils.credentialsPath(stormId); return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, callback != null), Credentials.class); }
@Override public SupervisorInfo supervisorInfo(String supervisorId) { String path = ClusterUtils.supervisorPath(supervisorId); return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, false), SupervisorInfo.class); }
@Override public List<NimbusSummary> nimbuses() { List<NimbusSummary> nimbusSummaries = new ArrayList<>(); List<String> nimbusIds = stateStorage.get_children(ClusterUtils.NIMBUSES_SUBTREE, false); for (String nimbusId : nimbusIds) { byte[] serialized = stateStorage.get_data(ClusterUtils.nimbusPath(nimbusId), false); // check for null which can exist because of a race condition in which nimbus nodes in zk may have been // removed when connections are reconnected after getting children in the above line if (serialized != null) { NimbusSummary nimbusSummary = ClusterUtils.maybeDeserialize(serialized, NimbusSummary.class); nimbusSummaries.add(nimbusSummary); } } return nimbusSummaries; }
@Override public void syncRemoteAssignments(Map<String, byte[]> remote) { if (null != remote) { this.assignmentsBackend.syncRemoteAssignments(remote); } else { Map<String, byte[]> tmp = new HashMap<>(); List<String> stormIds = this.stateStorage.get_children(ClusterUtils.ASSIGNMENTS_SUBTREE, false); for (String stormId : stormIds) { byte[] assignment = this.stateStorage.get_data(ClusterUtils.assignmentPath(stormId), false); tmp.put(stormId, assignment); } this.assignmentsBackend.syncRemoteAssignments(tmp); } }
@Override public ErrorInfo lastError(String stormId, String componentId) { String path = ClusterUtils.lastErrorPath(stormId, componentId); if (stateStorage.node_exists(path, false)) { ErrorInfo errorInfo = ClusterUtils.maybeDeserialize(stateStorage.get_data(path, false), ErrorInfo.class); return errorInfo; } return null; }
@Override public List<ErrorInfo> errors(String stormId, String componentId) { List<ErrorInfo> errorInfos = new ArrayList<>(); String path = ClusterUtils.errorPath(stormId, componentId); if (stateStorage.node_exists(path, false)) { List<String> childrens = stateStorage.get_children(path, false); for (String child : childrens) { String childPath = path + ClusterUtils.ZK_SEPERATOR + child; ErrorInfo errorInfo = ClusterUtils.maybeDeserialize(stateStorage.get_data(childPath, false), ErrorInfo.class); if (errorInfo != null) { errorInfos.add(errorInfo); } } } Collections.sort(errorInfos, new Comparator<ErrorInfo>() { public int compare(ErrorInfo arg0, ErrorInfo arg1) { return Integer.compare(arg1.get_error_time_secs(), arg0.get_error_time_secs()); } }); return errorInfos; }
@Override public List<ProfileRequest> getTopologyProfileRequests(String stormId) { List<ProfileRequest> profileRequests = new ArrayList<>(); String path = ClusterUtils.profilerConfigPath(stormId); if (stateStorage.node_exists(path, false)) { List<String> strs = stateStorage.get_children(path, false); for (String str : strs) { String childPath = path + ClusterUtils.ZK_SEPERATOR + str; byte[] raw = stateStorage.get_data(childPath, false); ProfileRequest request = ClusterUtils.maybeDeserialize(raw, ProfileRequest.class); if (request != null) { profileRequests.add(request); } } } return profileRequests; }
/** * Check whether a topology is in throttle-on status or not: if the backpresure/storm-id dir is not empty, this topology has * throttle-on, otherwise throttle-off. But if the backpresure/storm-id dir is not empty and has not been updated for more than * timeoutMs, we treat it as throttle-off. This will prevent the spouts from getting stuck indefinitely if something wrong happens. * * @param stormId The topology Id * @param timeoutMs How long until the backpressure znode is invalid. * @param callback The callback function * @return True is backpresure/storm-id dir is not empty and at least one of the backpressure znodes has not timed out; false otherwise. */ @Override public boolean topologyBackpressure(String stormId, long timeoutMs, Runnable callback) { if (callback != null) { backPressureCallback.put(stormId, callback); } String path = ClusterUtils.backpressureStormRoot(stormId); long mostRecentTimestamp = 0; if (stateStorage.node_exists(path, false)) { List<String> children = stateStorage.get_children(path, callback != null); mostRecentTimestamp = children.stream() .map(childPath -> stateStorage.get_data(ClusterUtils.backpressurePath(stormId, childPath), false)) .filter(data -> data != null) .mapToLong(data -> ByteBuffer.wrap(data).getLong()) .max() .orElse(0); } boolean ret = ((System.currentTimeMillis() - mostRecentTimestamp) < timeoutMs); LOG.debug("topology backpressure is {}", ret ? "on" : "off"); return ret; }
@Override public void removeExpiredPrivateWorkerKeys(String topologyId) { for (WorkerTokenServiceType type : WorkerTokenServiceType.values()) { String basePath = ClusterUtils.secretKeysPath(type, topologyId); try { for (String version : stateStorage.get_children(basePath, false)) { String fullPath = basePath + ClusterUtils.ZK_SEPERATOR + version; try { PrivateWorkerKey key = ClusterUtils.maybeDeserialize(stateStorage.get_data(fullPath, false), PrivateWorkerKey.class); if (Time.currentTimeMillis() > key.get_expirationTimeMillis()) { LOG.info("Removing expired worker key {}", fullPath); stateStorage.delete_node(fullPath); } } catch (RuntimeException e) { //This should never happen because only the primary nimbus is active, but just in case // declare the race safe, even if we lose it. if (!Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) { throw e; } } } } catch (RuntimeException e) { //No node for basePath is OK, nothing to remove if (!Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) { throw e; } } } }
@Override public Assignment assignmentInfo(String stormId, Runnable callback) { if (callback != null) { assignmentInfoCallback.put(stormId, callback); } byte[] serialized = stateStorage.get_data(ClusterUtils.assignmentPath(stormId), callback != null); return ClusterUtils.maybeDeserialize(serialized, Assignment.class); }
@Override public StormBase stormBase(String stormId, Runnable callback) { if (callback != null) { stormBaseCallback.put(stormId, callback); } return ClusterUtils.maybeDeserialize(stateStorage.get_data(ClusterUtils.stormPath(stormId), callback != null), StormBase.class); }
@Override public LogConfig topologyLogConfig(String stormId, Runnable cb) { if (cb != null){ logConfigCallback.put(stormId, cb); } String path = ClusterUtils.logConfigPath(stormId); return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, cb != null), LogConfig.class); }
@Override public SupervisorInfo supervisorInfo(String supervisorId) { String path = ClusterUtils.supervisorPath(supervisorId); return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, false), SupervisorInfo.class); }
@Override public ErrorInfo lastError(String stormId, String componentId) { String path = ClusterUtils.lastErrorPath(stormId, componentId); if (stateStorage.node_exists(path, false)) { ErrorInfo errorInfo = ClusterUtils.maybeDeserialize(stateStorage.get_data(path, false), ErrorInfo.class); return errorInfo; } return null; }