/** * map from worker id to heartbeat * * @param conf * @return * */ public static Map<String, LSWorkerHeartbeat> readWorkerHeartbeats(Map<String, Object> conf) { return _instance.readWorkerHeartbeatsImpl(conf); }
public static boolean isWorkerHbTimedOut(int now, LSWorkerHeartbeat whb, Map<String, Object> conf) { return _instance.isWorkerHbTimedOutImpl(now, whb, conf); }
public Map<String, LSWorkerHeartbeat> readWorkerHeartbeatsImpl(Map<String, Object> conf) { Map<String, LSWorkerHeartbeat> workerHeartbeats = new HashMap<>(); Collection<String> workerIds = SupervisorUtils.supervisorWorkerIds(conf); for (String workerId : workerIds) { LSWorkerHeartbeat whb = readWorkerHeartbeat(conf, workerId); // ATTENTION: whb can be null workerHeartbeats.put(workerId, whb); } return workerHeartbeats; }
/** * Retrieve the set of alive worker IDs. * * @param nowSecs current time in seconds */ public Set<String> getAliveIds(int nowSecs) { return SupervisorUtils.readWorkerHeartbeats(stormConf).entrySet().stream() .filter(entry -> Objects.nonNull(entry.getValue()) && !SupervisorUtils.isWorkerHbTimedOut(nowSecs, entry.getValue(), stormConf)) .map(Map.Entry::getKey) .collect(toCollection(TreeSet::new)); }
/** * Returns a list of LocalResources based on the blobstore-map passed in * * @param blobstoreMap * @return */ public static List<LocalResource> blobstoreMapToLocalresources(Map<String, Map<String, Object>> blobstoreMap) { List<LocalResource> localResourceList = new ArrayList<>(); if (blobstoreMap != null) { for (Map.Entry<String, Map<String, Object>> map : blobstoreMap.entrySet()) { Map<String, Object> blobConf = map.getValue(); LocalResource localResource = new LocalResource(map.getKey(), shouldUncompressBlob(blobConf), blobNeedsWorkerRestart(blobConf)); localResourceList.add(localResource); } } return localResourceList; }
/** * Build up workerid-workerlogdir map for the old workers' dirs. */ @Test public void testIdentifyWorkerLogDirs() throws Exception { File port1Dir = new MockDirectoryBuilder().setDirName("/workers-artifacts/topo1/port1").build(); File mockMetaFile = new MockFileBuilder().setFileName("worker.yaml").build(); String expId = "id12345"; SortedSet<File> expected = new TreeSet<>(); expected.add(port1Dir); try { SupervisorUtils mockedSupervisorUtils = mock(SupervisorUtils.class); SupervisorUtils.setInstance(mockedSupervisorUtils); Map<String, Object> stormConf = Utils.readStormConfig(); WorkerLogs workerLogs = new WorkerLogs(stormConf, port1Dir, new StormMetricsRegistry()) { @Override public Optional<File> getMetadataFileForWorkerLogDir(File logDir) throws IOException { return Optional.of(mockMetaFile); } @Override public String getWorkerIdFromMetadataFile(String metaFile) { return expId; } }; when(mockedSupervisorUtils.readWorkerHeartbeatsImpl(anyMapOf(String.class, Object.class))).thenReturn(null); assertEquals(expected, workerLogs.getLogDirs(Collections.singleton(port1Dir), (wid) -> true)); } finally { SupervisorUtils.resetInstance(); } }
private List<LocalResource> getLocalResources(PortAndAssignment pna) throws IOException { String topologyId = pna.getToplogyId(); Map<String, Object> topoConf = ConfigUtils.readSupervisorStormConf(conf, topologyId); @SuppressWarnings("unchecked") Map<String, Map<String, Object>> blobstoreMap = (Map<String, Map<String, Object>>) topoConf.get(Config.TOPOLOGY_BLOBSTORE_MAP); List<LocalResource> ret = new ArrayList<>(); if (blobstoreMap != null) { List<LocalResource> tmp = SupervisorUtils.blobstoreMapToLocalresources(blobstoreMap); if (tmp != null) { ret.addAll(tmp); } } StormTopology stormCode = ConfigUtils.readSupervisorTopology(conf, topologyId, fsOps); List<String> dependencies = new ArrayList<>(); if (stormCode.is_set_dependency_jars()) { dependencies.addAll(stormCode.get_dependency_jars()); } if (stormCode.is_set_dependency_artifacts()) { dependencies.addAll(stormCode.get_dependency_artifacts()); } for (String dependency : dependencies) { ret.add(new LocalResource(dependency, false, true)); } return ret; }
@Override protected boolean runProfilingCommand(List<String> command, Map<String, String> env, String logPrefix, File targetDir) throws IOException, InterruptedException { String user = this.getWorkerUser(); String td = targetDir.getAbsolutePath(); LOG.info("Running as user: {} command: {}", user, command); String containerFile = Utils.containerFilePath(td); if (Utils.checkFileExists(containerFile)) { SupervisorUtils.rmrAsUser(_conf, containerFile, containerFile); } String scriptFile = Utils.scriptFilePath(td); if (Utils.checkFileExists(scriptFile)) { SupervisorUtils.rmrAsUser(_conf, scriptFile, scriptFile); } String script = Utils.writeScript(td, command, env); List<String> args = Arrays.asList("profiler", td, script); int ret = SupervisorUtils.processLauncherAndWait(_conf, user, args, env, logPrefix); return ret == 0; }
/** * get worker heartbeat by workerId. * * @param conf * @param workerId * @return */ private static LSWorkerHeartbeat readWorkerHeartbeat(Map<String, Object> conf, String workerId) { return _instance.readWorkerHeartbeatImpl(conf, workerId); }
@Override public void close() throws Exception { SupervisorUtils.resetInstance(); } }
public MockedSupervisorUtils(SupervisorUtils inst) { SupervisorUtils.setInstance(inst); }
/** * Returns a list of LocalResources based on the blobstore-map passed in * * @param blobstoreMap * @return */ public static List<LocalResource> blobstoreMapToLocalresources(Map<String, Map<String, Object>> blobstoreMap) { List<LocalResource> localResourceList = new ArrayList<>(); if (blobstoreMap != null) { for (Map.Entry<String, Map<String, Object>> map : blobstoreMap.entrySet()) { LocalResource localResource = new LocalResource(map.getKey(), shouldUncompressBlob(map.getValue())); localResourceList.add(localResource); } } return localResourceList; }
private SupervisorWorkerHeartbeats getAndResetWorkerHeartbeats() { Map<String, LSWorkerHeartbeat> localHeartbeats; try { localHeartbeats = SupervisorUtils.readWorkerHeartbeats(this.conf); return getSupervisorWorkerHeartbeatsFromLocal(localHeartbeats); } catch (Exception e) { LOG.error("Read local worker heartbeats error, skipping heartbeats for this round, msg:{}", e.getMessage()); return null; } }
@Override protected boolean runProfilingCommand(List<String> command, Map<String, String> env, String logPrefix, File targetDir) throws IOException, InterruptedException { String user = this.getWorkerUser(); String td = targetDir.getAbsolutePath(); LOG.info("Running as user: {} command: {}", user, command); String containerFile = ServerUtils.containerFilePath(td); if (Utils.checkFileExists(containerFile)) { SupervisorUtils.rmrAsUser(_conf, containerFile, containerFile); } String scriptFile = ServerUtils.scriptFilePath(td); if (Utils.checkFileExists(scriptFile)) { SupervisorUtils.rmrAsUser(_conf, scriptFile, scriptFile); } String script = ServerUtils.writeScript(td, command, env); List<String> args = Arrays.asList("profiler", td, script); int ret = ClientSupervisorUtils.processLauncherAndWait(_conf, user, args, env, logPrefix); return ret == 0; }
SupervisorUtils.setInstance(mockedSupervisorUtils); when(mockedSupervisorUtils.readWorkerHeartbeatsImpl(anyMapOf(String.class, Object.class))).thenReturn(idToHb); assertEquals(Sets.newSet(expectedDir2, expectedDir3), logCleaner.getDeadWorkerDirs(nowSecs, logDirs)); } finally { SupervisorUtils.resetInstance();
/** * For each of the downloaded topologies, adds references to the blobs that the topologies are using. This is used to reconstruct the cache on restart. * * @param localizer * @param stormId * @param conf */ static void addBlobReferences(Localizer localizer, String stormId, Map<String, Object> conf, String user) throws IOException { Map<String, Object> stormConf = ConfigUtils.readSupervisorStormConf(conf, stormId); Map<String, Map<String, Object>> blobstoreMap = (Map<String, Map<String, Object>>) stormConf.get(Config.TOPOLOGY_BLOBSTORE_MAP); String topoName = (String) stormConf.get(Config.TOPOLOGY_NAME); List<LocalResource> localresources = SupervisorUtils.blobstoreMapToLocalresources(blobstoreMap); if (blobstoreMap != null) { localizer.addReferences(localresources, user, topoName); } }
/** * get worker heartbeat by workerId * * @param conf * @param workerId * @return * @throws IOException */ private static LSWorkerHeartbeat readWorkerHeartbeat(Map<String, Object> conf, String workerId) { return _instance.readWorkerHeartbeatImpl(conf, workerId); }
@Override public void close() throws Exception { SupervisorUtils.resetInstance(); } }
public MockedSupervisorUtils(SupervisorUtils inst) { SupervisorUtils.setInstance(inst); }
Map<String, Object> blobInfo = entry.getValue(); try { _localizer.removeBlobReference(key, user, topoName, SupervisorUtils.shouldUncompressBlob(blobInfo)); } catch (Exception e) { throw new IOException(e);