_type.assertFull(); String stormRoot = ConfigUtils.supervisorStormDistRoot(_conf, _topologyId); String workerRoot = ConfigUtils.workerRoot(_conf, _workerId);
_localState = localState; if (type.isRecovery() && !type.isOnlyKillable()) { synchronized (localState) { String wid = null;
_type.assertFull(); Map<String, Object> data = new HashMap<>(); data.put(Config.TOPOLOGY_SUBMITTER_USER, user);
_type.assertFull(); String stormRoot = ConfigUtils.supervisorStormDistRoot(_conf, _topologyId); String workerRoot = ConfigUtils.workerRoot(_conf, _workerId);
_assignment = assignment; if (_type.isOnlyKillable()) { assert(_assignment == null); assert(_port <= 0);
_type.assertFull(); if (!_ops.doRequiredTopoFilesExist(_conf, _topologyId)) { LOG.info("Missing topology storm code, so can't launch worker with assignment {} for this supervisor {} on port {} with id {}", _assignment,
@Override public void launch() throws IOException { _type.assertFull(); LOG.info("Launching worker with assignment {} for this supervisor {} on port {} with id {}", _assignment, _supervisorId, _port, _workerId);
@Override public boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException { _type.assertFull(); String targetDir = ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port); @SuppressWarnings("unchecked") Map<String, String> env = (Map<String, String>) _topoConf.get(Config.TOPOLOGY_ENVIRONMENT); if (env == null) { env = new HashMap<String, String>(); } String str = ConfigUtils.workerArtifactsPidPath(_conf, _topologyId, _port); String workerPid = _ops.slurpString(new File(str)).trim(); ProfileAction profileAction = request.get_action(); String logPrefix = "ProfilerAction process " + _topologyId + ":" + _port + " PROFILER_ACTION: " + profileAction + " "; List<String> command = mkProfileCommand(profileAction, stop, workerPid, targetDir); File targetFile = new File(targetDir); if (command.size() > 0) { return runProfilingCommand(command, env, logPrefix, targetFile); } LOG.warn("PROFILING REQUEST NOT SUPPORTED {} IGNORED...", request); return true; }
/** * Run the given command for profiling * * @param command * the command to run * @param env * the environment to run the command * @param logPrefix * the prefix to include in the logs * @param targetDir * the working directory to run the command in * @return true if it ran successfully, else false * @throws IOException * on any error * @throws InterruptedException * if interrupted wile waiting for the process to exit. */ protected boolean runProfilingCommand(List<String> command, Map<String, String> env, String logPrefix, File targetDir) throws IOException, InterruptedException { _type.assertFull(); Process p = SupervisorUtils.launchProcess(command, env, logPrefix, null, targetDir); int ret = p.waitFor(); return ret == 0; }
/** * Create a new worker ID for this process and store in in this object and * in the local state. Never call this if a worker is currently up and running. * We will lose track of the process. */ protected void createNewWorkerId() { _type.assertFull(); assert(_workerId == null); synchronized (_localState) { _workerId = Utils.uuid(); Map<String, Integer> workerToPort = _localState.getApprovedWorkers(); if (workerToPort == null) { workerToPort = new HashMap<>(1); } removeWorkersOn(workerToPort, _port); workerToPort.put(_workerId, _port); _localState.setApprovedWorkers(workerToPort); LOG.info("Created Worker ID {}", _workerId); } }
/** * Create symlink from the containers directory/artifacts to the artifacts directory * @throws IOException on any error */ protected void createArtifactsLink() throws IOException { _type.assertFull(); if (!_symlinksDisabled) { File workerDir = new File(ConfigUtils.workerRoot(_conf, _workerId)); File topoDir = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port)); if (_ops.fileExists(workerDir)) { LOG.debug("Creating symlinks for worker-id: {} topology-id: {} to its port artifacts directory", _workerId, _topologyId); _ops.createSymlink(new File(workerDir, "artifacts"), topoDir); } } }
@Override public void relaunch() throws IOException { _type.assertFull(); //We are launching it now... _type = ContainerType.LAUNCH; createNewWorkerId(); setup(); launch(); }
protected void saveWorkerUser(String user) throws IOException { _type.assertFull(); LOG.info("SET worker-user {} {}", _workerId, user); _ops.dump(new File(ConfigUtils.workerUserFile(_conf, _workerId)), user); }
_assignment = assignment; if (_type.isOnlyKillable()) { assert (_assignment == null); assert (_port <= 0);
_type.assertFull(); Map<String, Object> data = new HashMap<>(); data.put(Config.TOPOLOGY_SUBMITTER_USER, user);
@Override public void launch() throws IOException { _type.assertFull(); LOG.info("Launching worker with assignment {} for this supervisor {} on port {} with id {}", _assignment, _supervisorId, _port, _workerId);
/** * Setup the container to run. By default this creates the needed directories/links in the local file system PREREQUISITE: All needed * blobs and topology, jars/configs have been downloaded and placed in the appropriate locations * * @throws IOException on any error */ protected void setup() throws IOException { _type.assertFull(); if (!_ops.doRequiredTopoFilesExist(_conf, _topologyId)) { LOG.info("Missing topology storm code, so can't launch worker with assignment {} for this supervisor {} on port {} with id {}", _assignment, _supervisorId, _port, _workerId); throw new IllegalStateException("Not all needed files are here!!!!"); } LOG.info("Setting up {}:{}", _supervisorId, _workerId); _ops.forceMkdir(new File(ConfigUtils.workerPidsRoot(_conf, _workerId))); _ops.forceMkdir(new File(ConfigUtils.workerTmpRoot(_conf, _workerId))); _ops.forceMkdir(new File(ConfigUtils.workerHeartbeatsRoot(_conf, _workerId))); File workerArtifacts = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port)); if (!_ops.fileExists(workerArtifacts)) { _ops.forceMkdir(workerArtifacts); _ops.setupWorkerArtifactsDir(_assignment.get_owner(), workerArtifacts); } String user = getWorkerUser(); writeLogMetadata(user); saveWorkerUser(user); createArtifactsLink(); createBlobstoreLinks(); }
@Override public boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException { _type.assertFull(); String targetDir = ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port); @SuppressWarnings("unchecked") Map<String, String> env = (Map<String, String>) _topoConf.get(Config.TOPOLOGY_ENVIRONMENT); if (env == null) { env = new HashMap<>(); } String str = ConfigUtils.workerArtifactsPidPath(_conf, _topologyId, _port); String workerPid = _ops.slurpString(new File(str)).trim(); ProfileAction profileAction = request.get_action(); String logPrefix = "ProfilerAction process " + _topologyId + ":" + _port + " PROFILER_ACTION: " + profileAction + " "; List<String> command = mkProfileCommand(profileAction, stop, workerPid, targetDir); File targetFile = new File(targetDir); if (command.size() > 0) { return runProfilingCommand(command, env, logPrefix, targetFile); } LOG.warn("PROFILING REQUEST NOT SUPPORTED {} IGNORED...", request); return true; }
/** * Create a new worker ID for this process and store in in this object and in the local state. Never call this if a worker is currently * up and running. We will lose track of the process. */ protected void createNewWorkerId() { _type.assertFull(); assert (_workerId == null); synchronized (_localState) { _workerId = Utils.uuid(); Map<String, Integer> workerToPort = _localState.getApprovedWorkers(); if (workerToPort == null) { workerToPort = new HashMap<>(1); } removeWorkersOn(workerToPort, _port); workerToPort.put(_workerId, _port); _localState.setApprovedWorkers(workerToPort); LOG.info("Created Worker ID {}", _workerId); } }
/** * Create symlink from the containers directory/artifacts to the artifacts directory. * * @throws IOException on any error */ protected void createArtifactsLink() throws IOException { _type.assertFull(); if (!_symlinksDisabled) { File workerDir = new File(ConfigUtils.workerRoot(_conf, _workerId)); File topoDir = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port)); if (_ops.fileExists(workerDir)) { LOG.debug("Creating symlinks for worker-id: {} topology-id: {} to its port artifacts directory", _workerId, _topologyId); _ops.createSymlink(new File(workerDir, "artifacts"), topoDir); } } }