public static String workerArtifactsRoot(Map<String, Object> conf, String id, Integer port) { return (workerArtifactsRoot(conf, id) + FILE_SEPARATOR + port); }
public static String workerArtifactsPidPath(Map<String, Object> conf, String id, Integer port) { return (workerArtifactsRoot(conf, id, port) + FILE_SEPARATOR + "worker.pid"); }
public static String workerArtifactsRoot(Map<String, Object> conf, String id) { return (workerArtifactsRoot(conf) + FILE_SEPARATOR + id); }
public static File getLogMetaDataFile(Map<String, Object> conf, String id, Integer port) { String fname = ConfigUtils.workerArtifactsRoot(conf, id, port) + FILE_SEPARATOR + "worker.yaml"; return new File(fname); }
/** * Get a set of java properties that are common to both the log writer and the worker processes. These are mostly system properties that * are used by logging. * * @return a list of command line options */ private List<String> getCommonParams() { final String workersArtifacts = ConfigUtils.workerArtifactsRoot(_conf); String stormLogDir = ConfigUtils.getLogDir(); List<String> commonParams = new ArrayList<>(); commonParams.add("-Dlogging.sensitivity=" + OR((String) _topoConf.get(Config.TOPOLOGY_LOGGING_SENSITIVITY), "S3")); commonParams.add("-Dlogfile.name=worker.log"); commonParams.add("-Dstorm.home=" + OR(_stormHome, "")); commonParams.add("-Dworkers.artifacts=" + workersArtifacts); commonParams.add("-Dstorm.id=" + _topologyId); commonParams.add("-Dworker.id=" + _workerId); commonParams.add("-Dworker.port=" + _port); commonParams.add("-Dstorm.log.dir=" + stormLogDir); commonParams.add("-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector"); commonParams.add("-Dstorm.local.dir=" + _conf.get(Config.STORM_LOCAL_DIR)); if (memoryLimitMB > 0) { commonParams.add("-Dworker.memory_limit_mb=" + memoryLimitMB); } return commonParams; }
@Override public void prepare(Map<String, Object> conf, Map<String, Object> arguments, TopologyContext context) { String stormId = context.getStormId(); int port = context.getThisWorkerPort(); /* * Include the topology name & worker port in the file name so that * multiple event loggers can log independently. */ String workersArtifactRoot = ConfigUtils.workerArtifactsRoot(conf, stormId, port); Path path = Paths.get(workersArtifactRoot, "events.log"); File dir = path.toFile().getParentFile(); if (!dir.exists()) { dir.mkdirs(); } initLogWriter(path); setUpFlushTask(); }
/** * Create symlink from the containers directory/artifacts to the artifacts directory. * * @throws IOException on any error */ protected void createArtifactsLink() throws IOException { _type.assertFull(); if (!_symlinksDisabled) { File workerDir = new File(ConfigUtils.workerRoot(_conf, _workerId)); File topoDir = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port)); if (_ops.fileExists(workerDir)) { LOG.debug("Creating symlinks for worker-id: {} topology-id: {} to its port artifacts directory", _workerId, _topologyId); _ops.createSymlink(new File(workerDir, "artifacts"), topoDir); } } }
@Override public boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException { _type.assertFull(); String targetDir = ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port); @SuppressWarnings("unchecked") Map<String, String> env = (Map<String, String>) _topoConf.get(Config.TOPOLOGY_ENVIRONMENT); if (env == null) { env = new HashMap<>(); } String str = ConfigUtils.workerArtifactsPidPath(_conf, _topologyId, _port); String workerPid = _ops.slurpString(new File(str)).trim(); ProfileAction profileAction = request.get_action(); String logPrefix = "ProfilerAction process " + _topologyId + ":" + _port + " PROFILER_ACTION: " + profileAction + " "; List<String> command = mkProfileCommand(profileAction, stop, workerPid, targetDir); File targetFile = new File(targetDir); if (command.size() > 0) { return runProfilingCommand(command, env, logPrefix, targetFile); } LOG.warn("PROFILING REQUEST NOT SUPPORTED {} IGNORED...", request); return true; }
/** * @return the user that some operations should be done as. * * @throws IOException on any error */ protected String getWorkerUser() throws IOException { LOG.info("GET worker-user for {}", _workerId); File file = new File(ConfigUtils.workerUserFile(_conf, _workerId)); if (_ops.fileExists(file)) { return _ops.slurpString(file).trim(); } else if (_assignment != null && _assignment.is_set_owner()) { return _assignment.get_owner(); } if (ConfigUtils.isLocalMode(_conf)) { return System.getProperty("user.name"); } else { File f = new File(ConfigUtils.workerArtifactsRoot(_conf)); if (f.exists()) { return Files.getOwner(f.toPath()).getName(); } throw new IllegalStateException("Could not recover the user for " + _workerId); } }
/** * Constructor. */ public LogviewerApplication() { String logRoot = ConfigUtils.workerArtifactsRoot(stormConf); String daemonLogRoot = logRootDir(ObjectReader.getString(stormConf.get(LOGVIEWER_APPENDER_NAME))); ResourceAuthorizer resourceAuthorizer = new ResourceAuthorizer(stormConf); WorkerLogs workerLogs = new WorkerLogs(stormConf, new File(logRoot), metricsRegistry); LogviewerLogPageHandler logviewer = new LogviewerLogPageHandler(logRoot, daemonLogRoot, workerLogs, resourceAuthorizer, metricsRegistry); LogviewerProfileHandler profileHandler = new LogviewerProfileHandler(logRoot, resourceAuthorizer, metricsRegistry); LogviewerLogDownloadHandler logDownloadHandler = new LogviewerLogDownloadHandler(logRoot, daemonLogRoot, workerLogs, resourceAuthorizer, metricsRegistry); LogviewerLogSearchHandler logSearchHandler = new LogviewerLogSearchHandler(stormConf, logRoot, daemonLogRoot, resourceAuthorizer, metricsRegistry); IHttpCredentialsPlugin httpCredsHandler = ServerAuthUtils.getUiHttpCredentialsPlugin(stormConf); singletons.add(new LogviewerResource(logviewer, profileHandler, logDownloadHandler, logSearchHandler, httpCredsHandler, metricsRegistry)); singletons.add(new AuthorizationExceptionMapper()); }
/** * Setup the container to run. By default this creates the needed directories/links in the local file system PREREQUISITE: All needed * blobs and topology, jars/configs have been downloaded and placed in the appropriate locations * * @throws IOException on any error */ protected void setup() throws IOException { _type.assertFull(); if (!_ops.doRequiredTopoFilesExist(_conf, _topologyId)) { LOG.info("Missing topology storm code, so can't launch worker with assignment {} for this supervisor {} on port {} with id {}", _assignment, _supervisorId, _port, _workerId); throw new IllegalStateException("Not all needed files are here!!!!"); } LOG.info("Setting up {}:{}", _supervisorId, _workerId); _ops.forceMkdir(new File(ConfigUtils.workerPidsRoot(_conf, _workerId))); _ops.forceMkdir(new File(ConfigUtils.workerTmpRoot(_conf, _workerId))); _ops.forceMkdir(new File(ConfigUtils.workerHeartbeatsRoot(_conf, _workerId))); File workerArtifacts = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port)); if (!_ops.fileExists(workerArtifacts)) { _ops.forceMkdir(workerArtifacts); _ops.setupWorkerArtifactsDir(_assignment.get_owner(), workerArtifacts); } String user = getWorkerUser(); writeLogMetadata(user); saveWorkerUser(user); createArtifactsLink(); createBlobstoreLinks(); }
/** * Main method to start the server. */ public static void main(String [] args) throws Exception { Utils.setupDefaultUncaughtExceptionHandler(); Map<String, Object> conf = ConfigUtils.readStormConfig(); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); String logRoot = ConfigUtils.workerArtifactsRoot(conf); File logRootDir = new File(logRoot); logRootDir.mkdirs(); WorkerLogs workerLogs = new WorkerLogs(conf, logRootDir, metricsRegistry); DirectoryCleaner directoryCleaner = new DirectoryCleaner(metricsRegistry); try (LogviewerServer server = new LogviewerServer(conf, metricsRegistry); LogCleaner logCleaner = new LogCleaner(conf, workerLogs, directoryCleaner, logRootDir, metricsRegistry)) { metricsRegistry.startMetricsReporters(conf); Utils.addShutdownHookWithForceKillIn1Sec(() -> { server.meterShutdownCalls.mark(); metricsRegistry.stopMetricsReporters(); server.close(); }); logCleaner.start(); server.start(); server.awaitTermination(); } } }
public static String workerArtifactsRoot(Map conf, String id) { return (workerArtifactsRoot(conf) + FILE_SEPARATOR + id); }
public static String workerArtifactsRoot(Map conf, String id, Integer port) { return (workerArtifactsRoot(conf, id) + FILE_SEPARATOR + port); }
public static String workerArtifactsPidPath(Map conf, String id, Integer port) { return (workerArtifactsRoot(conf, id, port) + FILE_SEPARATOR + "worker.pid"); }
public static File getLogMetaDataFile(Map conf, String id, Integer port) { String fname = workerArtifactsRoot(conf, id, port) + FILE_SEPARATOR + "worker.yaml"; return new File(fname); }
/** * Get a set of java properties that are common to both the log writer and the worker processes. * These are mostly system properties that are used by logging. * @return a list of command line options */ private List<String> getCommonParams() { final String workersArtifacts = ConfigUtils.workerArtifactsRoot(_conf); String stormLogDir = ConfigUtils.getLogDir(); String log4jConfigurationFile = getWorkerLoggingConfigFile(); List<String> commonParams = new ArrayList<>(); commonParams.add("-Dlogging.sensitivity=" + OR((String) _topoConf.get(Config.TOPOLOGY_LOGGING_SENSITIVITY), "S3")); commonParams.add("-Dlogfile.name=worker.log"); commonParams.add("-Dstorm.home=" + OR(_stormHome, "")); commonParams.add("-Dworkers.artifacts=" + workersArtifacts); commonParams.add("-Dstorm.id=" + _topologyId); commonParams.add("-Dworker.id=" + _workerId); commonParams.add("-Dworker.port=" + _port); commonParams.add("-Dstorm.log.dir=" + stormLogDir); commonParams.add("-Dlog4j.configurationFile=" + log4jConfigurationFile); commonParams.add("-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector"); commonParams.add("-Dstorm.local.dir=" + _conf.get(Config.STORM_LOCAL_DIR)); return commonParams; }
/** * Create symlink from the containers directory/artifacts to the artifacts directory * @throws IOException on any error */ protected void createArtifactsLink() throws IOException { _type.assertFull(); if (!_symlinksDisabled) { File workerDir = new File(ConfigUtils.workerRoot(_conf, _workerId)); File topoDir = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port)); if (_ops.fileExists(workerDir)) { LOG.debug("Creating symlinks for worker-id: {} topology-id: {} to its port artifacts directory", _workerId, _topologyId); _ops.createSymlink(new File(workerDir, "artifacts"), topoDir); } } }
@Override public boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException { _type.assertFull(); String targetDir = ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port); @SuppressWarnings("unchecked") Map<String, String> env = (Map<String, String>) _topoConf.get(Config.TOPOLOGY_ENVIRONMENT); if (env == null) { env = new HashMap<String, String>(); } String str = ConfigUtils.workerArtifactsPidPath(_conf, _topologyId, _port); String workerPid = _ops.slurpString(new File(str)).trim(); ProfileAction profileAction = request.get_action(); String logPrefix = "ProfilerAction process " + _topologyId + ":" + _port + " PROFILER_ACTION: " + profileAction + " "; List<String> command = mkProfileCommand(profileAction, stop, workerPid, targetDir); File targetFile = new File(targetDir); if (command.size() > 0) { return runProfilingCommand(command, env, logPrefix, targetFile); } LOG.warn("PROFILING REQUEST NOT SUPPORTED {} IGNORED...", request); return true; }
/** * @return the user that some operations should be done as. * @throws IOException on any error */ protected String getWorkerUser() throws IOException { LOG.info("GET worker-user for {}", _workerId); File file = new File(ConfigUtils.workerUserFile(_conf, _workerId)); if (_ops.fileExists(file)) { return _ops.slurpString(file).trim(); } else if (_assignment != null && _assignment.is_set_owner()) { return _assignment.get_owner(); } if (ConfigUtils.isLocalMode(_conf)) { return System.getProperty("user.name"); } else { File f = new File(ConfigUtils.workerArtifactsRoot(_conf)); if (f.exists()) { return Files.getOwner(f.toPath()).getName(); } throw new IllegalStateException("Could not recover the user for " + _workerId); } }