@Override public void relaunch() throws IOException { _type.assertFull(); //We are launching it now... _type = ContainerType.LAUNCH; createNewWorkerId(); setup(); launch(); }
protected List<String> substituteChildopts(Object value) { return substituteChildopts(value, -1); }
@Override public Container recoverContainer(int port, LocalAssignment assignment, LocalState state) throws IOException { return new BasicContainer(ContainerType.RECOVER_FULL, _conf, _supervisorId, _supervisorPort, port, assignment, _resourceIsolationManager, state, null, metricsRegistry, containerMemoryTracker); }
/** * Compute the classpath for the worker process. * * @param stormJar the topology jar * @param dependencyLocations any dependencies from the topology * @param topoVersion the version of the storm framework to use * @return the full classpath */ protected String getWorkerClassPath(String stormJar, List<String> dependencyLocations, SimpleVersion topoVersion) { List<String> workercp = new ArrayList<>(); workercp.addAll(asStringList(_topoConf.get(Config.TOPOLOGY_CLASSPATH_BEGINNING))); workercp.addAll(frameworkClasspath(topoVersion)); workercp.add(stormJar); workercp.addAll(dependencyLocations); workercp.addAll(asStringList(_topoConf.get(Config.TOPOLOGY_CLASSPATH))); return CPJ.join(workercp); }
/** * Get parameters for the class path of the worker process. Also used by the log Writer. * * @param stormRoot the root dist dir for the topology * @return the classpath for the topology as command line arguments. * * @throws IOException on any error. */ private List<String> getClassPathParams(final String stormRoot, final SimpleVersion topoVersion) throws IOException { final String stormJar = ConfigUtils.supervisorStormJarPath(stormRoot); final List<String> dependencyLocations = getDependencyLocationsFor(_conf, _topologyId, _ops, stormRoot); final String workerClassPath = getWorkerClassPath(stormJar, dependencyLocations, topoVersion); List<String> classPathParams = new ArrayList<>(); classPathParams.add("-cp"); classPathParams.add(workerClassPath); return classPathParams; }
final String javaCmd = javaCmd("java"); final String stormOptions = ConfigUtils.concatIfNotNull(System.getProperty("storm.options")); final String topoConfFile = ConfigUtils.concatIfNotNull(System.getProperty("storm.conf.file")); final String workerTmpDir = ConfigUtils.workerTmpRoot(_conf, _workerId); String topoVersionString = getStormVersionFor(_conf, _topologyId, _ops, stormRoot); if (topoVersionString == null) { topoVersionString = (String) _conf.getOrDefault(Config.SUPERVISOR_WORKER_DEFAULT_VERSION, VersionInfo.getVersion()); List<String> classPathParams = getClassPathParams(stormRoot, topoVersion); List<String> commonParams = getCommonParams(); String log4jConfigurationFile = getWorkerLoggingConfigFile(); String workerLog4jConfig = log4jConfigurationFile; if (_topoConf.get(Config.TOPOLOGY_LOGGING_CONFIG_FILE) != null) { String logWriter = getWorkerLogWriter(topoVersion); if (logWriter != null) { commandList.addAll(substituteChildopts(_topoConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS))); commandList.addAll(commonParams); commandList.add("-Dlog4j.configurationFile=" + log4jConfigurationFile); commandList.addAll(commonParams); commandList.add("-Dlog4j.configurationFile=" + workerLog4jConfig); commandList.addAll(substituteChildopts(_conf.get(Config.WORKER_CHILDOPTS), memOnheap)); commandList.addAll(substituteChildopts(_topoConf.get(Config.TOPOLOGY_WORKER_CHILDOPTS), memOnheap)); commandList.addAll(substituteChildopts(Utils.OR( _topoConf.get(Config.TOPOLOGY_WORKER_GC_CHILDOPTS), _conf.get(Config.WORKER_GC_CHILDOPTS)), memOnheap));
final int memOnHeap = getMemOnHeap(resources); memoryLimitMB = calculateMemoryLimit(resources, memOnHeap); final String stormRoot = ConfigUtils.supervisorStormDistRoot(_conf, _topologyId); String jlp = javaLibraryPath(stormRoot, _conf); List<String> commandList = mkLaunchCommand(memOnHeap, stormRoot, jlp); launchWorkerProcess(commandList, topEnvironment, logPrefix, processExitCallback, new File(workerDir));
final String javaCmd = javaCmd("java"); final String stormOptions = ConfigUtils.concatIfNotNull(System.getProperty("storm.options")); final String stormConfFile = ConfigUtils.concatIfNotNull(System.getProperty("storm.conf.file")); final String workerTmpDir = ConfigUtils.workerTmpRoot(_conf, _workerId); List<String> classPathParams = getClassPathParams(stormRoot); List<String> commonParams = getCommonParams(); commandList.addAll(substituteChildopts(_topoConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS))); commandList.addAll(commonParams); commandList.add("org.apache.storm.LogWriter"); //The LogWriter in turn launches the actual worker. commandList.add("-server"); commandList.addAll(commonParams); commandList.addAll(substituteChildopts(_conf.get(Config.WORKER_CHILDOPTS), memOnheap)); commandList.addAll(substituteChildopts(_topoConf.get(Config.TOPOLOGY_WORKER_CHILDOPTS), memOnheap)); commandList.addAll(substituteChildopts(OR( _topoConf.get(Config.TOPOLOGY_WORKER_GC_CHILDOPTS), _conf.get(Config.WORKER_GC_CHILDOPTS)), memOnheap)); commandList.addAll(getWorkerProfilerChildOpts(memOnheap)); commandList.add("-Djava.library.path=" + jlp); commandList.add("-Dstorm.conf.file=" + stormConfFile);
@Override public void createNewWorkerId() { super.createNewWorkerId(); }
protected List<String> frameworkClasspath(SimpleVersion topoVersion) { File stormWorkerLibDir = new File(_stormHome, "lib-worker"); String topoConfDir = System.getenv("STORM_CONF_DIR") != null ? System.getenv("STORM_CONF_DIR") : new File(_stormHome, "conf").getAbsolutePath(); File stormExtlibDir = new File(_stormHome, "extlib"); String extcp = System.getenv("STORM_EXT_CLASSPATH"); List<String> pathElements = new LinkedList<>(); pathElements.add(getWildcardDir(stormWorkerLibDir)); pathElements.add(getWildcardDir(stormExtlibDir)); pathElements.add(extcp); pathElements.add(topoConfDir); NavigableMap<SimpleVersion, List<String>> classpaths = Utils.getConfiguredClasspathVersions(_conf, pathElements); return Utils.getCompatibleVersion(classpaths, topoVersion, "classpath", pathElements); }
final int memOnheap = getMemOnHeap(resources); final String stormRoot = ConfigUtils.supervisorStormDistRoot(_conf, _topologyId); final String jlp = javaLibraryPath(stormRoot, _conf); List<String> commandList = mkLaunchCommand(memOnheap, stormRoot, jlp); launchWorkerProcess(commandList, topEnvironment, logPrefix, processExitCallback, new File(workerDir));
/** * Compute the classpath for the worker process * @param stormJar the topology jar * @param dependencyLocations any dependencies from the topology * @return the full classpath */ protected String getWorkerClassPath(String stormJar, List<String> dependencyLocations) { List<String> workercp = new ArrayList<>(); workercp.addAll(frameworkClasspath()); workercp.add(stormJar); workercp.addAll(dependencyLocations); workercp.addAll(asStringList(_topoConf.get(Config.TOPOLOGY_CLASSPATH))); return CPJ.join(workercp); }
/** * Get parameters for the class path of the worker process. Also used by the * log Writer * @param stormRoot the root dist dir for the topology * @return the classpath for the topology as command line arguments. * @throws IOException on any error. */ private List<String> getClassPathParams(final String stormRoot) throws IOException { final String stormJar = ConfigUtils.supervisorStormJarPath(stormRoot); final List<String> dependencyLocations = getDependencyLocationsFor(_conf, _topologyId, _ops, stormRoot); final String workerClassPath = getWorkerClassPath(stormJar, dependencyLocations); List<String> classPathParams = new ArrayList<>(); classPathParams.add("-cp"); classPathParams.add(workerClassPath); return classPathParams; }
protected List<String> frameworkClasspath() { File stormLibDir = new File(_stormHome, "lib"); String stormConfDir = System.getenv("STORM_CONF_DIR") != null ? System.getenv("STORM_CONF_DIR") : new File(_stormHome, "conf").getAbsolutePath(); File stormExtlibDir = new File(_stormHome, "extlib"); String extcp = System.getenv("STORM_EXT_CLASSPATH"); List<String> pathElements = new LinkedList<>(); pathElements.add(getWildcardDir(stormLibDir)); pathElements.add(getWildcardDir(stormExtlibDir)); pathElements.add(extcp); pathElements.add(stormConfDir); return pathElements; }
@Override public void relaunch() throws IOException { _type.assertFull(); //We are launching it now... _type = ContainerType.LAUNCH; createNewWorkerId(); setup(); launch(); }
@Override public List<String> substituteChildopts(Object value, int memOnheap) { return super.substituteChildopts(value, memOnheap); }
@Override public Killable recoverContainer(String workerId, LocalState localState) throws IOException { return new BasicContainer(ContainerType.RECOVER_PARTIAL, _conf, _supervisorId, _supervisorPort, -1, null, _resourceIsolationManager, localState, workerId, metricsRegistry, containerMemoryTracker); } }
private List<String> getWorkerProfilerChildOpts(int memOnheap) { List<String> workerProfilerChildopts = new ArrayList<>(); if (ObjectReader.getBoolean(_conf.get(DaemonConfig.WORKER_PROFILER_ENABLED), false)) { workerProfilerChildopts = substituteChildopts(_conf.get(DaemonConfig.WORKER_PROFILER_CHILDOPTS), memOnheap); } return workerProfilerChildopts; }
@Override public Container launchContainer(int port, LocalAssignment assignment, LocalState state) throws IOException { Container container = new BasicContainer(ContainerType.LAUNCH, _conf, _supervisorId, _supervisorPort, port, assignment, _resourceIsolationManager, state, null, metricsRegistry, containerMemoryTracker); container.setup(); container.launch(); return container; }