public String supervisorStormDistRootImpl(Map<String, Object> conf, String stormId) throws IOException { return supervisorStormDistRoot(conf) + FILE_SEPARATOR + URLEncoder.encode(stormId, "UTF-8"); }
private void forEachTopologyDistDir(ConsumePathAndId consumer) throws IOException { Path stormCodeRoot = Paths.get(ConfigUtils.supervisorStormDistRoot(conf)); if (Files.exists(stormCodeRoot) && Files.isDirectory(stormCodeRoot)) { try (DirectoryStream<Path> children = Files.newDirectoryStream(stormCodeRoot)) { for (Path child : children) { if (Files.isDirectory(child)) { String topologyId = child.getFileName().toString(); consumer.accept(child, topologyId); } } } } }
public Map<String, Long> getCurrentBlobVersions() throws IOException { Map<String, Long> results = new HashMap<>(); Map<String, Map<String, Object>> blobstoreMap = (Map<String, Map<String, Object>>) workerState.getTopologyConf().get(Config.TOPOLOGY_BLOBSTORE_MAP); if (blobstoreMap != null) { String stormRoot = ConfigUtils.supervisorStormDistRoot(workerState.getTopologyConf(), workerState.getTopologyId()); for (Map.Entry<String, Map<String, Object>> entry : blobstoreMap.entrySet()) { String localFileName = entry.getKey(); Map<String, Object> blobInfo = entry.getValue(); if (blobInfo != null && blobInfo.containsKey("localname")) { localFileName = (String) blobInfo.get("localname"); } String blobWithVersion = new File(stormRoot, localFileName).getCanonicalFile().getName(); Matcher m = BLOB_VERSION_EXTRACTION.matcher(blobWithVersion); if (m.matches()) { results.put(localFileName, Long.valueOf(m.group(1))); } } } return results; }
public StormTopology readSupervisorTopologyImpl(Map<String, Object> conf, String stormId, AdvancedFSOps ops) throws IOException { String stormRoot = supervisorStormDistRoot(conf, stormId); String topologyPath = supervisorStormCodePath(stormRoot); return readSupervisorStormCodeGivenPath(topologyPath, ops); }
public Map<String, Object> readSupervisorStormConfImpl(Map<String, Object> conf, String stormId) throws IOException { String stormRoot = supervisorStormDistRoot(conf, stormId); String confPath = supervisorStormConfPath(stormRoot); return readSupervisorStormConfGivenPath(conf, confPath); }
@SuppressWarnings("unchecked") public static WorkerTopologyContext makeWorkerContext(Map<String, Object> workerData) { try { StormTopology stormTopology = (StormTopology) workerData.get(Constants.SYSTEM_TOPOLOGY); Map<String, Object> topoConf = (Map) workerData.get(Constants.STORM_CONF); Map<Integer, String> taskToComponent = (Map<Integer, String>) workerData.get(Constants.TASK_TO_COMPONENT); Map<String, List<Integer>> componentToSortedTasks = (Map<String, List<Integer>>) workerData.get(Constants.COMPONENT_TO_SORTED_TASKS); Map<String, Map<String, Fields>> componentToStreamToFields = (Map<String, Map<String, Fields>>) workerData.get(Constants.COMPONENT_TO_STREAM_TO_FIELDS); String stormId = (String) workerData.get(Constants.STORM_ID); Map<String, Object> conf = (Map) workerData.get(Constants.CONF); Integer port = (Integer) workerData.get(Constants.PORT); String codeDir = ConfigUtils.supervisorStormResourcesPath(ConfigUtils.supervisorStormDistRoot(conf, stormId)); String pidDir = ConfigUtils.workerPidsRoot(conf, stormId); List<Integer> workerTasks = (List<Integer>) workerData.get(Constants.TASK_IDS); Map<String, Object> defaultResources = (Map<String, Object>) workerData.get(Constants.DEFAULT_SHARED_RESOURCES); Map<String, Object> userResources = (Map<String, Object>) workerData.get(Constants.USER_SHARED_RESOURCES); return new WorkerTopologyContext(stormTopology, topoConf, taskToComponent, componentToSortedTasks, componentToStreamToFields, stormId, codeDir, pidDir, port, workerTasks, defaultResources, userResources); } catch (IOException e) { throw Utils.wrapInRuntime(e); } }
String stormRoot = ConfigUtils.supervisorStormDistRoot(_conf, _topologyId); String workerRoot = ConfigUtils.workerRoot(_conf, _workerId);
/** * Create a new LocallyCachedBlob. * @param topologyId the ID of the topology. * @param type the type of the blob. * @param owner the name of the user that owns this blob. */ protected LocallyCachedTopologyBlob(final String topologyId, final boolean isLocalMode, final Map<String, Object> conf, final AdvancedFSOps fsOps, final TopologyBlobType type, String owner, StormMetricsRegistry metricsRegistry) throws IOException { super(topologyId + " " + type.getFileName(), type.getKey(topologyId), metricsRegistry); this.topologyId = topologyId; this.type = type; this.isLocalMode = isLocalMode; this.fsOps = fsOps; this.owner = owner; topologyBasicBlobsRootDir = Paths.get(ConfigUtils.supervisorStormDistRoot(conf, topologyId)); readVersion(); updateSizeOnDisk(); }
public WorkerTopologyContext getWorkerTopologyContext() { try { String codeDir = ConfigUtils.supervisorStormResourcesPath(ConfigUtils.supervisorStormDistRoot(conf, topologyId)); String pidDir = ConfigUtils.workerPidsRoot(conf, topologyId); return new WorkerTopologyContext(systemTopology, topologyConf, taskToComponent, componentToSortedTasks, componentToStreamToFields, topologyId, codeDir, pidDir, port, localTaskIds, defaultSharedResources, userSharedResources, cachedTaskToNodePort, assignmentId); } catch (IOException e) { throw Utils.wrapInRuntime(e); } }
final int memOnHeap = getMemOnHeap(resources); memoryLimitMB = calculateMemoryLimit(resources, memOnHeap); final String stormRoot = ConfigUtils.supervisorStormDistRoot(_conf, _topologyId); String jlp = javaLibraryPath(stormRoot, _conf);
String topologyId = pna.getToplogyId(); String topoOwner = pna.getOwner(); String stormroot = ConfigUtils.supervisorStormDistRoot(conf, topologyId); Map<String, Object> topoConf = ConfigUtils.readSupervisorStormConf(conf, topologyId);
static boolean doRequiredTopoFilesExist(Map<String, Object> conf, String stormId) throws IOException { String stormroot = ConfigUtils.supervisorStormDistRoot(conf, stormId); String stormjarpath = ConfigUtils.supervisorStormJarPath(stormroot); String stormcodepath = ConfigUtils.supervisorStormCodePath(stormroot); String stormconfpath = ConfigUtils.supervisorStormConfPath(stormroot); if (!Utils.checkFileExists(stormroot)) { return false; } if (!Utils.checkFileExists(stormcodepath)) { return false; } if (!Utils.checkFileExists(stormconfpath)) { return false; } if (ConfigUtils.isLocalMode(conf) || Utils.checkFileExists(stormjarpath)) { return true; } return false; }
private TopologyContext mkTopologyContext(StormTopology topology) throws IOException { Map<String, Object> conf = workerData.getConf(); return new TopologyContext( topology, workerData.getTopologyConf(), workerData.getTaskToComponent(), workerData.getComponentToSortedTasks(), workerData.getComponentToStreamToFields(), // This is updated by the Worker and the topology has shared access to it workerData.getBlobToLastKnownVersion(), workerData.getTopologyId(), ConfigUtils.supervisorStormResourcesPath( ConfigUtils.supervisorStormDistRoot(conf, workerData.getTopologyId())), ConfigUtils.workerPidsRoot(conf, workerData.getWorkerId()), taskId, workerData.getPort(), workerData.getLocalTaskIds(), workerData.getDefaultSharedResources(), workerData.getUserSharedResources(), executor.getSharedExecutorData(), executor.getIntervalToTaskToMetricToRegistry(), executor.getOpenOrPrepareWasCalled()); }
public DownloadBaseBlobsDistributed(String topologyId, String owner) throws IOException { _topologyId = topologyId; _stormRoot = new File(ConfigUtils.supervisorStormDistRoot(_conf, _topologyId)); this.owner = owner; }
public String supervisorStormDistRootImpl(Map conf, String stormId) throws IOException { return supervisorStormDistRoot(conf) + FILE_SEPARATOR + URLEncoder.encode(stormId, "UTF-8"); }
public static Set<String> readDownloadedTopologyIds(Map<String, Object> conf) throws IOException { Set<String> stormIds = new HashSet<>(); String path = ConfigUtils.supervisorStormDistRoot(conf); Collection<String> rets = Utils.readDirContents(path); for (String ret : rets) { stormIds.add(URLDecoder.decode(ret)); } return stormIds; }
@Override public synchronized void cleanupUnusedTopologies() throws IOException { File distRoot = new File(ConfigUtils.supervisorStormDistRoot(_conf)); LOG.info("Cleaning up unused topologies in {}", distRoot); File[] children = distRoot.listFiles(); if (children != null) { for (File topoDir : children) { String topoId = URLDecoder.decode(topoDir.getName(), "UTF-8"); if (_basicPending.get(topoId) == null && _blobPending.get(topoId) == null) { _fsOps.deleteIfExists(topoDir, null, "rmr " + topoId); } } } }
public Map<String, Object> readSupervisorStormConfImpl(Map<String, Object> conf, String stormId) throws IOException { String stormRoot = supervisorStormDistRoot(conf, stormId); String confPath = supervisorStormConfPath(stormRoot); return readSupervisorStormConfGivenPath(conf, confPath); }
public StormTopology readSupervisorTopologyImpl(Map conf, String stormId, AdvancedFSOps ops) throws IOException { String stormRoot = supervisorStormDistRoot(conf, stormId); String topologyPath = supervisorStormCodePath(stormRoot); return readSupervisorStormCodeGivenPath(topologyPath, ops); }
static boolean doRequiredTopoFilesExist(Map<String, Object> conf, String stormId) throws IOException { String stormroot = ConfigUtils.supervisorStormDistRoot(conf, stormId); String stormjarpath = ConfigUtils.supervisorStormJarPath(stormroot); String stormcodepath = ConfigUtils.supervisorStormCodePath(stormroot); String stormconfpath = ConfigUtils.supervisorStormConfPath(stormroot); if (!Utils.checkFileExists(stormroot)) return false; if (!Utils.checkFileExists(stormcodepath)) return false; if (!Utils.checkFileExists(stormconfpath)) return false; if (ConfigUtils.isLocalMode(conf) || Utils.checkFileExists(stormjarpath)) return true; return false; }