public GeneralTopologyContext(StormTopology topology, Map<String, Object> topoConf, Map<Integer, String> taskToComponent, Map<String, List<Integer>> componentToSortedTasks, Map<String, Map<String, Fields>> componentToStreamToFields, String stormId) { _topology = topology; _topoConf = topoConf; _taskToComponent = taskToComponent; _stormId = stormId; _componentToTasks = componentToSortedTasks; _componentToStreamToFields = componentToStreamToFields; _doSanityCheck = ConfigUtils.isLocalMode(_topoConf); }
public static void validateDistributedMode(Map<String, Object> conf) { if (ConfigUtils.isLocalMode(conf)) { throw new IllegalArgumentException("Cannot start server in local mode!"); } }
static List<String> getKeyListFromId(Map<String, Object> conf, String id) { List<String> ret = new ArrayList<>(3); ret.add(ConfigUtils.masterStormCodeKey(id)); ret.add(ConfigUtils.masterStormConfKey(id)); if (!ConfigUtils.isLocalMode(conf)) { ret.add(ConfigUtils.masterStormJarKey(id)); } return ret; }
/** * Function for initialization. * * @param conf config */ public void prepare(Map conf) { this.conf = conf; this.random = new Random(47); this.threadsNum = ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_ASSIGNMENTS_SERVICE_THREADS), 10); this.queueSize = ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_ASSIGNMENTS_SERVICE_THREAD_QUEUE_SIZE), 100); this.assignmentsQueue = new HashMap<>(); for (int i = 0; i < threadsNum; i++) { this.assignmentsQueue.put(i, new LinkedBlockingQueue<NodeAssignments>(queueSize)); } //start the thread pool this.service = Executors.newFixedThreadPool(threadsNum); this.active = true; //start the threads for (int i = 0; i < threadsNum; i++) { this.service.submit(new DistributeTask(this, i)); } // for local cluster localSupervisors = new HashMap<>(); if (ConfigUtils.isLocalMode(conf)) { isLocalMode = true; } }
public static ClientBlobStore getClientBlobStoreForSupervisor(Map<String, Object> conf) { ClientBlobStore store; if (ConfigUtils.isLocalMode(conf)) { store = new LocalModeClientBlobStore(getNimbusBlobStore(conf, null, null)); } else { store = (ClientBlobStore) ReflectionUtils.newInstance( (String) conf.get(DaemonConfig.SUPERVISOR_BLOBSTORE)); } store.prepare(conf); return store; }
/** * @return the user that some operations should be done as. * * @throws IOException on any error */ protected String getWorkerUser() throws IOException { LOG.info("GET worker-user for {}", _workerId); File file = new File(ConfigUtils.workerUserFile(_conf, _workerId)); if (_ops.fileExists(file)) { return _ops.slurpString(file).trim(); } else if (_assignment != null && _assignment.is_set_owner()) { return _assignment.get_owner(); } if (ConfigUtils.isLocalMode(_conf)) { return System.getProperty("user.name"); } else { File f = new File(ConfigUtils.workerArtifactsRoot(_conf)); if (f.exists()) { return Files.getOwner(f.toPath()).getName(); } throw new IllegalStateException("Could not recover the user for " + _workerId); } }
if (!ConfigUtils.isLocalMode(superConf)) { throw new IllegalArgumentException("Cannot start server in distrubuted mode!");
private void reportWorkerHeartbeats(SupervisorWorkerHeartbeats supervisorWorkerHeartbeats) { if (supervisorWorkerHeartbeats == null) { // error/exception thrown, just skip return; } // if it is local mode, just get the local nimbus instance and set the heartbeats if (ConfigUtils.isLocalMode(conf)) { try { this.supervisor.getLocalNimbus().sendSupervisorWorkerHeartbeats(supervisorWorkerHeartbeats); } catch (TException tex) { LOG.error("Send local supervisor heartbeats error", tex); } } else { try (NimbusClient master = NimbusClient.getConfiguredClient(conf)) { master.getClient().sendSupervisorWorkerHeartbeats(supervisorWorkerHeartbeats); } catch (Exception t) { LOG.error("Send worker heartbeats to master exception", t); } } } }
IContext sharedContext, StormMetricsRegistry metricsRegistry, ContainerMemoryTracker containerMemoryTracker) throws IOException { if (ConfigUtils.isLocalMode(conf)) { return new LocalContainerLauncher(conf, supervisorId, supervisorPort, sharedContext, metricsRegistry, containerMemoryTracker);
public void prepare(Map<String, Object> topoConf, TopologyContext context, final OutputCollector collector) { if (ConfigUtils.isLocalMode(topoConf)) { _isLocalMode = true;
@VisibleForTesting AsyncLocalizer(Map<String, Object> conf, AdvancedFSOps ops, String baseDir, StormMetricsRegistry metricsRegistry) throws IOException { this.conf = conf; this.singleBlobLocalizationDuration = metricsRegistry.registerTimer("supervisor:single-blob-localization-duration"); this.blobCacheUpdateDuration = metricsRegistry.registerTimer("supervisor:blob-cache-update-duration"); this.blobLocalizationDuration = metricsRegistry.registerTimer("supervisor:blob-localization-duration"); this.numBlobUpdateVersionChanged = metricsRegistry.registerMeter("supervisor:num-blob-update-version-changed"); this.metricsRegistry = metricsRegistry; isLocalMode = ConfigUtils.isLocalMode(conf); fsOps = ops; localBaseDir = Paths.get(baseDir); // default cache size 10GB, converted to Bytes cacheTargetSize = ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_TARGET_SIZE_MB), 10 * 1024).longValue() << 20; // default 30 seconds. (we cache the size so it is cheap to do) cacheCleanupPeriod = ObjectReader.getInt(conf.get( DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS), 30 * 1000).longValue(); // if we needed we could make config for update thread pool size int threadPoolSize = ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_BLOBSTORE_DOWNLOAD_THREAD_COUNT), 5); blobDownloadRetries = ObjectReader.getInt(conf.get( DaemonConfig.SUPERVISOR_BLOBSTORE_DOWNLOAD_MAX_RETRIES), 3); execService = Executors.newScheduledThreadPool(threadPoolSize, new ThreadFactoryBuilder().setNameFormat("AsyncLocalizer Executor - %d").build()); reconstructLocalizedResources(); symlinksDisabled = (boolean) conf.getOrDefault(Config.DISABLE_SYMLINKS, false); blobPending = new ConcurrentHashMap<>(); }
/** * Used by {@link Supervisor} to fetch assignments when start up. * @param conf config * @param clusterState {@link IStormClusterState} * @param node id of node */ public void getAssignmentsFromMaster(Map conf, IStormClusterState clusterState, String node) { if (ConfigUtils.isLocalMode(conf)) { try { SupervisorAssignments assignments = this.supervisor.getLocalNimbus().getSupervisorAssignments(node); assignedAssignmentsToLocal(clusterState, assignments); } catch (TException e) { LOG.error("Get assignments from local master exception", e); } } else { try (NimbusClient master = NimbusClient.getConfiguredClient(conf)) { SupervisorAssignments assignments = master.getClient().getSupervisorAssignments(node); LOG.debug("Sync an assignments from master, will start to sync with assignments: {}", assignments); assignedAssignmentsToLocal(clusterState, assignments); } catch (Exception t) { LOG.error("Get assignments from master exception", t); } } } }
/** * start distribute supervisor. */ public void launchDaemon() { LOG.info("Starting supervisor for storm version '{}'.", VersionInfo.getVersion()); try { Map<String, Object> conf = getConf(); if (ConfigUtils.isLocalMode(conf)) { throw new IllegalArgumentException("Cannot start server in local mode!"); } launch(); metricsRegistry.registerGauge("supervisor:num-slots-used-gauge", () -> SupervisorUtils.supervisorWorkerIds(conf).size()); //This will only get updated once metricsRegistry.registerMeter("supervisor:num-launched").mark(); metricsRegistry.registerMeter("supervisor:num-shell-exceptions", ShellUtils.numShellExceptions); metricsRegistry.startMetricsReporters(conf); Utils.addShutdownHookWithForceKillIn1Sec(() -> { metricsRegistry.stopMetricsReporters(); this.close(); }); // blocking call under the hood, must invoke after launch cause some services must be initialized launchSupervisorThriftServer(conf); } catch (Exception e) { LOG.error("Failed to start supervisor\n", e); System.exit(1); } }
int maxWaitTime = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_MAX_REPLICATION_WAIT_TIME_SEC)); int jarCount = minReplicationCount; if (!ConfigUtils.isLocalMode(topoConf)) { jarCount = getBlobReplicationCount(ConfigUtils.masterStormJarKey(topoId)); Time.sleepSecs(1); totalWaitTime++; if (!ConfigUtils.isLocalMode(topoConf)) { jarCount = getBlobReplicationCount(ConfigUtils.masterStormJarKey(topoId));
/** * Send a heartbeat to local supervisor first to check if supervisor is ok for heartbeating. */ private void heartbeatToMasterIfLocalbeatFail(LSWorkerHeartbeat lsWorkerHeartbeat) { if (ConfigUtils.isLocalMode(this.conf)) { return; } //In distributed mode, send heartbeat directly to master if local supervisor goes down. SupervisorWorkerHeartbeat workerHeartbeat = new SupervisorWorkerHeartbeat(lsWorkerHeartbeat.get_topology_id(), lsWorkerHeartbeat.get_executors(), lsWorkerHeartbeat.get_time_secs()); try (SupervisorClient client = SupervisorClient.getConfiguredClient(conf, Utils.hostname(), supervisorPort)) { client.getClient().sendSupervisorWorkerHeartbeat(workerHeartbeat); } catch (Exception tr1) { //If any error/exception thrown, report directly to nimbus. LOG.warn("Exception when send heartbeat to local supervisor", tr1.getMessage()); try (NimbusClient nimbusClient = NimbusClient.getConfiguredClient(conf)) { nimbusClient.getClient().sendSupervisorWorkerHeartbeat(workerHeartbeat); } catch (Exception tr2) { //if any error/exception thrown, just ignore. LOG.error("Exception when send heartbeat to master", tr2.getMessage()); } } }
private Assignment getLocalAssignment(Map<String, Object> conf, IStormClusterState stormClusterState, String topologyId) { if (!ConfigUtils.isLocalMode(conf)) { try (SupervisorClient supervisorClient = SupervisorClient.getConfiguredClient(conf, Utils.hostname(), supervisorPort)) { Assignment assignment = supervisorClient.getClient().getLocalAssignmentForStorm(topologyId); return assignment; } catch (Throwable tr1) { //if any error/exception thrown, fetch it from zookeeper return stormClusterState.remoteAssignmentInfo(topologyId, null); } } else { return stormClusterState.remoteAssignmentInfo(topologyId, null); } }
if (ConfigUtils.isLocalMode(topologyConf)) { Executor executor = LocalExecutor.mkExecutor(workerState, e, initCreds); execs.add(executor);
if (!ConfigUtils.isLocalMode(conf)) {
static boolean doRequiredTopoFilesExist(Map<String, Object> conf, String stormId) throws IOException { String stormroot = ConfigUtils.supervisorStormDistRoot(conf, stormId); String stormjarpath = ConfigUtils.supervisorStormJarPath(stormroot); String stormcodepath = ConfigUtils.supervisorStormCodePath(stormroot); String stormconfpath = ConfigUtils.supervisorStormConfPath(stormroot); if (!Utils.checkFileExists(stormroot)) { return false; } if (!Utils.checkFileExists(stormcodepath)) { return false; } if (!Utils.checkFileExists(stormconfpath)) { return false; } if (ConfigUtils.isLocalMode(conf) || Utils.checkFileExists(stormjarpath)) { return true; } return false; }
AsyncLocalizer(Map<String, Object> conf, Localizer localizer, AdvancedFSOps ops) { _conf = conf; _symlinksDisabled = (boolean)OR(conf.get(Config.DISABLE_SYMLINKS), false); _isLocalMode = ConfigUtils.isLocalMode(conf); _localizer = localizer; _execService = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder() .setNameFormat("Async Localizer") .build()); _basicPending = new HashMap<>(); _blobPending = new HashMap<>(); _fsOps = ops; }