public void downloadResourcesAsSupervisorImpl(String key, String localFile, ClientBlobStore cb) throws AuthorizationException, KeyNotFoundException, IOException { final int MAX_RETRY_ATTEMPTS = 2; final int ATTEMPTS_INTERVAL_TIME = 100; for (int retryAttempts = 0; retryAttempts < MAX_RETRY_ATTEMPTS; retryAttempts++) { if (downloadResourcesAsSupervisorAttempt(cb, key, localFile)) { break; } Utils.sleep(ATTEMPTS_INTERVAL_TIME); } } }
private static void rotateTopologyCleanupMap(long deletionDelay) { if (Time.currentTimeMillis() - topologyCleanupRotationTime > deletionDelay) { topologyCleanupDetected.rotate(); topologyCleanupRotationTime = Time.currentTimeMillis(); } }
/** * Execute cb with a configured nimbus client that will be closed once cb returns. * @param cb the callback to send to nimbus. * @param conf the conf to use instead of reading the global storm conf. * @throws Exception on any kind of error. */ public static void withConfiguredClient(WithNimbus cb, Map<String, Object> conf) throws Exception { try (NimbusClient client = getConfiguredClientAs(conf, null)) { cb.run(client.getClient()); } }
private static long getTopologyCleanupDetectedTime(String topologyId) { Long firstDetectedForDeletion = topologyCleanupDetected.get(topologyId); if (firstDetectedForDeletion == null) { firstDetectedForDeletion = Time.currentTimeMillis(); topologyCleanupDetected.put(topologyId, firstDetectedForDeletion); } return firstDetectedForDeletion; }
public String createVersion() throws IOException { Long mostRecent = mostRecentVersion(); long version = Time.currentTimeMillis(); if (mostRecent != null && version <= mostRecent) { version = mostRecent + 1; } return createVersion(version); }
/** * Execute cb with a configured nimbus client that will be closed once cb returns. * @param cb the callback to send to nimbus. * @throws Exception on any kind of error. */ public static void withConfiguredClient(WithNimbus cb) throws Exception { withConfiguredClient(cb, ConfigUtils.readStormConfig()); }
public LocalState supervisorStateImpl(Map<String, Object> conf) throws IOException { return new LocalState((ConfigUtils.supervisorLocalDir(conf) + FILE_SEPARATOR + "localstate"), true); }
/** * Deletes topologies from history older than mins minutes. * * @param mins the number of mins for old topologies */ private void cleanTopologyHistory(int mins) { int cutoffAgeSecs = Time.currentTimeSecs() - (mins * 60); synchronized (topologyHistoryLock) { LocalState state = topologyHistoryState; state.filterOldTopologies(cutoffAgeSecs); } }
public LocalState nimbusTopoHistoryStateImpl(Map<String, Object> conf) throws IOException { return new LocalState((masterLocalDir(conf) + FILE_SEPARATOR + "history"), true); } }
protected String getWorkerMain(SimpleVersion topoVersion) { String defaultWorkerGuess = "org.apache.storm.daemon.worker.Worker"; if (topoVersion.getMajor() == 0) { //Prior to the org.apache change defaultWorkerGuess = "backtype.storm.daemon.worker"; } else if (topoVersion.getMajor() == 1) { //Have not moved to a java worker yet defaultWorkerGuess = "org.apache.storm.daemon.worker"; } NavigableMap<SimpleVersion, String> mains = Utils.getConfiguredWorkerMainVersions(_conf); return Utils.getCompatibleVersion(mains, topoVersion, "worker main class", defaultWorkerGuess); }
protected String getWorkerLogWriter(SimpleVersion topoVersion) { String defaultGuess = "org.apache.storm.LogWriter"; if (topoVersion.getMajor() == 0) { //Prior to the org.apache change defaultGuess = "backtype.storm.LogWriter"; } NavigableMap<SimpleVersion, String> mains = Utils.getConfiguredWorkerLogWriterVersions(_conf); return Utils.getCompatibleVersion(mains, topoVersion, "worker log writer class", defaultGuess); }
private static double getQueueLoad(JCQueue q) { JCQueue.QueueMetrics qMetrics = q.getMetrics(); return ((double) qMetrics.population()) / qMetrics.capacity(); }
/** * if(batchSz>1) : Blocking call. Does not return until at least 1 element is drained or Thread.interrupt() is received if(batchSz==1) * : NO-OP. Returns immediately. doesnt throw. */ public void flush() throws InterruptedException { Inserter inserter = getInserter(); inserter.flush(); }
/** * if(batchSz>1) : Non-Blocking call. Tries to flush as many as it can. Returns true if flushed at least 1. if(batchSz==1) : This is a * NO-OP. Returns true immediately. */ public boolean tryFlush() { Inserter inserter = getInserter(); return inserter.tryFlush(); }
/** * Non-blocking call, returns false if full. **/ public boolean tryPublish(Object obj) { Inserter inserter = getInserter(); return inserter.tryPublish(obj); }
/** * Blocking call. Retries till it can successfully publish the obj. Can be interrupted via Thread.interrupt(). */ public void publish(Object obj) throws InterruptedException { Inserter inserter = getInserter(); inserter.publish(obj); }
public void recordMsgDrop() { getMetrics().notifyDroppedMsg(); }
private void rotateIfNeeded() { long nowMs = Time.currentTimeMillis(); if (nowMs >= lastRotationMs + timeoutMs) { //Rotate once per timeout period that has passed since last time this was called. //This is necessary since this method may be called at arbitrary intervals. int rotationsToDo = (int) ((nowMs - lastRotationMs) / timeoutMs); for (int i = 0; i < rotationsToDo; i++) { cachedGroups.rotate(); } lastRotationMs = nowMs; } }