public TimeTriggerPolicy(long millis, TriggerHandler handler, EvictionPolicy<T, ?> evictionPolicy) { this.duration = millis; this.handler = handler; ThreadFactory threadFactory = new ThreadFactoryBuilder() .setNameFormat("time-trigger-policy-%d") .setDaemon(true) .build(); this.executor = Executors.newSingleThreadScheduledExecutor(threadFactory); this.evictionPolicy = evictionPolicy; }
@VisibleForTesting AsyncLocalizer(Map<String, Object> conf, AdvancedFSOps ops, String baseDir, StormMetricsRegistry metricsRegistry) throws IOException { this.conf = conf; this.singleBlobLocalizationDuration = metricsRegistry.registerTimer("supervisor:single-blob-localization-duration"); this.blobCacheUpdateDuration = metricsRegistry.registerTimer("supervisor:blob-cache-update-duration"); this.blobLocalizationDuration = metricsRegistry.registerTimer("supervisor:blob-localization-duration"); this.numBlobUpdateVersionChanged = metricsRegistry.registerMeter("supervisor:num-blob-update-version-changed"); this.metricsRegistry = metricsRegistry; isLocalMode = ConfigUtils.isLocalMode(conf); fsOps = ops; localBaseDir = Paths.get(baseDir); // default cache size 10GB, converted to Bytes cacheTargetSize = ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_TARGET_SIZE_MB), 10 * 1024).longValue() << 20; // default 30 seconds. (we cache the size so it is cheap to do) cacheCleanupPeriod = ObjectReader.getInt(conf.get( DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS), 30 * 1000).longValue(); // if we needed we could make config for update thread pool size int threadPoolSize = ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_BLOBSTORE_DOWNLOAD_THREAD_COUNT), 5); blobDownloadRetries = ObjectReader.getInt(conf.get( DaemonConfig.SUPERVISOR_BLOBSTORE_DOWNLOAD_MAX_RETRIES), 3); execService = Executors.newScheduledThreadPool(threadPoolSize, new ThreadFactoryBuilder().setNameFormat("AsyncLocalizer Executor - %d").build()); reconstructLocalizedResources(); symlinksDisabled = (boolean) conf.getOrDefault(Config.DISABLE_SYMLINKS, false); blobPending = new ConcurrentHashMap<>(); }
public void startCleaner() { _cacheCleanupService = new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder() .setNameFormat("Localizer Cache Cleanup") .build()); _cacheCleanupService.scheduleWithFixedDelay(new Runnable() { @Override public void run() { handleCacheCleanup(); } }, _cacheCleanupPeriod, _cacheCleanupPeriod, TimeUnit.MILLISECONDS); }
/** * Creates a new WatermarkEventGenerator. * * @param windowManager The window manager this generator will submit watermark events to * @param intervalMs The generator will check if it should generate a watermark event with this interval * @param eventTsLagMs The max allowed lag behind the last watermark event before an event is considered late * @param inputStreams The input streams this generator is expected to handle */ public WaterMarkEventGenerator(WindowManager<T> windowManager, int intervalMs, int eventTsLagMs, Set<GlobalStreamId> inputStreams) { this.windowManager = windowManager; streamToTs = new ConcurrentHashMap<>(); ThreadFactory threadFactory = new ThreadFactoryBuilder() .setNameFormat("watermark-event-generator-%d") .setDaemon(true) .build(); executorService = Executors.newSingleThreadScheduledExecutor(threadFactory); this.interval = intervalMs; this.eventTsLag = eventTsLagMs; this.inputStreams = inputStreams; }
AsyncLocalizer(Map<String, Object> conf, Localizer localizer, AdvancedFSOps ops) { _conf = conf; _symlinksDisabled = (boolean)OR(conf.get(Config.DISABLE_SYMLINKS), false); _isLocalMode = ConfigUtils.isLocalMode(conf); _localizer = localizer; _execService = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder() .setNameFormat("Async Localizer") .build()); _basicPending = new HashMap<>(); _blobPending = new HashMap<>(); _fsOps = ops; }
private void setUpFlushTask() { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setNameFormat("event-logger-flush-%d") .setDaemon(true) .build(); flushScheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); Runnable runnable = new Runnable() { @Override public void run() { try { if (dirty) { eventLogWriter.flush(); dirty = false; } } catch (IOException ex) { LOG.error("Error flushing " + eventLogPath, ex); throw new RuntimeException(ex); } } }; flushScheduler.scheduleAtFixedRate(runnable, FLUSH_INTERVAL_MILLIS, FLUSH_INTERVAL_MILLIS, TimeUnit.MILLISECONDS); }
public TimeTriggerPolicy(long millis, TriggerHandler handler, EvictionPolicy<T> evictionPolicy) { this.duration = millis; this.handler = handler; ThreadFactory threadFactory = new ThreadFactoryBuilder() .setNameFormat("time-trigger-policy-%d") .setDaemon(true) .build(); this.executor = Executors.newSingleThreadScheduledExecutor(threadFactory); this.evictionPolicy = evictionPolicy; }
/** * Creates a new WatermarkEventGenerator. * @param windowManager The window manager this generator will submit watermark events to * @param intervalMs The generator will check if it should generate a watermark event with this interval * @param eventTsLagMs The max allowed lag behind the last watermark event before an event is considered late * @param inputStreams The input streams this generator is expected to handle */ public WaterMarkEventGenerator(WindowManager<T> windowManager, int intervalMs, int eventTsLagMs, Set<GlobalStreamId> inputStreams) { this.windowManager = windowManager; streamToTs = new ConcurrentHashMap<>(); ThreadFactory threadFactory = new ThreadFactoryBuilder() .setNameFormat("watermark-event-generator-%d") .setDaemon(true) .build(); executorService = Executors.newSingleThreadScheduledExecutor(threadFactory); this.interval = intervalMs; this.eventTsLag = eventTsLagMs; this.inputStreams = inputStreams; }
private void setUpFlushTask() { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setNameFormat("event-logger-flush-%d") .setDaemon(true) .build(); flushScheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); Runnable runnable = new Runnable() { @Override public void run() { try { if (dirty) { eventLogWriter.flush(); dirty = false; } } catch (IOException ex) { LOG.error("Error flushing " + eventLogPath, ex); throw new RuntimeException(ex); } } }; flushScheduler.scheduleAtFixedRate(runnable, FLUSH_INTERVAL_MILLIS, FLUSH_INTERVAL_MILLIS, TimeUnit.MILLISECONDS); }
public FlusherPool() { _exec = new ThreadPoolExecutor(1, getNumFlusherPoolThreads(), 10, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(1024), new ThreadPoolExecutor.DiscardPolicy()); ThreadFactory threadFactory = new ThreadFactoryBuilder() .setDaemon(true) .setNameFormat(THREAD_PREFIX + "-task-pool") .build(); _exec.setThreadFactory(threadFactory); }