/** * Returns a {@link ExecutorServiceFactory} which creates threadpool executors with the given base * name. Created threads will be daemonic. * * @param name the base name for executor thread names * @return the {@link ExecutorServiceFactory} */ public static ExecutorServiceFactory cachedThreadPool(String name) { return () -> Executors.newCachedThreadPool(ThreadFactoryUtils.build(name + "-%d", true)); }
/** * Returns a {@link ExecutorServiceFactory} which creates threadpool executors with the given base * name and number of threads. Created threads will be daemonic. * * @param name the base name for executor thread names * @param nThreads the number of threads to create executors with * @return the {@link ExecutorServiceFactory} */ public static ExecutorServiceFactory fixedThreadPool(String name, int nThreads) { return () -> Executors.newFixedThreadPool(nThreads, ThreadFactoryUtils.build(name + "-%d", true)); }
/** * Creates a new instance of {@link AsyncUfsAbsentPathCache}. * * @param mountTable the mount table * @param numThreads the maximum number of threads for the async thread pool */ public AsyncUfsAbsentPathCache(MountTable mountTable, int numThreads) { mMountTable = mountTable; mCurrentPaths = new ConcurrentHashMap<>(8, 0.95f, 8); mCache = CacheBuilder.newBuilder().maximumSize(MAX_PATHS).build(); mThreads = numThreads; mPool = new ThreadPoolExecutor(mThreads, mThreads, THREAD_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), ThreadFactoryUtils.build("UFS-Absent-Path-Cache-%d", true)); mPool.allowCoreThreadTimeOut(true); }
/** * Constructor of AsyncBlockRemover. * @param worker block worker */ public AsyncBlockRemover(BlockWorker worker) { mBlockWorker = worker; mBlocksToRemove = new LinkedBlockingQueue<>(); mRemovingBlocks = Collections.newSetFromMap(new ConcurrentHashMap<>()); mRemoverPool = Executors.newFixedThreadPool(DEFAULT_BLOCK_REMOVER_POOL_SIZE, ThreadFactoryUtils.build("block-removal-service-%d", true)); for (int i = 0; i < DEFAULT_BLOCK_REMOVER_POOL_SIZE; i++) { mRemoverPool.execute(new BlockRemover()); } }
/** * Creates a new instance of {@link FileWorkerMasterSyncExecutor}. * * @param fileDataManager a {@link FileDataManager} handle * @param masterClient a {@link FileSystemMasterClient} * @param workerId a reference to the id for this worker */ public FileWorkerMasterSyncExecutor(FileDataManager fileDataManager, FileSystemMasterClient masterClient, AtomicReference<Long> workerId) { mFileDataManager = Preconditions.checkNotNull(fileDataManager, "fileDataManager"); mMasterClient = Preconditions.checkNotNull(masterClient, "masterClient"); mWorkerId = Preconditions.checkNotNull(workerId, "workerId"); mPersistFileService = Executors.newFixedThreadPool( ServerConfiguration.getInt(PropertyKey.WORKER_FILE_PERSIST_POOL_SIZE), ThreadFactoryUtils.build("persist-file-service-%d", true)); }
/** * Creates a Netty {@link EventLoopGroup} based on {@link ChannelType}. * * @param type Selector for which form of low-level IO we should use * @param numThreads target number of threads * @param threadPrefix name pattern for each thread. should contain '%d' to distinguish between * threads. * @param isDaemon if true, the {@link java.util.concurrent.ThreadFactory} will create daemon * threads. * @return EventLoopGroup matching the ChannelType */ public static EventLoopGroup createEventLoop(ChannelType type, int numThreads, String threadPrefix, boolean isDaemon) { ThreadFactory threadFactory = ThreadFactoryUtils.build(threadPrefix, isDaemon); switch (type) { case NIO: return new NioEventLoopGroup(numThreads, threadFactory); case EPOLL: return new EpollEventLoopGroup(numThreads, threadFactory); default: throw new IllegalArgumentException("Unknown io type: " + type); } }
/** * Creates {@link DefaultAuthenticationServer} instance. * * @param conf Alluxio configuration */ public DefaultAuthenticationServer(AlluxioConfiguration conf) { checkSupported(conf.getEnum(PropertyKey.SECURITY_AUTHENTICATION_TYPE, AuthType.class)); mCleanupIntervalMs = conf.getMs(PropertyKey.AUTHENTICATION_INACTIVE_CHANNEL_REAUTHENTICATE_PERIOD); mConfiguration = conf; mChannels = new ConcurrentHashMap<>(); mScheduler = Executors.newScheduledThreadPool(1, ThreadFactoryUtils.build("auth-cleanup", true)); mScheduler.scheduleAtFixedRate(() -> { cleanupStaleClients(); }, mCleanupIntervalMs, mCleanupIntervalMs, TimeUnit.MILLISECONDS); }
/** * Creates a file system context with a subject. * * @param ctx the parent subject, set to null if not present */ private FileSystemContext(ClientContext ctx) { mClientContext = ctx; mExecutorService = Executors.newFixedThreadPool(1, ThreadFactoryUtils.build("metrics-master-heartbeat-%d", true)); mClosed = new AtomicBoolean(false); mAppId = ctx.getConf().isSet(PropertyKey.USER_APP_ID) ? ctx.getConf().get(PropertyKey.USER_APP_ID) : IdUtils.createFileSystemContextId(); LOG.info("Created filesystem context with id {}. This ID will be used for identifying info " + "from the client, such as metrics. It can be set manually through the {} property", mAppId, PropertyKey.Name.USER_APP_ID); }
Executors.newFixedThreadPool(parallelism, ThreadFactoryUtils.build("persist-cli-%d", true)); final Object progressLock = new Object(); AtomicInteger completedFiles = new AtomicInteger(0);
/** * Creates a new instance of {@link JobWorker}. * * @param ufsManager the ufs manager */ JobWorker(UfsManager ufsManager) { super( Executors.newFixedThreadPool(1, ThreadFactoryUtils.build("job-worker-heartbeat-%d", true))); mUfsManager = ufsManager; mJobMasterClient = JobMasterClient.Factory.create(JobMasterClientContext .newBuilder(ClientContext.create(ServerConfiguration.global())).build()); mTaskExecutorManager = new TaskExecutorManager(); }
/** * Constructs a default block worker. * * @param blockMasterClientPool a client pool for talking to the block master * @param fileSystemMasterClient a client for talking to the file system master * @param sessions an object for tracking and cleaning up client sessions * @param blockStore an Alluxio block store * @param ufsManager ufs manager */ DefaultBlockWorker(BlockMasterClientPool blockMasterClientPool, FileSystemMasterClient fileSystemMasterClient, Sessions sessions, BlockStore blockStore, UfsManager ufsManager) { super(Executors .newFixedThreadPool(4, ThreadFactoryUtils.build("block-worker-heartbeat-%d", true))); mBlockMasterClientPool = blockMasterClientPool; mBlockMasterClient = mBlockMasterClientPool.acquire(); mFileSystemMasterClient = fileSystemMasterClient; mHeartbeatReporter = new BlockHeartbeatReporter(); mMetricsReporter = new BlockMetricsReporter(); mSessions = sessions; mBlockStore = blockStore; mWorkerId = new AtomicReference<>(-1L); mBlockStore.registerBlockStoreEventListener(mHeartbeatReporter); mBlockStore.registerBlockStoreEventListener(mMetricsReporter); mUfsManager = ufsManager; mUnderFileSystemBlockStore = new UnderFileSystemBlockStore(mBlockStore, ufsManager); Metrics.registerGauges(this); }
/** * Creates a new DefaultFileSystemWorker. * * @param blockWorker the block worker handle * @param ufsManager the ufs manager */ DefaultFileSystemWorker(BlockWorker blockWorker, UfsManager ufsManager) { super(Executors.newFixedThreadPool(3, ThreadFactoryUtils.build("file-system-worker-heartbeat-%d", true))); mWorkerId = blockWorker.getWorkerId(); mUfsManager = ufsManager; mFileDataManager = new FileDataManager(Preconditions.checkNotNull(blockWorker, "blockWorker"), RateLimiter.create(ServerConfiguration .getBytes(PropertyKey.WORKER_FILE_PERSIST_RATE_LIMIT)), mUfsManager); // Setup AbstractMasterClient mFileSystemMasterWorkerClient = new FileSystemMasterClient(MasterClientContext .newBuilder(ClientContext.create(ServerConfiguration.global())).build()); }
mClientMasterSync = new ClientMasterSync(mMetricsMasterClient, mAppId); mExecutorService = Executors.newFixedThreadPool(1, ThreadFactoryUtils.build("metrics-master-heartbeat-%d", true)); mExecutorService .submit(new HeartbeatThread(HeartbeatContext.MASTER_METRICS_SYNC, mClientMasterSync,
@Before public void before() throws Exception { mRegistry = new MasterRegistry(); mClock = new ManualClock(); mExecutorService = Executors.newFixedThreadPool(2, ThreadFactoryUtils.build("TestMetricsMaster-%d", true)); mMetricsMaster = new DefaultMetricsMaster(MasterTestUtils.testMasterContext(), mClock, ExecutorServiceFactories.constantExecutorServiceFactory(mExecutorService)); mRegistry.add(MetricsMaster.class, mMetricsMaster); mRegistry.start(true); }
/** * Sets up the dependencies before a test runs. */ @Before public void before() throws Exception { mRegistry = new MasterRegistry(); mSafeModeManager = new TestSafeModeManager(); mStartTimeMs = System.currentTimeMillis(); mPort = ServerConfiguration.getInt(PropertyKey.MASTER_RPC_PORT); mMetrics = Lists.newArrayList(); JournalSystem journalSystem = new NoopJournalSystem(); CoreMasterContext masterContext = MasterTestUtils.testMasterContext(); mMetricsMaster = new MetricsMasterFactory().create(mRegistry, masterContext); mClock = new ManualClock(); mExecutorService = Executors.newFixedThreadPool(2, ThreadFactoryUtils.build("TestBlockMaster-%d", true)); mBlockMaster = new DefaultBlockMaster(mMetricsMaster, masterContext, mClock, ExecutorServiceFactories.constantExecutorServiceFactory(mExecutorService)); mRegistry.add(BlockMaster.class, mBlockMaster); mRegistry.start(true); }
ThreadFactoryUtils.build("DailyMetadataBackup-%d", true)), mUfs); mDailyBackup.start();
PERSIST_CHECKER_POOL_THREADS, 1, java.util.concurrent.TimeUnit.MINUTES, new LinkedBlockingQueue<Runnable>(), alluxio.util.ThreadFactoryUtils.build("Persist-Checker-%d", true)); mPersistCheckerPool.allowCoreThreadTimeOut(true); getExecutorService().submit(
mBlockMaster = new BlockMasterFactory().create(mRegistry, masterContext); mExecutorService = Executors.newFixedThreadPool(4, ThreadFactoryUtils.build("DefaultFileSystemMasterTest-%d", true)); mFileSystemMaster = new DefaultFileSystemMaster(mBlockMaster, masterContext, ExecutorServiceFactories.constantExecutorServiceFactory(mExecutorService));
@Override public ExecutorService create() { return Executors.newFixedThreadPool(nThreads, ThreadFactoryUtils.build(name + "-%d", true)); } };
/** * @param args the arguments for creating the server socket */ public SocketTrackingTServerSocket(ServerSocketTransportArgs args) throws TTransportException { super(args); mExecutor = Executors .newSingleThreadScheduledExecutor(ThreadFactoryUtils.build("socket-closer-thread", true)); mExecutor.scheduleAtFixedRate(this::removeClosedSockets, CLEANUP_INTERVAL_MS, CLEANUP_INTERVAL_MS, TimeUnit.MILLISECONDS); }