/** * Same as {#newDaemonThreadFactory(String, UncaughtExceptionHandler)}, * without setting the exception handler. */ public static ThreadFactory newDaemonThreadFactory(final String prefix) { return newDaemonThreadFactory(prefix, null); }
private static ExecutorService createThreadPool(Configuration conf) { int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS); return new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck")); }
@SuppressWarnings("deprecation") public IncrementCoalescer(ThriftHBaseServiceHandler hand) { this.handler = hand; LinkedBlockingQueue<Runnable> queue = new LinkedBlockingQueue<>(); pool = new ThreadPoolExecutor(CORE_POOL_SIZE, CORE_POOL_SIZE, 50, TimeUnit.MILLISECONDS, queue, Threads.newDaemonThreadFactory("IncrementCoalescer")); MBeans.register("thrift", "Thrift", this); }
private void createBatchPool(Configuration conf) { // Use the same config for keep alive as in ConnectionImplementation.getBatchPool(); int maxThreads = conf.getInt("hbase.multihconnection.threads.max", 256); if (maxThreads == 0) { maxThreads = Runtime.getRuntime().availableProcessors() * 8; } long keepAliveTime = conf.getLong("hbase.multihconnection.threads.keepalivetime", 60); LinkedBlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(maxThreads * conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS, HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS)); ThreadPoolExecutor tpe = new ThreadPoolExecutor(maxThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, workQueue, Threads.newDaemonThreadFactory("MultiHConnection" + "-shared-")); tpe.allowCoreThreadTimeOut(true); this.batchPool = tpe; }
/** * Creates a thread pool. * @param conf the Configuration * @return A thread pool. */ public static ExecutorService createMobCompactorThreadPool(Configuration conf) { int maxThreads = conf.getInt(MobConstants.MOB_COMPACTION_THREADS_MAX, MobConstants.DEFAULT_MOB_COMPACTION_THREADS_MAX); if (maxThreads == 0) { maxThreads = 1; } final SynchronousQueue<Runnable> queue = new SynchronousQueue<>(); ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, 60, TimeUnit.SECONDS, queue, Threads.newDaemonThreadFactory("MobCompactor"), new RejectedExecutionHandler() { @Override public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { try { // waiting for a thread to pick up instead of throwing exceptions. queue.put(r); } catch (InterruptedException e) { throw new RejectedExecutionException(e); } } }); ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true); return pool; }
@InterfaceAudience.Private public static ThreadPoolExecutor getDefaultExecutor(Configuration conf) { int maxThreads = conf.getInt("hbase.htable.threads.max", Integer.MAX_VALUE); if (maxThreads == 0) { maxThreads = 1; // is there a better default? } int corePoolSize = conf.getInt("hbase.htable.threads.coresize", 1); long keepAliveTime = conf.getLong("hbase.htable.threads.keepalivetime", 60); // Using the "direct handoff" approach, new threads will only be created // if it is necessary and will grow unbounded. This could be bad but in HCM // we only create as many Runnables as there are region servers. It means // it also scales when new region servers are added. ThreadPoolExecutor pool = new ThreadPoolExecutor(corePoolSize, maxThreads, keepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<>(), Threads.newDaemonThreadFactory("htable")); pool.allowCoreThreadTimeOut(true); return pool; }
/** * Returns a Thread pool for the RPC's to region replicas. Similar to * Connection's thread pool. */ private ExecutorService getDefaultThreadPool(Configuration conf) { int maxThreads = conf.getInt("hbase.region.replica.replication.threads.max", 256); if (maxThreads == 0) { maxThreads = Runtime.getRuntime().availableProcessors() * 8; } long keepAliveTime = conf.getLong("hbase.region.replica.replication.threads.keepalivetime", 60); LinkedBlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(maxThreads * conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS, HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS)); ThreadPoolExecutor tpe = new ThreadPoolExecutor( maxThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, workQueue, Threads.newDaemonThreadFactory(this.getClass().getSimpleName() + "-rpc-shared-")); tpe.allowCoreThreadTimeOut(true); return tpe; }
private ExecutorService getThreadPool(int maxThreads, int coreThreads, String nameHint, BlockingQueue<Runnable> passedWorkQueue) { // shared HTable thread executor not yet initialized if (maxThreads == 0) { maxThreads = Runtime.getRuntime().availableProcessors() * 8; } if (coreThreads == 0) { coreThreads = Runtime.getRuntime().availableProcessors() * 8; } long keepAliveTime = conf.getLong("hbase.hconnection.threads.keepalivetime", 60); BlockingQueue<Runnable> workQueue = passedWorkQueue; if (workQueue == null) { workQueue = new LinkedBlockingQueue<>(maxThreads * conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS, HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS)); coreThreads = maxThreads; } ThreadPoolExecutor tpe = new ThreadPoolExecutor( coreThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, workQueue, Threads.newDaemonThreadFactory(toString() + nameHint)); tpe.allowCoreThreadTimeOut(true); return tpe; }
public boolean start() { if (running.getAndSet(true)) { LOG.warn("Already running"); return false; } LOG.info("Instantiated, coreThreads={} (allowCoreThreadTimeOut=true), queueMaxSize={}, " + "operationDelay={}", this.corePoolSize, this.queueMaxSize, this.operationDelay); // Create the timeout executor timeoutExecutor = new TimeoutExecutorThread(); timeoutExecutor.start(); // Create the thread pool that will execute RPCs threadPool = Threads.getBoundedCachedThreadPool(corePoolSize, 60L, TimeUnit.SECONDS, Threads.newDaemonThreadFactory(this.getClass().getSimpleName(), getUncaughtExceptionHandler())); return true; }
private ExecutorService createThreadPool() { int maxThreads = 256; int coreThreads = 128; long keepAliveTime = 60; BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<Runnable>( maxThreads * HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS); ThreadPoolExecutor tpe = new ThreadPoolExecutor(coreThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, workQueue, Threads.newDaemonThreadFactory(toString() + "-shared")); tpe.allowCoreThreadTimeOut(true); return tpe; }
/** * Get a pool that has only ever one thread. A second action added to the pool (running * concurrently), will cause an exception. * @return */ private ExecutorService getPool() { int maxThreads = 1; long keepAliveTime = 60; ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, keepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<>(), Threads.newDaemonThreadFactory("hbase-table")); pool.allowCoreThreadTimeOut(true); return pool; }
synchronized void start(UncaughtExceptionHandler eh) { ThreadFactory flusherThreadFactory = Threads.newDaemonThreadFactory( server.getServerName().toShortString() + "-MemStoreFlusher", eh); for (int i = 0; i < flushHandlers.length; i++) { flushHandlers[i] = new FlushHandler("MemStoreFlusher." + i); flusherThreadFactory.newThread(flushHandlers[i]); flushHandlers[i].start(); } }
private static ExecutorService createThreadPool() { int maxThreads = 10; long keepAliveTime = 60; final SynchronousQueue<Runnable> queue = new SynchronousQueue<>(); ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, keepAliveTime, TimeUnit.SECONDS, queue, Threads.newDaemonThreadFactory("MobFileCompactionChore"), new RejectedExecutionHandler() { @Override public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { try { // waiting for a thread to pick up instead of throwing exceptions. queue.put(r); } catch (InterruptedException e) { throw new RejectedExecutionException(e); } } }); ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true); return pool; }
public static ExecutorService getCoprocessorPool() { if (coprocessorPool != null) { return coprocessorPool; } synchronized (HBaseConnection.class) { if (coprocessorPool != null) { return coprocessorPool; } KylinConfig config = KylinConfig.getInstanceFromEnv(); // copy from HConnectionImplementation.getBatchPool() int maxThreads = config.getHBaseMaxConnectionThreads(); int coreThreads = config.getHBaseCoreConnectionThreads(); long keepAliveTime = config.getHBaseConnectionThreadPoolAliveSeconds(); LinkedBlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<Runnable>(maxThreads * 100); ThreadPoolExecutor tpe = new ThreadPoolExecutor(coreThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, workQueue, // Threads.newDaemonThreadFactory("kylin-coproc-")); tpe.allowCoreThreadTimeOut(true); logger.info("Creating coprocessor thread pool with max of {}, core of {}", maxThreads, coreThreads); coprocessorPool = tpe; return coprocessorPool; } }
private static ExecutorService createThreadPool(Configuration conf) { int maxThreads = 10; long keepAliveTime = 60; final SynchronousQueue<Runnable> queue = new SynchronousQueue<>(); ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, keepAliveTime, TimeUnit.SECONDS, queue, Threads.newDaemonThreadFactory("MobFileCompactionChore"), new RejectedExecutionHandler() { @Override public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { try { // waiting for a thread to pick up instead of throwing exceptions. queue.put(r); } catch (InterruptedException e) { throw new RejectedExecutionException(e); } } }); pool.allowCoreThreadTimeOut(true); return pool; }
@BeforeClass public static void setUp() throws Exception { threadPool = Threads.getBoundedCachedThreadPool(32, 60L, TimeUnit.SECONDS, Threads.newDaemonThreadFactory("ProcedureDispatcher", new UncaughtExceptionHandler() { @Override public void uncaughtException(Thread t, Throwable e) { LOG.warn("Failed thread " + t.getName(), e); } })); executorService = new ExecutorCompletionService(threadPool); }
@BeforeClass public static void setUpBeforeClass() throws Exception { GROUP = new NioEventLoopGroup(1, Threads.newDaemonThreadFactory("TestAsyncWALReplay")); CHANNEL_CLASS = NioSocketChannel.class; Configuration conf = AbstractTestWALReplay.TEST_UTIL.getConfiguration(); conf.set(WALFactory.WAL_PROVIDER, "asyncfs"); AbstractTestWALReplay.setUpBeforeClass(); }
@BeforeClass public static void setUpBeforeClass() throws Exception { GROUP = new NioEventLoopGroup(1, Threads.newDaemonThreadFactory("TestAsyncFSWAL")); CHANNEL_CLASS = NioSocketChannel.class; AbstractTestFSWAL.setUpBeforeClass(); }
@BeforeClass public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncCoprocessor.class.getName()); conf.setInt("hbase.regionserver.handler.count", 2); conf.setInt("hbase.regionserver.metahandler.count", 30); conf.setInt("hbase.htable.threads.max", POOL_SIZE); conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE); conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT); conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT); TEST_UTIL.startMiniCluster(1); tableExecutorService = new ThreadPoolExecutor(1, POOL_SIZE, 60, TimeUnit.SECONDS, new SynchronousQueue<>(), Threads.newDaemonThreadFactory("testhbck")); hbfsckExecutorService = new ScheduledThreadPoolExecutor(POOL_SIZE); AssignmentManager assignmentManager = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(); regionStates = assignmentManager.getRegionStates(); connection = (ClusterConnection) TEST_UTIL.getConnection(); admin = connection.getAdmin(); admin.setBalancerRunning(false, true); TEST_UTIL.waitUntilAllRegionsAssigned(TableName.META_TABLE_NAME); }
AtomicBoolean stop = new AtomicBoolean(false); ExecutorService executor = Executors.newFixedThreadPool(numThreads, Threads.newDaemonThreadFactory("TestAsyncGet-")); List<Future<?>> futures = new ArrayList<>(); IntStream.range(0, numThreads).forEach(i -> futures.add(executor.submit(() -> {