@Override public void setupPool(HiveConf conf) throws Exception { conf.setVar(ConfVars.LLAP_TASK_SCHEDULER_AM_REGISTRY_NAME, ""); super.setupPool(conf); }
private void initAndStartTezSessionPoolManager(final WMFullResourcePlan resourcePlan) { // starting Tez session pool in start here to let parent session state initialize on CliService state, to avoid // SessionState.get() return null during createTezDir try { // will be invoked anyway in TezTask. Doing it early to initialize triggers for non-pool tez session. LOG.info("Initializing tez session pool manager"); tezSessionPoolManager = TezSessionPoolManager.getInstance(); HiveConf hiveConf = getHiveConf(); if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS)) { tezSessionPoolManager.setupPool(hiveConf); } else { tezSessionPoolManager.setupNonPool(hiveConf); } tezSessionPoolManager.startPool(hiveConf, resourcePlan); LOG.info("Tez session pool manager initialized."); } catch (Exception e) { throw new ServiceException("Unable to setup tez session pool", e); } }
@Test public void testLlapSessionQueuing() { try { random = new Random(1000); conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_LLAP_CONCURRENT_QUERIES, 2); poolManager = new TestTezSessionPoolManager(); poolManager.setupPool(conf); poolManager.startPool(conf, null); } catch (Exception e) { LOG.error("Initialization error", e); fail(); } List<Thread> threadList = new ArrayList<Thread>(); for (int i = 0; i < 15; i++) { Thread t = new Thread(new SessionThread(true)); threadList.add(t); t.start(); } for (Thread t : threadList) { try { t.join(); } catch (InterruptedException e) { e.printStackTrace(); fail(); } } }
@Test public void testReturn() { conf.set("tez.queue.name", ""); random = new Random(1000); conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_DEFAULT_QUEUES, "a,b,c"); conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE, 2); try { poolManager = new TestTezSessionPoolManager(); poolManager.setupPool(conf); poolManager.startPool(conf, null); } catch (Exception e) { e.printStackTrace(); fail(); } List<Thread> threadList = new ArrayList<Thread>(); for (int i = 0; i < 15; i++) { Thread t = new Thread(new SessionThread(false)); threadList.add(t); t.start(); } for (Thread t : threadList) { try { t.join(); } catch (InterruptedException e) { e.printStackTrace(); fail(); } } }
@Test public void testSessionPoolThreads() { // Make sure we get a correct number of sessions in each queue and that we don't crash. try { conf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); conf.setVar(ConfVars.HIVE_SERVER2_TEZ_DEFAULT_QUEUES, "0,1,2"); conf.setIntVar(ConfVars.HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE, 4); conf.setIntVar(ConfVars.HIVE_SERVER2_TEZ_SESSION_MAX_INIT_THREADS, 16); poolManager = new TestTezSessionPoolManager(); poolManager.setupPool(conf); poolManager.startPool(conf, null); TezSessionState[] sessions = new TezSessionState[12]; int[] queueCounts = new int[3]; for (int i = 0; i < sessions.length; ++i) { sessions[i] = poolManager.getSession(null, conf, true, false); queueCounts[Integer.parseInt(sessions[i].getQueueName())] += 1; } for (int i = 0; i < queueCounts.length; ++i) { assertEquals(4, queueCounts[i]); } for (int i = 0; i < sessions.length; ++i) { poolManager.returnSession(sessions[i]); } } catch (Exception e) { e.printStackTrace(); fail(); } }
poolManager.setupPool(conf); poolManager.startPool(conf, null);
private void initAndStartTezSessionPoolManager(final WMFullResourcePlan resourcePlan) { // starting Tez session pool in start here to let parent session state initialize on CliService state, to avoid // SessionState.get() return null during createTezDir try { // will be invoked anyway in TezTask. Doing it early to initialize triggers for non-pool tez session. LOG.info("Initializing tez session pool manager"); tezSessionPoolManager = TezSessionPoolManager.getInstance(); HiveConf hiveConf = getHiveConf(); if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS)) { tezSessionPoolManager.setupPool(hiveConf); } else { tezSessionPoolManager.setupNonPool(hiveConf); } tezSessionPoolManager.startPool(hiveConf, resourcePlan); LOG.info("Tez session pool manager initialized."); } catch (Exception e) { throw new ServiceException("Unable to setup tez session pool", e); } }
sessionPool.setupPool(hiveConf); sessionPool.startPool();
sessionPool.setupPool(hiveConf); sessionPool.startPool();