private void initAndStartTezSessionPoolManager(final WMFullResourcePlan resourcePlan) { // starting Tez session pool in start here to let parent session state initialize on CliService state, to avoid // SessionState.get() return null during createTezDir try { // will be invoked anyway in TezTask. Doing it early to initialize triggers for non-pool tez session. LOG.info("Initializing tez session pool manager"); tezSessionPoolManager = TezSessionPoolManager.getInstance(); HiveConf hiveConf = getHiveConf(); if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS)) { tezSessionPoolManager.setupPool(hiveConf); } else { tezSessionPoolManager.setupNonPool(hiveConf); } tezSessionPoolManager.startPool(hiveConf, resourcePlan); LOG.info("Tez session pool manager initialized."); } catch (Exception e) { throw new ServiceException("Unable to setup tez session pool", e); } }
private static TezSessionState getUnmanagedSession( TezSessionState session, HiveConf conf, Set<String> desiredCounters, boolean isWorkLlapNode, final WmContext wmContext) throws Exception { TezSessionPoolManager pm = TezSessionPoolManager.getInstance(); session = pm.getSession(session, conf, false, isWorkLlapNode); desiredCounters.addAll(pm.getTriggerCounterNames()); wmContext.setSubscribedCounters(desiredCounters); session.setWmContext(wmContext); return session; }
public void returnToSessionManager() throws Exception { // By default, TezSessionPoolManager handles this for both pool and non-pool session. TezSessionPoolManager.getInstance().returnSession(this); }
public TezSessionState reopen() throws Exception { // By default, TezSessionPoolManager handles this for both pool and non-pool session. return TezSessionPoolManager.getInstance().reopen(this); }
public void destroy() throws Exception { // By default, TezSessionPoolManager handles this for both pool and non-pool session. TezSessionPoolManager.getInstance().destroy(this); }
poolManager.setupPool(conf); poolManager.startPool(conf, null); TezSessionState sessionState = poolManager.getSession(null, conf, true, false); assertEquals("a", sessionState.getQueueName()); poolManager.returnSession(sessionState); sessionState = poolManager.getSession(null, conf, true, false); assertEquals("a", sessionState.getQueueName()); poolManager.returnSession(sessionState); TezSessionState first = poolManager.getSession(null, conf, true, false); TezSessionState second = poolManager.getSession(null, conf, true, false); assertEquals("a", first.getQueueName()); assertEquals("b", second.getQueueName()); poolManager.returnSession(first); poolManager.returnSession(second); TezSessionState third = poolManager.getSession(null, conf, true, false); assertEquals("b", third.getQueueName()); poolManager.returnSession(third); first = poolManager.getSession(null, conf, true, false); second = poolManager.getSession(null, conf, true, false); third = poolManager.getSession(null, conf, true, false); poolManager.returnSession(first); poolManager.returnSession(second); poolManager.returnSession(third);
LOG.warn("The session: " + session + " has not been opened"); session = TezSessionPoolManager.getInstance().getSession( session, conf, false, getWork().getLlapMode()); ss.setTezSession(session); TezSessionPoolManager.getInstance() .returnSession(session, getWork().getLlapMode()); } catch (Exception e) { LOG.error("Failed to return session: {} to pool", session, e);
TezSessionPoolManager sessionPool = TezSessionPoolManager.getInstance(); sessionPool.setupPool(hiveConf); sessionPool.startPool();
public TezSessionState getSession( TezSessionState session, HiveConf conf, boolean doOpen, boolean llap) throws Exception { if (llap && (this.numConcurrentLlapQueries > 0)) { llapQueue.acquire(); // blocks if no more llap queries can be submitted. } if (canWorkWithSameSession(session, conf)) { return session; } if (session != null) { closeIfNotDefault(session, false); } return getSession(conf, doOpen); }
TezSessionPoolManager.getInstance().stop(); } catch (Exception e) { LOG.error("Tez session pool manager stop had an error during stop of HiveServer2. "
@Override public void run() { TezJobMonitor.killRunningJobs(); try { // TODO: why does this only kill non-default sessions? // Nothing for workload management since that only deals with default ones. TezSessionPoolManager.getInstance().closeNonDefaultSessions(); } catch (Exception e) { // ignore } } });
@Test public void testLlapSessionQueuing() { try { random = new Random(1000); conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_LLAP_CONCURRENT_QUERIES, 2); poolManager = new TestTezSessionPoolManager(); poolManager.setupPool(conf); poolManager.startPool(conf, null); } catch (Exception e) { LOG.error("Initialization error", e); fail(); } List<Thread> threadList = new ArrayList<Thread>(); for (int i = 0; i < 15; i++) { Thread t = new Thread(new SessionThread(true)); threadList.add(t); t.start(); } for (Thread t : threadList) { try { t.join(); } catch (InterruptedException e) { e.printStackTrace(); fail(); } } }
TezSessionPoolManager.getInstance().close(tezSessionState, false);
TezSessionPoolManager.getInstance().reopenSession( sessionState, this.conf, inputOutputJars, true); console.printInfo("Session re-established."); console.printInfo("Dag submit failed due to " + e.getMessage() + " stack trace: " + Arrays.toString(e.getStackTrace()) + " retrying..."); TezSessionPoolManager.getInstance().reopenSession(sessionState, this.conf, inputOutputJars, true); dagClient = sessionState.getSession().submitDAG(dag); } catch (Exception retryException) { TezSessionPoolManager.getInstance().destroySession(sessionState); throw retryException;
@Override public void run() { try { HiveConf tmpConf = new HiveConf(conf); if (random.nextDouble() > 0.5) { tmpConf.set("tez.queue.name", "default"); } else { tmpConf.set("tez.queue.name", ""); } TezSessionState session = poolManager.getSession(null, tmpConf, true, llap); Thread.sleep((random.nextInt(9) % 10) * 1000); session.setLegacyLlapMode(llap); poolManager.returnSession(session); } catch (Exception e) { e.printStackTrace(); } } }
&& ss != null) { TezSessionState session = ss.getTezSession(); TezSessionPoolManager.getInstance().closeIfNotDefault(session, true);
public TezSessionState getSession(TezSessionState session, HiveConf conf, boolean doOpen, boolean forceCreate) throws Exception { if (canWorkWithSameSession(session, conf)) { return session; } if (session != null) { close(session, false); } return getSession(conf, doOpen, forceCreate); }
@Override public void run() { for (DAGClient c: shutdownList) { TezJobMonitor.killRunningJobs(); } try { for (TezSessionState s : TezSessionPoolManager.getInstance().getOpenSessions()) { System.err.println("Shutting down tez session."); TezSessionPoolManager.getInstance().close(s, false); } } catch (Exception e) { // ignore } } });
DAGClient submit(JobConf conf, DAG dag, Path scratchDir, LocalResource appJarLr, TezSessionState sessionState, List<LocalResource> additionalLr, String[] inputOutputJars, Map<String,LocalResource> inputOutputLocalResources) throws Exception { perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_SUBMIT_DAG); DAGClient dagClient = null; Map<String, LocalResource> resourceMap = new HashMap<String, LocalResource>(); if (additionalLr != null) { for (LocalResource lr: additionalLr) { if (lr.getType() == LocalResourceType.FILE) { // TEZ AM will only localize FILE (no script operators in the AM) resourceMap.put(utils.getBaseName(lr), lr); } } } try { // ready to start execution on the cluster sessionState.getSession().addAppMasterLocalFiles(resourceMap); dagClient = sessionState.getSession().submitDAG(dag); } catch (SessionNotRunning nr) { console.printInfo("Tez session was closed. Reopening..."); // close the old one, but keep the tmp files around TezSessionPoolManager.getInstance().closeAndOpen(sessionState, this.conf, inputOutputJars, true); console.printInfo("Session re-established."); dagClient = sessionState.getSession().submitDAG(dag); } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_SUBMIT_DAG); return dagClient; }
Mockito.when(session.getConf()).thenReturn(conf); poolManager.reopen(session); poolManager.reopen(session); assertEquals("tezq1", poolManager.getSession(null, conf, false, false).getQueueName()); poolManager.reopen(session); assertEquals("default", poolManager.getSession(null, conf, false, false).getQueueName()); poolManager.reopen(session); assertEquals("tezq1", poolManager.getSession(null, conf, false, false).getQueueName()); poolManager.reopen(session); assertEquals("default", poolManager.getSession(null, conf, false, false).getQueueName()); poolManager.reopen(session); assertEquals("tezq1", poolManager.getSession(null, conf, false, false).getQueueName()); } catch (Exception e) { e.printStackTrace();