private void closeClient(TezClient client) throws TezException, IOException { try { client.stop(); } catch (SessionNotRunning nr) { // ignore } }
/** * Ensures that the Tez Session is open and the AM has all necessary jars configured. */ void updateSession(TezSessionState session, JobConf jobConf, Path scratchDir, String[] inputOutputJars, Map<String,LocalResource> extraResources) throws Exception { final boolean missingLocalResources = !session .hasResources(inputOutputJars); TezClient client = session.getSession(); // TODO null can also mean that this operation was interrupted. Should we really try to re-create the session in that case ? if (client == null) { // can happen if the user sets the tez flag after the session was // established LOG.info("Tez session hasn't been created yet. Opening session"); session.open(conf, inputOutputJars); } else { LOG.info("Session is already open"); // Ensure the open session has the necessary resources (StorageHandler) if (missingLocalResources) { LOG.info("Tez session missing resources," + " adding additional necessary resources"); client.addAppMasterLocalFiles(extraResources); } session.refreshLocalResourcesFromConf(conf); } }
private TezClient startSessionAndContainers(TezClient session, HiveConf conf, Map<String, LocalResource> commonLocalResources, TezConfiguration tezConfig, boolean isOnThread) throws TezException, IOException { session.start(); boolean isSuccessful = false; try { tezConfig, n, commonLocalResources); try { session.preWarm(prewarmVertex); } catch (IOException ie) { if (!isOnThread && ie.getMessage().contains("Interrupted while waiting")) { session.waitTillReady(); } catch (InterruptedException ie) { if (isOnThread) {
try { sessionState.getSession().addAppMasterLocalFiles(resourceMap); dagClient = sessionState.getSession().submitDAG(dag); } catch (SessionNotRunning nr) { console.printInfo("Tez session was closed. Reopening..."); console.printInfo("Session re-established."); dagClient = sessionState.getSession().submitDAG(dag); TezSessionPoolManager.getInstance().reopenSession(sessionState, this.conf, inputOutputJars, true); dagClient = sessionState.getSession().submitDAG(dag); } catch (Exception retryException) {
@Test public void testSubmit() throws Exception { DAG dag = DAG.create("test"); task.submit(conf, dag, Ref.from(sessionState)); // validate close/reopen verify(sessionState, times(1)).reopen(); verify(session, times(2)).submitDAG(any(DAG.class)); }
final TezClient session = TezClient.newBuilder("HIVE-" + sessionId, tezConfig) .setIsSession(true).setLocalResources(commonLocalResources) .setCredentials(llapCredentials).setServicePluginDescriptor(servicePluginsDescriptor) .build();
CallerContext callerContext = CallerContext.create( "HIVE", queryPlan.getQueryId(), "HIVE_QUERY_ID", queryPlan.getQueryStr());
/** * Create a new TezClient. Session or non-session execution mode will be * inferred from configuration. * @param name * Name of the client. Used for logging etc. This will also be used * as app master name is session mode * @param tezConf * Configuration for the framework */ public static TezClient create(String name, TezConfiguration tezConf) { return new TezClient(name, tezConf); }
@Override public void run() { proxy = sendAMHeartbeat(proxy); } }, pollPeriod, pollPeriod, TimeUnit.MILLISECONDS);
private TezClient startSessionAndContainers(TezClient session, HiveConf conf, Map<String, LocalResource> commonLocalResources, TezConfiguration tezConfig, boolean isOnThread) throws TezException, IOException { session.start(); boolean isSuccessful = false; try { tezConfig, n, commonLocalResources); try { session.preWarm(prewarmVertex); } catch (IOException ie) { if (!isOnThread && ie.getMessage().contains("Interrupted while waiting")) { session.waitTillReady(); } catch (InterruptedException ie) { if (isOnThread) throw new IOException(ie);
private void closeClient(TezClient client) throws TezException, IOException { try { client.stop(); } catch (SessionNotRunning nr) { // ignore } }
try { dagClient = sessionState.getSession().submitDAG(dag); } catch (SessionNotRunning nr) { console.printInfo("Tez session was closed. Reopening..."); sessionStateRef.value = sessionState = getNewTezSessionOnError(sessionState); console.printInfo("Session re-established."); dagClient = sessionState.getSession().submitDAG(dag); sessionStateRef.value = null; sessionStateRef.value = sessionState = getNewTezSessionOnError(sessionState); dagClient = sessionState.getSession().submitDAG(dag); } catch (Exception retryException) {
final TezClient session = TezClient.newBuilder("HIVE-" + sessionId, tezConfig) .setIsSession(true).setLocalResources(commonLocalResources) .setCredentials(llapCredentials).setServicePluginDescriptor(servicePluginsDescriptor) .build();
CallerContext callerContext = CallerContext.create( "HIVE", queryPlan.getQueryId(), "HIVE_QUERY_ID", queryPlan.getQueryStr());
session.addAppMasterLocalFiles(DagUtils.createTezLrMap(null, newResources)); session.addAppMasterLocalFiles( DagUtils.getResourcesUpdatableForAm(resources.localizedResources));
/** * Build the actual instance of the {@link TezClient} * @return an instance of {@link TezClient} */ public TezClient build() { return new TezClient(name, tezConf, isSession, localResourceMap, credentials, servicePluginsDescriptor); } }
private static void closeAndIgnoreExceptions(TezClient session) { try { session.stop(); } catch (SessionNotRunning nr) { // Ignore. } catch (IOException | TezException ex) { LOG.info("Failed to close Tez session after failure to initialize: " + ex.getMessage()); } }
when(sessionState.getSession()).thenReturn(session); when(sessionState.reopen()).thenReturn(sessionState); when(session.submitDAG(any(DAG.class))) .thenThrow(new SessionNotRunning("")) .thenReturn(mock(DAGClient.class));
private static void closeAndIgnoreExceptions(TezClient session) { try { session.stop(); } catch (SessionNotRunning nr) { // Ignore. } catch (IOException | TezException ex) { LOG.info("Failed to close Tez session after failure to initialize: " + ex.getMessage()); } }
@Override public Boolean call() throws Exception { tezClient.stop(); return true; } }, 5, TimeUnit.MINUTES );