try { dagClient = sessionState.getSession().submitDAG(dag); } catch (SessionNotRunning nr) { console.printInfo("Tez session was closed. Reopening..."); sessionStateRef.value = sessionState = getNewTezSessionOnError(sessionState); console.printInfo("Session re-established."); dagClient = sessionState.getSession().submitDAG(dag); sessionStateRef.value = null; sessionStateRef.value = sessionState = getNewTezSessionOnError(sessionState); dagClient = sessionState.getSession().submitDAG(dag); } catch (Exception retryException) {
@Test public void testSubmit() throws Exception { DAG dag = DAG.create("test"); task.submit(conf, dag, Ref.from(sessionState)); // validate close/reopen verify(sessionState, times(1)).reopen(); verify(session, times(2)).submitDAG(any(DAG.class)); }
dagClient = sessionState.getSession().submitDAG(dag); } catch (SessionNotRunning nr) { console.printInfo("Tez session was closed. Reopening..."); console.printInfo("Session re-established."); dagClient = sessionState.getSession().submitDAG(dag); TezSessionPoolManager.getInstance().reopenSession(sessionState, this.conf, inputOutputJars, true); dagClient = sessionState.getSession().submitDAG(dag); } catch (Exception retryException) {
when(sessionState.getSession()).thenReturn(session); when(sessionState.reopen()).thenReturn(sessionState); when(session.submitDAG(any(DAG.class))) .thenThrow(new SessionNotRunning("")) .thenReturn(mock(DAGClient.class));
DAGClient dagClient = tezClient.submitDAG(dag); Set<StatusGetOpts> getOpts = Sets.newHashSet(); if (isCountersLog()) {
dagClient = tezSession.submitDAG(dag);
try { for (; numDags > 0; --numDags) { DAGClient dagClient = tezSession.submitDAG(dag); dagClient.waitForCompletion(); if (!dagClient.getDAGStatus(null).getState().equals(DAGStatus.State.SUCCEEDED)) {
DAGClient submit(JobConf conf, DAG dag, Path scratchDir, LocalResource appJarLr, TezSessionState sessionState, List<LocalResource> additionalLr, String[] inputOutputJars, Map<String,LocalResource> inputOutputLocalResources) throws Exception { perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_SUBMIT_DAG); DAGClient dagClient = null; Map<String, LocalResource> resourceMap = new HashMap<String, LocalResource>(); if (additionalLr != null) { for (LocalResource lr: additionalLr) { if (lr.getType() == LocalResourceType.FILE) { // TEZ AM will only localize FILE (no script operators in the AM) resourceMap.put(utils.getBaseName(lr), lr); } } } try { // ready to start execution on the cluster sessionState.getSession().addAppMasterLocalFiles(resourceMap); dagClient = sessionState.getSession().submitDAG(dag); } catch (SessionNotRunning nr) { console.printInfo("Tez session was closed. Reopening..."); // close the old one, but keep the tmp files around TezSessionPoolManager.getInstance().closeAndOpen(sessionState, this.conf, inputOutputJars, true); console.printInfo("Session re-established."); dagClient = sessionState.getSession().submitDAG(dag); } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_SUBMIT_DAG); return dagClient; }
dag.getCredentials(), tezJobConf); log.info("Submitting DAG " + dag.getName()); dagClient = tezClient.submitDAG(dag); appId = tezClient.getAppMasterApplicationId(); log.info("Submitted DAG " + dag.getName() + ". Application id: " + appId);
dagClient = tezSession.submitDAG(dag);
submitDAG(dag); } else { throw new SessionNotReady("Tez AM not ready, could not submit DAG");
waitForTezSessionReady(tezSession); LOG.info("Submitting DAG to Tez Session, dagIndex=" + dagIndex); dagClient = tezSession.submitDAG(dag); LOG.info("Submitted DAG to Tez Session, dagIndex=" + dagIndex); } else { LOG.info("Submitting DAG as a new Tez Application"); dagClient = tezSession.submitDAG(dag);
DAGClient dagClient = tezSession.submitDAG(dag); LOG.info("Submitted DAG to Tez Session");
DAGClient dagClient = tezClientInternal.submitDAG(dag); Set<StatusGetOpts> getOpts = Sets.newHashSet(); if (printCounters) {
DAGClient dagClient = tezSession.submitDAG(dag); LOG.info("Submitted DAG to Tez Session");
protected void internalNonBlockingStart() throws IOException { try { if( !isTimelineServiceEnabled( jobConfiguration ) ) flowStep.logWarn( "'" + YarnConfiguration.TIMELINE_SERVICE_ENABLED + "' is disabled, please enable to capture detailed metrics of completed flows, this may require starting the YARN timeline server daemon" ); TezConfiguration workingConf = new TezConfiguration( jobConfiguration ); // this could be problematic flowStep.logInfo( "tez session mode enabled: " + workingConf.getBoolean( TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT ) ); prepareEnsureStagingDir( workingConf ); tezClient = TezClient.create( flowStep.getName(), workingConf, ( (Hadoop2TezFlowStep) flowStep ).getAllLocalResources(), null ); tezClient.start(); dagClient = tezClient.submitDAG( dag ); dagId = Util.returnInstanceFieldIfExistsSafe( dagClient, "dagId" ); flowStep.logInfo( "submitted tez dag to app master: {}, with dag id: {}", tezClient.getAppMasterApplicationId(), dagId ); } catch( TezException exception ) { this.throwable = exception; throw new CascadingException( exception ); } }
protected void internalNonBlockingStart() throws IOException { try { if( !isTimelineServiceEnabled( jobConfiguration ) ) flowStep.logWarn( "'" + YarnConfiguration.TIMELINE_SERVICE_ENABLED + "' is disabled, please enable to capture detailed metrics of completed flows, this may require starting the YARN timeline server daemon" ); TezConfiguration workingConf = new TezConfiguration( jobConfiguration ); // this could be problematic flowStep.logInfo( "tez session mode enabled: " + workingConf.getBoolean( TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT ) ); prepareEnsureStagingDir( workingConf ); tezClient = TezClient.create( flowStep.getName(), workingConf, ( (Hadoop2TezFlowStep) flowStep ).getAllLocalResources(), null ); tezClient.start(); dagClient = tezClient.submitDAG( dag ); dagId = Util.returnInstanceFieldIfExistsSafe( dagClient, "dagId" ); flowStep.logInfo( "submitted tez dag to app master: {}, with dag id: {}", tezClient.getAppMasterApplicationId(), dagId ); } catch( TezException exception ) { this.throwable = exception; throw new CascadingException( exception ); } }