private TezClient startSessionAndContainers(TezClient session, HiveConf conf, Map<String, LocalResource> commonLocalResources, TezConfiguration tezConfig, boolean isOnThread) throws TezException, IOException { session.start(); boolean isSuccessful = false; try {
private TezClient startSessionAndContainers(TezClient session, HiveConf conf, Map<String, LocalResource> commonLocalResources, TezConfiguration tezConfig, boolean isOnThread) throws TezException, IOException { session.start(); boolean isSuccessful = false; try {
private TezClient createTezClient(TezConfiguration tezConf) throws IOException, TezException { TezClient tezClient = TezClient.create(getClass().getSimpleName(), tezConf); tezClient.start(); return tezClient; }
tezSession.start(); try { for (; numDags > 0; --numDags) {
tezSession.start();
tezSession.start();
private static SessionInfo createSession(TezConfiguration amConf, Map<String, LocalResource> requestedAMResources, Credentials creds, TezJobConfig tezJobConf) throws TezException, IOException, InterruptedException { MRToTezHelper.translateMRSettingsForTezAM(amConf); TezScriptState ss = TezScriptState.get(); ss.addDAGSettingsToConf(amConf); if (amConf.getBoolean(PigConfiguration.PIG_TEZ_CONFIGURE_AM_MEMORY, true)) { adjustAMConfig(amConf, tezJobConf); } String jobName = amConf.get(PigContext.JOB_NAME, "pig"); TezClient tezClient = TezClient.create(jobName, amConf, true, requestedAMResources, creds); try { tezClient.start(); TezAppMasterStatus appMasterStatus = tezClient.getAppMasterStatus(); if (appMasterStatus.equals(TezAppMasterStatus.SHUTDOWN)) { throw new RuntimeException("TezSession has already shutdown"); } tezClient.waitTillReady(); } catch (Throwable e) { log.error("Exception while waiting for Tez client to be ready", e); tezClient.stop(); throw new RuntimeException(e); } return new SessionInfo(tezClient, amConf, requestedAMResources); }
tezSession.start(); if (tezSession.getAppMasterApplicationId() != null) { TezUtilsInternal.setHadoopCallerContext(hadoopShim, tezSession.getAppMasterApplicationId());
tezSession.start(); // Why do I need to start the TezSession.
tezSession.start(); // Why do I need to start the TezSession.
protected void internalNonBlockingStart() throws IOException { try { if( !isTimelineServiceEnabled( jobConfiguration ) ) flowStep.logWarn( "'" + YarnConfiguration.TIMELINE_SERVICE_ENABLED + "' is disabled, please enable to capture detailed metrics of completed flows, this may require starting the YARN timeline server daemon" ); TezConfiguration workingConf = new TezConfiguration( jobConfiguration ); // this could be problematic flowStep.logInfo( "tez session mode enabled: " + workingConf.getBoolean( TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT ) ); prepareEnsureStagingDir( workingConf ); tezClient = TezClient.create( flowStep.getName(), workingConf, ( (Hadoop2TezFlowStep) flowStep ).getAllLocalResources(), null ); tezClient.start(); dagClient = tezClient.submitDAG( dag ); dagId = Util.returnInstanceFieldIfExistsSafe( dagClient, "dagId" ); flowStep.logInfo( "submitted tez dag to app master: {}, with dag id: {}", tezClient.getAppMasterApplicationId(), dagId ); } catch( TezException exception ) { this.throwable = exception; throw new CascadingException( exception ); } }
protected void internalNonBlockingStart() throws IOException { try { if( !isTimelineServiceEnabled( jobConfiguration ) ) flowStep.logWarn( "'" + YarnConfiguration.TIMELINE_SERVICE_ENABLED + "' is disabled, please enable to capture detailed metrics of completed flows, this may require starting the YARN timeline server daemon" ); TezConfiguration workingConf = new TezConfiguration( jobConfiguration ); // this could be problematic flowStep.logInfo( "tez session mode enabled: " + workingConf.getBoolean( TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT ) ); prepareEnsureStagingDir( workingConf ); tezClient = TezClient.create( flowStep.getName(), workingConf, ( (Hadoop2TezFlowStep) flowStep ).getAllLocalResources(), null ); tezClient.start(); dagClient = tezClient.submitDAG( dag ); dagId = Util.returnInstanceFieldIfExistsSafe( dagClient, "dagId" ); flowStep.logInfo( "submitted tez dag to app master: {}, with dag id: {}", tezClient.getAppMasterApplicationId(), dagId ); } catch( TezException exception ) { this.throwable = exception; throw new CascadingException( exception ); } }