private TezClient createTezClient(TezConfiguration tezConf) throws IOException, TezException { TezClient tezClient = TezClient.create(getClass().getSimpleName(), tezConf); tezClient.start(); return tezClient; }
iReduceSleepTime, iReduceSleepCount, writeSplitsToDfs, generateSplitsInAM); TezClient tezSession = TezClient.create("MRRSleep", conf, isSession, null, credentials); tezSession.start(); try {
TezClient tezSession = TezClient.create("UnionExampleSession", tezConf); tezSession.start();
tezSession = TezClient.create("broadcastAndOneToOneExample", tezConf); tezSession.start();
private static SessionInfo createSession(TezConfiguration amConf, Map<String, LocalResource> requestedAMResources, Credentials creds, TezJobConfig tezJobConf) throws TezException, IOException, InterruptedException { MRToTezHelper.translateMRSettingsForTezAM(amConf); TezScriptState ss = TezScriptState.get(); ss.addDAGSettingsToConf(amConf); if (amConf.getBoolean(PigConfiguration.PIG_TEZ_CONFIGURE_AM_MEMORY, true)) { adjustAMConfig(amConf, tezJobConf); } String jobName = amConf.get(PigContext.JOB_NAME, "pig"); TezClient tezClient = TezClient.create(jobName, amConf, true, requestedAMResources, creds); try { tezClient.start(); TezAppMasterStatus appMasterStatus = tezClient.getAppMasterStatus(); if (appMasterStatus.equals(TezAppMasterStatus.SHUTDOWN)) { throw new RuntimeException("TezSession has already shutdown"); } tezClient.waitTillReady(); } catch (Throwable e) { log.error("Exception while waiting for Tez client to be ready", e); tezClient.stop(); throw new RuntimeException(e); } return new SessionInfo(tezClient, amConf, requestedAMResources); }
tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false); TezClient tezSession = TezClient.create("OrderedWordCountSession", tezConf, null, instance.credentials); tezSession.start();
TezClient tezSession = TezClient.create("FilterLinesByWordSession", tezConf, commonLocalResources, null);
session = TezClient.create("HIVE-" + sessionId, tezConfig, true, commonLocalResources, null);
TezClient tezSession = TezClient.create("FilterLinesByWordSession", tezConf, commonLocalResources, credentials);
protected void internalNonBlockingStart() throws IOException { try { if( !isTimelineServiceEnabled( jobConfiguration ) ) flowStep.logWarn( "'" + YarnConfiguration.TIMELINE_SERVICE_ENABLED + "' is disabled, please enable to capture detailed metrics of completed flows, this may require starting the YARN timeline server daemon" ); TezConfiguration workingConf = new TezConfiguration( jobConfiguration ); // this could be problematic flowStep.logInfo( "tez session mode enabled: " + workingConf.getBoolean( TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT ) ); prepareEnsureStagingDir( workingConf ); tezClient = TezClient.create( flowStep.getName(), workingConf, ( (Hadoop2TezFlowStep) flowStep ).getAllLocalResources(), null ); tezClient.start(); dagClient = tezClient.submitDAG( dag ); dagId = Util.returnInstanceFieldIfExistsSafe( dagClient, "dagId" ); flowStep.logInfo( "submitted tez dag to app master: {}, with dag id: {}", tezClient.getAppMasterApplicationId(), dagId ); } catch( TezException exception ) { this.throwable = exception; throw new CascadingException( exception ); } }
protected void internalNonBlockingStart() throws IOException { try { if( !isTimelineServiceEnabled( jobConfiguration ) ) flowStep.logWarn( "'" + YarnConfiguration.TIMELINE_SERVICE_ENABLED + "' is disabled, please enable to capture detailed metrics of completed flows, this may require starting the YARN timeline server daemon" ); TezConfiguration workingConf = new TezConfiguration( jobConfiguration ); // this could be problematic flowStep.logInfo( "tez session mode enabled: " + workingConf.getBoolean( TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT ) ); prepareEnsureStagingDir( workingConf ); tezClient = TezClient.create( flowStep.getName(), workingConf, ( (Hadoop2TezFlowStep) flowStep ).getAllLocalResources(), null ); tezClient.start(); dagClient = tezClient.submitDAG( dag ); dagId = Util.returnInstanceFieldIfExistsSafe( dagClient, "dagId" ); flowStep.logInfo( "submitted tez dag to app master: {}, with dag id: {}", tezClient.getAppMasterApplicationId(), dagId ); } catch( TezException exception ) { this.throwable = exception; throw new CascadingException( exception ); } }