AMConfiguration(TezConfiguration tezConf, Map<String, LocalResource> localResources, Credentials credentials) { this.tezConf = tezConf; if (localResources != null) { addAMLocalResources(localResources); } if (credentials != null) { setCredentials(credentials); } }
/** * If the next DAG App Master needs different local files, then use this * method to clear the local files and then add the new local files * using {@link #addAppMasterLocalFiles(Map)}. This method is a no-op in session mode, * after start() is called. */ public synchronized void clearAppMasterLocalFiles() { amConfig.clearAMLocalResources(); }
static DAGPlan prepareAndCreateDAGPlan(DAG dag, AMConfiguration amConfig, Map<String, LocalResource> tezJarResources, boolean tezLrsAsArchive, Credentials credentials, ServicePluginsDescriptor servicePluginsDescriptor, JavaOptsChecker javaOptsChecker) throws IOException { Credentials dagCredentials = setupDAGCredentials(dag, credentials, amConfig.getTezConfiguration()); return dag.createDag(amConfig.getTezConfiguration(), dagCredentials, tezJarResources, amConfig.getBinaryConfLR(), tezLrsAsArchive, servicePluginsDescriptor, javaOptsChecker); }
TezConfiguration conf = amConfig.getTezConfiguration(); amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB_DEFAULT)); capability.setVirtualCores( amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES_DEFAULT)); if (LOG.isDebugEnabled()) { if (amConfig.getCredentials() != null) { amLaunchCredentials.addAll(amConfig.getCredentials()); String amOpts = constructAMLaunchOpts(amConfig.getTezConfiguration(), capability); vargs.add(amOpts); String amLogLevelString = amConfig.getTezConfiguration().get( TezConfiguration.TEZ_AM_LOG_LEVEL, TezConfiguration.TEZ_AM_LOG_LEVEL_DEFAULT); if (amConfig.getAMLocalResources() != null) { amLocalResources.putAll(amConfig.getAMLocalResources()); TezConfiguration tezConf = amConfig.getTezConfiguration(); tezConf = new TezConfiguration(amConfig.getTezConfiguration()); dag.getDagAccessControls().mergeIntoAmAcls(tezConf); binaryConfPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION);
amConfig.setYarnConfiguration(new YarnConfiguration(amConfig.getTezConfiguration())); frameworkClient.init(amConfig.getTezConfiguration(), amConfig.getYarnConfiguration()); frameworkClient.start(); if (this.amConfig.getTezConfiguration().getBoolean( TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED, TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED_DEFAULT)) { String javaOptsCheckerClassName = this.amConfig.getTezConfiguration().get( TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_CLASS, ""); if (!javaOptsCheckerClassName.isEmpty()) { LOG.info("Session mode. Starting session."); TezClientUtils.processTezLocalCredentialsFile(sessionCredentials, amConfig.getTezConfiguration()); clientTimeout = amConfig.getTezConfiguration().getInt( TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS, TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT); if (!amConfig.getTezConfiguration().getBoolean( TezConfiguration.DAG_RECOVERY_ENABLED, TezConfiguration.DAG_RECOVERY_ENABLED_DEFAULT)) { TezCommonUtils.getAMClientHeartBeatTimeoutMillis(amConfig.getTezConfiguration()); getAMClientHeartBeatPollIntervalMillis(amConfig.getTezConfiguration(), amClientKeepAliveTimeoutIntervalMillis, 10);
Credentials credentials = amConfig.getCredentials(); if (credentials == null) { credentials = new Credentials(); amConfig.getTezConfiguration()); return getDAGClient(appId, amConfig.getTezConfiguration(), amConfig.getYarnConfiguration(), frameworkClient);
private synchronized Map<String, LocalResource> getTezJarResources(Credentials credentials) throws IOException { if (cachedTezJarResources == null) { cachedTezJarResources = new HashMap<String, LocalResource>(); usingTezArchiveDeploy = TezClientUtils.setupTezJarsLocalResources( amConfig.getTezConfiguration(), credentials, cachedTezJarResources); } return cachedTezJarResources; }
Path dagPlanPath = new Path(TezCommonUtils.getTezSystemStagingPath(amConfig.getTezConfiguration(), sessionAppId.toString()), TezConstants.TEZ_PB_PLAN_BINARY_NAME + serializedSubmitDAGPlanRequestCounter.incrementAndGet()); + ", dag plan size=" + request.getSerializedSize() + ", max dag plan size through IPC=" + maxSubmitDAGRequestSizeThroughIPC + ", max IPC message size= " + amConfig.getTezConfiguration().getInt( CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT)); request.writeTo(fsDataOutputStream); + ", dagName=" + dag.getName()); return new DAGClientImpl(sessionAppId, dagId, amConfig.getTezConfiguration(), amConfig.getYarnConfiguration(), frameworkClient);
@VisibleForTesting // for testing protected DAGClientAMProtocolBlockingPB getAMProxy(ApplicationId appId) throws TezException, IOException { return TezClientUtils.getAMProxy( frameworkClient, amConfig.getYarnConfiguration(), appId); }
/** * Add local files for the DAG App Master. These may be files, archives, * jars etc.<br> * <p> * In non-session mode these will be added to the files of the App Master * to be launched for the next DAG. Files added via this method will * accumulate and be used for every new App Master until * {@link #clearAppMasterLocalFiles()} is invoked. <br> * <p> * In session mode, the recommended usage is to add all files before * calling start() so that all needed files are available to the app * master before it starts. When called after start(), these local files * will be re-localized to the running session DAG App Master and will be * added to its classpath for execution of this DAG. * <p> * Caveats for invoking this method after start() in session mode: files * accumulate across DAG submissions and are never removed from the classpath. * Only LocalResourceType.FILE is supported. All files will be treated as * private. * * @param localFiles the files to be made available in the AM */ public synchronized void addAppMasterLocalFiles(Map<String, LocalResource> localFiles) { Preconditions.checkNotNull(localFiles); if (isSession && sessionStarted.get()) { additionalLocalResources.putAll(localFiles); } amConfig.addAMLocalResources(localFiles); }
@Private protected TezClient(String name, TezConfiguration tezConf, boolean isSession, @Nullable Map<String, LocalResource> localResources, @Nullable Credentials credentials, ServicePluginsDescriptor servicePluginsDescriptor) { this.clientName = name; this.isSession = isSession; // Set in conf for local mode AM to figure out whether in session mode or not tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, isSession); try { InetAddress ip = InetAddress.getLocalHost(); if (ip != null) { tezConf.set(TezConfigurationConstants.TEZ_SUBMIT_HOST, ip.getCanonicalHostName()); tezConf.set(TezConfigurationConstants.TEZ_SUBMIT_HOST_ADDRESS, ip.getHostAddress()); } } catch (UnknownHostException e) { LOG.warn("The host name of the client the tez application was submitted from was unable to be retrieved", e); } this.amConfig = new AMConfiguration(tezConf, localResources, credentials); this.apiVersionInfo = new TezApiVersionInfo(); this.servicePluginsDescriptor = servicePluginsDescriptor; this.maxSubmitDAGRequestSizeThroughIPC = tezConf.getInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT) - tezConf.getInt(TezConfiguration.TEZ_IPC_PAYLOAD_RESERVED_BYTES, TezConfiguration.TEZ_IPC_PAYLOAD_RESERVED_BYTES_DEFAULT); Limits.setConfiguration(tezConf); LOG.info("Tez Client Version: " + apiVersionInfo.toString()); }
/** * Sets the history log level for this session. It will be in effect for DAGs submitted after this * call. * * @param historyLogLevel The log level to be used. */ public synchronized void setHistoryLogLevel(HistoryLogLevel historyLogLevel) { amConfig.getTezConfiguration().setEnum(TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL, historyLogLevel); }
@VisibleForTesting // for testing @Private protected FrameworkClient createFrameworkClient() { return FrameworkClient.createFrameworkClient(amConfig.getTezConfiguration()); }
@VisibleForTesting // this method will set the app priority only if the priority config has been defined public static void setApplicationPriority(ApplicationSubmissionContext context, AMConfiguration amConfig) { if (amConfig.getTezConfiguration().get(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY) != null) { // since we already checked not null condition, we are guaranteed that default value // (0 in this case for getInt) will not be returned/used. // The idea is to not use any default priority from TEZ side, if none provided in config; // let YARN determine the priority of the submitted application int priority = amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY, 0); context.setPriority(Priority.newInstance(priority)); if (LOG.isDebugEnabled()) { LOG.debug("Settting TEZ application priority, applicationId= " + context.getApplicationId() + ", priority= " + context.getPriority().getPriority()); } } }
proxy.shutdownSession(null, request); sessionShutdownSuccessful = true; boolean asynchronousStop = amConfig.getTezConfiguration().getBoolean( TezConfiguration.TEZ_CLIENT_ASYNCHRONOUS_STOP, TezConfiguration.TEZ_CLIENT_ASYNCHRONOUS_STOP_DEFAULT); long currentTimeMillis = System.currentTimeMillis(); long timeKillIssued = currentTimeMillis; long killTimeOut = amConfig.getTezConfiguration().getLong( TezConfiguration.TEZ_CLIENT_HARD_KILL_TIMEOUT_MS, TezConfiguration.TEZ_CLIENT_HARD_KILL_TIMEOUT_MS_DEFAULT);