String env = conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV); if (conf.get(MRJobConfig.MR_AM_ENV) != null) { env = (env == null) ? conf.get(MRJobConfig.MR_AM_ENV) : env + "," + conf.get(MRJobConfig.MR_AM_ENV); org.apache.tez.mapreduce.hadoop.MRHelpers.getJavaOptsForMRAM(conf)); String queueName = conf.get(JobContext.QUEUE_NAME, YarnConfiguration.DEFAULT_QUEUE_NAME); conf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName); conf.get(MRJobConfig.JOB_ACL_VIEW_JOB, MRJobConfig.DEFAULT_JOB_ACL_VIEW_JOB)); conf.get(MRJobConfig.JOB_ACL_MODIFY_JOB, MRJobConfig.DEFAULT_JOB_ACL_MODIFY_JOB)); if (conf.get(dep.getKey()) != null) { final String mrValue = conf.get(dep.getKey()); final String tezValue = conf.get(dep.getValue()); conf.unset(dep.getKey()); if (tezValue == null) { + mrValue + ", tez(original):" + dep.getValue() + "=" + tezValue + ", tez(final):" + dep.getValue() + "=" + conf.get(dep.getValue()));
@Override public Object getProperty( String key ) { return configuration.get( key ); }
@Private @VisibleForTesting static String constructAMLaunchOpts(TezConfiguration tezConf, Resource capability) { String defaultOpts = tezConf.get(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS, TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT); Path tmpDir = new Path(Environment.PWD.$(), YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR); String amOpts = "-Djava.io.tmpdir=" + tmpDir + " "; if (defaultOpts != null && !defaultOpts.isEmpty()) { amOpts = amOpts + defaultOpts + " "; } amOpts = amOpts + tezConf.get(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT); amOpts = maybeAddDefaultMemoryJavaOpts(amOpts, capability, tezConf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION, TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT)); return amOpts; }
cleanupFrameworkClient = true; isATSEnabled = conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "") .equals("org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService") && conf.getBoolean(TezConfiguration.TEZ_DAG_HISTORY_LOGGING_ENABLED,
private static SessionInfo createSession(TezConfiguration amConf, Map<String, LocalResource> requestedAMResources, Credentials creds, TezJobConfig tezJobConf) throws TezException, IOException, InterruptedException { MRToTezHelper.translateMRSettingsForTezAM(amConf); TezScriptState ss = TezScriptState.get(); ss.addDAGSettingsToConf(amConf); if (amConf.getBoolean(PigConfiguration.PIG_TEZ_CONFIGURE_AM_MEMORY, true)) { adjustAMConfig(amConf, tezJobConf); } String jobName = amConf.get(PigContext.JOB_NAME, "pig"); TezClient tezClient = TezClient.create(jobName, amConf, true, requestedAMResources, creds); try { tezClient.start(); TezAppMasterStatus appMasterStatus = tezClient.getAppMasterStatus(); if (appMasterStatus.equals(TezAppMasterStatus.SHUTDOWN)) { throw new RuntimeException("TezSession has already shutdown"); } tezClient.waitTillReady(); } catch (Throwable e) { log.error("Exception while waiting for Tez client to be ready", e); tezClient.stop(); throw new RuntimeException(e); } return new SessionInfo(tezClient, amConf, requestedAMResources); }
@Override public void initialize() throws Exception { configuration = new TezConfiguration( TezUtils.createConfFromUserPayload( getContext().getUserPayload() ) ); TezUtil.setMRProperties( getContext(), configuration, true ); try { HadoopUtil.initLog4j( configuration ); LOG.info( "cascading version: {}", configuration.get( "cascading.version", "" ) ); currentProcess = new Hadoop2TezFlowProcess( new FlowSession(), getContext(), configuration ); flowNode = deserializeBase64( configuration.getRaw( FlowNode.CASCADING_FLOW_NODE ), configuration, BaseFlowNode.class ); LOG.info( "flow node id: {}, ordinal: {}", flowNode.getID(), flowNode.getOrdinal() ); logMemory( LOG, "flow node id: " + flowNode.getID() + ", mem on start" ); } catch( Throwable throwable ) { if( throwable instanceof CascadingException ) throw (CascadingException) throwable; throw new FlowException( "internal error during processor configuration", throwable ); } }
@Override public Object getProperty( String key ) { return configuration.get( key ); }
@VisibleForTesting // this method will set the app priority only if the priority config has been defined public static void setApplicationPriority(ApplicationSubmissionContext context, AMConfiguration amConfig) { if (amConfig.getTezConfiguration().get(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY) != null) { // since we already checked not null condition, we are guaranteed that default value // (0 in this case for getInt) will not be returned/used. // The idea is to not use any default priority from TEZ side, if none provided in config; // let YARN determine the priority of the submitted application int priority = amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY, 0); context.setPriority(Priority.newInstance(priority)); if (LOG.isDebugEnabled()) { LOG.debug("Settting TEZ application priority, applicationId= " + context.getApplicationId() + ", priority= " + context.getPriority().getPriority()); } } }
@Override public void initialize() throws Exception { configuration = new TezConfiguration( TezUtils.createConfFromUserPayload( getContext().getUserPayload() ) ); TezUtil.setMRProperties( getContext(), configuration, true ); try { HadoopUtil.initLog4j( configuration ); LOG.info( "cascading version: {}", configuration.get( "cascading.version", "" ) ); currentProcess = new Hadoop2TezFlowProcess( new FlowSession(), getContext(), configuration ); flowNode = deserializeBase64( configuration.getRaw( FlowNode.CASCADING_FLOW_NODE ), configuration, BaseFlowNode.class ); LOG.info( "flow node id: {}, ordinal: {}", flowNode.getID(), flowNode.getOrdinal() ); logMemory( LOG, "flow node id: " + flowNode.getID() + ", mem on start" ); } catch( Throwable throwable ) { if( throwable instanceof CascadingException ) throw (CascadingException) throwable; throw new FlowException( "internal error during processor configuration", throwable ); } }
String getQueueName() { return this.tezConf.get(TezConfiguration.TEZ_QUEUE_NAME); }
/** * Method getProperty returns the value associated with the given key from the underlying properties system. * * @param key of type String * @return String */ public String getProperty( String key ) { return getConfig().get( key ); }
/** * Method getProperty returns the value associated with the given key from the underlying properties system. * * @param key of type String * @return String */ public String getProperty( String key ) { return getConfig().get( key ); }
private TezConfiguration getDAGAMConfFromMRConf() { TezConfiguration finalConf = new TezConfiguration(this.tezConf); Map<String, String> mrParamToDAGParamMap = DeprecatedKeys .getMRToDAGParamMap(); for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) { if (finalConf.get(entry.getKey()) != null) { finalConf.set(entry.getValue(), finalConf.get(entry.getKey())); finalConf.unset(entry.getKey()); if (LOG.isDebugEnabled()) { LOG.debug("MR->DAG Translating MR key: " + entry.getKey() + " to Tez key: " + entry.getValue() + " with value " + finalConf.get(entry.getValue())); } } } return finalConf; }
public static void translateMRSettingsForTezAM(TezConfiguration dagAMConf) { convertMRToTezConf(dagAMConf, dagAMConf, DeprecatedKeys.getMRToDAGParamMap()); convertMRToTezConf(dagAMConf, dagAMConf, mrAMParamToTezAMParamMap); String env = dagAMConf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV); if (dagAMConf.get(MRJobConfig.MR_AM_ENV) != null) { env = (env == null) ? dagAMConf.get(MRJobConfig.MR_AM_ENV) : env + "," + dagAMConf.get(MRJobConfig.MR_AM_ENV); } if (env != null) { dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_ENV, env); } dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, org.apache.tez.mapreduce.hadoop.MRHelpers .getJavaOptsForMRAM(dagAMConf)); String queueName = dagAMConf.get(JobContext.QUEUE_NAME, YarnConfiguration.DEFAULT_QUEUE_NAME); dagAMConf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName); dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_VIEW_ACLS, dagAMConf.get(MRJobConfig.JOB_ACL_VIEW_JOB, MRJobConfig.DEFAULT_JOB_ACL_VIEW_JOB)); dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_MODIFY_ACLS, dagAMConf.get(MRJobConfig.JOB_ACL_MODIFY_JOB, MRJobConfig.DEFAULT_JOB_ACL_MODIFY_JOB)); // Hardcoding at AM level instead of setting per vertex till TEZ-2710 is available dagAMConf.setIfUnset(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, "0.5"); removeUnwantedSettings(dagAMConf, true); }
public DAGClientTimelineImpl(ApplicationId appId, String dagId, TezConfiguration conf, FrameworkClient frameworkClient, int connTimeout) throws TezException { if (!TimelineReaderFactory.isTimelineClientSupported()) { throw new TezException("Reading from secure timeline is supported only for hadoop 2.6 and above."); } this.appId = appId; this.dagId = dagId; this.frameworkClient = frameworkClient; String scheme; String webAppAddress; boolean useHttps = webappHttpsOnly(conf); if (useHttps) { scheme = HTTPS_SCHEME; webAppAddress = conf.get(ATSConstants.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS_CONF_NAME); } else { scheme = HTTP_SCHEME; webAppAddress = conf.get(ATSConstants.TIMELINE_SERVICE_WEBAPP_HTTP_ADDRESS_CONF_NAME); } if (webAppAddress == null) { throw new TezException("Failed to get ATS webapp address"); } baseUri = Joiner.on("").join(scheme, webAppAddress, ATSConstants.RESOURCE_URI_BASE); timelineReaderStrategy = TimelineReaderFactory.getTimelineReaderStrategy(conf, useHttps, connTimeout); }
String createStepStagingPath() { String result = ""; if( HadoopUtil.isLocal( jobConfiguration ) ) result = jobConfiguration.get( "hadoop.tmp.dir" ) + Path.SEPARATOR; String flowStagingPath = ( (Hadoop2TezFlow) flowStep.getFlow() ).getFlowStagingPath(); return result + flowStagingPath + Path.SEPARATOR + flowStep.getID(); }
String createStepStagingPath() { String result = ""; if( HadoopUtil.isLocal( jobConfiguration ) ) result = jobConfiguration.get( "hadoop.tmp.dir" ) + Path.SEPARATOR; String flowStagingPath = ( (Hadoop2TezFlow) flowStep.getFlow() ).getFlowStagingPath(); return result + flowStagingPath + Path.SEPARATOR + flowStep.getID(); }
String appJarPath = stepConf.get( AppProps.APP_JAR_PATH );
String appJarPath = stepConf.get( AppProps.APP_JAR_PATH );
String stagingDirStr = tezConf.get(TezConfiguration.TEZ_AM_STAGING_DIR, TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT) + Path.SEPARATOR + "BroadcastAndOneToOneExample" + Path.SEPARATOR + Long.toString(System.currentTimeMillis());