conf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName); int amMemMB = conf.getInt(MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB); conf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, "" + amMemMB); int amCores = conf.getInt(MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES); conf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, "" + amCores); + conf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS));
n = Math.max(tezConfig.getInt( TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT), n);
n = Math.max(tezConfig.getInt( TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT), n);
static int getMaxConcurrentSteps( TezConfiguration jobConf ) { return jobConf.getInt( MAX_CONCURRENT_STEPS, 0 ); }
static int getMaxConcurrentSteps( TezConfiguration jobConf ) { return jobConf.getInt( MAX_CONCURRENT_STEPS, 0 ); }
/** * Method getCurrentTaskNum returns the task number of this task. Task 0 is the first task. * * @return int */ @Override public int getCurrentSliceNum() { return getConfiguration().getInt( "mapred.task.partition", 0 ); // TODO: likely incorrect }
/** * Method getCurrentTaskNum returns the task number of this task. Task 0 is the first task. * * @return int */ @Override public int getCurrentSliceNum() { return getConfiguration().getInt( "mapred.task.partition", 0 ); // TODO: likely incorrect }
public ClientServiceDelegate(Configuration conf, ResourceMgrDelegate rm, JobID jobId) { this.conf = new TezConfiguration(conf); // Cloning for modifying. // For faster redirects from AM to HS. this.conf.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, this.conf.getInt(MRJobConfig.MR_CLIENT_TO_AM_IPC_MAX_RETRIES, MRJobConfig.DEFAULT_MR_CLIENT_TO_AM_IPC_MAX_RETRIES)); }
@Override public TupleEntryCollector openTrapForWrite( Tap trap ) throws IOException { TezConfiguration jobConf = new TezConfiguration( getConfiguration() ); int stepNum = jobConf.getInt( "cascading.flow.step.num", 0 ); int nodeNum = jobConf.getInt( "cascading.flow.node.num", 0 ); String partname = String.format( "-%05d-%05d-", stepNum, nodeNum ); jobConf.set( "cascading.tapcollector.partname", "%s%spart" + partname + "%05d" ); return trap.openForWrite( new Hadoop2TezFlowProcess( this, jobConf ), null ); // do not honor sinkmode as this may be opened across tasks }
@Override public TupleEntryCollector openTrapForWrite( Tap trap ) throws IOException { TezConfiguration jobConf = new TezConfiguration( getConfiguration() ); int stepNum = jobConf.getInt( "cascading.flow.step.num", 0 ); int nodeNum = jobConf.getInt( "cascading.flow.node.num", 0 ); String partname = String.format( "-%05d-%05d-", stepNum, nodeNum ); jobConf.set( "cascading.tapcollector.partname", "%s%spart" + partname + "%05d" ); return trap.openForWrite( new Hadoop2TezFlowProcess( this, jobConf ), null ); // do not honor sinkmode as this may be opened across tasks }
@Private protected TezClient(String name, TezConfiguration tezConf, boolean isSession, @Nullable Map<String, LocalResource> localResources, @Nullable Credentials credentials, ServicePluginsDescriptor servicePluginsDescriptor) { this.clientName = name; this.isSession = isSession; // Set in conf for local mode AM to figure out whether in session mode or not tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, isSession); try { InetAddress ip = InetAddress.getLocalHost(); if (ip != null) { tezConf.set(TezConfigurationConstants.TEZ_SUBMIT_HOST, ip.getCanonicalHostName()); tezConf.set(TezConfigurationConstants.TEZ_SUBMIT_HOST_ADDRESS, ip.getHostAddress()); } } catch (UnknownHostException e) { LOG.warn("The host name of the client the tez application was submitted from was unable to be retrieved", e); } this.amConfig = new AMConfiguration(tezConf, localResources, credentials); this.apiVersionInfo = new TezApiVersionInfo(); this.servicePluginsDescriptor = servicePluginsDescriptor; this.maxSubmitDAGRequestSizeThroughIPC = tezConf.getInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT) - tezConf.getInt(TezConfiguration.TEZ_IPC_PAYLOAD_RESERVED_BYTES, TezConfiguration.TEZ_IPC_PAYLOAD_RESERVED_BYTES_DEFAULT); Limits.setConfiguration(tezConf); LOG.info("Tez Client Version: " + apiVersionInfo.toString()); }
@VisibleForTesting // this method will set the app priority only if the priority config has been defined public static void setApplicationPriority(ApplicationSubmissionContext context, AMConfiguration amConfig) { if (amConfig.getTezConfiguration().get(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY) != null) { // since we already checked not null condition, we are guaranteed that default value // (0 in this case for getInt) will not be returned/used. // The idea is to not use any default priority from TEZ side, if none provided in config; // let YARN determine the priority of the submitted application int priority = amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY, 0); context.setPriority(Priority.newInstance(priority)); if (LOG.isDebugEnabled()) { LOG.debug("Settting TEZ application priority, applicationId= " + context.getApplicationId() + ", priority= " + context.getPriority().getPriority()); } } }
DAG dag = DAG.create( getStepDisplayName( initializedConfig.getInt( "cascading.display.id.truncate", Util.ID_LENGTH ) ) );
DAG dag = DAG.create( getStepDisplayName( initializedConfig.getInt( "cascading.display.id.truncate", Util.ID_LENGTH ) ) );
+ ", dag plan size=" + request.getSerializedSize() + ", max dag plan size through IPC=" + maxSubmitDAGRequestSizeThroughIPC + ", max IPC message size= " + amConfig.getTezConfiguration().getInt( CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT)); request.writeTo(fsDataOutputStream);
n = Math.max(tezConfig.getInt( TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT), n);
amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB_DEFAULT)); capability.setVirtualCores( amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES_DEFAULT)); if (LOG.isDebugEnabled()) { amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS_DEFAULT));
TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT); int configuredAMMaxHeap = Utils.extractHeapSizeInMB(amLaunchOpts); int configuredAMResourceMB = amConf.getInt( TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB_DEFAULT);
clientTimeout = amConfig.getTezConfiguration().getInt( TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS, TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT);
CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION, CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION_DEFAULT)); builder.setMaxParallelism(conf.getInt( CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM, CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM_DEFAULT));