Best code snippets using org.apache.tez.dag.api.TezConfiguration(Showing top 15 results out of 315)

/**
 * Creates and initializes a JobConf object that can be used to execute
 * the DAG. The configuration object will contain configurations from mapred-site
 * overlaid with key/value pairs from the conf object. Finally it will also
 * contain some hive specific configurations that do not change from DAG to DAG.
 *
 * @param hiveConf Current conf for the execution
 * @return JobConf base configuration for job execution
 * @throws IOException
 */
public JobConf createConfiguration(HiveConf hiveConf) throws IOException {
 hiveConf.setBoolean("mapred.mapper.new-api", false);
 JobConf conf = new JobConf(new TezConfiguration(hiveConf));
 conf.set("mapred.output.committer.class", NullOutputCommitter.class.getName());
 conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false);
 conf.setBoolean("mapred.committer.job.task.cleanup.needed", false);
 conf.setClass("mapred.output.format.class", HiveOutputFormatImpl.class, OutputFormat.class);
 conf.set(MRJobConfig.OUTPUT_KEY_CLASS, HiveKey.class.getName());
 conf.set(MRJobConfig.OUTPUT_VALUE_CLASS, BytesWritable.class.getName());
 conf.set("mapred.partitioner.class", HiveConf.getVar(conf, HiveConf.ConfVars.HIVEPARTITIONER));
 conf.set("tez.runtime.partitioner.class", MRPartitioner.class.getName());
 // Removing job credential entry/ cannot be set on the tasks
 conf.unset("mapreduce.job.credentials.binary");
 hiveConf.stripHiddenConfigurations(conf);
 return conf;
}

String env = conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV);
if (conf.get(MRJobConfig.MR_AM_ENV) != null) {
 env = (env == null) ? conf.get(MRJobConfig.MR_AM_ENV) : env + "," + conf.get(MRJobConfig.MR_AM_ENV);
 conf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_ENV, env);
conf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
  org.apache.tez.mapreduce.hadoop.MRHelpers.getJavaOptsForMRAM(conf));
String queueName = conf.get(JobContext.QUEUE_NAME, YarnConfiguration.DEFAULT_QUEUE_NAME);
conf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName);
int amMemMB = conf.getInt(MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB);
conf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, "" + amMemMB);
int amCores = conf.getInt(MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES);
conf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, "" + amCores);
conf.setIfUnset(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, ""
  + conf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS));
conf.setIfUnset(TezConfiguration.TEZ_AM_VIEW_ACLS,
  conf.get(MRJobConfig.JOB_ACL_VIEW_JOB, MRJobConfig.DEFAULT_JOB_ACL_VIEW_JOB));
conf.setIfUnset(TezConfiguration.TEZ_AM_MODIFY_ACLS,
  conf.get(MRJobConfig.JOB_ACL_MODIFY_JOB, MRJobConfig.DEFAULT_JOB_ACL_MODIFY_JOB));
  if (conf.get(dep.getKey()) != null) {

v.setVertexManagerPlugin(
 VertexManagerPluginDescriptor.create(CartesianProductVertexManager.class.getName())
  .setUserPayload(cpConfig.toUserPayload(new TezConfiguration(conf))));

edgeManagerDescriptor.setUserPayload(cpConfig.toUserPayload(new TezConfiguration(conf)));
UnorderedPartitionedKVEdgeConfig cpEdgeConf =
 UnorderedPartitionedKVEdgeConfig.newBuilder(keyClass, valClass,

final TezConfiguration tezConfig = new TezConfiguration(true);
tezConfig.addResource(conf);
tezConfig.set(TezConfiguration.TEZ_AM_STAGING_DIR, tezScratchDir.toUri().toString());
conf.stripHiddenConfigurations(tezConfig);
 n = Math.max(tezConfig.getInt(
   TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS,
   TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT), n);
 tezConfig.setInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, n);

@Private
TezClient(String name, TezConfiguration tezConf,
     @Nullable Map<String, LocalResource> localResources,
     @Nullable Credentials credentials) {
 this(name, tezConf, tezConf.getBoolean(
   TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT),
   localResources, credentials);
}

/**
 * Sets the history log level for this session. It will be in effect for DAGs submitted after this
 * call.
 *
 * @param historyLogLevel The log level to be used.
 */
public synchronized void setHistoryLogLevel(HistoryLogLevel historyLogLevel) {
 amConfig.getTezConfiguration().setEnum(TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL,
   historyLogLevel);
}

public TezJob(TezConfiguration conf, DAG dag,
    Map<String, LocalResource> requestAMResources,
    TezOperPlan tezPlan) throws IOException {
  this.conf = conf;
  this.dag = dag;
  this.requestAMResources = requestAMResources;
  this.reuseSession = conf.getBoolean(PigConfiguration.PIG_TEZ_SESSION_REUSE, true);
  this.statusGetOpts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
  tezJobConf = new TezJobConfig(tezPlan);
}

public TezConfiguration(boolean loadDefaults) {
 super(loadDefaults);
 if (loadDefaults) {
  addResource(TEZ_SITE_XML);
 }
}

@Override
public void init(TezConfiguration tezConf, YarnConfiguration yarnConf) {
 this.conf = tezConf;
 tezConf.set("fs.defaultFS", "file:///");
 // Tez libs already in the client's classpath
 this.conf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true);
 this.conf.set(TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS, localModeDAGSchedulerClassName);
 isSession = tezConf.getBoolean(TezConfiguration.TEZ_AM_SESSION_MODE,
   TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT);
 // disable web service for local mode.
 this.conf.setBoolean(TezConfiguration.TEZ_AM_WEBSERVICE_ENABLE, false);
}

public ClientServiceDelegate(Configuration conf, ResourceMgrDelegate rm,
  JobID jobId) {
 this.conf = new TezConfiguration(conf); // Cloning for modifying.
 // For faster redirects from AM to HS.
 this.conf.setInt(
   CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
   this.conf.getInt(MRJobConfig.MR_CLIENT_TO_AM_IPC_MAX_RETRIES,
     MRJobConfig.DEFAULT_MR_CLIENT_TO_AM_IPC_MAX_RETRIES));
}

public static FrameworkClient createFrameworkClient(TezConfiguration tezConf) {
 boolean isLocal = tezConf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
 if (isLocal) {
  try {
   return ReflectionUtils.createClazzInstance("org.apache.tez.client.LocalClient");
  } catch (TezReflectionException e) {
   throw new TezUncheckedException("Fail to create LocalClient", e);
  }
 }
 return new TezYarnClient(YarnClient.createYarnClient());
}

@Private
TezClient(String name, TezConfiguration tezConf,
     @Nullable Map<String, LocalResource> localResources,
     @Nullable Credentials credentials) {
 this(name, tezConf, tezConf.getBoolean(
   TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT),
   localResources, credentials);
}

public static FrameworkClient createFrameworkClient(TezConfiguration tezConf) {
 boolean isLocal = tezConf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
 if (isLocal) {
  try {
   return ReflectionUtils.createClazzInstance("org.apache.tez.client.LocalClient");
  } catch (TezReflectionException e) {
   throw new TezUncheckedException("Fail to create LocalClient", e);
  }
 }
 return new TezYarnClient(YarnClient.createYarnClient());
}

/**
 * Sets the history log level for this session. It will be in effect for DAGs submitted after this
 * call.
 *
 * @param historyLogLevel The log level to be used.
 */
public synchronized void setHistoryLogLevel(HistoryLogLevel historyLogLevel) {
 amConfig.getTezConfiguration().setEnum(TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL,
   historyLogLevel);
}