public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode, boolean usingLlap) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); conf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512); // Overrides values from the hive/tez-site. conf.setInt("hive.tez.container.size", 128); conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 128); conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 128); conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 24); conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 10); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); mr.init(conf); mr.start(); this.conf = mr.getConfig(); this.isLlap = usingLlap; }
public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); mr.init(conf); mr.start(); this.conf = mr.getConfig(); }
public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); mr.init(conf); mr.start(); this.conf = mr.getConfig(); }
public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); mr.init(conf); mr.start(); this.conf = mr.getConfig(); }
public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode, boolean usingLlap) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); conf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512); // Overrides values from the hive/tez-site. conf.setInt("hive.tez.container.size", 128); conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 128); conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 128); conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 24); conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 10); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); mr.init(conf); mr.start(); this.conf = mr.getConfig(); this.isLlap = usingLlap; }
miniTezCluster = new MiniTezCluster( getClass().getName(), 4, 1, 1 ); // todo: set to 4 miniTezCluster.init( tezConf ); miniTezCluster.start();
miniTezCluster = new MiniTezCluster( getClass().getName(), 4, 1, 1 ); // todo: set to 4 miniTezCluster.init( tezConf ); miniTezCluster.start();