@Override public void shutdown() throws IOException { mr.stop(); }
public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode, boolean usingLlap) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); conf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512); // Overrides values from the hive/tez-site. conf.setInt("hive.tez.container.size", 128); conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 128); conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 128); conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 24); conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 10); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); mr.init(conf); mr.start(); this.conf = mr.getConfig(); this.isLlap = usingLlap; }
@Override public void setupConfiguration(Configuration conf) { Configuration config = mr.getConfig(); for (Map.Entry<String, String> pair: config) { conf.set(pair.getKey(), pair.getValue()); } // Overrides values from the hive/tez-site. conf.setInt("hive.tez.container.size", 128); conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 128); conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 128); conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 24); conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 10); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f); if (isLlap) { conf.set("hive.llap.execution.mode", "all"); } } }
public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); mr.init(conf); mr.start(); this.conf = mr.getConfig(); }
@Override public void setupConfiguration(Configuration conf) { Configuration config = mr.getConfig(); for (Map.Entry<String, String> pair: config) { conf.set(pair.getKey(), pair.getValue()); } // Overrides values from the hive/tez-site. conf.setInt("hive.tez.container.size", 128); conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 128); conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 128); conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 24); conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 10); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f); if (isLlap) { conf.set("hive.llap.execution.mode", "all"); } } }
@Override public void shutdown() throws IOException { mr.stop(); }
public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); mr.init(conf); mr.start(); this.conf = mr.getConfig(); }
@Override public void setupConfiguration(Configuration conf) { Configuration config = mr.getConfig(); for (Map.Entry<String, String> pair: config) { conf.set(pair.getKey(), pair.getValue()); } Path jarPath = new Path("hdfs:///user/hive"); Path hdfsPath = new Path("hdfs:///user/"); try { FileSystem fs = cluster.getFileSystem(); jarPath = fs.makeQualified(jarPath); conf.set("hive.jar.directory", jarPath.toString()); fs.mkdirs(jarPath); hdfsPath = fs.makeQualified(hdfsPath); conf.set("hive.user.install.directory", hdfsPath.toString()); fs.mkdirs(hdfsPath); } catch (Exception e) { e.printStackTrace(); } } }
@Override public void shutdown() throws IOException { mr.stop(); }
public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); mr.init(conf); mr.start(); this.conf = mr.getConfig(); }
@Override public void setupConfiguration(Configuration conf) { Configuration config = mr.getConfig(); for (Map.Entry<String, String> pair: config) { conf.set(pair.getKey(), pair.getValue()); } Path jarPath = new Path("hdfs:///user/hive"); Path hdfsPath = new Path("hdfs:///user/"); try { FileSystem fs = cluster.getFileSystem(); jarPath = fs.makeQualified(jarPath); conf.set("hive.jar.directory", jarPath.toString()); fs.mkdirs(jarPath); hdfsPath = fs.makeQualified(hdfsPath); conf.set("hive.user.install.directory", hdfsPath.toString()); fs.mkdirs(hdfsPath); } catch (Exception e) { e.printStackTrace(); } } }
@Override public void shutdown() throws IOException { mr.stop(); }
public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode, boolean usingLlap) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); conf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512); // Overrides values from the hive/tez-site. conf.setInt("hive.tez.container.size", 128); conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 128); conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 128); conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 24); conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 10); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); mr.init(conf); mr.start(); this.conf = mr.getConfig(); this.isLlap = usingLlap; }
@Override public void setupConfiguration(Configuration conf) { Configuration config = mr.getConfig(); for (Map.Entry<String, String> pair: config) { conf.set(pair.getKey(), pair.getValue()); } Path jarPath = new Path("hdfs:///user/hive"); Path hdfsPath = new Path("hdfs:///user/"); try { FileSystem fs = cluster.getFileSystem(); jarPath = fs.makeQualified(jarPath); conf.set("hive.jar.directory", jarPath.toString()); fs.mkdirs(jarPath); hdfsPath = fs.makeQualified(hdfsPath); conf.set("hive.user.install.directory", hdfsPath.toString()); fs.mkdirs(hdfsPath); } catch (Exception e) { e.printStackTrace(); } } }
@Override public void shutdown() throws IOException { mr.stop(); }
miniTezCluster = new MiniTezCluster( getClass().getName(), 4, 1, 1 ); // todo: set to 4 miniTezCluster.init( tezConf ); miniTezCluster.start(); configuration = miniTezCluster.getConfig();
miniTezCluster = new MiniTezCluster( getClass().getName(), 4, 1, 1 ); // todo: set to 4 miniTezCluster.init( tezConf ); miniTezCluster.start(); configuration = miniTezCluster.getConfig();