int maxHeapSize = org.apache.twill.internal.utils.Resources.computeMaxHeapSize(memoryMB, reservedMemory, minHeapRatio);
int maxHeapSize = org.apache.twill.internal.utils.Resources.computeMaxHeapSize(memoryMB, reservedMemory, minHeapRatio);
int memory = Resources.computeMaxHeapSize(appMasterInfo.getMemoryMB(), twillRuntimeSpec.getAMReservedMemory(), twillRuntimeSpec.getAMMinHeapRatio());
int memory = Resources.computeMaxHeapSize(appMasterInfo.getMemoryMB(), twillRuntimeSpec.getAMReservedMemory(), twillRuntimeSpec.getAMMinHeapRatio());
private RunningContainers createRunningContainers(ContainerId appMasterContainerId, String appMasterHost) throws Exception { int containerMemoryMB = Integer.parseInt(System.getenv(EnvKeys.YARN_CONTAINER_MEMORY_MB)); // We can't get the -Xmx easily, so just recompute the -Xmx in the same way that the client does int maxHeapMemoryMB = Resources.computeMaxHeapSize(containerMemoryMB, twillRuntimeSpec.getAMReservedMemory(), twillRuntimeSpec.getAMMinHeapRatio()); TwillRunResources appMasterResources = new DefaultTwillRunResources( 0, appMasterContainerId.toString(), Integer.parseInt(System.getenv(EnvKeys.YARN_CONTAINER_VIRTUAL_CORES)), containerMemoryMB, maxHeapMemoryMB, appMasterHost, null); String appId = appMasterContainerId.getApplicationAttemptId().getApplicationId().toString(); return new RunningContainers(twillRuntimeSpec, appId, appMasterResources, zkClient, applicationLocation, twillSpec.getRunnables(), eventHandler); }
private RunningContainers createRunningContainers(ContainerId appMasterContainerId, String appMasterHost) throws Exception { int containerMemoryMB = Integer.parseInt(System.getenv(EnvKeys.YARN_CONTAINER_MEMORY_MB)); // We can't get the -Xmx easily, so just recompute the -Xmx in the same way that the client does int maxHeapMemoryMB = Resources.computeMaxHeapSize(containerMemoryMB, twillRuntimeSpec.getAMReservedMemory(), twillRuntimeSpec.getAMMinHeapRatio()); TwillRunResources appMasterResources = new DefaultTwillRunResources( 0, appMasterContainerId.toString(), Integer.parseInt(System.getenv(EnvKeys.YARN_CONTAINER_VIRTUAL_CORES)), containerMemoryMB, maxHeapMemoryMB, appMasterHost, null); String appId = appMasterContainerId.getApplicationAttemptId().getApplicationId().toString(); return new RunningContainers(twillRuntimeSpec, appId, appMasterResources, zkClient, applicationLocation, twillSpec.getRunnables(), eventHandler); }
@Test public void testProgrammatic() { CConfiguration cConf = CConfiguration.create(); cConf.setInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB, 300); Configuration hConf = new Configuration(); hConf.setInt(Job.MAP_MEMORY_MB, 3000); hConf.setInt(Job.MAP_CPU_VCORES, 5); // Always use configurations setup programmatically via job conf. MapReduceRuntimeService.TaskType.MAP.configure(hConf, cConf, Collections.emptyMap(), null); int maxHeapSize = org.apache.twill.internal.utils.Resources.computeMaxHeapSize( 3000, cConf.getInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB), 0); validateResources(cConf, hConf, 3000, 5, maxHeapSize); // Even resources is provided via context, it is ignored. hConf = new Configuration(); hConf.setInt(Job.MAP_MEMORY_MB, 3000); hConf.setInt(Job.MAP_CPU_VCORES, 5); MapReduceRuntimeService.TaskType.MAP.configure(hConf, cConf, Collections.emptyMap(), new Resources(1234)); maxHeapSize = org.apache.twill.internal.utils.Resources.computeMaxHeapSize( 3000, cConf.getInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB), 0); validateResources(cConf, hConf, 3000, 5, maxHeapSize); // Set the reserved memory via task arguments hConf = new Configuration(); hConf.setInt(Job.MAP_MEMORY_MB, 3000); hConf.setInt(Job.MAP_CPU_VCORES, 5); MapReduceRuntimeService.TaskType.MAP.configure( hConf, cConf, Collections.singletonMap("system.resources.reserved.memory.override", "2000"), null); validateResources(cConf, hConf, 3000, 5, 3000 - 2000); }
@Test public void testDefault() { CConfiguration cConf = CConfiguration.create(); Configuration hConf = new Configuration(); // Nothing is set. It should be using the default. MapReduceRuntimeService.TaskType.MAP.configure(hConf, cConf, Collections.emptyMap(), null); int maxHeapSize = org.apache.twill.internal.utils.Resources.computeMaxHeapSize( Job.DEFAULT_MAP_MEMORY_MB, cConf.getInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB), cConf.getDouble(Configs.Keys.HEAP_RESERVED_MIN_RATIO)); validateResources(cConf, hConf, Job.DEFAULT_MAP_MEMORY_MB, Job.DEFAULT_MAP_CPU_VCORES, maxHeapSize); // Sets the reserved memory via task arguments hConf = new Configuration(); int reservedMemory = Job.DEFAULT_MAP_MEMORY_MB / 2; MapReduceRuntimeService.TaskType.MAP.configure( hConf, cConf, Collections.singletonMap("system.resources.reserved.memory.override", String.valueOf(reservedMemory)), null); validateResources(cConf, hConf, Job.DEFAULT_MAP_MEMORY_MB, Job.DEFAULT_MAP_CPU_VCORES, Job.DEFAULT_MAP_MEMORY_MB - reservedMemory); }
maxHeapSizeMB = Resources.computeMaxHeapSize(containerInfo.getMemoryMB(), reservedMemory, minHeapRatio); commandBuilder.add("-Djava.io.tmpdir=tmp", "-Dyarn.container=$" + EnvKeys.YARN_CONTAINER_ID,
int memory = Resources.computeMaxHeapSize(runtimeSpec.getResourceSpecification().getMemorySize(), twillRuntimeSpec.getReservedMemory(runnableName), twillRuntimeSpec.getMinHeapRatio(runnableName));
maxHeapSizeMB = Resources.computeMaxHeapSize(containerInfo.getMemoryMB(), reservedMemory, minHeapRatio); commandBuilder.add("-Djava.io.tmpdir=tmp", "-Dyarn.container=$" + EnvKeys.YARN_CONTAINER_ID,
int memory = Resources.computeMaxHeapSize(runtimeSpec.getResourceSpecification().getMemorySize(), twillRuntimeSpec.getReservedMemory(runnableName), twillRuntimeSpec.getMinHeapRatio(runnableName));
@Test public void testContextResources() { CConfiguration cConf = CConfiguration.create(); cConf.setInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB, 300); Configuration hConf = new Configuration(); // Resources is set through context object // At runtime time, it is either from spec, from runtime arg of the program // or from the context.setResources call in the initialize() method) Resources resources = new Resources(2345, 8); MapReduceRuntimeService.TaskType.MAP.configure(hConf, cConf, Collections.emptyMap(), resources); int maxHeapSize = org.apache.twill.internal.utils.Resources.computeMaxHeapSize( 2345, cConf.getInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB), 0); validateResources(cConf, hConf, 2345, 8, maxHeapSize); // Set the reserved memory via task arguments hConf = new Configuration(); MapReduceRuntimeService.TaskType.MAP.configure( hConf, cConf, Collections.singletonMap("system.resources.reserved.memory.override", "1000"), resources); validateResources(cConf, hConf, 2345, 8, 2345 - 1000); }
Assert.assertNotNull(resourceReport); Assert.assertEquals(256, resourceReport.getAppMasterResources().getMemoryMB()); Assert.assertEquals(Resources.computeMaxHeapSize(256, Configs.Defaults.YARN_AM_RESERVED_MEMORY_MB, 0.65d), resourceReport.getAppMasterResources().getMaxHeapMemoryMB()); Assert.assertFalse(runnableResources.isEmpty()); TwillRunResources resources = runnableResources.iterator().next(); Assert.assertEquals(Resources.computeMaxHeapSize(resources.getMemoryMB(), 1024, 0.8d), resources.getMaxHeapMemoryMB()); resources = runnableResources.iterator().next(); Assert.assertEquals( Resources.computeMaxHeapSize(resources.getMemoryMB(), Configs.Defaults.YARN_AM_RESERVED_MEMORY_MB, 0.65d), resources.getMaxHeapMemoryMB()); } finally {
Resources.computeMaxHeapSize(offHeapMemory + onHeapMemory, offHeapMemory, yarnConfig.getDouble(HEAP_RESERVED_MIN_RATIO, 0))); assertEquals(onHeapMemory, yarnConfig.getInt(DacDaemonYarnApplication.YARN_MEMORY_ON_HEAP, 0)); assertEquals(offHeapMemory, yarnConfig.getInt(DacDaemonYarnApplication.YARN_MEMORY_OFF_HEAP, 0));