/** * Determines the buffer size for one pipeline. */ private long getBufferSize(int numberOfPipelines, CConfiguration cConf, int partitions) { long bufferSize = cConf.getLong(Constants.Logging.PIPELINE_BUFFER_SIZE); if (bufferSize > 0) { return bufferSize; } double bufferRatio = cConf.getDouble(Constants.Logging.PIPELINE_AUTO_BUFFER_RATIO); Preconditions.checkArgument(bufferRatio > 0 && bufferRatio < 1, "Config %s must be between 0 and 1", Constants.Logging.PIPELINE_AUTO_BUFFER_RATIO); int kafkaFetchSize = cConf.getInt(Constants.Logging.PIPELINE_KAFKA_FETCH_SIZE) * partitions; // Try to derive it from the total memory size, the number of pipelines and the kafka fetch size bufferSize = (long) ((Runtime.getRuntime().maxMemory() * bufferRatio - kafkaFetchSize) / numberOfPipelines); // The size has to be > 0 for it to make any progress. This is just to safe guard return bufferSize > 0 ? bufferSize : 1L; } }
/** * Determines the buffer size for one pipeline. */ private long getBufferSize(int numberOfPipelines, CConfiguration cConf, int partitions) { long bufferSize = cConf.getLong(Constants.Logging.PIPELINE_BUFFER_SIZE); if (bufferSize > 0) { return bufferSize; } double bufferRatio = cConf.getDouble(Constants.Logging.PIPELINE_AUTO_BUFFER_RATIO); Preconditions.checkArgument(bufferRatio > 0 && bufferRatio < 1, "Config %s must be between 0 and 1", Constants.Logging.PIPELINE_AUTO_BUFFER_RATIO); int kafkaFetchSize = cConf.getInt(Constants.Logging.PIPELINE_KAFKA_FETCH_SIZE) * partitions; // Try to derive it from the total memory size, the number of pipelines and the kafka fetch size bufferSize = (long) ((Runtime.getRuntime().maxMemory() * bufferRatio - kafkaFetchSize) / numberOfPipelines); // The size has to be > 0 for it to make any progress. This is just to safe guard return bufferSize > 0 ? bufferSize : 1L; } }
double minHeapRatio = cConf.getDouble(Configs.Keys.HEAP_RESERVED_MIN_RATIO); String maxHeapSource = "from system configuration " + Configs.Keys.JAVA_RESERVED_MEMORY_MB +
double minHeapRatio = cConf.getDouble(Configs.Keys.HEAP_RESERVED_MIN_RATIO); String maxHeapSource = "from system configuration " + Configs.Keys.JAVA_RESERVED_MEMORY_MB +
@Test public void testDefault() { CConfiguration cConf = CConfiguration.create(); Configuration hConf = new Configuration(); // Nothing is set. It should be using the default. MapReduceRuntimeService.TaskType.MAP.configure(hConf, cConf, Collections.emptyMap(), null); int maxHeapSize = org.apache.twill.internal.utils.Resources.computeMaxHeapSize( Job.DEFAULT_MAP_MEMORY_MB, cConf.getInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB), cConf.getDouble(Configs.Keys.HEAP_RESERVED_MIN_RATIO)); validateResources(cConf, hConf, Job.DEFAULT_MAP_MEMORY_MB, Job.DEFAULT_MAP_CPU_VCORES, maxHeapSize); // Sets the reserved memory via task arguments hConf = new Configuration(); int reservedMemory = Job.DEFAULT_MAP_MEMORY_MB / 2; MapReduceRuntimeService.TaskType.MAP.configure( hConf, cConf, Collections.singletonMap("system.resources.reserved.memory.override", String.valueOf(reservedMemory)), null); validateResources(cConf, hConf, Job.DEFAULT_MAP_MEMORY_MB, Job.DEFAULT_MAP_CPU_VCORES, Job.DEFAULT_MAP_MEMORY_MB - reservedMemory); }