private ClusterSpecification createClusterSpecification(Configuration configuration, CommandLine cmd) { if (cmd.hasOption(container.getOpt())) { // number of containers is required option! LOG.info("The argument {} is deprecated in will be ignored.", container.getOpt()); } // TODO: The number of task manager should be deprecated soon final int numberTaskManagers; if (cmd.hasOption(container.getOpt())) { numberTaskManagers = Integer.valueOf(cmd.getOptionValue(container.getOpt())); } else { numberTaskManagers = 1; } // JobManager Memory final int jobManagerMemoryMB = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes(); // Task Managers memory final int taskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes(); int slotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS); return new ClusterSpecification.ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMB) .setTaskManagerMemoryMB(taskManagerMemoryMB) .setNumberTaskManagers(numberTaskManagers) .setSlotsPerTaskManager(slotsPerTaskManager) .createClusterSpecification(); }
.setMasterMemoryMB(jobManagerMemoryMb) .setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(clusterSpecification.getNumberTaskManagers()) .setSlotsPerTaskManager(clusterSpecification.getSlotsPerTaskManager()) .createClusterSpecification();
.setTaskManagerMemoryMB(1024) .setSlotsPerTaskManager(1) .setNumberTaskManagers(1) .createClusterSpecification();
.setMasterMemoryMB(jobManagerMemoryMb) .setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(clusterSpecification.getNumberTaskManagers()) .setSlotsPerTaskManager(clusterSpecification.getSlotsPerTaskManager()) .createClusterSpecification();
.setMasterMemoryMB(jobManagerMemoryMb) .setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(clusterSpecification.getNumberTaskManagers()) .setSlotsPerTaskManager(clusterSpecification.getSlotsPerTaskManager()) .createClusterSpecification();
.setMasterMemoryMB(jobmanagerMemoryMb) .setTaskManagerMemoryMB(taskmanagerMemoryMb) .setNumberTaskManagers(numberTaskManagers) .setSlotsPerTaskManager(slotsPerTaskManager) .createClusterSpecification();
private ClusterSpecification createClusterSpecification(Configuration configuration, CommandLine cmd) { if (cmd.hasOption(container.getOpt())) { // number of containers is required option! LOG.info("The argument {} is deprecated in will be ignored.", container.getOpt()); } // TODO: The number of task manager should be deprecated soon final int numberTaskManagers; if (cmd.hasOption(container.getOpt())) { numberTaskManagers = Integer.valueOf(cmd.getOptionValue(container.getOpt())); } else { numberTaskManagers = 1; } // JobManager Memory final int jobManagerMemoryMB = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes(); // Task Managers memory final int taskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes(); int slotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS); return new ClusterSpecification.ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMB) .setTaskManagerMemoryMB(taskManagerMemoryMB) .setNumberTaskManagers(numberTaskManagers) .setSlotsPerTaskManager(slotsPerTaskManager) .createClusterSpecification(); }
private ClusterSpecification createClusterSpecification(Configuration configuration, CommandLine cmd) { if (cmd.hasOption(container.getOpt())) { // number of containers is required option! LOG.info("The argument {} is deprecated in will be ignored.", container.getOpt()); } // TODO: The number of task manager should be deprecated soon final int numberTaskManagers; if (cmd.hasOption(container.getOpt())) { numberTaskManagers = Integer.valueOf(cmd.getOptionValue(container.getOpt())); } else { numberTaskManagers = 1; } // JobManager Memory final int jobManagerMemoryMB = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes(); // Task Managers memory final int taskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes(); int slotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS); return new ClusterSpecification.ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMB) .setTaskManagerMemoryMB(taskManagerMemoryMB) .setNumberTaskManagers(numberTaskManagers) .setSlotsPerTaskManager(slotsPerTaskManager) .createClusterSpecification(); }
public static ClusterSpecification fromConfiguration(Configuration configuration) { int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1); int jobManagerMemoryMb = configuration.getInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY); int taskManagerMemoryMb = configuration.getInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY); return new ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMb) .setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(1) .setSlotsPerTaskManager(slots) .createClusterSpecification(); }
public static ClusterSpecification fromConfiguration(Configuration configuration) { int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1); int jobManagerMemoryMb = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes(); int taskManagerMemoryMb = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes(); return new ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMb) .setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(1) .setSlotsPerTaskManager(slots) .createClusterSpecification(); }
.setMasterMemoryMB(1) .setTaskManagerMemoryMB(1) .setNumberTaskManagers(1) .setSlotsPerTaskManager(1) .createClusterSpecification();
@Test public void testFailIfTaskSlotsHigherThanMaxVcores() throws ClusterDeploymentException { final Configuration flinkConfiguration = new Configuration(); flinkConfiguration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0); YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor( flinkConfiguration, yarnConfiguration, temporaryFolder.getRoot().getAbsolutePath(), yarnClient, true); clusterDescriptor.setLocalJarPath(new Path(flinkJar.getPath())); ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder() .setMasterMemoryMB(1) .setTaskManagerMemoryMB(1) .setNumberTaskManagers(1) .setSlotsPerTaskManager(Integer.MAX_VALUE) .createClusterSpecification(); try { clusterDescriptor.deploySessionCluster(clusterSpecification); fail("The deploy call should have failed."); } catch (ClusterDeploymentException e) { // we expect the cause to be an IllegalConfigurationException if (!(e.getCause() instanceof IllegalConfigurationException)) { throw e; } } finally { clusterDescriptor.close(); } }
public static ClusterSpecification fromConfiguration(Configuration configuration) { int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1); int jobManagerMemoryMb = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes(); int taskManagerMemoryMb = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes(); return new ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMb) .setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(1) .setSlotsPerTaskManager(slots) .createClusterSpecification(); }