int configuredVcores = flinkConfiguration.getInteger(YarnConfigOptions.VCORES, clusterSpecification.getSlotsPerTaskManager());
clusterSpecification.getSlotsPerTaskManager()); appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_HOME_DIR, homeDir.toString()); appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_SHIP_FILES, envShipFileList.toString()); appMasterEnv.put(YarnConfigKeys.ENV_SLOTS, String.valueOf(clusterSpecification.getSlotsPerTaskManager())); appMasterEnv.put(YarnConfigKeys.ENV_DETACHED, String.valueOf(detached)); appMasterEnv.put(YarnConfigKeys.ENV_ZOOKEEPER_NAMESPACE, getZookeeperNamespace());
.setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(clusterSpecification.getNumberTaskManagers()) .setSlotsPerTaskManager(clusterSpecification.getSlotsPerTaskManager()) .createClusterSpecification();
clusterSpecification.getNumberTaskManagers() * clusterSpecification.getSlotsPerTaskManager(), yarnClusterDescriptor.getDynamicPropertiesEncoded()); } catch (Exception e) {
/** * Tests that the command line arguments override the configuration settings * when the {@link ClusterSpecification} is created. */ @Test public void testCommandLineClusterSpecification() throws Exception { final Configuration configuration = new Configuration(); final int jobManagerMemory = 1337; final int taskManagerMemory = 7331; final int slotsPerTaskManager = 30; configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m"); configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m"); configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager); final String[] args = {"-yjm", String.valueOf(jobManagerMemory) + "m", "-ytm", String.valueOf(taskManagerMemory) + "m", "-ys", String.valueOf(slotsPerTaskManager)}; final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( configuration, tmp.getRoot().getAbsolutePath(), "y", "yarn"); CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false); final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine); assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory)); assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory)); assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager)); }
this, validClusterSpecification.getNumberTaskManagers(), validClusterSpecification.getSlotsPerTaskManager(), report, flinkConfiguration,
/** * Tests that the configuration settings are used to create the * {@link ClusterSpecification}. */ @Test public void testConfigurationClusterSpecification() throws Exception { final Configuration configuration = new Configuration(); final int jobManagerMemory = 1337; configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m"); final int taskManagerMemory = 7331; configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m"); final int slotsPerTaskManager = 42; configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager); final String[] args = {}; final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( configuration, tmp.getRoot().getAbsolutePath(), "y", "yarn"); CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false); final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine); assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory)); assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory)); assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager)); }
@Test public void testCorrectSettingOfMaxSlots() throws Exception { String[] params = new String[] {"-yn", "2", "-ys", "3"}; FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli( new Configuration(), tmp.getRoot().getAbsolutePath(), "y", "yarn"); final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true); AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine); final ClusterSpecification clusterSpecification = yarnCLI.getClusterSpecification(commandLine); // each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased. assertEquals(3, clusterSpecification.getSlotsPerTaskManager()); assertEquals(2, clusterSpecification.getNumberTaskManagers()); }
int configuredVcores = flinkConfiguration.getInteger(YarnConfigOptions.VCORES, clusterSpecification.getSlotsPerTaskManager());
int configuredVcores = flinkConfiguration.getInteger(YarnConfigOptions.VCORES, clusterSpecification.getSlotsPerTaskManager());
clusterSpecification.getSlotsPerTaskManager()); appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_HOME_DIR, homeDir.toString()); appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_SHIP_FILES, envShipFileList.toString()); appMasterEnv.put(YarnConfigKeys.ENV_SLOTS, String.valueOf(clusterSpecification.getSlotsPerTaskManager())); appMasterEnv.put(YarnConfigKeys.ENV_DETACHED, String.valueOf(detached)); appMasterEnv.put(YarnConfigKeys.ENV_ZOOKEEPER_NAMESPACE, getZookeeperNamespace());
clusterSpecification.getSlotsPerTaskManager()); appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_HOME_DIR, homeDir.toString()); appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_SHIP_FILES, envShipFileList.toString()); appMasterEnv.put(YarnConfigKeys.ENV_SLOTS, String.valueOf(clusterSpecification.getSlotsPerTaskManager())); appMasterEnv.put(YarnConfigKeys.ENV_DETACHED, String.valueOf(detached)); appMasterEnv.put(YarnConfigKeys.ENV_ZOOKEEPER_NAMESPACE, getZookeeperNamespace());
.setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(clusterSpecification.getNumberTaskManagers()) .setSlotsPerTaskManager(clusterSpecification.getSlotsPerTaskManager()) .createClusterSpecification();
.setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(clusterSpecification.getNumberTaskManagers()) .setSlotsPerTaskManager(clusterSpecification.getSlotsPerTaskManager()) .createClusterSpecification();
clusterSpecification.getNumberTaskManagers() * clusterSpecification.getSlotsPerTaskManager(), yarnClusterDescriptor.getDynamicPropertiesEncoded()); } catch (Exception e) {
clusterSpecification.getNumberTaskManagers() * clusterSpecification.getSlotsPerTaskManager(), yarnClusterDescriptor.getDynamicPropertiesEncoded()); } catch (Exception e) {
this, validClusterSpecification.getNumberTaskManagers(), validClusterSpecification.getSlotsPerTaskManager(), report, flinkConfiguration,
this, validClusterSpecification.getNumberTaskManagers(), validClusterSpecification.getSlotsPerTaskManager(), report, flinkConfiguration,