ClusterResourceDescription freeClusterResources) throws YarnDeploymentException { int taskManagerCount = clusterSpecification.getNumberTaskManagers(); int jobManagerMemoryMb = clusterSpecification.getMasterMemoryMB(); int taskManagerMemoryMb = clusterSpecification.getTaskManagerMemoryMB(); .setMasterMemoryMB(jobManagerMemoryMb) .setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(clusterSpecification.getNumberTaskManagers()) .setSlotsPerTaskManager(clusterSpecification.getSlotsPerTaskManager()) .createClusterSpecification();
public ClusterSpecification createClusterSpecification() { return new ClusterSpecification( masterMemoryMB, taskManagerMemoryMB, numberTaskManagers, slotsPerTaskManager); } }
private void isReadyForDeployment(ClusterSpecification clusterSpecification) throws YarnDeploymentException { if (clusterSpecification.getNumberTaskManagers() <= 0) { throw new YarnDeploymentException("Taskmanager count must be positive"); int configuredVcores = flinkConfiguration.getInteger(YarnConfigOptions.VCORES, clusterSpecification.getSlotsPerTaskManager());
/** * Tests the specifying heap memory with config default value for job manager and task manager. */ @Test public void testHeapMemoryPropertyWithConfigDefaultValue() throws Exception { final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( new Configuration(), tmp.getRoot().getAbsolutePath(), "y", "yarn"); final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false); final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine); assertThat(clusterSpecification.getMasterMemoryMB(), is(1024)); assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(1024)); }
/** * Tests that the command line arguments override the configuration settings * when the {@link ClusterSpecification} is created. */ @Test public void testCommandLineClusterSpecification() throws Exception { final Configuration configuration = new Configuration(); final int jobManagerMemory = 1337; final int taskManagerMemory = 7331; final int slotsPerTaskManager = 30; configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m"); configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m"); configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager); final String[] args = {"-yjm", String.valueOf(jobManagerMemory) + "m", "-ytm", String.valueOf(taskManagerMemory) + "m", "-ys", String.valueOf(slotsPerTaskManager)}; final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( configuration, tmp.getRoot().getAbsolutePath(), "y", "yarn"); CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false); final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine); assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory)); assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory)); assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager)); }
/** * Method to validate cluster specification before deploy it, it will throw * an {@link FlinkException} if the {@link ClusterSpecification} is invalid. * * @param clusterSpecification cluster specification to check against the configuration of the * AbstractYarnClusterDescriptor * @throws FlinkException if the cluster cannot be started with the provided {@link ClusterSpecification} */ private void validateClusterSpecification(ClusterSpecification clusterSpecification) throws FlinkException { try { final long taskManagerMemorySize = clusterSpecification.getTaskManagerMemoryMB(); // We do the validation by calling the calculation methods here // Internally these methods will check whether the cluster can be started with the provided // ClusterSpecification and the configured memory requirements final long cutoff = ContaineredTaskManagerParameters.calculateCutoffMB(flinkConfiguration, taskManagerMemorySize); TaskManagerServices.calculateHeapSizeMB(taskManagerMemorySize - cutoff, flinkConfiguration); } catch (IllegalArgumentException iae) { throw new FlinkException("Cannot fulfill the minimum memory requirements with the provided " + "cluster specification. Please increase the memory of the cluster.", iae); } }
clusterSpecification.getNumberTaskManagers() * clusterSpecification.getSlotsPerTaskManager(), yarnClusterDescriptor.getDynamicPropertiesEncoded()); } catch (Exception e) {
/** * Tests the specifying heap memory with arbitrary unit for job manager and task manager. */ @Test public void testHeapMemoryPropertyWithArbitraryUnit() throws Exception { final String[] args = new String[] { "-yn", "2", "-yjm", "1g", "-ytm", "2g" }; final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( new Configuration(), tmp.getRoot().getAbsolutePath(), "y", "yarn"); final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false); final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine); assertThat(clusterSpecification.getMasterMemoryMB(), is(1024)); assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048)); }
/** * Tests that the configuration settings are used to create the * {@link ClusterSpecification}. */ @Test public void testConfigurationClusterSpecification() throws Exception { final Configuration configuration = new Configuration(); final int jobManagerMemory = 1337; configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m"); final int taskManagerMemory = 7331; configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m"); final int slotsPerTaskManager = 42; configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager); final String[] args = {}; final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( configuration, tmp.getRoot().getAbsolutePath(), "y", "yarn"); CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false); final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine); assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory)); assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory)); assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager)); }
/** * Method to validate cluster specification before deploy it, it will throw * an {@link FlinkException} if the {@link ClusterSpecification} is invalid. * * @param clusterSpecification cluster specification to check against the configuration of the * AbstractYarnClusterDescriptor * @throws FlinkException if the cluster cannot be started with the provided {@link ClusterSpecification} */ private void validateClusterSpecification(ClusterSpecification clusterSpecification) throws FlinkException { try { final long taskManagerMemorySize = clusterSpecification.getTaskManagerMemoryMB(); // We do the validation by calling the calculation methods here // Internally these methods will check whether the cluster can be started with the provided // ClusterSpecification and the configured memory requirements final long cutoff = ContaineredTaskManagerParameters.calculateCutoffMB(flinkConfiguration, taskManagerMemorySize); TaskManagerServices.calculateHeapSizeMB(taskManagerMemorySize - cutoff, flinkConfiguration); } catch (IllegalArgumentException iae) { throw new FlinkException("Cannot fulfill the minimum memory requirements with the provided " + "cluster specification. Please increase the memory of the cluster.", iae); } }
clusterSpecification.getSlotsPerTaskManager()); clusterSpecification.getTaskManagerMemoryMB() + "m"); hasLog4j, hasKrb5, clusterSpecification.getMasterMemoryMB()); appMasterEnv.put(YarnConfigKeys.ENV_TM_COUNT, String.valueOf(clusterSpecification.getNumberTaskManagers())); appMasterEnv.put(YarnConfigKeys.ENV_TM_MEMORY, String.valueOf(clusterSpecification.getTaskManagerMemoryMB())); appMasterEnv.put(YarnConfigKeys.FLINK_JAR_PATH, remotePathJar.toString()); appMasterEnv.put(YarnConfigKeys.ENV_APP_ID, appId.toString()); appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_HOME_DIR, homeDir.toString()); appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_SHIP_FILES, envShipFileList.toString()); appMasterEnv.put(YarnConfigKeys.ENV_SLOTS, String.valueOf(clusterSpecification.getSlotsPerTaskManager())); appMasterEnv.put(YarnConfigKeys.ENV_DETACHED, String.valueOf(detached)); appMasterEnv.put(YarnConfigKeys.ENV_ZOOKEEPER_NAMESPACE, getZookeeperNamespace()); capability.setMemory(clusterSpecification.getMasterMemoryMB()); capability.setVirtualCores(1);
@Test public void testCorrectSettingOfMaxSlots() throws Exception { String[] params = new String[] {"-yn", "2", "-ys", "3"}; FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli( new Configuration(), tmp.getRoot().getAbsolutePath(), "y", "yarn"); final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true); AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine); final ClusterSpecification clusterSpecification = yarnCLI.getClusterSpecification(commandLine); // each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased. assertEquals(3, clusterSpecification.getSlotsPerTaskManager()); assertEquals(2, clusterSpecification.getNumberTaskManagers()); }
/** * Tests the specifying heap memory without unit for job manager and task manager. */ @Test public void testHeapMemoryPropertyWithoutUnit() throws Exception { final String[] args = new String[] { "-yn", "2", "-yjm", "1024", "-ytm", "2048" }; final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( new Configuration(), tmp.getRoot().getAbsolutePath(), "y", "yarn"); final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false); final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine); assertThat(clusterSpecification.getMasterMemoryMB(), is(1024)); assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048)); }
/** * Method to validate cluster specification before deploy it, it will throw * an {@link FlinkException} if the {@link ClusterSpecification} is invalid. * * @param clusterSpecification cluster specification to check against the configuration of the * AbstractYarnClusterDescriptor * @throws FlinkException if the cluster cannot be started with the provided {@link ClusterSpecification} */ private void validateClusterSpecification(ClusterSpecification clusterSpecification) throws FlinkException { try { final long taskManagerMemorySize = clusterSpecification.getTaskManagerMemoryMB(); // We do the validation by calling the calculation methods here // Internally these methods will check whether the cluster can be started with the provided // ClusterSpecification and the configured memory requirements final long cutoff = ContaineredTaskManagerParameters.calculateCutoffMB(flinkConfiguration, taskManagerMemorySize); TaskManagerServices.calculateHeapSizeMB(taskManagerMemorySize - cutoff, flinkConfiguration); } catch (IllegalArgumentException iae) { throw new FlinkException("Cannot fulfill the minimum memory requirements with the provided " + "cluster specification. Please increase the memory of the cluster.", iae); } }
public ClusterSpecification createClusterSpecification() { return new ClusterSpecification( masterMemoryMB, taskManagerMemoryMB, numberTaskManagers, slotsPerTaskManager); } }
ClusterResourceDescription freeClusterResources) throws YarnDeploymentException { int taskManagerCount = clusterSpecification.getNumberTaskManagers(); int jobManagerMemoryMb = clusterSpecification.getMasterMemoryMB(); int taskManagerMemoryMb = clusterSpecification.getTaskManagerMemoryMB(); .setMasterMemoryMB(jobManagerMemoryMb) .setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(clusterSpecification.getNumberTaskManagers()) .setSlotsPerTaskManager(clusterSpecification.getSlotsPerTaskManager()) .createClusterSpecification();
validClusterSpecification.getNumberTaskManagers(), validClusterSpecification.getSlotsPerTaskManager(), report, flinkConfiguration,
/** * Tests the specifying heap memory with unit (MB) for job manager and task manager. */ @Test public void testHeapMemoryPropertyWithUnitMB() throws Exception { final String[] args = new String[] { "-yn", "2", "-yjm", "1024m", "-ytm", "2048m" }; final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( new Configuration(), tmp.getRoot().getAbsolutePath(), "y", "yarn"); final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false); final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine); assertThat(clusterSpecification.getMasterMemoryMB(), is(1024)); assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048)); }
public ClusterSpecification createClusterSpecification() { return new ClusterSpecification( masterMemoryMB, taskManagerMemoryMB, numberTaskManagers, slotsPerTaskManager); } }
ClusterResourceDescription freeClusterResources) throws YarnDeploymentException { int taskManagerCount = clusterSpecification.getNumberTaskManagers(); int jobManagerMemoryMb = clusterSpecification.getMasterMemoryMB(); int taskManagerMemoryMb = clusterSpecification.getTaskManagerMemoryMB(); .setMasterMemoryMB(jobManagerMemoryMb) .setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(clusterSpecification.getNumberTaskManagers()) .setSlotsPerTaskManager(clusterSpecification.getSlotsPerTaskManager()) .createClusterSpecification();