String[] tmpDirPaths = ConfigurationUtils.parseTempDirectories(flinkConfig); File tmpDir = new File(tmpDirPaths[0], "hadoop-" + System.getProperty("user.name")); hadoopConfig.set("hadoop.tmp.dir", tmpDir.getPath());
@Test public void testPropertiesToConfiguration() { final Properties properties = new Properties(); final int entries = 10; for (int i = 0; i < entries; i++) { properties.setProperty("key" + i, "value" + i); } final Configuration configuration = ConfigurationUtils.createConfiguration(properties); for (String key : properties.stringPropertyNames()) { assertThat(configuration.getString(key, ""), is(equalTo(properties.getProperty(key)))); } assertThat(configuration.toMap().size(), is(properties.size())); }
private ClusterSpecification createClusterSpecification(Configuration configuration, CommandLine cmd) { if (cmd.hasOption(container.getOpt())) { // number of containers is required option! LOG.info("The argument {} is deprecated in will be ignored.", container.getOpt()); } // TODO: The number of task manager should be deprecated soon final int numberTaskManagers; if (cmd.hasOption(container.getOpt())) { numberTaskManagers = Integer.valueOf(cmd.getOptionValue(container.getOpt())); } else { numberTaskManagers = 1; } // JobManager Memory final int jobManagerMemoryMB = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes(); // Task Managers memory final int taskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes(); int slotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS); return new ClusterSpecification.ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMB) .setTaskManagerMemoryMB(taskManagerMemoryMB) .setNumberTaskManagers(numberTaskManagers) .setSlotsPerTaskManager(slotsPerTaskManager) .createClusterSpecification(); }
final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration); String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration); memoryFraction, timerServiceShutdownTimeout, ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));
final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration); String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration);
/** * Extracts the task manager directories for temporary files as defined by * {@link org.apache.flink.configuration.CoreOptions#TMP_DIRS}. * * @param configuration configuration object * @return array of configured directories (in order) */ @Nonnull public static String[] parseTempDirectories(Configuration configuration) { return splitPaths(configuration.getString(CoreOptions.TMP_DIRS)); }
this.defaultTaskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(flinkConfig).getMebiBytes(); this.defaultCpus = flinkConfig.getInteger(YarnConfigOptions.VCORES, numberOfTaskSlots); this.resource = Resource.newInstance(defaultTaskManagerMemoryMB, defaultCpus);
metricRegistry, "localhost", ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));
final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration); String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration); memoryFraction, timerServiceShutdownTimeout, ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));
/** * Extracts the local state directories as defined by * {@link CheckpointingOptions#LOCAL_RECOVERY_TASK_MANAGER_STATE_ROOT_DIRS}. * * @param configuration configuration object * @return array of configured directories (in order) */ @Nonnull public static String[] parseLocalStateDirectories(Configuration configuration) { String configValue = configuration.getString(CheckpointingOptions.LOCAL_RECOVERY_TASK_MANAGER_STATE_ROOT_DIRS, ""); return splitPaths(configValue); }
this.defaultTaskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(flinkConfig).getMebiBytes(); this.defaultCpus = flinkConfig.getInteger(YarnConfigOptions.VCORES, numberOfTaskSlots);
metricRegistry, "localhost", ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));
final String[] localTmpDirectories = ConfigurationUtils.parseTempDirectories(flinkConfig); Preconditions.checkArgument(localTmpDirectories.length > 0); final String localTmpDirectory = localTmpDirectories[0];
public static ClusterSpecification fromConfiguration(Configuration configuration) { int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1); int jobManagerMemoryMb = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes(); int taskManagerMemoryMb = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes(); return new ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMb) .setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(1) .setSlotsPerTaskManager(slots) .createClusterSpecification(); }
private static Configuration loadConfiguration(String[] args) throws FlinkParseException { final CommandLineParser<ClusterConfiguration> commandLineParser = new CommandLineParser<>(new ClusterConfigurationParserFactory()); final ClusterConfiguration clusterConfiguration; try { clusterConfiguration = commandLineParser.parse(args); } catch (FlinkParseException e) { LOG.error("Could not parse the command line options.", e); commandLineParser.printHelp(TaskManagerRunner.class.getSimpleName()); throw e; } final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(clusterConfiguration.getDynamicProperties()); return GlobalConfiguration.loadConfiguration(clusterConfiguration.getConfigDir(), dynamicProperties); }
/** * Extracts the working state directories as defined by {@link CheckpointingOptions#WORKING_DIRS}. * * @param configuration configuration object * @return array of configured directories (in order) */ @Nonnull public static String[] parseWorkingDirectories(Configuration configuration) { String configValue = configuration.getString(CheckpointingOptions.WORKING_DIRS, ""); return splitPaths(configValue); }
this.defaultTaskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(flinkConfig).getMebiBytes(); this.defaultCpus = flinkConfig.getInteger(YarnConfigOptions.VCORES, numberOfTaskSlots); this.resource = Resource.newInstance(defaultTaskManagerMemoryMB, defaultCpus);
metricRegistry, rpcService.getAddress(), ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));
final String[] tmpDirPaths = ConfigurationUtils.parseTempDirectories(config); baseDir = new File(tmpDirPaths[RANDOM.nextInt(tmpDirPaths.length)]);
private ClusterSpecification createClusterSpecification(Configuration configuration, CommandLine cmd) { if (cmd.hasOption(container.getOpt())) { // number of containers is required option! LOG.info("The argument {} is deprecated in will be ignored.", container.getOpt()); } // TODO: The number of task manager should be deprecated soon final int numberTaskManagers; if (cmd.hasOption(container.getOpt())) { numberTaskManagers = Integer.valueOf(cmd.getOptionValue(container.getOpt())); } else { numberTaskManagers = 1; } // JobManager Memory final int jobManagerMemoryMB = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes(); // Task Managers memory final int taskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes(); int slotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS); return new ClusterSpecification.ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMB) .setTaskManagerMemoryMB(taskManagerMemoryMB) .setNumberTaskManagers(numberTaskManagers) .setSlotsPerTaskManager(slotsPerTaskManager) .createClusterSpecification(); }