Refine search
LOG.info("Loading config from directory " + baseDirectory); Configuration configuration = GlobalConfiguration.loadConfiguration(baseDirectory); configuration.setString(property.getKey(), property.getValue()); configuration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, cliZKNamespace); if (configuration.getInteger(WebOptions.PORT, 0) >= 0) { configuration.setInteger(WebOptions.PORT, 0);
final Configuration config = new Configuration(); LOG.info("Loading configuration property: {}, {}", key, isSensitive(key) ? HIDDEN_CONTENT : value); config.setString(key, value);
LOG.info("TM: remote keytab principal obtained {}", remoteKeytabPrincipal); final Configuration configuration = GlobalConfiguration.loadConfiguration(currDir); FileSystem.initialize(configuration); configuration.setBoolean(AkkaOptions.JVM_EXIT_ON_FATAL_ERROR, true); configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath); configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
public static ClusterClient createStandaloneClient(LauncherOptions launcherOptions) throws Exception { String flinkConfDir = launcherOptions.getFlinkconf(); Configuration config = GlobalConfiguration.loadConfiguration(flinkConfDir); StandaloneClusterDescriptor standaloneClusterDescriptor = new StandaloneClusterDescriptor(config); RestClusterClient clusterClient = standaloneClusterDescriptor.retrieve(StandaloneClusterId.getInstance()); LeaderConnectionInfo connectionInfo = clusterClient.getClusterConnectionInfo(); InetSocketAddress address = AkkaUtils.getInetSocketAddressFromAkkaURL(connectionInfo.getAddress()); config.setString(JobManagerOptions.ADDRESS, address.getAddress().getHostName()); config.setInteger(JobManagerOptions.PORT, address.getPort()); clusterClient.setDetached(true); return clusterClient; }
/** * Loads the configuration files from the specified directory. * * <p>YAML files are supported as configuration files. * * @param configDir * the directory which contains the configuration files */ public static Configuration loadConfiguration(final String configDir) { return loadConfiguration(configDir, null); }
/** * Loads the global configuration and adds the given dynamic properties * configuration. * * @param dynamicProperties The given dynamic properties * @return Returns the loaded global configuration with dynamic properties */ public static Configuration loadConfigurationWithDynamicProperties(Configuration dynamicProperties) { final String configDir = System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR); if (configDir == null) { return new Configuration(dynamicProperties); } return loadConfiguration(configDir, dynamicProperties); }
protected StreamPlanEnvironment(ExecutionEnvironment env) { super(); this.env = env; int parallelism = env.getParallelism(); if (parallelism > 0) { setParallelism(parallelism); } else { // determine parallelism setParallelism(GlobalConfiguration.loadConfiguration().getInteger(CoreOptions.DEFAULT_PARALLELISM)); } }
private FiniteDuration getTimeout() { final Configuration configuration = GlobalConfiguration.loadConfiguration(); if (this.timeout != null) { configuration.setString(AkkaOptions.ASK_TIMEOUT, this.timeout); } return AkkaUtils.getClientTimeout(configuration); }
public static ClusterConfigurationInfo from(Configuration config) { ClusterConfigurationInfo clusterConfig = new ClusterConfigurationInfo(config.keySet().size()); for (String key : config.keySet()) { String value = config.getString(key, null); // Mask key values which contain sensitive information if (value != null && GlobalConfiguration.isSensitive(key)) { value = GlobalConfiguration.HIDDEN_CONTENT; } clusterConfig.add(new ClusterConfigurationInfoEntry(key, value)); } return clusterConfig; }
final Configuration flinkConfig = GlobalConfiguration.loadConfiguration(); if (!stormConf.containsKey(Config.NIMBUS_HOST)) { stormConf.put(Config.NIMBUS_HOST, flinkConfig.getString(JobManagerOptions.ADDRESS, "localhost")); new Integer(flinkConfig.getInteger(JobManagerOptions.PORT)));
Configuration configuration = GlobalConfiguration.loadConfiguration(configDirectory.getAbsolutePath()); File yarnPropertiesFile = FlinkYarnSessionCli.getYarnPropertiesLocation(configuration.getString(YarnConfigOptions.PROPERTIES_FILE_LOCATION)); if (yarnPropertiesFile.exists()) { LOG.info("testDetachedPerJobYarnClusterInternal: Cleaning up temporary Yarn address reference: {}", yarnPropertiesFile.getAbsolutePath());
Configuration conf = GlobalConfiguration.loadConfiguration(tmpDir.getAbsolutePath()); assertEquals(6, conf.keySet().size()); assertEquals("myvalue1", conf.getString("mykey1", null)); assertEquals("myvalue2", conf.getString("mykey2", null)); assertEquals("null", conf.getString("mykey3", "null")); assertEquals("myvalue4", conf.getString("mykey4", null));
protected static Configuration loadConfiguration(ClusterConfiguration clusterConfiguration) { final Configuration configuration = GlobalConfiguration.loadConfiguration(clusterConfiguration.getConfigDir()); final int restPort = clusterConfiguration.getRestPort(); if (restPort >= 0) { configuration.setInteger(RestOptions.PORT, restPort); } return configuration; }
public FlinkPlanner(List<String> classPath) { super(); this.classPath = classPath; env.getConfig().disableSysoutLogging(); if (env.getParallelism() <= 0) { // load the default parallelism from config GlobalConfiguration.loadConfiguration(new File(CliFrontend.getConfigurationDirectoryFromEnv()).getAbsolutePath()); org.apache.flink.configuration.Configuration configuration = GlobalConfiguration.getConfiguration(); int parallelism = configuration.getInteger(ConfigConstants.DEFAULT_PARALLELISM_KEY, -1); if (parallelism <= 0) { throw new RuntimeException("Please set the default parallelism via the -p command-line flag"); } else { env.setParallelism(parallelism); } } }
public AbstractYarnClusterDescriptor() { // for unit tests only if (System.getenv("IN_TESTS") != null) { try { conf.addResource(new File(System.getenv("YARN_CONF_DIR") + "/yarn-site.xml").toURI().toURL()); } catch (Throwable t) { throw new RuntimeException("Error", t); } } // tries to load the config through the environment, if it fails it can //still be set through the setters try { this.configurationDirectory = CliFrontend. getConfigurationDirectoryFromEnv(); GlobalConfiguration.loadConfiguration(configurationDirectory); this.flinkConfiguration = GlobalConfiguration.getConfiguration(); File confFile = new File(configurationDirectory + File.separator + CONFIG_FILE_NAME); if (!confFile.exists()) { throw new RuntimeException("Unable to locate configuration file in " + confFile); } flinkConfigurationPath = new Path(confFile.getAbsolutePath()); } catch (Exception e) { LOG.debug("Config couldn't be loaded from environment variable."); } }
@Test public void testHiddenKey() { assertTrue(GlobalConfiguration.isSensitive("password123")); assertTrue(GlobalConfiguration.isSensitive("123pasSword")); assertTrue(GlobalConfiguration.isSensitive("PasSword")); assertTrue(GlobalConfiguration.isSensitive("Secret")); assertFalse(GlobalConfiguration.isSensitive("Hello")); } }
Configuration flinkConfig = GlobalConfiguration.loadConfiguration(); flinkConfig.setBoolean(SecurityOptions.ZOOKEEPER_SASL_DISABLE, false); flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, testKeytab); flinkConfig.setBoolean(SecurityOptions.KERBEROS_LOGIN_USETICKETCACHE, false); flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, testPrincipal); flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_CONTEXTS, "Client,KafkaClient");
protected static Configuration loadConfiguration(EntrypointClusterConfiguration entrypointClusterConfiguration) { final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(entrypointClusterConfiguration.getDynamicProperties()); final Configuration configuration = GlobalConfiguration.loadConfiguration(entrypointClusterConfiguration.getConfigDir(), dynamicProperties); final int restPort = entrypointClusterConfiguration.getRestPort(); if (restPort >= 0) { configuration.setInteger(RestOptions.PORT, restPort); } final String hostname = entrypointClusterConfiguration.getHostname(); if (hostname != null) { configuration.setString(JobManagerOptions.ADDRESS, hostname); } return configuration; }
protected DelimitedInputFormat(Path filePath, Configuration configuration) { super(filePath); if (configuration == null) { configuration = GlobalConfiguration.loadConfiguration(); } loadConfigParameters(configuration); }
/** * Loads the global configuration from the environment. Fails if an error occurs during loading. Returns an * empty configuration object if the environment variable is not set. In production this variable is set but * tests and local execution/debugging don't have this environment variable set. That's why we should fail * if it is not set. * @return Returns the Configuration */ public static Configuration loadConfiguration() { final String configDir = System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR); if (configDir == null) { return new Configuration(); } return loadConfiguration(configDir, null); }