private void logAllSettings(Config config) { StringBuilder b = new StringBuilder(); b.append("Elasticsearch System settings: "); b.append("\n"); for (Map.Entry<String, String> entry : config.entrySet()) { b.append('\t'); b.append(entry.getKey()); b.append(" = "); b.append(entry.getValue()); b.append("\n"); } LOGGER.info(b.toString()); } }
public Config regexSubset(String regex) { Map<String, String> out = new HashMap<String, String>(); Pattern pattern = Pattern.compile(regex); for (Entry<String, String> entry : entrySet()) { String k = entry.getKey(); Matcher matcher = pattern.matcher(k); if (matcher.find()) { out.put(k, entry.getValue()); } } return new MapConfig(out); }
public Config regexSubset(String regex) { Map<String, String> out = new HashMap<String, String>(); Pattern pattern = Pattern.compile(regex); for (Entry<String, String> entry : entrySet()) { String k = entry.getKey(); Matcher matcher = pattern.matcher(k); if (matcher.find()) { out.put(k, entry.getValue()); } } return new MapConfig(out); }
public Config subset(String prefix, boolean stripPrefix) { Map<String, String> out = new HashMap<String, String>(); for (Entry<String, String> entry : entrySet()) { String k = entry.getKey(); if (k != null && k.startsWith(prefix)) { if (stripPrefix) { k = k.substring(prefix.length()); } out.put(k, entry.getValue()); } } return new MapConfig(out); }
public Config subset(String prefix, boolean stripPrefix) { Map<String, String> out = new HashMap<String, String>(); for (Entry<String, String> entry : entrySet()) { String k = entry.getKey(); if (k != null && k.startsWith(prefix)) { if (stripPrefix) { k = k.substring(prefix.length()); } out.put(k, entry.getValue()); } } return new MapConfig(out); }
/** * Get a list of system names. * * @return A list system names */ public List<String> getSystemNames() { Config subConf = subset(SYSTEM_PREFIX, true); ArrayList<String> systemNames = new ArrayList<String>(); for (Map.Entry<String, String> entry : subConf.entrySet()) { String key = entry.getKey(); if (key.endsWith(SYSTEM_FACTORY_SUFFIX)) { systemNames.add(key.replace(SYSTEM_FACTORY_SUFFIX, EMPTY)); } } return systemNames; }
/** * Get a list of system names. * * @return A list system names */ public List<String> getSystemNames() { Config subConf = subset(SYSTEM_PREFIX, true); ArrayList<String> systemNames = new ArrayList<String>(); for (Map.Entry<String, String> entry : subConf.entrySet()) { String key = entry.getKey(); if (key.endsWith(SYSTEM_FACTORY_SUFFIX)) { systemNames.add(key.replace(SYSTEM_FACTORY_SUFFIX, EMPTY)); } } return systemNames; }
/** * Get a list of system names. * * @return A list system names */ public List<String> getSystemNames() { Config subConf = subset(SYSTEM_PREFIX, true); ArrayList<String> systemNames = new ArrayList<String>(); for (Map.Entry<String, String> entry : subConf.entrySet()) { String key = entry.getKey(); if (key.endsWith(SYSTEM_FACTORY_SUFFIX)) { systemNames.add(key.replace(SYSTEM_FACTORY_SUFFIX, EMPTY)); } } return systemNames; }
/** * Get a list of system names. * * @return A list system names */ public List<String> getSystemNames() { Config subConf = subset(SYSTEM_PREFIX, true); ArrayList<String> systemNames = new ArrayList<String>(); for (Map.Entry<String, String> entry : subConf.entrySet()) { String key = entry.getKey(); if (key.endsWith(SYSTEM_FACTORY_SUFFIX)) { systemNames.add(key.replace(SYSTEM_FACTORY_SUFFIX, EMPTY)); } } return systemNames; }
/** * Get a list of system names. * * @return A list system names */ public List<String> getSystemNames() { Config subConf = subset(SYSTEM_PREFIX, true); ArrayList<String> systemNames = new ArrayList<String>(); for (Map.Entry<String, String> entry : subConf.entrySet()) { String key = entry.getKey(); if (key.endsWith(SYSTEM_FACTORY_SUFFIX)) { systemNames.add(key.replace(SYSTEM_FACTORY_SUFFIX, EMPTY)); } } return systemNames; }
protected Config rewrite(Config config, Map<String, String> env) { Map<String, String> envConfig = new HashMap<>(); for (Map.Entry<String, String> entry: env.entrySet()) { if (entry.getKey().startsWith(SAMZA_KEY_PREFIX)) { String key = renameKey(entry.getKey()); String value = entry.getValue(); if (config.containsKey(key)) { LOGGER.info(String.format("Overriding property from environment: %s=%s", key, value)); } else { LOGGER.info(String.format("Setting property from environment: %s=%s", key, value)); } envConfig.put(key, value); } } for (Map.Entry<String, String> entry: config.entrySet()) { String key = entry.getKey(); if (!envConfig.containsKey(key) && envConfig.containsKey(key.toLowerCase())) { throw new IllegalArgumentException(String.format( "Can't override property from environment with mixed case: %s", key)); } } return new MapConfig(Arrays.asList(config, envConfig)); }
protected Config rewrite(Config config, Map<String, String> env) { Map<String, String> envConfig = new HashMap<>(); for (Map.Entry<String, String> entry: env.entrySet()) { if (entry.getKey().startsWith(SAMZA_KEY_PREFIX)) { String key = renameKey(entry.getKey()); String value = entry.getValue(); if (config.containsKey(key)) { LOGGER.info(String.format("Overriding property from environment: %s=%s", key, value)); } else { LOGGER.info(String.format("Setting property from environment: %s=%s", key, value)); } envConfig.put(key, value); } } for (Map.Entry<String, String> entry: config.entrySet()) { String key = entry.getKey(); if (!envConfig.containsKey(key) && envConfig.containsKey(key.toLowerCase())) { throw new IllegalArgumentException(String.format( "Can't override property from environment with mixed case: %s", key)); } } return new MapConfig(Arrays.asList(config, envConfig)); }
protected Config rewrite(Config config, Map<String, String> env) { Map<String, String> envConfig = new HashMap<>(); for (Map.Entry<String, String> entry: env.entrySet()) { if (entry.getKey().startsWith(SAMZA_KEY_PREFIX)) { String key = renameKey(entry.getKey()); String value = entry.getValue(); if (config.containsKey(key)) { LOGGER.info(String.format("Overriding property from environment: %s=%s", key, value)); } else { LOGGER.info(String.format("Setting property from environment: %s=%s", key, value)); } envConfig.put(key, value); } } for (Map.Entry<String, String> entry: config.entrySet()) { String key = entry.getKey(); if (!envConfig.containsKey(key) && envConfig.containsKey(key.toLowerCase())) { throw new IllegalArgumentException(String.format( "Can't override property from environment with mixed case: %s", key)); } } return new MapConfig(Arrays.asList(config, envConfig)); }
/** * Helper method that sends a series of SetConfig messages to the coordinator * stream. * * @param source * An identifier to denote which source is sending a message. This * can be any arbitrary string. * @param config * The config object to store in the coordinator stream. */ public void writeConfig(String source, Config config) { log.debug("Writing config: {}", config); for (Map.Entry<String, String> configPair : config.entrySet()) { send(new SetConfig(source, configPair.getKey(), configPair.getValue())); } systemProducer.flush(source); }
/** * Helper method that sends a series of SetConfig messages to the coordinator * stream. * * @param source * An identifier to denote which source is sending a message. This * can be any arbitrary string. * @param config * The config object to store in the coordinator stream. */ public void writeConfig(String source, Config config) { log.debug("Writing config: {}", config); for (Map.Entry<String, String> configPair : config.entrySet()) { send(new SetConfig(source, configPair.getKey(), configPair.getValue())); } systemProducer.flush(source); }
/** * Helper method that sends a series of SetConfig messages to the coordinator * stream. * * @param source * An identifier to denote which source is sending a message. This * can be any arbitrary string. * @param config * The config object to store in the coordinator stream. */ public void writeConfig(String source, Config config) { log.debug("Writing config: {}", config); for (Map.Entry<String, String> configPair : config.entrySet()) { send(new SetConfig(source, configPair.getKey(), configPair.getValue())); } systemProducer.flush(source); }
/** * Helper method that sends a series of SetConfig messages to the coordinator * stream. * * @param source * An identifier to denote which source is sending a message. This * can be any arbitrary string. * @param config * The config object to store in the coordinator stream. */ public void writeConfig(String source, Config config) { log.debug("Writing config: {}", config); for (Map.Entry<String, String> configPair : config.entrySet()) { send(new SetConfig(source, configPair.getKey(), configPair.getValue())); } systemProducer.flush(source); }
/** * Helper method that sends a series of SetConfig messages to the coordinator * stream. * * @param source * An identifier to denote which source is sending a message. This * can be any arbitrary string. * @param config * The config object to store in the coordinator stream. */ public void writeConfig(String source, Config config) { log.debug("Writing config: {}", config); for (Map.Entry<String, String> configPair : config.entrySet()) { send(new SetConfig(source, configPair.getKey(), configPair.getValue())); } systemProducer.flush(source); }
private Map<String, Serde> validateAndGetDeserializedSerdes(Config jobConfig, int numSerdes) { Config serializers = jobConfig.subset("serializers.registry.", true); // make sure that the serializers deserialize correctly SerializableSerde<Serde> serializableSerde = new SerializableSerde<>(); assertEquals(numSerdes, serializers.size()); return serializers.entrySet().stream().collect(Collectors.toMap( e -> e.getKey().replace(SerializerConfig.SERIALIZED_INSTANCE_SUFFIX(), ""), e -> serializableSerde.fromBytes(Base64.getDecoder().decode(e.getValue().getBytes())) )); }
/** * Stores the configuration of the job in the coordinator stream. */ private void storeConfigInCoordinatorStream() { MetadataStore metadataStore = null; try { // Creates the coordinator stream if it does not exists. createCoordinatorStream(); MetadataStoreFactory metadataStoreFactory = Util.getObj(new JobConfig(config).getMetadataStoreFactory(), MetadataStoreFactory.class); metadataStore = metadataStoreFactory.getMetadataStore(SetConfig.TYPE, config, metrics.getMetricsRegistry()); metadataStore.init(); CoordinatorStreamValueSerde jsonSerde = new CoordinatorStreamValueSerde(SetConfig.TYPE); for (Map.Entry<String, String> entry : config.entrySet()) { byte[] serializedValue = jsonSerde.toBytes(entry.getValue()); metadataStore.put(entry.getKey(), serializedValue); } } finally { if (metadataStore != null) { LOG.info("Stopping the coordinator system producer."); metadataStore.close(); } } }