public static void writeGlobalConfigToZookeeper(Map<String, Object> globalConfig, String zookeeperUrl) throws Exception { try(CuratorFramework client = getClient(zookeeperUrl)) { client.start(); writeGlobalConfigToZookeeper(globalConfig, client); } } public static void writeGlobalConfigToZookeeper(Map<String, Object> globalConfig, CuratorFramework client) throws Exception {
public static void writeGlobalConfigToZookeeper(byte[] globalConfig, String zookeeperUrl) throws Exception { try(CuratorFramework client = getClient(zookeeperUrl)) { client.start(); writeGlobalConfigToZookeeper(globalConfig, client); } }
@Override public Map<String, Object> save(Map<String, Object> globalConfig) throws RestException { try { ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig, client); } catch (Exception e) { throw new RestException(e); } return globalConfig; }
public static void writeGlobalConfigToZookeeper(Map<String, Object> globalConfig, CuratorFramework client) throws Exception { writeGlobalConfigToZookeeper(JSONUtils.INSTANCE.toJSONPretty(globalConfig), client); }
protected void pushConfig(Path srcPath, Path dstPath, String configName, String zookeeper) { System.out.println("Beginning update of global configs"); try (CuratorFramework client = ConfigurationsUtils.getClient(zookeeper)) { client.start(); // Use the parent and place a new file. Has to be a new file so we can update the configs and trigger updates. // Fetch the global configuration Map<String, Object> global = JSONUtils.INSTANCE.load( new ByteArrayInputStream(ConfigurationsUtils.readGlobalConfigBytesFromZookeeper(client)), JSONUtils.MAP_SUPPLIER); // Update the global config and push it back global.put(configName, dstPath.toString() + "/" + srcPath.getName()); ConfigurationsUtils.writeGlobalConfigToZookeeper(global, client); } catch (Exception e) { System.err.println("Unable to load new GeoLite2 config for " + configName + " into HDFS: " + e); e.printStackTrace(); System.exit(2); } System.out.println("Finished update of global configs"); }
if (globalConfig.length > 0) { setupStellarStatically(client, Optional.of(new String(globalConfig))); writeGlobalConfigToZookeeper(globalConfig, client);
writeGlobalConfigToZookeeper(value.getBytes(), client);
/** * Uploads configuration to Zookeeper. * @throws Exception */ private void upload() throws Exception { assert zookeeperURL != null; try(CuratorFramework client = getClient(zookeeperURL)) { if(client.getState() != CuratorFrameworkState.STARTED) { client.start(); } if (globals != null) { writeGlobalConfigToZookeeper(globals, client); } } }
public void update() throws UnableToStartException { try { final String zookeeperUrl = connectionString == null?topologyProperties.getProperty(ZKServerComponent.ZOOKEEPER_PROPERTY):connectionString; if(globalConfigPath != null || parserConfigsPath != null || enrichmentConfigsPath != null || indexingConfigsPath != null || profilerConfigPath != null ) { uploadConfigsToZookeeper(globalConfigPath, parserConfigsPath, enrichmentConfigsPath, indexingConfigsPath, profilerConfigPath, zookeeperUrl); } for(Map.Entry<String, SensorParserConfig> kv : parserSensorConfigs.entrySet()) { writeSensorParserConfigToZookeeper(kv.getKey(), kv.getValue(), zookeeperUrl); } if(globalConfig.isPresent()) { writeGlobalConfigToZookeeper(globalConfig.get().getBytes(), zookeeperUrl); } if(postStartCallback.isPresent()) { postStartCallback.get().accept(this); } } catch (Exception e) { throw new UnableToStartException(e.getMessage(), e); } }
if (globalConfig.length > 0) { setupStellarStatically(client, Optional.of(new String(globalConfig))); ConfigurationsUtils.writeGlobalConfigToZookeeper(readGlobalConfigFromFile(globalConfigPath), client);
ConfigurationsUtils.writeGlobalConfigToZookeeper(config, client);
@Before public void setupConfiguration() throws Exception { TestingServer testZkServer = new TestingServer(true); this.zookeeperUrl = testZkServer.getConnectString(); byte[] globalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH); ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig, zookeeperUrl); parserConfigurationTypes.add(ConfigurationType.GLOBAL.getTypeName()); Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH); for (String sensorType : sensorEnrichmentConfigs.keySet()) { ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(sensorType, sensorEnrichmentConfigs.get(sensorType), zookeeperUrl); } Map<String, byte[]> sensorParserConfigs = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH); for (String sensorType : sensorParserConfigs.keySet()) { ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorType, sensorParserConfigs.get(sensorType), zookeeperUrl); parserConfigurationTypes.add(sensorType); } }
@Before public void setupConfiguration() throws Exception { TestingServer testZkServer = new TestingServer(true); this.zookeeperUrl = testZkServer.getConnectString(); byte[] globalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH); ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig, zookeeperUrl); enrichmentConfigurationTypes.add(ConfigurationType.GLOBAL.getTypeName()); Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH); for (String sensorType : sensorEnrichmentConfigs.keySet()) { ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(sensorType, sensorEnrichmentConfigs.get(sensorType), zookeeperUrl); enrichmentConfigurationTypes.add(sensorType); } Map<String, byte[]> sensorParserConfigs = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH); for (String sensorType : sensorParserConfigs.keySet()) { ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorType, sensorParserConfigs.get(sensorType), zookeeperUrl); } }
@Test public void test() throws Exception { Assert.assertTrue(expectedGlobalConfig.length > 0); ConfigurationsUtils.writeGlobalConfigToZookeeper(expectedGlobalConfig, zookeeperUrl); byte[] actualGlobalConfigBytes = ConfigurationsUtils.readGlobalConfigBytesFromZookeeper(client); Assert.assertTrue(Arrays.equals(expectedGlobalConfig, actualGlobalConfigBytes)); Assert.assertTrue(expectedSensorParserConfigMap.size() > 0); String testSensorType = "yaf"; byte[] expectedSensorParserConfigBytes = expectedSensorParserConfigMap.get(testSensorType); ConfigurationsUtils.writeSensorParserConfigToZookeeper(testSensorType, expectedSensorParserConfigBytes, zookeeperUrl); byte[] actualSensorParserConfigBytes = ConfigurationsUtils.readSensorParserConfigBytesFromZookeeper(testSensorType, client); Assert.assertTrue(Arrays.equals(expectedSensorParserConfigBytes, actualSensorParserConfigBytes)); Assert.assertTrue(expectedSensorEnrichmentConfigMap.size() > 0); byte[] expectedSensorEnrichmentConfigBytes = expectedSensorEnrichmentConfigMap.get(testSensorType); ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(testSensorType, expectedSensorEnrichmentConfigBytes, zookeeperUrl); byte[] actualSensorEnrichmentConfigBytes = ConfigurationsUtils.readSensorEnrichmentConfigBytesFromZookeeper(testSensorType, client); Assert.assertTrue(Arrays.equals(expectedSensorEnrichmentConfigBytes, actualSensorEnrichmentConfigBytes)); String name = "testConfig"; Map<String, Object> testConfig = new HashMap<>(); testConfig.put("stringField", "value"); testConfig.put("intField", 1); testConfig.put("doubleField", 1.1); ConfigurationsUtils.writeConfigToZookeeper(name, testConfig, zookeeperUrl); byte[] readConfigBytes = ConfigurationsUtils.readConfigBytesFromZookeeper(name, client); Assert.assertTrue(Arrays.equals(JSONUtils.INSTANCE.toJSONPretty(testConfig), readConfigBytes)); }
Map<String, Object> sampleGlobalConfig = sampleConfigurations.getGlobalConfig(); sampleGlobalConfig.put("newGlobalField", "newGlobalValue"); ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl); waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName()); Assert.assertEquals("Add global config field", sampleConfigurations.getGlobalConfig(), configuredBolt.getConfigurations().getGlobalConfig()); ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl); waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName()); Assert.assertEquals("Remove global config field", sampleConfigurations, configuredBolt.getConfigurations());
Map<String, Object> sampleGlobalConfig = sampleConfigurations.getGlobalConfig(); sampleGlobalConfig.put("newGlobalField", "newGlobalValue"); ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl); waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName()); Assert.assertEquals("Add global config field", sampleConfigurations.getGlobalConfig(), configuredBolt.getConfigurations().getGlobalConfig()); ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl); waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName()); Assert.assertEquals("Remove global config field", sampleConfigurations, configuredBolt.getConfigurations());
@Test public void validateUpdate() throws Exception { ConfigurationsUtils.writeSensorIndexingConfigToZookeeper("test", testIndexingConfig.getBytes(), client); ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig.getBytes(), client); ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper("test", testEnrichmentConfig.getBytes(), client); ConfigurationsUtils.writeSensorParserConfigToZookeeper("bro", testParserConfig.getBytes(), client);