public HandlerResources(BenderConfig config) throws ClassNotFoundException { setSources(config.getSources()); setSerializerProcessor(config.getSerializerConfig()); setTransportFactory(config.getTransportConfig()); setWrapperFactory(config.getWrapperConfig()); setReporters(config.getReporters()); }
public static BenderConfig load(String filename, String data, ObjectMapper mapper, boolean validate) { String swappedData = swapEnvironmentVariables(data); if (validate) { BenderConfig.validate(swappedData, mapper); } BenderConfig config = null; try { config = mapper.readValue(swappedData, BenderConfig.class); } catch (IOException e) { throw new ConfigurationException("invalid config file", e); } return config; }
public void setSources(List<SourceConfig> sourceConfigs) throws ClassNotFoundException { for (SourceConfig sourceConfig : sourceConfigs) { Source source = new Source(sourceConfig); logger.info("Using source: " + source.toString()); sources.put(source.getSourceName(), source); } }
@Test public void testLoadConfg() { BenderConfig config = BenderConfig.load("/com/nextdoor/bender/partition/partition_config.json"); PartitionOperationConfig op = (PartitionOperationConfig) config.getSources().get(0).getOperationConfigs().get(0); PartitionSpec spec = op.getPartitionSpecs().get(0); assertEquals("type", spec.getName()); assertEquals(Interpreter.STRING, spec.getInterpreter()); List<String> expected = Arrays.asList("one", "two", "three"); assertEquals(expected, spec.getSources()); }
@Test public void testStringTemplatesInConfig() throws ConfigurationException, ClassNotFoundException { envVars.set("CUSTOM_SOURCE_NAME", "TestSourceName"); BenderConfig config = BenderConfig.load("/config/config_with_env.json"); List<SourceConfig> sources = config.getSources(); assertEquals(sources.size(), 1); assertEquals(sources.get(0).getName(), "TestSourceName"); }
public static void main(String[] args) throws JsonGenerationException, JsonMappingException, IOException { String filename = args[0]; new File(filename).getParentFile().mkdirs(); BenderSchema schema = new BenderSchema(); ObjectMapper mapper = new ObjectMapper(); mapper.enable(SerializationFeature.INDENT_OUTPUT); JsonNode node = schema.getSchema(); mapper.writeValue(new File(filename), node); } }
public Subtypes() { long start = System.nanoTime(); try { subtypes.addAll(ClassScanner.getSubtypes(abstractConfigClasses)); } catch (InterruptedException | ExecutionException e) { throw new ConfigurationException("unable to find config classes", e); } logger.debug( "Generating config subtype list took " + ((System.nanoTime() - start) / 1000000) + "ms"); /* * Sort the subtypes so that the order is deterministic. Without this locally generated * schemas differ in order from those generated by CircleCI. */ subtypes.sort(new Comparator<Class>() { @Override public int compare(Class o1, Class o2) { return o1.getName().compareToIgnoreCase(o2.getName()); } }); }
@SuppressWarnings("unchecked") public T getFactory(ConfigurableFactoryConfig<?> factoryConfig) throws ClassNotFoundException { ConfigurableFactory factory = (ConfigurableFactory) ReflectionUtils.newInstance(factoryConfig.getFactoryClass()); factory.setConf(factoryConfig); logger.trace(factory.getClass().getName() + " initialized"); return (T) factory; } }
@Test(expected = ConfigurationException.class) public void testMissingStringTemplatesInConfig() throws ConfigurationException, ClassNotFoundException { BenderConfig.load("/config/config_with_env.json"); }
public BenderSchema(File schemaFile) { ObjectMapper objectMapper = new ObjectMapper(); objectMapper.registerSubtypes(BenderConfig.subtypes); try { this.schema = objectMapper.readTree(schemaFile); } catch (IOException e) { throw new ConfigurationException("unable to load schema file", e); } }
public HandlerResources(BenderConfig config) throws ClassNotFoundException { setSources(config.getSources()); setSerializerProcessor(config.getSerializerConfig()); setTransportFactory(config.getTransportConfig()); setWrapperFactory(config.getWrapperConfig()); setReporters(config.getReporters()); }
@Test public void testStringFormatUpper() { BenderConfig config = BenderConfig.load("/com/nextdoor/bender/partition/partition_config_format.json"); PartitionOperationConfig op = (PartitionOperationConfig) config.getSources().get(0).getOperationConfigs().get(0); PartitionSpec spec = op.getPartitionSpecs().get(0); assertEquals("type", spec.getName()); assertEquals(Interpreter.STRING, spec.getInterpreter()); assertEquals(StringFormat.TOUPPER, spec.getStringFormat()); assertEquals("THISISATEST", spec.interpret("ThisIsATest")); }
@Test public void testYaml() throws ConfigurationException, ClassNotFoundException { envVars.set("CUSTOM_SOURCE_NAME", "TestSourceName"); BenderConfig config = BenderConfig.load("/config/config_with_env.yaml"); List<SourceConfig> sources = config.getSources(); assertEquals(sources.size(), 1); assertEquals(sources.get(0).getName(), "TestSourceName"); } }
public static BenderConfig load(String filename, String data, ObjectMapper mapper, boolean validate) { String swappedData = swapEnvironmentVariables(data); if (validate) { BenderConfig.validate(swappedData, mapper); } BenderConfig config = null; try { config = mapper.readValue(swappedData, BenderConfig.class); } catch (IOException e) { throw new ConfigurationException("invalid config file", e); } return config; }
public void setSources(List<SourceConfig> sourceConfigs) throws ClassNotFoundException { for (SourceConfig sourceConfig : sourceConfigs) { Source source = new Source(sourceConfig); logger.info("Using source: " + source.toString()); sources.put(source.getSourceName(), source); } }
public static void main(String[] args) throws JsonGenerationException, JsonMappingException, IOException { String filename = args[0]; new File(filename).getParentFile().mkdirs(); BenderSchema schema = new BenderSchema(); ObjectMapper mapper = new ObjectMapper(); mapper.enable(SerializationFeature.INDENT_OUTPUT); JsonNode node = schema.getSchema(); mapper.writeValue(new File(filename), node); } }
@SuppressWarnings("unchecked") public T getFactory(ConfigurableFactoryConfig<?> factoryConfig) throws ClassNotFoundException { ConfigurableFactory factory = (ConfigurableFactory) ReflectionUtils.newInstance(factoryConfig.getFactoryClass()); factory.setConf(factoryConfig); logger.trace(factory.getClass().getName() + " initialized"); return (T) factory; } }
public BenderSchema(File schemaFile) { ObjectMapper objectMapper = new ObjectMapper(); objectMapper.registerSubtypes(BenderConfig.subtypes); try { this.schema = objectMapper.readTree(schemaFile); } catch (IOException e) { throw new ConfigurationException("unable to load schema file", e); } }
@Test public void testStringFormatLower() { BenderConfig config = BenderConfig.load("/com/nextdoor/bender/partition/partition_config_format.json"); PartitionOperationConfig op = (PartitionOperationConfig) config.getSources().get(0).getOperationConfigs().get(0); PartitionSpec spec = op.getPartitionSpecs().get(0); spec.setStringFormat(StringFormat.TOLOWER); assertEquals("type", spec.getName()); assertEquals(Interpreter.STRING, spec.getInterpreter()); assertEquals("thisisatest", spec.interpret("ThisIsATest")); } }
@Test public void testLoadConfgTime() { BenderConfig config = BenderConfig.load("/com/nextdoor/bender/partition/partition_config_time.json"); PartitionOperationConfig op = (PartitionOperationConfig) config.getSources().get(0).getOperationConfigs().get(0); PartitionSpec spec = op.getPartitionSpecs().get(0); assertEquals("dt", spec.getName()); assertEquals(Interpreter.SECONDS, spec.getInterpreter()); List<String> expected = Arrays.asList("epoch"); assertEquals(expected, spec.getSources()); assertEquals("2017-01-19 05:05:59", spec.interpret("1484802359")); }