@Override public Map<String, Object> getSensorConfig(String sensorName) { return config.getSensorParserConfig(sensorName).getParserConfig(); }
@Override public String getIndex(String sensorName) { if(config != null && config.getSensorParserConfig(sensorName) != null && config.getSensorParserConfig(sensorName).getParserConfig() != null ) { Object indexObj = config.getSensorParserConfig(sensorName).getParserConfig().get(IndexingConfigurations.INDEX_CONF); if(indexObj != null) { return indexObj.toString(); } return null; } return sensorName; }
@Override public int getBatchTimeout(String sensorName) { if(config != null && config.getSensorParserConfig(sensorName) != null && config.getSensorParserConfig(sensorName).getParserConfig() != null ) { Object batchObj = config.getSensorParserConfig(sensorName).getParserConfig().get(IndexingConfigurations.BATCH_TIMEOUT_CONF); return batchObj == null ? 0 : ConversionUtils.convert(batchObj, Integer.class); } return 0; }
@Override public int getBatchSize(String sensorName) { if(config != null && config.getSensorParserConfig(sensorName) != null && config.getSensorParserConfig(sensorName).getParserConfig() != null ) { Object batchObj = config.getSensorParserConfig(sensorName).getParserConfig().get(IndexingConfigurations.BATCH_SIZE_CONF); return batchObj == null ? ParserConfigurations.DEFAULT_KAFKA_BATCH_SIZE : ConversionUtils.convert(batchObj, Integer.class); } return 1; }
@Override public boolean isEnabled(String sensorName) { if(config != null && config.getSensorParserConfig(sensorName) != null && config.getSensorParserConfig(sensorName).getParserConfig() != null ) { Object enabledObj = config.getSensorParserConfig(sensorName).getParserConfig().get(IndexingConfigurations.ENABLED_CONF); return enabledObj == null ? true : ConversionUtils.convert(enabledObj, Boolean.class); } return true; }
@Override public ParserConfigurations getConfigurations() { config.getSensorParserConfig(sensorType).getParserConfig().put(IndexingConfigurations.BATCH_SIZE_CONF, 1); return config; }
@Override public Map<String, Object> applyTransformations(SensorParserContext sensorParserContext) { JSONObject sampleJson = new JSONObject(sensorParserContext.getSampleData()); sensorParserContext.getSensorParserConfig().getFieldTransformations().forEach(fieldTransformer -> { fieldTransformer.transformAndUpdate(sampleJson, Context.EMPTY_CONTEXT(), sensorParserContext.getSensorParserConfig().getParserConfig()); } ); return sampleJson; }
parserConfig.getParserConfig().putIfAbsent("stellarContext", stellarContext); if (!StringUtils.isEmpty(parserConfig.getFilterClassName())) { filter = Filters.get( parserConfig.getFilterClassName(), parserConfig.getParserConfig() ); parser.configure(parserConfig.getParserConfig()); parser.init(); sensorToParserComponentMap.put(sensorType, new ParserComponent(parser, filter));
/** * Applies Stellar field transformations defined in the sensor parser config. * @param message Message parsed by the MessageParser * @param rawMessage Raw message including metadata * @param sensorParserConfig Sensor parser config */ private void applyFieldTransformations(JSONObject message, RawMessage rawMessage, SensorParserConfig sensorParserConfig) { for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) { if (handler != null) { if (!sensorParserConfig.getMergeMetadata()) { //if we haven't merged metadata, then we need to pass them along as configuration params. handler.transformAndUpdate( message, stellarContext, sensorParserConfig.getParserConfig(), rawMessage.getMetadata() ); } else { handler.transformAndUpdate( message, stellarContext, sensorParserConfig.getParserConfig() ); } } } }
String name = parseMessageRequest.getSensorParserConfig().getSensorTopic(); temporaryGrokPath = grokService.saveTemporary(parseMessageRequest.getGrokStatement(), name); sensorParserConfig.getParserConfig() .put(MetronRestConstants.GROK_PATH_KEY, new Path(temporaryGrokPath, name).toString()); parser.configure(sensorParserConfig.getParserConfig()); parser.init();
@Test public void testSimpleMapping() throws IOException { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(config)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); Assert.assertNotNull(handler); Assert.assertEquals(ImmutableMap.of("protocol", "TCP") ,handler.transform(new JSONObject(ImmutableMap.of("protocol", 6)) , Context.EMPTY_CONTEXT() , c.getParserConfig() ) ); } }
@Test public void testComplexMapping() throws IOException { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(complexConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); Assert.assertNotNull(handler); Assert.assertEquals(ImmutableMap.of("output", "field1=value1,field2=value2") ,handler.transform(new JSONObject(ImmutableMap.of("field1", "value1" ,"field2", "value2" ) ) , Context.EMPTY_CONTEXT() , c.getParserConfig() ) ); } @Test
SensorParserConfig yafConfig = mock(SensorParserConfig.class); when(yafConfig.getSensorTopic()).thenReturn("yafTopic"); when(yafConfig.getParserConfig()).thenReturn(new HashMap<String, Object>() {{ put(IndexingConfigurations.BATCH_SIZE_CONF, 10); }});
@Before public void setup() throws IOException { parserConfigurations = new ParserConfigurations(); SensorParserConfig broConfig = SensorParserConfig.fromBytes(broConfigString.getBytes()); SensorParserConfig snortConfig = SensorParserConfig.fromBytes(snortConfigString.getBytes()); parserConfigurations.updateSensorParserConfig("bro", broConfig); parserConfigurations.updateSensorParserConfig("snort", snortConfig); parserConfigurations.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfigString, JSONUtils.MAP_SUPPLIER)); parserRunner = new ParserRunnerImpl(new HashSet<>(Arrays.asList("bro", "snort"))); broParser = mock(MessageParser.class); snortParser = mock(MessageParser.class); stellarFilter = mock(StellarFilter.class); mockStatic(ReflectionUtils.class); mockStatic(Filters.class); when(ReflectionUtils.createInstance("org.apache.metron.parsers.bro.BasicBroParser")).thenReturn(broParser); when(ReflectionUtils.createInstance("org.apache.metron.parsers.snort.BasicSnortParser")).thenReturn(snortParser); when(Filters.get("org.apache.metron.parsers.filters.StellarFilter", broConfig.getParserConfig())) .thenReturn(stellarFilter); }
@Test public void shouldInit() throws Exception { Context stellarContext = mock(Context.class); Map<String, Object> broParserConfig = parserConfigurations.getSensorParserConfig("bro").getParserConfig(); Map<String, Object> snortParserConfig = parserConfigurations.getSensorParserConfig("snort").getParserConfig();
assertThat(actualSensorConfig.getCacheConfig(), not(new HashMap<>())); assertThat(actualSensorConfig.getCacheConfig().get("stellar.cache.maxSize"), equalTo(20000)); assertThat(actualSensorConfig.getParserConfig(), not(new HashMap<>())); assertThat(actualSensorConfig.getParserConfig().get("parser"), equalTo("config")); assertThat(actualSensorConfig.getFieldTransformations(), not(new ArrayList<>())); assertThat(actualSensorConfig.getFieldTransformations().get(0), not(nullValue()));
parser.configure(config.getParserConfig());
put("dc", "london"); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT(), c.getParserConfig()); long expected = 1452013350000L; Assert.assertEquals(expected, input.get("utc_timestamp")); put("url", "https://caseystella.com/blog"); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT(), c.getParserConfig()); long expected = 1452013350000L; Assert.assertEquals(expected, input.get("utc_timestamp"));