/** * Applies Stellar field transformations defined in the sensor parser config. * @param message Message parsed by the MessageParser * @param rawMessage Raw message including metadata * @param sensorParserConfig Sensor parser config */ private void applyFieldTransformations(JSONObject message, RawMessage rawMessage, SensorParserConfig sensorParserConfig) { for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) { if (handler != null) { if (!sensorParserConfig.getMergeMetadata()) { //if we haven't merged metadata, then we need to pass them along as configuration params. handler.transformAndUpdate( message, stellarContext, sensorParserConfig.getParserConfig(), rawMessage.getMetadata() ); } else { handler.transformAndUpdate( message, stellarContext, sensorParserConfig.getParserConfig() ); } } } }
public SensorParserConfig getSensorParserConfig() { if (sensorParserConfig == null) { return new SensorParserConfig(); } return sensorParserConfig; }
String inputTopic = parserConfig.getSensorTopic() != null ? parserConfig.getSensorTopic() : sensorType; kafkaSpoutConfigOptions.putIfAbsent( SpoutConfiguration.FIRST_POLL_OFFSET_STRATEGY.key , KafkaSpoutConfig.FirstPollOffsetStrategy.UNCOMMITTED_EARLIEST.name()
.append(writerClassName, that.writerClassName) .append(errorWriterClassName, that.errorWriterClassName) .append(getReadMetadata(), that.getReadMetadata()) .append(getMergeMetadata(), that.getMergeMetadata()) .append(numWorkers, that.numWorkers) .append(numAckers, that.numAckers)
sensorType, parserConfig.getParserClassName(), parserConfig.getFilterClassName()); .createInstance(parserConfig.getParserClassName()); parserConfig.getParserConfig().putIfAbsent("stellarContext", stellarContext); if (!StringUtils.isEmpty(parserConfig.getFilterClassName())) { filter = Filters.get( parserConfig.getFilterClassName(), parserConfig.getParserConfig() ); parser.configure(parserConfig.getParserConfig()); parser.init(); sensorToParserComponentMap.put(sensorType, new ParserComponent(parser, filter));
parserConfigs.updateSensorParserConfig("test-sensor", parserConfig.getBytes()); SensorParserConfig actualSensorConfig = parserConfigs.getSensorParserConfig("test-sensor"); assertThat(actualSensorConfig.getParserClassName(), equalTo("parser-class")); assertThat(actualSensorConfig.getFilterClassName(), equalTo("filter-class")); assertThat(actualSensorConfig.getSensorTopic(), equalTo("sensor-topic")); assertThat(actualSensorConfig.getOutputTopic(), equalTo("output-topic")); assertThat(actualSensorConfig.getErrorTopic(), equalTo("error-topic")); assertThat(actualSensorConfig.getWriterClassName(), equalTo("writer-class")); assertThat(actualSensorConfig.getErrorWriterClassName(), equalTo("error-writer-class")); assertThat(actualSensorConfig.getReadMetadata(), equalTo(true)); assertThat(actualSensorConfig.getMergeMetadata(), equalTo(true)); assertThat(actualSensorConfig.getNumWorkers(), equalTo(40)); assertThat(actualSensorConfig.getNumAckers(), equalTo(40)); assertThat(actualSensorConfig.getSpoutParallelism(), equalTo(40)); assertThat(actualSensorConfig.getSpoutNumTasks(), equalTo(40)); assertThat(actualSensorConfig.getParserParallelism(), equalTo(40)); assertThat(actualSensorConfig.getParserNumTasks(), equalTo(40)); assertThat(actualSensorConfig.getErrorWriterParallelism(), equalTo(40)); assertThat(actualSensorConfig.getErrorWriterNumTasks(), equalTo(40)); assertThat(actualSensorConfig.getSecurityProtocol(), equalTo("security-protocol")); assertThat(actualSensorConfig.getSpoutConfig(), not(new HashMap<>())); assertThat(actualSensorConfig.getSpoutConfig().get("foo"), equalTo("bar")); assertThat(actualSensorConfig.getStormConfig(), not(new HashMap<>())); assertThat(actualSensorConfig.getStormConfig().get("storm"), equalTo("config")); assertThat(actualSensorConfig.getCacheConfig(), not(new HashMap<>())); assertThat(actualSensorConfig.getCacheConfig().get("stellar.cache.maxSize"), equalTo(20000)); assertThat(actualSensorConfig.getParserConfig(), not(new HashMap<>())); assertThat(actualSensorConfig.getParserConfig().get("parser"), equalTo("config")); assertThat(actualSensorConfig.getFieldTransformations(), not(new ArrayList<>()));
@Test public void testValidSerde_simple() throws IOException { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(config)); Assert.assertEquals(1, c.getFieldTransformations().size()); Assert.assertEquals(IPProtocolTransformation.class, c.getFieldTransformations().get(0).getFieldTransformation().getClass()); Assert.assertEquals(ImmutableList.of("protocol"), c.getFieldTransformations().get(0).getInput()); }
spoutParallelisms.add(parserConfig.getSpoutParallelism()); numTasks.add(parserConfig.getSpoutNumTasks()); Integer configValue = config.getParserParallelism(); retValue = configValue == null ? retValue : configValue; Integer configValue = config.getParserNumTasks(); retValue = configValue == null ? retValue : configValue; Integer configValue = config.getErrorWriterParallelism(); retValue = configValue == null ? retValue : configValue; Integer configValue = config.getErrorWriterNumTasks(); retValue = configValue == null ? retValue : configValue; retValue.add(config.getSpoutConfig()); String configSp = config.getSecurityProtocol(); if (!SecurityProtocol.PLAINTEXT.name.equals(configSp)) { Map<String, Object> c = parserConfig.getStormConfig(); if (c != null && !c.isEmpty()) { finalConfig.putAll(c); if (parserConfig.getNumAckers() != null) { Config.setNumAckers(finalConfig, parserConfig.getNumAckers()); if (parserConfig.getNumWorkers() != null) {
@Test public void testSimpleMapping() throws IOException { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(config)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); Assert.assertNotNull(handler); Assert.assertEquals(ImmutableMap.of("protocol", "TCP") ,handler.transform(new JSONObject(ImmutableMap.of("protocol", 6)) , Context.EMPTY_CONTEXT() , c.getParserConfig() ) ); } }
@Override public Map<String, Object> getSensorConfig(String sensorName) { return config.getSensorParserConfig(sensorName).getParserConfig(); }
if (sensorParserConfig == null) { throw new RestException("SensorParserConfig is missing from ParseMessageRequest"); } else if (sensorParserConfig.getParserClassName() == null) { throw new RestException("SensorParserConfig must have a parserClassName"); } else { MessageParser<JSONObject> parser; try { parser = (MessageParser<JSONObject>) Class.forName(sensorParserConfig.getParserClassName()) .newInstance(); } catch (Exception e) { String name = parseMessageRequest.getSensorParserConfig().getSensorTopic(); temporaryGrokPath = grokService.saveTemporary(parseMessageRequest.getGrokStatement(), name); sensorParserConfig.getParserConfig() .put(MetronRestConstants.GROK_PATH_KEY, new Path(temporaryGrokPath, name).toString()); parser.configure(sensorParserConfig.getParserConfig()); parser.init();
Map stormConf = mock(Map.class); SensorParserConfig yafConfig = mock(SensorParserConfig.class); when(yafConfig.getSensorTopic()).thenReturn("yafTopic"); when(yafConfig.getParserConfig()).thenReturn(new HashMap<String, Object>() {{ put(IndexingConfigurations.BATCH_SIZE_CONF, 10); }}); verify(yafConfig, times(1)).init(); Map<String, String> topicToSensorMap = parserBolt.getTopicToSensorMap(); Assert.assertEquals(1, topicToSensorMap.size());
@Override public Map<String, Object> applyTransformations(SensorParserContext sensorParserContext) { JSONObject sampleJson = new JSONObject(sensorParserContext.getSampleData()); sensorParserContext.getSensorParserConfig().getFieldTransformations().forEach(fieldTransformer -> { fieldTransformer.transformAndUpdate(sampleJson, Context.EMPTY_CONTEXT(), sensorParserContext.getSensorParserConfig().getParserConfig()); } ); return sampleJson; }
@Before public void setup() throws IOException { parserConfigurations = new ParserConfigurations(); SensorParserConfig broConfig = SensorParserConfig.fromBytes(broConfigString.getBytes()); SensorParserConfig snortConfig = SensorParserConfig.fromBytes(snortConfigString.getBytes()); parserConfigurations.updateSensorParserConfig("bro", broConfig); parserConfigurations.updateSensorParserConfig("snort", snortConfig); parserConfigurations.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfigString, JSONUtils.MAP_SUPPLIER)); parserRunner = new ParserRunnerImpl(new HashSet<>(Arrays.asList("bro", "snort"))); broParser = mock(MessageParser.class); snortParser = mock(MessageParser.class); stellarFilter = mock(StellarFilter.class); mockStatic(ReflectionUtils.class); mockStatic(Filters.class); when(ReflectionUtils.createInstance("org.apache.metron.parsers.bro.BasicBroParser")).thenReturn(broParser); when(ReflectionUtils.createInstance("org.apache.metron.parsers.snort.BasicSnortParser")).thenReturn(snortParser); when(Filters.get("org.apache.metron.parsers.filters.StellarFilter", broConfig.getParserConfig())) .thenReturn(stellarFilter); }
public ParserDriver(String sensorType, String parserConfig, String globalConfig) throws IOException { SensorParserConfig sensorParserConfig = SensorParserConfig.fromBytes(parserConfig.getBytes()); this.sensorType = sensorType == null ? sensorParserConfig.getSensorTopic() : sensorType; config = new ParserConfigurations(); config.updateSensorParserConfig(this.sensorType, SensorParserConfig.fromBytes(parserConfig.getBytes())); config.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER)); parserRunner = new ParserRunnerImpl(new HashSet<String>() {{ add(sensorType); }}); }
SensorParserConfig testSensorConfig = new SensorParserConfig(); testSensorConfig.setParserClassName("className"); testSensorConfig.setSensorTopic("sensorTopic"); testSensorConfig.setParserConfig(new HashMap<String, Object>() {{ put("configName", "configObject"); }});
Optional<ProcessResult> processResult = Optional.empty(); SensorParserConfig sensorParserConfig = parserConfigurations.getSensorParserConfig(sensorType); sensorParserConfig.getRawMessageStrategy().mergeMetadata( message, rawMessage.getMetadata(), sensorParserConfig.getMergeMetadata(), sensorParserConfig.getRawMessageStrategyConfig() ); message.put(Constants.SENSOR_TYPE, sensorType);
ParserConfigurations parserConfigurations = getConfigurations(); SensorParserConfig sensorParserConfig = parserConfigurations.getSensorParserConfig(sensorType); RawMessage rawMessage = RawMessageUtil.INSTANCE.getRawMessage( sensorParserConfig.getRawMessageStrategy() , tuple , originalMessage , sensorParserConfig.getReadMetadata() , sensorParserConfig.getRawMessageStrategyConfig() ); ParserRunnerResults<JSONObject> parserRunnerResults = parserRunner.execute(sensorType, rawMessage, parserConfigurations);
config.init(); topicToSensorMap.put(config.getSensorTopic(), sensor); } else { throw new IllegalStateException(
public void init() { for(FieldTransformer h : getFieldTransformations()) { h.initAndValidate(); } }