private ParserConfigurations create(byte[] sensorConfig) { try { ParserConfigurations result = new ParserConfigurations(); result.updateSensorParserConfig(sensorType, SensorParserConfig.fromBytes(sensorConfig)); return result; } catch(IOException e) { throw new IllegalArgumentException(e); } }
/** * Reads the Parser configuration from Zookeeper. * * @param sensorType The type of sensor. * @param client The Zookeeper client. * @return The Parser configuration for the given sensor type, if one exists. Otherwise, null. * @throws Exception */ public static SensorParserConfig readSensorParserConfigFromZookeeper(String sensorType, CuratorFramework client) throws Exception { SensorParserConfig config = null; Optional<byte[]> bytes = readFromZookeeperSafely(PARSER.getZookeeperRoot() + "/" + sensorType, client); if(bytes.isPresent()) { config = SensorParserConfig.fromBytes(bytes.get()); } return config; }
@Test(expected = IllegalStateException.class) public void testInValidSerde_missingMapping() throws IOException { SensorParserConfig.fromBytes(Bytes.toBytes(badConfigMissingMapping)); }
@Test public void testSerDe() throws IOException { for(File parserConfig : new File(new File(TestConstants.PARSER_CONFIGS_PATH), "parsers").listFiles()) { SensorParserConfig config = null; try (BufferedReader br = new BufferedReader(new FileReader(parserConfig))) { String parserStr = IOUtils.toString(br); config = SensorParserConfig.fromBytes(parserStr.getBytes()); } SensorParserConfig config2 = SensorParserConfig.fromBytes(config.toJSON().getBytes()); Assert.assertEquals(config2, config); } } }
private String transform(String in, String config) throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(config)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ put("in_field", in); put("dummy_field", "dummy"); //this is added to ensure that it looks like something approaching a real message }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); return (String) input.get("out_field"); }
@Test(expected=IllegalStateException.class) public void testStellarBadConfig() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(badConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(); try { handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); } catch(IllegalStateException ex) { Assert.assertTrue(ex.getMessage().contains("URL_TO_HOST")); Assert.assertTrue(ex.getMessage().contains("123")); throw ex; } }
@Test public void testValidSerde_simple() throws IOException { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(config)); Assert.assertEquals(1, c.getFieldTransformations().size()); Assert.assertEquals(IPProtocolTransformation.class, c.getFieldTransformations().get(0).getFieldTransformation().getClass()); Assert.assertEquals(ImmutableList.of("protocol"), c.getFieldTransformations().get(0).getInput()); }
@Test public void testSimpleMapping() throws IOException { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(config)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); Assert.assertNotNull(handler); Assert.assertEquals(ImmutableMap.of("protocol", "TCP") ,handler.transform(new JSONObject(ImmutableMap.of("protocol", 6)) , Context.EMPTY_CONTEXT() , c.getParserConfig() ) ); } }
public ParserDriver(String sensorType, String parserConfig, String globalConfig) throws IOException { SensorParserConfig sensorParserConfig = SensorParserConfig.fromBytes(parserConfig.getBytes()); this.sensorType = sensorType == null ? sensorParserConfig.getSensorTopic() : sensorType; config = new ParserConfigurations(); config.updateSensorParserConfig(this.sensorType, SensorParserConfig.fromBytes(parserConfig.getBytes())); config.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER)); parserRunner = new ParserRunnerImpl(new HashSet<String>() {{ add(sensorType); }}); }
@Test public void testUnconditionalRemove() throws Exception{ SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(removeUnconditionalConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ put("field1", "foo"); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertFalse(input.containsKey("field1")); }
@Test public void testComplexMapping() throws IOException { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(complexConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); Assert.assertNotNull(handler); Assert.assertEquals(ImmutableMap.of("output", "field1=value1,field2=value2") ,handler.transform(new JSONObject(ImmutableMap.of("field1", "value1" ,"field2", "value2" ) ) , Context.EMPTY_CONTEXT() , c.getParserConfig() ) ); } @Test
@Before public void setup() throws IOException { parserConfigurations = new ParserConfigurations(); SensorParserConfig broConfig = SensorParserConfig.fromBytes(broConfigString.getBytes()); SensorParserConfig snortConfig = SensorParserConfig.fromBytes(snortConfigString.getBytes()); parserConfigurations.updateSensorParserConfig("bro", broConfig); parserConfigurations.updateSensorParserConfig("snort", snortConfig); parserConfigurations.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfigString, JSONUtils.MAP_SUPPLIER)); parserRunner = new ParserRunnerImpl(new HashSet<>(Arrays.asList("bro", "snort"))); broParser = mock(MessageParser.class); snortParser = mock(MessageParser.class); stellarFilter = mock(StellarFilter.class); mockStatic(ReflectionUtils.class); mockStatic(Filters.class); when(ReflectionUtils.createInstance("org.apache.metron.parsers.bro.BasicBroParser")).thenReturn(broParser); when(ReflectionUtils.createInstance("org.apache.metron.parsers.snort.BasicSnortParser")).thenReturn(snortParser); when(Filters.get("org.apache.metron.parsers.filters.StellarFilter", broConfig.getParserConfig())) .thenReturn(stellarFilter); }
@Test public void testIntermediateValues() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(intermediateValuesConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); int expected = 3; Assert.assertEquals(expected, input.get("final_value")); Assert.assertFalse(input.containsKey("value1")); Assert.assertFalse(input.containsKey("value2")); }
@Test public void testConfigAll() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(configAll)); JSONObject input = new JSONObject(); input.put("source.type", "test"); for (FieldTransformer handler : c.getFieldTransformations()) { handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); } Assert.assertEquals(2, input.size()); Assert.assertTrue(input.containsKey("new_field")); Assert.assertEquals("test", input.get("new_field")); }
/** * Test the happy path. This ensures that a simple transformation, converting a timestamp in a yyyy-MM-dd HH:mm:ss * format can be converted to the expected UTC MS since Epoch. */ @Test public void testStellar() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(stellarConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ put("timestamp", "2016-01-05 17:02:30"); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); long expected = 1452013350000L; Assert.assertEquals(expected, input.get("utc_timestamp")); Assert.assertTrue(input.containsKey("timestamp")); }
@Test public void testStellarNumericDomain() throws Exception { /* Despite the domain being weird, URL_TO_HOST should allow it to pass through. However, because it does NOT form a proper domain (no TLD), DOMAIN_REMOVE_SUBDOMAINS returns null indicating that the input is semantically incorrect. */ SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(configNumericDomain)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertTrue(input.containsKey("full_hostname")); Assert.assertEquals("1234567890123456789012345678901234567890123456789012345678901234567890", input.get("full_hostname")); Assert.assertFalse(input.containsKey("domain_without_subdomains")); }
@Test public void testStellarSpecialCharacters() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(stellarConfigEspecial)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ put("timestamp", "2016-01-05 17:02:30"); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); long expected = 1452013350000L; Assert.assertEquals(expected, input.get("utc_timestamp")); Assert.assertTrue(input.containsKey("timestamp")); Assert.assertTrue(input.containsKey("newStellarField")); }
/** * Ensures that if we try to transform with a field which does not exist, it does not * 1. throw an exception * 2. do any transformation. */ @Test public void testStellar_negative() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(stellarConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); //no input fields => no transformation JSONObject input = new JSONObject(new HashMap<String, Object>() {{ }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertFalse(input.containsKey("utc_timestamp")); Assert.assertTrue(input.isEmpty()); }
@Test public void renameMissingField() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(renameMissingField)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ for(int i = 2;i <= 10;++i) { put("old_field" + i, "f" + i); } }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertFalse(input.containsKey("new_field1")); for(int i = 2;i <= 10;++i) { Assert.assertEquals("f" + i, input.get("old_field" + i)); } Assert.assertEquals(9, input.size()); } }
@Test public void testSingleFieldReturned() throws Exception { SensorParserConfig sensorConfig = SensorParserConfig.fromBytes(Bytes.toBytes(selectSingleFieldConfig)); FieldTransformer handler = Iterables.getFirst(sensorConfig.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() { { put("field1", "foo"); put("field2", "bar"); } }); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertTrue(input.containsKey("field1")); Assert.assertFalse(input.containsKey("field2")); Assert.assertEquals(1, input.size()); }