public List<Map<String, Object>> cleanDocs(ProcessorResult<List<Map<String, Object>>> result) { List<Map<String,Object>> docs = result.getResult(); StringBuffer buffer = new StringBuffer(); boolean failed = false; List<Map<String, Object>> ret = new ArrayList<>(); if(result.failed()) { failed = true; result.getBadResults(buffer); buffer.append(String.format("%d Valid messages processed", docs.size())).append("\n"); for (Map<String, Object> doc : docs) { Map<String, Object> msg = new HashMap<>(); for (Map.Entry<String, Object> kv : doc.entrySet()) { //for writers like solr who modify the keys, we want to undo that if we can buffer.append(cleanField(kv.getKey())).append(kv.getValue().toString()).append("\n"); } } Assert.fail(buffer.toString()); }else { for (Map<String, Object> doc : docs) { Map<String, Object> msg = new HashMap<>(); for (Map.Entry<String, Object> kv : doc.entrySet()) { //for writers like solr who modify the keys, we want to undo that if we can msg.put(cleanField(kv.getKey()), kv.getValue()); } ret.add(msg); } } return ret; }
public void testCiscoPixEnvelopingCisco302020(ParserDriver syslogDriver, ParserDriver driver) throws Exception { byte[] envelopedData = null; String inputRecord = "Mar 29 2004 09:54:18: %PIX-6-302005: Built UDP connection for faddr 198.207.223.240/53337 gaddr 10.0.0.187/53 laddr 192.168.0.2/53"; ProcessorResult<List<byte[]>> syslogResult = syslogDriver.run(ImmutableList.of(inputRecord.getBytes())); Assert.assertFalse(syslogResult.failed()); List<byte[]> syslogResultList = syslogResult.getResult(); envelopedData = syslogResultList.get(0); ProcessorResult<List<byte[]>> results = driver.run(ImmutableList.of(envelopedData)); Assert.assertFalse(results.failed()); List<byte[]> resultList = results.getResult(); Assert.assertEquals(1, resultList.size()); Map<String, Object> result = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER); Assert.assertEquals("UDP", result.get("protocol")); Assert.assertTrue((long) result.get("timestamp") > 1000); } }
public void runTest(ParserDriver driver) throws Exception { String sensorType = driver.getSensorType(); inputMessages = TestUtils.readSampleData(SampleDataUtils.getSampleDataPath("..", sensorType, TestDataType.RAW)); ProcessorResult<List<byte[]>> result = driver.run(inputMessages); List<byte[]> outputMessages = result.getResult(); StringBuffer buffer = new StringBuffer(); if (result.failed()){ result.getBadResults(buffer); buffer.append(String.format("%d Valid Messages Processed", outputMessages.size())).append("\n"); dumpParsedMessages(outputMessages,buffer); Assert.fail(buffer.toString()); } else { List<ParserValidation> validations = getValidations(); if (validations == null || validations.isEmpty()) { buffer.append("No validations configured for sensorType ").append(sensorType).append(". Dumping parsed messages").append("\n"); dumpParsedMessages(outputMessages,buffer); Assert.fail(buffer.toString()); } else { for (ParserValidation validation : validations) { System.out.println("Running " + validation.getName() + " on sensorType " + sensorType); validation.validate(sensorType, outputMessages); } } } }
public void testEnvelopedData(ParserDriver driver) throws IOException { Map<String, Object> inputRecord = new HashMap<String, Object>() {{ put(Constants.Fields.ORIGINAL.getName(), "real_original_string"); put("data", "field1_val,100"); put("metadata_field", "metadata_val"); }}; ProcessorResult<List<byte[]>> results = driver.run(ImmutableList.of(JSONUtils.INSTANCE.toJSONPretty(inputRecord))); Assert.assertFalse(results.failed()); List<byte[]> resultList = results.getResult(); Assert.assertEquals(1, resultList.size()); Map<String, Object> outputRecord = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER); Assert.assertEquals("field1_val", outputRecord.get("field1")); Assert.assertEquals(inputRecord.get(Constants.Fields.ORIGINAL.getName()), outputRecord.get(Constants.Fields.ORIGINAL.getName())); Assert.assertEquals(inputRecord.get(MetadataUtil.METADATA_PREFIX + ".metadata_field"), outputRecord.get("metadata_field")); }
public void testEnvelopedData_withMetadataPrefix(ParserDriver driver) throws IOException { Map<String, Object> inputRecord = new HashMap<String, Object>() {{ put(Constants.Fields.ORIGINAL.getName(), "real_original_string"); put("data", "field1_val,100"); put("metadata_field", "metadata_val"); }}; ProcessorResult<List<byte[]>> results = driver.run(ImmutableList.of(JSONUtils.INSTANCE.toJSONPretty(inputRecord))); Assert.assertFalse(results.failed()); List<byte[]> resultList = results.getResult(); Assert.assertEquals(1, resultList.size()); Map<String, Object> outputRecord = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER); Assert.assertEquals("field1_val", outputRecord.get("field1")); Assert.assertEquals(inputRecord.get(Constants.Fields.ORIGINAL.getName()), outputRecord.get(Constants.Fields.ORIGINAL.getName())); Assert.assertEquals(inputRecord.get("metadata_field"), outputRecord.get("metadata_field")); } /**
public void testEnvelopedData_noMergeMetadata(ParserDriver driver) throws IOException { Map<String, Object> inputRecord = new HashMap<String, Object>() {{ put(Constants.Fields.ORIGINAL.getName(), "real_original_string"); put("data", "field1_val,100"); put("metadata_field", "metadata_val"); }}; ProcessorResult<List<byte[]>> results = driver.run(ImmutableList.of(JSONUtils.INSTANCE.toJSONPretty(inputRecord))); Assert.assertFalse(results.failed()); List<byte[]> resultList = results.getResult(); Assert.assertEquals(1, resultList.size()); Map<String, Object> outputRecord = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER); Assert.assertEquals("field1_val", outputRecord.get("field1")); Assert.assertEquals(inputRecord.get(Constants.Fields.ORIGINAL.getName()), outputRecord.get(Constants.Fields.ORIGINAL.getName())); Assert.assertFalse(outputRecord.containsKey(MetadataUtil.METADATA_PREFIX + ".metadata_field")); }