public void init() { for(FieldTransformer h : getFieldTransformations()) { h.initAndValidate(); } }
private static void pruneEmptyStellarTransformers(SensorParserConfig config) { List<FieldTransformer> toRemove = new ArrayList<>(); List<FieldTransformer> fieldTransformations = config.getFieldTransformations(); for(FieldTransformer transformer : fieldTransformations) { if(transformer.getFieldTransformation().getClass().getName() .equals(FieldTransformations.STELLAR.getMappingClass().getName()) && transformer.getConfig().isEmpty() ) { toRemove.add(transformer); } } for(FieldTransformer t : toRemove) { fieldTransformations.remove(t); } } private static FieldTransformer getStellarTransformer(SensorParserConfig config) {
private static FieldTransformer getStellarTransformer(SensorParserConfig config) { List<FieldTransformer> fieldTransformations = config.getFieldTransformations(); FieldTransformer stellarTransformer = null; for(FieldTransformer transformer : fieldTransformations) { if(transformer.getFieldTransformation().getClass().getName() .equals(FieldTransformations.STELLAR.getMappingClass().getName())) { stellarTransformer = transformer; } } if(stellarTransformer == null) { stellarTransformer = new FieldTransformer(); stellarTransformer.setConfig(new LinkedHashMap<>()); stellarTransformer.setTransformation(FieldTransformations.STELLAR.toString()); fieldTransformations.add(stellarTransformer); } return stellarTransformer; }
@Override public Map<String, Object> applyTransformations(SensorParserContext sensorParserContext) { JSONObject sampleJson = new JSONObject(sensorParserContext.getSampleData()); sensorParserContext.getSensorParserConfig().getFieldTransformations().forEach(fieldTransformer -> { fieldTransformer.transformAndUpdate(sampleJson, Context.EMPTY_CONTEXT(), sensorParserContext.getSensorParserConfig().getParserConfig()); } ); return sampleJson; }
@Test public void testValidSerde_simple() throws IOException { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(config)); Assert.assertEquals(1, c.getFieldTransformations().size()); Assert.assertEquals(IPProtocolTransformation.class, c.getFieldTransformations().get(0).getFieldTransformation().getClass()); Assert.assertEquals(ImmutableList.of("protocol"), c.getFieldTransformations().get(0).getInput()); }
private String transform(String in, String config) throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(config)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ put("in_field", in); put("dummy_field", "dummy"); //this is added to ensure that it looks like something approaching a real message }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); return (String) input.get("out_field"); }
@Test(expected=IllegalStateException.class) public void testStellarBadConfig() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(badConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(); try { handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); } catch(IllegalStateException ex) { Assert.assertTrue(ex.getMessage().contains("URL_TO_HOST")); Assert.assertTrue(ex.getMessage().contains("123")); throw ex; } }
@Test public void testSimpleMapping() throws IOException { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(config)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); Assert.assertNotNull(handler); Assert.assertEquals(ImmutableMap.of("protocol", "TCP") ,handler.transform(new JSONObject(ImmutableMap.of("protocol", 6)) , Context.EMPTY_CONTEXT() , c.getParserConfig() ) ); } }
@Test public void testUnconditionalRemove() throws Exception{ SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(removeUnconditionalConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ put("field1", "foo"); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertFalse(input.containsKey("field1")); }
@Test public void testComplexMapping() throws IOException { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(complexConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); Assert.assertNotNull(handler); Assert.assertEquals(ImmutableMap.of("output", "field1=value1,field2=value2") ,handler.transform(new JSONObject(ImmutableMap.of("field1", "value1" ,"field2", "value2" ) ) , Context.EMPTY_CONTEXT() , c.getParserConfig() ) ); } @Test
@Test public void testIntermediateValues() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(intermediateValuesConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); int expected = 3; Assert.assertEquals(expected, input.get("final_value")); Assert.assertFalse(input.containsKey("value1")); Assert.assertFalse(input.containsKey("value2")); }
@Test public void testConfigAll() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(configAll)); JSONObject input = new JSONObject(); input.put("source.type", "test"); for (FieldTransformer handler : c.getFieldTransformations()) { handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); } Assert.assertEquals(2, input.size()); Assert.assertTrue(input.containsKey("new_field")); Assert.assertEquals("test", input.get("new_field")); }
/** * Test the happy path. This ensures that a simple transformation, converting a timestamp in a yyyy-MM-dd HH:mm:ss * format can be converted to the expected UTC MS since Epoch. */ @Test public void testStellar() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(stellarConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ put("timestamp", "2016-01-05 17:02:30"); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); long expected = 1452013350000L; Assert.assertEquals(expected, input.get("utc_timestamp")); Assert.assertTrue(input.containsKey("timestamp")); }
@Test public void testStellarNumericDomain() throws Exception { /* Despite the domain being weird, URL_TO_HOST should allow it to pass through. However, because it does NOT form a proper domain (no TLD), DOMAIN_REMOVE_SUBDOMAINS returns null indicating that the input is semantically incorrect. */ SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(configNumericDomain)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertTrue(input.containsKey("full_hostname")); Assert.assertEquals("1234567890123456789012345678901234567890123456789012345678901234567890", input.get("full_hostname")); Assert.assertFalse(input.containsKey("domain_without_subdomains")); }
@Test public void testStellarSpecialCharacters() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(stellarConfigEspecial)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ put("timestamp", "2016-01-05 17:02:30"); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); long expected = 1452013350000L; Assert.assertEquals(expected, input.get("utc_timestamp")); Assert.assertTrue(input.containsKey("timestamp")); Assert.assertTrue(input.containsKey("newStellarField")); }
/** * Ensures that if we try to transform with a field which does not exist, it does not * 1. throw an exception * 2. do any transformation. */ @Test public void testStellar_negative() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(stellarConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); //no input fields => no transformation JSONObject input = new JSONObject(new HashMap<String, Object>() {{ }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertFalse(input.containsKey("utc_timestamp")); Assert.assertTrue(input.isEmpty()); }
@Test public void renameMissingField() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(renameMissingField)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ for(int i = 2;i <= 10;++i) { put("old_field" + i, "f" + i); } }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertFalse(input.containsKey("new_field1")); for(int i = 2;i <= 10;++i) { Assert.assertEquals("f" + i, input.get("old_field" + i)); } Assert.assertEquals(9, input.size()); } }
@Test public void testSingleFieldReturned() throws Exception { SensorParserConfig sensorConfig = SensorParserConfig.fromBytes(Bytes.toBytes(selectSingleFieldConfig)); FieldTransformer handler = Iterables.getFirst(sensorConfig.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() { { put("field1", "foo"); put("field2", "bar"); } }); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertTrue(input.containsKey("field1")); Assert.assertFalse(input.containsKey("field2")); Assert.assertEquals(1, input.size()); }
@Test public void testMulitpleFieldReturned() throws Exception { SensorParserConfig sensorConfig = SensorParserConfig.fromBytes(Bytes.toBytes(selectMultiFieldConfig)); FieldTransformer handler = Iterables.getFirst(sensorConfig.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() { { put("field1", "foo"); put("field2", "bar"); put("field3", "bar2"); } }); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertTrue(input.containsKey("field1")); Assert.assertTrue(input.containsKey("field2")); Assert.assertFalse(input.containsKey("field3")); Assert.assertEquals(2, input.size()); }
@Test public void testPreserveSystemFields() throws Exception { SensorParserConfig sensorConfig = SensorParserConfig.fromBytes(Bytes.toBytes(selectSingleFieldConfig)); FieldTransformer handler = Iterables.getFirst(sensorConfig.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() { { put("timestamp", 12345); put("original_string", "foo,bar"); put("source.type", "test"); put("field1", "foo"); put("field2", "bar"); } }); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertTrue(input.containsKey("timestamp")); Assert.assertTrue(input.containsKey("original_string")); Assert.assertTrue(input.containsKey("source.type")); Assert.assertTrue(input.containsKey("field1")); Assert.assertFalse(input.containsKey("field2")); Assert.assertEquals(4, input.size()); }