@Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; FieldTransformer that = (FieldTransformer) o; if (getInput() != null ? !getInput().equals(that.getInput()) : that.getInput() != null) return false; if (getOutput() != null ? !getOutput().equals(that.getOutput()) : that.getOutput() != null) return false; if (getTransformation() != null ? !getTransformation().equals(that.getTransformation()) : that.getTransformation() != null) return false; return getConfig() != null ? getConfig().equals(that.getConfig()) : that.getConfig() == null; }
private static FieldTransformer getStellarTransformer(SensorParserConfig config) { List<FieldTransformer> fieldTransformations = config.getFieldTransformations(); FieldTransformer stellarTransformer = null; for(FieldTransformer transformer : fieldTransformations) { if(transformer.getFieldTransformation().getClass().getName() .equals(FieldTransformations.STELLAR.getMappingClass().getName())) { stellarTransformer = transformer; } } if(stellarTransformer == null) { stellarTransformer = new FieldTransformer(); stellarTransformer.setConfig(new LinkedHashMap<>()); stellarTransformer.setTransformation(FieldTransformations.STELLAR.toString()); fieldTransformations.add(stellarTransformer); } return stellarTransformer; }
public void transformAndUpdate(JSONObject message, Context context, Map<String, Object>... sensorConfig) { Map<String, Object> currentValue = transform(message, context, sensorConfig); if(currentValue != null) { for(Map.Entry<String, Object> kv : currentValue.entrySet()) { if(kv.getValue() == null) { message.remove(kv.getKey()); } else { message.put(kv.getKey(), kv.getValue()); } } } }
public Map<String, Object> transform(JSONObject input, Context context, Map<String, Object>... sensorConfig) { if(getInput() == null || getInput().isEmpty()) { return transformation.map(input, getOutput(), config, context, sensorConfig); } else { Map<String, Object> in = new HashMap<>(); for(String inputField : getInput()) { in.put(inputField, input.get(inputField)); } return transformation.map(in, getOutput(), config, context, sensorConfig); } }
@Override public Map<String, Object> applyTransformations(SensorParserContext sensorParserContext) { JSONObject sampleJson = new JSONObject(sensorParserContext.getSampleData()); sensorParserContext.getSensorParserConfig().getFieldTransformations().forEach(fieldTransformer -> { fieldTransformer.transformAndUpdate(sampleJson, Context.EMPTY_CONTEXT(), sensorParserContext.getSensorParserConfig().getParserConfig()); } ); return sampleJson; }
private static void pruneEmptyStellarTransformers(SensorParserConfig config) { List<FieldTransformer> toRemove = new ArrayList<>(); List<FieldTransformer> fieldTransformations = config.getFieldTransformations(); for(FieldTransformer transformer : fieldTransformations) { if(transformer.getFieldTransformation().getClass().getName() .equals(FieldTransformations.STELLAR.getMappingClass().getName()) && transformer.getConfig().isEmpty() ) { toRemove.add(transformer); } } for(FieldTransformer t : toRemove) { fieldTransformations.remove(t); } } private static FieldTransformer getStellarTransformer(SensorParserConfig config) {
@Test public void testValidSerde_simple() throws IOException { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(config)); Assert.assertEquals(1, c.getFieldTransformations().size()); Assert.assertEquals(IPProtocolTransformation.class, c.getFieldTransformations().get(0).getFieldTransformation().getClass()); Assert.assertEquals(ImmutableList.of("protocol"), c.getFieldTransformations().get(0).getInput()); }
@Override public Object apply(List<Object> args, Context context) throws ParseException { String config = (String) args.get(0); if(config == null) { return null; } SensorParserConfig configObj = (SensorParserConfig) PARSER.deserialize(config); FieldTransformer stellarTransformer = getStellarTransformer(configObj); List<String> removals = (List<String>)args.get(1); if(removals == null || removals.isEmpty()) { return config; } for(String removal : removals) { stellarTransformer.getConfig().remove(removal); } List<String> output = new ArrayList<>(); output.addAll(stellarTransformer.getConfig().keySet()); stellarTransformer.setOutput(output); pruneEmptyStellarTransformers(configObj); try { return JSONUtils.INSTANCE.toJSON(configObj, true); } catch (JsonProcessingException e) { LOG.error("Unable to convert object to JSON: {}", configObj, e); return config; } }
assertThat(actualSensorConfig.getFieldTransformations().get(0), not(nullValue())); assertThat( ((FieldTransformer) actualSensorConfig.getFieldTransformations().get(0)).getInput().size(), equalTo(1)); assertThat( ((FieldTransformer) actualSensorConfig.getFieldTransformations().get(0)).getInput().get(0), equalTo("input-field")); assertThat(((FieldTransformer) actualSensorConfig.getFieldTransformations().get(0)) .getTransformation(), equalTo("REMOVE"));
public void initAndValidate() { if(!initialized) { if (getTransformation() == null) { throw new IllegalStateException("Mapping cannot be null."); } if (output == null || output.isEmpty()) { if (input == null || input.isEmpty()) { //both are empty, so let's set them both to null output = null; input = null; } else { output = input; } } initialized = true; } }
@Override public Object apply(List<Object> args, Context context) throws ParseException { String config = (String) args.get(0); if(config == null) { return null; } SensorParserConfig configObj = (SensorParserConfig) PARSER.deserialize(config); FieldTransformer stellarTransformer = getStellarTransformer(configObj); String[] headers = new String[] { "Field", "Transformation"}; String[][] data = new String[stellarTransformer.getConfig().size()][2]; int i = 0; for(Map.Entry<String, Object> kv : stellarTransformer.getConfig().entrySet()) { data[i++] = new String[] {kv.getKey(), kv.getValue().toString()}; } return FlipTable.of(headers, data); }
public void init() { for(FieldTransformer h : getFieldTransformations()) { h.initAndValidate(); } }
/** * Applies Stellar field transformations defined in the sensor parser config. * @param message Message parsed by the MessageParser * @param rawMessage Raw message including metadata * @param sensorParserConfig Sensor parser config */ private void applyFieldTransformations(JSONObject message, RawMessage rawMessage, SensorParserConfig sensorParserConfig) { for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) { if (handler != null) { if (!sensorParserConfig.getMergeMetadata()) { //if we haven't merged metadata, then we need to pass them along as configuration params. handler.transformAndUpdate( message, stellarContext, sensorParserConfig.getParserConfig(), rawMessage.getMetadata() ); } else { handler.transformAndUpdate( message, stellarContext, sensorParserConfig.getParserConfig() ); } } } }
@Override public Object apply(List<Object> args, Context context) throws ParseException { String config = (String) args.get(0); if(config == null) { return null; } SensorParserConfig configObj = (SensorParserConfig) PARSER.deserialize(config); FieldTransformer stellarTransformer = getStellarTransformer(configObj); Map<String, String> additionalTransforms = (Map<String, String>) args.get(1); if(additionalTransforms == null || additionalTransforms.isEmpty()) { return config; } for(Map.Entry<String, String> kv : additionalTransforms.entrySet()) { stellarTransformer.getConfig().put(kv.getKey(), kv.getValue()); } List<String> output = new ArrayList<>(); output.addAll(stellarTransformer.getConfig().keySet()); stellarTransformer.setOutput(output); try { return JSONUtils.INSTANCE.toJSON(configObj, true); } catch (JsonProcessingException e) { LOG.error("Unable to convert object to JSON: {}", configObj, e); return config; } }
private String transform(String in, String config) throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(config)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ put("in_field", in); put("dummy_field", "dummy"); //this is added to ensure that it looks like something approaching a real message }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); return (String) input.get("out_field"); }
@Override public int hashCode() { int result = getInput() != null ? getInput().hashCode() : 0; result = 31 * result + (getOutput() != null ? getOutput().hashCode() : 0); result = 31 * result + (getTransformation() != null ? getTransformation().hashCode() : 0); result = 31 * result + (getConfig() != null ? getConfig().hashCode() : 0); return result; } }
@Test public void testSimpleMapping() throws IOException { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(config)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); Assert.assertNotNull(handler); Assert.assertEquals(ImmutableMap.of("protocol", "TCP") ,handler.transform(new JSONObject(ImmutableMap.of("protocol", 6)) , Context.EMPTY_CONTEXT() , c.getParserConfig() ) ); } }
@Test(expected=IllegalStateException.class) public void testStellarBadConfig() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(badConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(); try { handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); } catch(IllegalStateException ex) { Assert.assertTrue(ex.getMessage().contains("URL_TO_HOST")); Assert.assertTrue(ex.getMessage().contains("123")); throw ex; } }
@Test public void testComplexMapping() throws IOException { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(complexConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); Assert.assertNotNull(handler); Assert.assertEquals(ImmutableMap.of("output", "field1=value1,field2=value2") ,handler.transform(new JSONObject(ImmutableMap.of("field1", "value1" ,"field2", "value2" ) ) , Context.EMPTY_CONTEXT() , c.getParserConfig() ) ); } @Test
@Test public void testUnconditionalRemove() throws Exception{ SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(removeUnconditionalConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ put("field1", "foo"); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertFalse(input.containsKey("field1")); }