@Override public FieldDependency get() { FieldDependencyBuilder builder = new FieldDependencyBuilder(); eventMappers.stream().forEach(mapper -> mapper.addFieldDependency(builder)); return builder.build(); } }
@Override public void addFieldDependency(FieldDependencyBuilder builder) { List<SchemaField> fields = CITY_DATABASE_ATTRIBUTES.stream() .map(attr -> new SchemaField("_" + attr, getType(attr))) .collect(Collectors.toList()); builder.addFields("_ip", fields); }
public void addFields(List<SchemaField> fields) { checkFields(fields); constantFields.addAll(fields); }
@Test(dataProvider = "google-ips") public void testIspEventMapper(Map<String, Object> props, InetAddress address) throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig() .setAttributes("") .setIspDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-ISP-Test.mmdb"))); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); Record properties = new Record(Schema.createRecord(ImmutableList.of( new Schema.Field("_ip", Schema.create(NULL), null, null), new Schema.Field("__ip", Schema.create(STRING), null, null), new Schema.Field("_isp", Schema.create(STRING), null, null)))); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, address, null); assertTrue(resp == null); assertEquals(event.getAttribute("_isp"), "Level 3 Communications"); GenericData.get().validate(properties.getSchema(), properties); }
@Test(dataProvider = "google-ips") public void testConnectionTypeEventMapper(Map<String, Object> props, InetAddress address) throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig() .setAttributes("") .setConnectionTypeDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-Connection-Type-Test.mmdb"))); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); Record properties = new Record(Schema.createRecord(ImmutableList.of( new Schema.Field("_ip", Schema.create(NULL), null, null), new Schema.Field("__ip", Schema.create(STRING), null, null), new Schema.Field("_connection_type", Schema.create(STRING), null, null)))); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, address, null); assertTrue(resp == null); // TODO: find a reliable ip that can be mapped. assertNull(event.getAttribute("connection_type")); GenericData.get().validate(properties.getSchema(), properties); }
@ApiOperation(value = "List event mappers", authorizations = @Authorization(value = "master_key") ) @GET @Path("/event_mappers") @JsonRequest public List<EventMapperDescription> getEventMappers() { return eventMappers.stream().map(mapper -> { Mapper annotation = mapper.getClass().getAnnotation(Mapper.class); String name; String description; if (annotation != null) { name = annotation.name(); description = annotation.description(); } else { name = mapper.getClass().getSimpleName(); description = ""; } FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); FieldDependencyBuilder.FieldDependency build = builder.build(); return new EventMapperDescription(name, description, build.dependentFields.isEmpty() ? null : build.dependentFields, build.constantFields.isEmpty() ? null : build.constantFields); }).collect(Collectors.toList()); }
@Test(dataProvider = "google-ips") public void testIspEventMapper(Map<String, Object> props, InetAddress address) throws Exception { GeoIPEventMapper mapper = new GeoIPEventMapper(new GeoIPModuleConfig() .setAttributes("") .setIspDatabaseUrl("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-ISP-Test.mmdb")); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); Record properties = new Record(Schema.createRecord(ImmutableList.of( new Schema.Field("_ip", Schema.create(NULL), null, null), new Schema.Field("_isp", Schema.create(STRING), null, null)))); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, address, null); assertTrue(resp == null); assertEquals(event.getAttribute("_isp"), "Level 3 Communications"); GenericData.get().validate(properties.getSchema(), properties); }
@Override public void addFieldDependency(FieldDependencyBuilder builder) { builder.addFields(of(new SchemaField(projectConfig.getTimeColumn(), FieldType.TIMESTAMP))); }
public void addFields(String dependentField, List<SchemaField> fields) { checkFields(fields); dependentFields.put(dependentField, fields); }
@Test public void testFieldDependency() throws Exception { MaxmindGeoIPModuleConfig config = new MaxmindGeoIPModuleConfig().setAttributes(list.stream().collect(Collectors.joining(","))); MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(config); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); FieldDependencyBuilder.FieldDependency build = builder.build(); assertEquals(0, build.constantFields.size()); assertEquals(1, build.dependentFields.size()); assertEquals(list.stream().map(a -> "_" + a).collect(Collectors.toSet()), build.dependentFields.get("_ip").stream().map(SchemaField::getName) .collect(Collectors.toSet())); }
@Test(dataProvider = "google-ips") public void testConnectionTypeEventMapper(Map<String, Object> props, InetAddress address) throws Exception { GeoIPEventMapper mapper = new GeoIPEventMapper(new GeoIPModuleConfig() .setAttributes("") .setConnectionTypeDatabaseUrl("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-Connection-Type-Test.mmdb")); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); Record properties = new Record(Schema.createRecord(ImmutableList.of( new Schema.Field("_ip", Schema.create(NULL), null, null), new Schema.Field("_connection_type", Schema.create(STRING), null, null)))); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, address, null); assertTrue(resp == null); // TODO: find a reliable ip that can be mapped. assertNull(event.getAttribute("connection_type")); GenericData.get().validate(properties.getSchema(), properties); }
@Override public void addFieldDependency(FieldDependencyBuilder builder) { List<SchemaField> fields = Arrays.stream(attributes) .map(attr -> new SchemaField("_" + attr, getType(attr))) .collect(Collectors.toList()); if (ispLookup != null) { fields.add(new SchemaField("_isp", STRING)); } if (connectionTypeLookup != null) { fields.add(new SchemaField("_connection_type", STRING)); } fields.add(new SchemaField("__ip", STRING)); builder.addFields("_ip", fields); }
@Test public void testFieldDependencyWithAll() throws Exception { MaxmindGeoIPModuleConfig config = new MaxmindGeoIPModuleConfig() .setAttributes(list.stream().collect(Collectors.joining(","))) .setConnectionTypeDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-Connection-Type-Test.mmdb")) .setIspDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-ISP-Test.mmdb")); MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(config); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); FieldDependencyBuilder.FieldDependency build = builder.build(); assertEquals(0, build.constantFields.size()); assertEquals(1, build.dependentFields.size()); assertEquals(ImmutableSet.builder().addAll(list.stream().map(e -> "_" + e).collect(Collectors.toList())).add("_isp", "_connection_type").build(), build.dependentFields.get("_ip").stream().map(SchemaField::getName).collect(Collectors.toSet())); } }
@Override public void addFieldDependency(FieldDependencyBuilder builder) { builder.addFields("_user_agent", ImmutableList.of( new SchemaField("_user_agent_family", FieldType.STRING), new SchemaField("_user_agent_version", FieldType.STRING), new SchemaField("_os", FieldType.STRING), new SchemaField("_os_version", FieldType.STRING), new SchemaField("_device_family", FieldType.STRING) )); }
@Test() public void testReferrerNotExists() throws Exception { ReferrerEventMapper mapper = new ReferrerEventMapper(); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_referrer"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_referrer", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_referrer", true); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertNull(resp); for (SchemaField field : fields) { assertNull(event.getAttribute(field.getName())); } }
@Override public void addFieldDependency(FieldDependencyBuilder builder) { builder.addFields("_referrer", ImmutableList.of( new SchemaField("_referrer_medium", STRING), new SchemaField("_referrer_source", STRING), new SchemaField("_referrer_term", STRING), new SchemaField("_referrer_domain", STRING), new SchemaField("_referrer_path", STRING) )); } }
@Test() public void testUnknownReferrer() throws Exception { ReferrerEventMapper mapper = new ReferrerEventMapper(); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_referrer"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_referrer", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_referrer", "http://test.com"); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertNull(resp); assertNull(event.getAttribute("_referrer_source")); assertNull(event.getAttribute("_referrer_term")); assertEquals("unknown", event.getAttribute("_referrer_medium")); assertEquals("test.com", event.getAttribute("_referrer_domain")); assertEquals("", event.getAttribute("_referrer_path")); }
@Override public void addFieldDependency(FieldDependencyBuilder builder) { ImmutableList<SchemaField> list = ImmutableList.of( new SchemaField("_referrer_medium", FieldType.STRING), new SchemaField("_referrer_source", FieldType.STRING), new SchemaField("_referrer_term", FieldType.STRING), new SchemaField("_referrer_domain", FieldType.STRING), new SchemaField("_referrer_path", FieldType.STRING) ); builder.addFields("_referrer", list); builder.addFields("_host", list); } }
@Test(dataProvider = "chrome-user-agent") public void testUserAgentMapper(Map<String, Object> props, EventMapper.RequestParams headers) throws Exception { UserAgentEventMapper mapper = new UserAgentEventMapper(new WebsiteMapperConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(builder.build().dependentFields.get("_user_agent").stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_user_agent", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, headers, InetAddress.getLocalHost(), null); assertEquals("Chrome", event.getAttribute("_user_agent_family")); assertEquals("Mac OS X", event.getAttribute("_os")); assertEquals("10", event.getAttribute("_os_version")); assertEquals("Other", event.getAttribute("_device_family")); assertNull(resp); GenericData.get().validate(properties.getSchema(), properties); }
@Override public void addFieldDependency(FieldDependencyBuilder builder) { builder.addFields("_user_agent", ImmutableList.of( new SchemaField("_user_agent_family", FieldType.STRING), new SchemaField("_user_agent_version", FieldType.LONG), new SchemaField("_os", FieldType.STRING), new SchemaField("_os_version", FieldType.LONG), new SchemaField("_device_family", FieldType.STRING) )); }