@Override public Object get(String attr) { return event.getAttribute(attr); }
private String getPartitionKey(Event event) { Object user = event.getAttribute("_user"); return event.project() + "|" + (user == null ? event.collection() : user.toString()); }
@Override public User get() { if (user == null) { String userAttr = event.getAttribute("_user"); if (userAttr != null) { user = userStorage.getUser(new RequestContext(event.project(), null), userAttr).join(); } } return user; } };
@Test public void testInvalidMapRecursiveType() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test0", "test0", "test1", ImmutableMap.of("test", ImmutableList.of("test4")), "test2", false))); Event event = mapper.readValue(bytes, Event.class); Map test1 = event.getAttribute("test1"); assertEquals(test1.get("test"), "[\"test4\"]"); }
@Test() public void testInvalidArrayRecursiveType() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test0", "test", "test1", ImmutableList.of("test", ImmutableMap.of("test", 2)), "test2", false))); Event event = mapper.readValue(bytes, Event.class); GenericArray test1 = event.getAttribute("test1"); assertEquals(test1.get(0), "test"); assertEquals(test1.get(1), "{\"test\":2}"); }
@Test() public void testObjectSentToInvalidScalarValue() throws Exception { metastore.getOrCreateCollectionFields("test", "test", ImmutableSet.of(new SchemaField("test", FieldType.INTEGER))); Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> props = ImmutableMap.of( "test", ImmutableList.of("test")); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "api", api, "collection", "test", "properties", props)); Event event = mapper.readValue(bytes, Event.class); assertNull(event.getAttribute("test")); }
@Test() public void testUnknownReferrer() throws Exception { ReferrerEventMapper mapper = new ReferrerEventMapper(); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_referrer"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_referrer", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_referrer", "http://test.com"); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertNull(resp); assertNull(event.getAttribute("_referrer_source")); assertNull(event.getAttribute("_referrer_term")); assertEquals("unknown", event.getAttribute("_referrer_medium")); assertEquals("test.com", event.getAttribute("_referrer_domain")); assertEquals("", event.getAttribute("_referrer_path")); }
@Test public void testScalarSentToObjectValue() throws Exception { metastore.getOrCreateCollectionFields("test", "test", ImmutableSet.of(new SchemaField("test", FieldType.ARRAY_BOOLEAN))); Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> props = ImmutableMap.of( "test", "test"); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "api", api, "collection", "test", "properties", props)); Event event = mapper.readValue(bytes, Event.class); assertEquals(event.getAttribute("test"), ImmutableList.of(false)); }
@Test(dataProvider = "google-referrer") public void testReferrer(Map<String, Object> props, EventMapper.RequestParams headers) throws Exception { ReferrerEventMapper mapper = new ReferrerEventMapper(); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(builder.build().dependentFields.get("_referrer").stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_referrer", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, headers, InetAddress.getLocalHost(), null); assertEquals("Google", event.getAttribute("_referrer_source")); assertEquals("test", event.getAttribute("_referrer_term")); assertEquals("search", event.getAttribute("_referrer_medium")); assertEquals("google.com", event.getAttribute("_referrer_domain")); assertEquals("/?q=test", event.getAttribute("_referrer_path")); assertNull(resp); GenericData.get().validate(properties.getSchema(), properties); }
@Test() public void testUnknownUserAgent() throws Exception { UserAgentEventMapper mapper = new UserAgentEventMapper(new WebsiteMapperConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_user_agent"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_user_agent", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_user_agent", "unknown user agent"); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertNull(resp); assertEquals("Other", event.getAttribute("_user_agent_family")); assertNull(event.getAttribute("_user_agent_version")); assertEquals("Other", event.getAttribute("_os")); assertNull(event.getAttribute("_os_version")); assertEquals("Other", event.getAttribute("_device_family")); }
@Test(dataProvider = "chrome-user-agent") public void testUserAgentMapper(Map<String, Object> props, EventMapper.RequestParams headers) throws Exception { UserAgentEventMapper mapper = new UserAgentEventMapper(new WebsiteMapperConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(builder.build().dependentFields.get("_user_agent").stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_user_agent", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, headers, InetAddress.getLocalHost(), null); assertEquals("Chrome", event.getAttribute("_user_agent_family")); assertEquals("Mac OS X", event.getAttribute("_os")); assertEquals("10", event.getAttribute("_os_version")); assertEquals("Other", event.getAttribute("_device_family")); assertNull(resp); GenericData.get().validate(properties.getSchema(), properties); }
@Test() public void testReferrerNotExists() throws Exception { ReferrerEventMapper mapper = new ReferrerEventMapper(); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_referrer"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_referrer", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_referrer", true); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertNull(resp); for (SchemaField field : fields) { assertNull(event.getAttribute(field.getName())); } }
@Test public void testNotTrackFlagIpEventMapper() throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig() .setConnectionTypeDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-Connection-Type-Test.mmdb")) .setIspDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-ISP-Test.mmdb"))); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); List<SchemaField> ip = builder.build().dependentFields.get("_ip"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(ip.stream().map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_ip", Schema.create(NULL), null, null)) .build(); Record properties = new Record(Schema.createRecord(build)); properties.put("_ip", false); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getByName("8.8.8.8"), null); assertTrue(resp == null); for (SchemaField schemaField : ip) { assertNull(event.getAttribute(schemaField.getName())); } }
@Test() public void testUserAgentNotExists() throws Exception { UserAgentEventMapper mapper = new UserAgentEventMapper(new WebsiteMapperConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_user_agent"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_user_agent", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_user_agent", true); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertNull(resp); for (SchemaField field : fields) { assertNull(event.getAttribute(field.getName())); } }
@Test public void testNotFoundIpEventMapper() throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig() .setConnectionTypeDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-Connection-Type-Test.mmdb")) .setIspDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-ISP-Test.mmdb"))); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); List<SchemaField> ip = builder.build().dependentFields.get("_ip"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(ip.stream().map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_ip", Schema.create(NULL), null, null)) .build(); Record properties = new Record(Schema.createRecord(build)); properties.put("_ip", "127.0.0.1"); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertTrue(resp == null); for (SchemaField schemaField : ip) { if (!schemaField.getName().equals("__ip")) { assertNull(event.getAttribute(schemaField.getName())); } } }
@Test() public void testDisableReferrer() throws Exception { ReferrerEventMapper mapper = new ReferrerEventMapper(); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_referrer"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_referrer", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_referrer", false); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, () -> { return new DefaultHttpHeaders().set("Referrer", "https://google.com/?q=test"); }, InetAddress.getLocalHost(), null); assertNull(resp); for (SchemaField field : fields) { assertNull(event.getAttribute(field.getName())); } }
@Test() public void testDisableUserAgent() throws Exception { UserAgentEventMapper mapper = new UserAgentEventMapper(new WebsiteMapperConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_user_agent"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_user_agent", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_user_agent", false); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, () -> new DefaultHttpHeaders().set("User-Agent", USER_AGENT), InetAddress.getLocalHost(), null); assertNull(resp); for (SchemaField field : fields) { assertNull(event.getAttribute(field.getName())); } } }
@Test(dataProvider = "google-ips") public void testEventMapper(Map<String, Object> props, InetAddress address) throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(builder.build().dependentFields.get("_ip").stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_ip", Schema.create(NULL), null, null)) .build(); Record properties = new Record(Schema.createRecord(build)); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, address, null); assertTrue(resp == null); assertTrue(event.properties().getSchema().getField("_country_code") != null); assertTrue(event.properties().getSchema().getField("_city") != null); assertTrue(event.properties().getSchema().getField("_timezone") != null); assertTrue(event.getAttribute("_latitude") instanceof Double); assertTrue(event.properties().getSchema().getField("_region") != null); assertTrue(event.getAttribute("_longitude") instanceof Double); GenericData.get().validate(properties.getSchema(), properties); }
@Test(dataProvider = "google-ips") public void testIspEventMapper(Map<String, Object> props, InetAddress address) throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig() .setAttributes("") .setIspDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-ISP-Test.mmdb"))); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); Record properties = new Record(Schema.createRecord(ImmutableList.of( new Schema.Field("_ip", Schema.create(NULL), null, null), new Schema.Field("__ip", Schema.create(STRING), null, null), new Schema.Field("_isp", Schema.create(STRING), null, null)))); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, address, null); assertTrue(resp == null); assertEquals(event.getAttribute("_isp"), "Level 3 Communications"); GenericData.get().validate(properties.getSchema(), properties); }
@Test(dataProvider = "google-ips") public void testConnectionTypeEventMapper(Map<String, Object> props, InetAddress address) throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig() .setAttributes("") .setConnectionTypeDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-Connection-Type-Test.mmdb"))); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); Record properties = new Record(Schema.createRecord(ImmutableList.of( new Schema.Field("_ip", Schema.create(NULL), null, null), new Schema.Field("__ip", Schema.create(STRING), null, null), new Schema.Field("_connection_type", Schema.create(STRING), null, null)))); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, address, null); assertTrue(resp == null); // TODO: find a reliable ip that can be mapped. assertNull(event.getAttribute("connection_type")); GenericData.get().validate(properties.getSchema(), properties); }