public String process(String project, Supplier<User> user, SendEventAction sendEventAction) { new Event(project, sendEventAction.collection, null, null, null); return null; }
public void mergeInternal(String project, Object user, Object anonymousId, Instant createdAt, Instant mergedAt) { FieldType config = configManager.getConfig(project, USER_TYPE.name(), FieldType.class); GenericData.Record properties = new GenericData.Record(ANONYMOUS_USER_MAPPING_SCHEMA.get(config)); properties.put(0, anonymousId.toString()); try { if (config == FieldType.STRING) { properties.put(1, user.toString()); } else if (config == FieldType.LONG) { properties.put(1, Long.parseLong(user.toString())); } else if (config == FieldType.INTEGER) { properties.put(1, Integer.parseInt(user.toString())); } else { throw new IllegalStateException(); } } catch (NumberFormatException e) { throw new RakamException("User type doesn't match", HttpResponseStatus.BAD_REQUEST); } properties.put(2, createdAt.toEpochMilli()); properties.put(3, mergedAt.toEpochMilli()); eventStore.store(new Event(project, ANONYMOUS_ID_MAPPING, null, ANONYMOUS_USER_MAPPING.get(config), properties)); }
public void merge(String project, Object user, Object anonymousId, Instant createdAt, Instant mergedAt) { if (!config.getEnableUserMapping()) { throw new RakamException(NOT_IMPLEMENTED); } GenericData.Record properties = new GenericData.Record(ANONYMOUS_USER_MAPPING_SCHEMA); properties.put(0, anonymousId); properties.put(1, user); properties.put(2, createdAt.toEpochMilli()); properties.put(3, mergedAt.toEpochMilli()); eventStore.store(new Event(project, ANONYMOUS_ID_MAPPING, null, null, properties)); } }
public EventList deserialize(String project, String collection, SliceInput slice) throws IOException { String json = slice.readSlice(slice.readInt()).toStringUtf8(); Schema schema = new Schema.Parser().parse(json); int records = slice.readInt(); BinaryDecoder binaryDecoder = DecoderFactory.get().directBinaryDecoder(slice, null); List<SchemaField> fields = metastore.getCollection(project, collection); Schema avroSchema = AvroUtil.convertAvroSchema(fields); GenericDatumReader<GenericRecord> reader = new GenericDatumReader(schema, avroSchema); List<Event> list = new ArrayList<>(records); for (int i = 0; i < records; i++) { GenericRecord record = reader.read(null, binaryDecoder); list.add(new Event(project, collection, null, fields, record)); } return new EventList(Event.EventContext.empty(), project, list); } }
public Event createEvent(String collection, Map<String, Object> properties) { List<SchemaField> cache = fieldCache.get(collection); List<SchemaField> fields; List<SchemaField> generatedSchema = generateSchema(properties); if (cache == null || !generatedSchema.stream().allMatch(f -> cache.contains(f))) { fields = metastore.getOrCreateCollectionFields(project, collection, ImmutableSet.copyOf(generatedSchema)); fieldCache.put(collection, fields); } else { fields = cache; } try { GenericData.Record record = new GenericData.Record(AvroUtil.convertAvroSchema(fields)); properties.forEach((key, value) -> record.put(key, cast(value, record.getSchema().getField(key).schema().getTypes().get(1).getType()))); return new Event(project, collection, Event.EventContext.empty(), fields, record); } catch (Exception e) { throw Throwables.propagate(e); } }
@Test() public void testReferrerNotExists() throws Exception { ReferrerEventMapper mapper = new ReferrerEventMapper(); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_referrer"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_referrer", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_referrer", true); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertNull(resp); for (SchemaField field : fields) { assertNull(event.getAttribute(field.getName())); } }
@Test() public void testUnknownReferrer() throws Exception { ReferrerEventMapper mapper = new ReferrerEventMapper(); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_referrer"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_referrer", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_referrer", "http://test.com"); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertNull(resp); assertNull(event.getAttribute("_referrer_source")); assertNull(event.getAttribute("_referrer_term")); assertEquals("unknown", event.getAttribute("_referrer_medium")); assertEquals("test.com", event.getAttribute("_referrer_domain")); assertEquals("", event.getAttribute("_referrer_path")); }
@Test public void testNotTrackFlagIpEventMapper() throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig() .setConnectionTypeDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-Connection-Type-Test.mmdb")) .setIspDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-ISP-Test.mmdb"))); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); List<SchemaField> ip = builder.build().dependentFields.get("_ip"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(ip.stream().map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_ip", Schema.create(NULL), null, null)) .build(); Record properties = new Record(Schema.createRecord(build)); properties.put("_ip", false); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getByName("8.8.8.8"), null); assertTrue(resp == null); for (SchemaField schemaField : ip) { assertNull(event.getAttribute(schemaField.getName())); } }
@Test(dataProvider = "chrome-user-agent") public void testUserAgentMapper(Map<String, Object> props, EventMapper.RequestParams headers) throws Exception { UserAgentEventMapper mapper = new UserAgentEventMapper(new WebsiteMapperConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(builder.build().dependentFields.get("_user_agent").stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_user_agent", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, headers, InetAddress.getLocalHost(), null); assertEquals("Chrome", event.getAttribute("_user_agent_family")); assertEquals("Mac OS X", event.getAttribute("_os")); assertEquals("10", event.getAttribute("_os_version")); assertEquals("Other", event.getAttribute("_device_family")); assertNull(resp); GenericData.get().validate(properties.getSchema(), properties); }
@Test() public void testUserAgentNotExists() throws Exception { UserAgentEventMapper mapper = new UserAgentEventMapper(new WebsiteMapperConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_user_agent"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_user_agent", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_user_agent", true); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertNull(resp); for (SchemaField field : fields) { assertNull(event.getAttribute(field.getName())); } }
@Test public void testNotFoundIpEventMapper() throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig() .setConnectionTypeDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-Connection-Type-Test.mmdb")) .setIspDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-ISP-Test.mmdb"))); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); List<SchemaField> ip = builder.build().dependentFields.get("_ip"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(ip.stream().map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_ip", Schema.create(NULL), null, null)) .build(); Record properties = new Record(Schema.createRecord(build)); properties.put("_ip", "127.0.0.1"); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertTrue(resp == null); for (SchemaField schemaField : ip) { if (!schemaField.getName().equals("__ip")) { assertNull(event.getAttribute(schemaField.getName())); } } }
@Test(dataProvider = "google-referrer") public void testReferrer(Map<String, Object> props, EventMapper.RequestParams headers) throws Exception { ReferrerEventMapper mapper = new ReferrerEventMapper(); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(builder.build().dependentFields.get("_referrer").stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_referrer", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, headers, InetAddress.getLocalHost(), null); assertEquals("Google", event.getAttribute("_referrer_source")); assertEquals("test", event.getAttribute("_referrer_term")); assertEquals("search", event.getAttribute("_referrer_medium")); assertEquals("google.com", event.getAttribute("_referrer_domain")); assertEquals("/?q=test", event.getAttribute("_referrer_path")); assertNull(resp); GenericData.get().validate(properties.getSchema(), properties); }
@Test() public void testDisableReferrer() throws Exception { ReferrerEventMapper mapper = new ReferrerEventMapper(); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_referrer"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_referrer", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_referrer", false); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, () -> { return new DefaultHttpHeaders().set("Referrer", "https://google.com/?q=test"); }, InetAddress.getLocalHost(), null); assertNull(resp); for (SchemaField field : fields) { assertNull(event.getAttribute(field.getName())); } }
@Test() public void testUnknownUserAgent() throws Exception { UserAgentEventMapper mapper = new UserAgentEventMapper(new WebsiteMapperConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_user_agent"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_user_agent", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_user_agent", "unknown user agent"); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertNull(resp); assertEquals("Other", event.getAttribute("_user_agent_family")); assertNull(event.getAttribute("_user_agent_version")); assertEquals("Other", event.getAttribute("_os")); assertNull(event.getAttribute("_os_version")); assertEquals("Other", event.getAttribute("_device_family")); }
@Test() public void testDisableUserAgent() throws Exception { UserAgentEventMapper mapper = new UserAgentEventMapper(new WebsiteMapperConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_user_agent"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_user_agent", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_user_agent", false); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, () -> new DefaultHttpHeaders().set("User-Agent", USER_AGENT), InetAddress.getLocalHost(), null); assertNull(resp); for (SchemaField field : fields) { assertNull(event.getAttribute(field.getName())); } } }
@Test() public void testInternalReferrer() throws Exception { ReferrerEventMapper mapper = new ReferrerEventMapper(); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_referrer"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_referrer", Schema.create(NULL), null, null), new Schema.Field("_host", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_referrer", "https://test.com/"); properties.put("_host", "test.com"); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertNull(resp); assertNull(event.getAttribute("_referrer_source")); assertNull(event.getAttribute("_referrer_term")); assertEquals("internal", event.getAttribute("_referrer_medium")); GenericData.get().validate(properties.getSchema(), properties); }
@Test(dataProvider = "google-ips") public void testEventMapper(Map<String, Object> props, InetAddress address) throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(builder.build().dependentFields.get("_ip").stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_ip", Schema.create(NULL), null, null)) .build(); Record properties = new Record(Schema.createRecord(build)); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, address, null); assertTrue(resp == null); assertTrue(event.properties().getSchema().getField("_country_code") != null); assertTrue(event.properties().getSchema().getField("_city") != null); assertTrue(event.properties().getSchema().getField("_timezone") != null); assertTrue(event.getAttribute("_latitude") instanceof Double); assertTrue(event.properties().getSchema().getField("_region") != null); assertTrue(event.getAttribute("_longitude") instanceof Double); GenericData.get().validate(properties.getSchema(), properties); }
@Test(dataProvider = "google-ips") public void testIspEventMapper(Map<String, Object> props, InetAddress address) throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig() .setAttributes("") .setIspDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-ISP-Test.mmdb"))); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); Record properties = new Record(Schema.createRecord(ImmutableList.of( new Schema.Field("_ip", Schema.create(NULL), null, null), new Schema.Field("__ip", Schema.create(STRING), null, null), new Schema.Field("_isp", Schema.create(STRING), null, null)))); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, address, null); assertTrue(resp == null); assertEquals(event.getAttribute("_isp"), "Level 3 Communications"); GenericData.get().validate(properties.getSchema(), properties); }
@Test(dataProvider = "google-ips") public void testConnectionTypeEventMapper(Map<String, Object> props, InetAddress address) throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig() .setAttributes("") .setConnectionTypeDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-Connection-Type-Test.mmdb"))); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); Record properties = new Record(Schema.createRecord(ImmutableList.of( new Schema.Field("_ip", Schema.create(NULL), null, null), new Schema.Field("__ip", Schema.create(STRING), null, null), new Schema.Field("_connection_type", Schema.create(STRING), null, null)))); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, address, null); assertTrue(resp == null); // TODO: find a reliable ip that can be mapped. assertNull(event.getAttribute("connection_type")); GenericData.get().validate(properties.getSchema(), properties); }