@Override public FieldDependency get() { FieldDependencyBuilder builder = new FieldDependencyBuilder(); eventMappers.stream().forEach(mapper -> mapper.addFieldDependency(builder)); return builder.build(); } }
@ApiOperation(value = "List event mappers", authorizations = @Authorization(value = "master_key") ) @GET @Path("/event_mappers") @JsonRequest public List<EventMapperDescription> getEventMappers() { return eventMappers.stream().map(mapper -> { Mapper annotation = mapper.getClass().getAnnotation(Mapper.class); String name; String description; if (annotation != null) { name = annotation.name(); description = annotation.description(); } else { name = mapper.getClass().getSimpleName(); description = ""; } FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); FieldDependencyBuilder.FieldDependency build = builder.build(); return new EventMapperDescription(name, description, build.dependentFields.isEmpty() ? null : build.dependentFields, build.constantFields.isEmpty() ? null : build.constantFields); }).collect(Collectors.toList()); }
@Test public void testFieldDependency() throws Exception { MaxmindGeoIPModuleConfig config = new MaxmindGeoIPModuleConfig().setAttributes(list.stream().collect(Collectors.joining(","))); MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(config); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); FieldDependencyBuilder.FieldDependency build = builder.build(); assertEquals(0, build.constantFields.size()); assertEquals(1, build.dependentFields.size()); assertEquals(list.stream().map(a -> "_" + a).collect(Collectors.toSet()), build.dependentFields.get("_ip").stream().map(SchemaField::getName) .collect(Collectors.toSet())); }
@Test public void testFieldDependencyWithAll() throws Exception { MaxmindGeoIPModuleConfig config = new MaxmindGeoIPModuleConfig() .setAttributes(list.stream().collect(Collectors.joining(","))) .setConnectionTypeDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-Connection-Type-Test.mmdb")) .setIspDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-ISP-Test.mmdb")); MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(config); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); FieldDependencyBuilder.FieldDependency build = builder.build(); assertEquals(0, build.constantFields.size()); assertEquals(1, build.dependentFields.size()); assertEquals(ImmutableSet.builder().addAll(list.stream().map(e -> "_" + e).collect(Collectors.toList())).add("_isp", "_connection_type").build(), build.dependentFields.get("_ip").stream().map(SchemaField::getName).collect(Collectors.toSet())); } }
@Test() public void testReferrerNotExists() throws Exception { ReferrerEventMapper mapper = new ReferrerEventMapper(); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_referrer"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_referrer", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_referrer", true); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertNull(resp); for (SchemaField field : fields) { assertNull(event.getAttribute(field.getName())); } }
@Test() public void testUserAgentNotExists() throws Exception { UserAgentEventMapper mapper = new UserAgentEventMapper(new WebsiteMapperConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_user_agent"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_user_agent", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_user_agent", true); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertNull(resp); for (SchemaField field : fields) { assertNull(event.getAttribute(field.getName())); } }
@Test(dataProvider = "chrome-user-agent") public void testUserAgentMapper(Map<String, Object> props, EventMapper.RequestParams headers) throws Exception { UserAgentEventMapper mapper = new UserAgentEventMapper(new WebsiteMapperConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(builder.build().dependentFields.get("_user_agent").stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_user_agent", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, headers, InetAddress.getLocalHost(), null); assertEquals("Chrome", event.getAttribute("_user_agent_family")); assertEquals("Mac OS X", event.getAttribute("_os")); assertEquals("10", event.getAttribute("_os_version")); assertEquals("Other", event.getAttribute("_device_family")); assertNull(resp); GenericData.get().validate(properties.getSchema(), properties); }
@Test() public void testDisableUserAgent() throws Exception { UserAgentEventMapper mapper = new UserAgentEventMapper(new WebsiteMapperConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_user_agent"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_user_agent", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_user_agent", false); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, () -> new DefaultHttpHeaders().set("User-Agent", USER_AGENT), InetAddress.getLocalHost(), null); assertNull(resp); for (SchemaField field : fields) { assertNull(event.getAttribute(field.getName())); } } }
@Test public void testNotTrackFlagIpEventMapper() throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig() .setConnectionTypeDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-Connection-Type-Test.mmdb")) .setIspDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-ISP-Test.mmdb"))); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); List<SchemaField> ip = builder.build().dependentFields.get("_ip"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(ip.stream().map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_ip", Schema.create(NULL), null, null)) .build(); Record properties = new Record(Schema.createRecord(build)); properties.put("_ip", false); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getByName("8.8.8.8"), null); assertTrue(resp == null); for (SchemaField schemaField : ip) { assertNull(event.getAttribute(schemaField.getName())); } }
@BeforeSuite public void setUp() throws Exception { FieldDependencyBuilder.FieldDependency fieldDependency = new FieldDependencyBuilder().build(); apiKeyService = new InMemoryApiKeyService(); metastore = new InMemoryMetastore(apiKeyService); SchemaChecker schemaChecker = new SchemaChecker(metastore, fieldDependency); eventDeserializer = new JsonEventDeserializer(metastore, apiKeyService, new TestingConfigManager(), schemaChecker, new ProjectConfig(), fieldDependency); EventListDeserializer eventListDeserializer = new EventListDeserializer(apiKeyService, eventDeserializer); mapper = JsonHelper.getMapper(); mapper.registerModule(new SimpleModule() .addDeserializer(Event.class, eventDeserializer) .addDeserializer(EventList.class, eventListDeserializer)); eventBuilder = new EventBuilder("test", metastore); }
@BeforeSuite public void setup() throws Exception { testingPostgresqlServer = new TestingEnvironmentPg9(); JDBCPoolDataSource dataSource = JDBCPoolDataSource.getOrCreateDataSource(testingPostgresqlServer.getPostgresqlConfig(), "set time zone 'UTC'"); FieldDependencyBuilder.FieldDependency build = new FieldDependencyBuilder().build(); EventBus eventBus = new EventBus(); metastore = new PostgresqlMetastore(dataSource, new PostgresqlModule.PostgresqlVersion(dataSource), eventBus, new ProjectConfig()); queryExecutor = new PostgresqlQueryExecutor(new ProjectConfig(), dataSource, metastore, new CustomDataSourceService(dataSource), false); eventStore = new PostgresqlEventStore(dataSource, new PostgresqlModule.PostgresqlVersion(dataSource), build); super.setup(); }
@BeforeSuite public void setup() throws Exception { testingPostgresqlServer = new TestingEnvironmentPg10(); JDBCPoolDataSource dataSource = JDBCPoolDataSource.getOrCreateDataSource(testingPostgresqlServer.getPostgresqlConfig(), "set time zone 'UTC'"); FieldDependencyBuilder.FieldDependency build = new FieldDependencyBuilder().build(); EventBus eventBus = new EventBus(); metastore = new PostgresqlMetastore(dataSource, new PostgresqlModule.PostgresqlVersion(dataSource), eventBus, new ProjectConfig()); queryExecutor = new PostgresqlQueryExecutor(new ProjectConfig(), dataSource, metastore, new CustomDataSourceService(dataSource), false); eventStore = new PostgresqlEventStore(dataSource, new PostgresqlModule.PostgresqlVersion(dataSource), build); super.setup(); }
@Override @BeforeSuite public void setup() throws Exception { testingPostgresqlServer = new TestingEnvironmentPg10(); InMemoryQueryMetadataStore queryMetadataStore = new InMemoryQueryMetadataStore(); JDBCPoolDataSource dataSource = JDBCPoolDataSource.getOrCreateDataSource(testingPostgresqlServer.getPostgresqlConfig(), "set time zone 'UTC'"); FieldDependencyBuilder.FieldDependency build = new FieldDependencyBuilder().build(); EventBus eventBus = new EventBus(); metastore = new PostgresqlMetastore(dataSource, new PostgresqlModule.PostgresqlVersion(dataSource), eventBus, new ProjectConfig()); PostgresqlQueryExecutor queryExecutor = new PostgresqlQueryExecutor(new ProjectConfig(), dataSource, metastore, new CustomDataSourceService(dataSource), false); PostgresqlMaterializedViewService postgresqlMaterializedViewService = new PostgresqlMaterializedViewService(queryExecutor, queryMetadataStore, Clock.systemUTC()); eventStore = new PostgresqlEventStore(dataSource, new PostgresqlModule.PostgresqlVersion(dataSource), build); PostgresqlMaterializedViewService materializedViewService = postgresqlMaterializedViewService; eventExplorer = new PostgresqlEventExplorer( new ProjectConfig(), new QueryExecutorService(queryExecutor, metastore, materializedViewService, '"'), materializedViewService); super.setup(); }
@Override @BeforeSuite public void setup() throws Exception { testingPostgresqlServer = new TestingEnvironmentPg9(); InMemoryQueryMetadataStore queryMetadataStore = new InMemoryQueryMetadataStore(); JDBCPoolDataSource dataSource = JDBCPoolDataSource.getOrCreateDataSource(testingPostgresqlServer.getPostgresqlConfig(), "set time zone 'UTC'"); FieldDependencyBuilder.FieldDependency build = new FieldDependencyBuilder().build(); EventBus eventBus = new EventBus(); metastore = new PostgresqlMetastore(dataSource, new PostgresqlModule.PostgresqlVersion(dataSource), eventBus, new ProjectConfig()); PostgresqlQueryExecutor queryExecutor = new PostgresqlQueryExecutor(new ProjectConfig(), dataSource, metastore, new CustomDataSourceService(dataSource), false); PostgresqlMaterializedViewService postgresqlMaterializedViewService = new PostgresqlMaterializedViewService(queryExecutor, queryMetadataStore, Clock.systemUTC()); eventStore = new PostgresqlEventStore(dataSource, new PostgresqlModule.PostgresqlVersion(dataSource), build); PostgresqlMaterializedViewService materializedViewService = postgresqlMaterializedViewService; eventExplorer = new PostgresqlEventExplorer( new ProjectConfig(), new QueryExecutorService(queryExecutor, metastore, materializedViewService, '"'), materializedViewService); super.setup(); }
@BeforeSuite public void setup() throws Exception { testingPostgresqlServer = new TestingEnvironmentPg10(); JDBCPoolDataSource dataSource = JDBCPoolDataSource.getOrCreateDataSource(testingPostgresqlServer.getPostgresqlConfig()); FieldDependencyBuilder.FieldDependency build = new FieldDependencyBuilder().build(); metastore = new PostgresqlMetastore(dataSource, new PostgresqlVersion(dataSource), new EventBus(), new ProjectConfig()); PostgresqlQueryExecutor queryExecutor = new PostgresqlQueryExecutor(new ProjectConfig(), dataSource, metastore, new CustomDataSourceService(dataSource), false); eventStore = new PostgresqlEventStore(dataSource, new PostgresqlVersion(dataSource), build); retentionQueryExecutor = new PostgresqlRetentionQueryExecutor(new ProjectConfig(), queryExecutor, metastore); retentionQueryExecutor.setup(); super.setup(); }
@BeforeSuite public void setup() throws Exception { testingPostgresqlServer = new TestingEnvironmentPg9(); JDBCPoolDataSource dataSource = JDBCPoolDataSource.getOrCreateDataSource(testingPostgresqlServer.getPostgresqlConfig()); FieldDependencyBuilder.FieldDependency build = new FieldDependencyBuilder().build(); metastore = new PostgresqlMetastore(dataSource, new PostgresqlVersion(dataSource), new EventBus(), new ProjectConfig()); PostgresqlQueryExecutor queryExecutor = new PostgresqlQueryExecutor(new ProjectConfig(), dataSource, metastore, new CustomDataSourceService(dataSource), false); eventStore = new PostgresqlEventStore(dataSource, new PostgresqlVersion(dataSource), build); retentionQueryExecutor = new PostgresqlRetentionQueryExecutor(new ProjectConfig(), queryExecutor, metastore); retentionQueryExecutor.setup(); super.setup(); }
@BeforeSuite @Override public void setUp() throws Exception { testingPostgresqlServer = new TestingEnvironmentPg10(); InMemoryQueryMetadataStore queryMetadataStore = new InMemoryQueryMetadataStore(); JDBCPoolDataSource dataSource = JDBCPoolDataSource.getOrCreateDataSource(testingPostgresqlServer.getPostgresqlConfig(), "set time zone 'UTC'"); EventBus eventBus = new EventBus(); metastore = new PostgresqlMetastore(dataSource, new PostgresqlModule.PostgresqlVersion(dataSource), eventBus, new ProjectConfig()); PostgresqlQueryExecutor queryExecutor = new PostgresqlQueryExecutor(new ProjectConfig(), dataSource, metastore, new CustomDataSourceService(dataSource), false); PostgresqlMaterializedViewService materializedViewService = new PostgresqlMaterializedViewService(queryExecutor, queryMetadataStore, Clock.systemUTC()); QueryExecutorService queryExecutorService = new QueryExecutorService(queryExecutor, metastore, materializedViewService, '"'); configManager = new PostgresqlConfigManager(dataSource); configManager.setup(); PostgresqlUserStorage userStorage = new PostgresqlUserStorage(queryExecutorService, materializedViewService, configManager, queryExecutor); PostgresqlEventStore postgresqlEventStore = new PostgresqlEventStore(dataSource, new PostgresqlModule.PostgresqlVersion(dataSource), new FieldDependencyBuilder().build()); userService = new PostgresqlUserService(new ProjectConfig(), configManager, postgresqlEventStore, userStorage, metastore, queryExecutor); super.setUp(); }
@BeforeSuite @Override public void setUp() throws Exception { testingPostgresqlServer = new TestingEnvironmentPg9(); InMemoryQueryMetadataStore queryMetadataStore = new InMemoryQueryMetadataStore(); JDBCPoolDataSource dataSource = JDBCPoolDataSource.getOrCreateDataSource(testingPostgresqlServer.getPostgresqlConfig(), "set time zone 'UTC'"); EventBus eventBus = new EventBus(); metastore = new PostgresqlMetastore(dataSource, new PostgresqlModule.PostgresqlVersion(dataSource), eventBus, new ProjectConfig()); PostgresqlQueryExecutor queryExecutor = new PostgresqlQueryExecutor(new ProjectConfig(), dataSource, metastore, new CustomDataSourceService(dataSource), false); PostgresqlMaterializedViewService materializedViewService = new PostgresqlMaterializedViewService(queryExecutor, queryMetadataStore, Clock.systemUTC()); QueryExecutorService queryExecutorService = new QueryExecutorService(queryExecutor, metastore, materializedViewService, '"'); configManager = new PostgresqlConfigManager(dataSource); configManager.setup(); PostgresqlUserStorage userStorage = new PostgresqlUserStorage(queryExecutorService, materializedViewService, configManager, queryExecutor); PostgresqlEventStore postgresqlEventStore = new PostgresqlEventStore(dataSource, new PostgresqlModule.PostgresqlVersion(dataSource), new FieldDependencyBuilder().build()); userService = new PostgresqlUserService(new ProjectConfig(), configManager, postgresqlEventStore, userStorage, metastore, queryExecutor); super.setUp(); }
@BeforeSuite @Override public void setup() throws Exception { testingPostgresqlServer = new TestingEnvironmentPg9(); JDBCPoolDataSource dataSource = JDBCPoolDataSource.getOrCreateDataSource(testingPostgresqlServer.getPostgresqlConfig()); FieldDependencyBuilder.FieldDependency build = new FieldDependencyBuilder().build(); metastore = new PostgresqlMetastore(dataSource, new PostgresqlModule.PostgresqlVersion(dataSource), new EventBus(), new ProjectConfig()); PostgresqlQueryExecutor queryExecutor = new PostgresqlQueryExecutor(new ProjectConfig(), dataSource, metastore, new CustomDataSourceService(dataSource), false); eventStore = new PostgresqlEventStore(dataSource, new PostgresqlModule.PostgresqlVersion(dataSource), build); FastGenericFunnelQueryExecutor exec = new FastGenericFunnelQueryExecutor(new QueryExecutorService(queryExecutor, metastore, null, '"'), new ProjectConfig(), metastore); funnelQueryExecutor = new PostgresqlFunnelQueryExecutor(exec, new ProjectConfig(), metastore, queryExecutor); funnelQueryExecutor.setup(); super.setup(); }
@BeforeSuite @Override public void setup() throws Exception { testingPostgresqlServer = new TestingEnvironmentPg10(); JDBCPoolDataSource dataSource = JDBCPoolDataSource.getOrCreateDataSource(testingPostgresqlServer.getPostgresqlConfig()); FieldDependencyBuilder.FieldDependency build = new FieldDependencyBuilder().build(); metastore = new PostgresqlMetastore(dataSource, new PostgresqlModule.PostgresqlVersion(dataSource), new EventBus(), new ProjectConfig()); PostgresqlQueryExecutor queryExecutor = new PostgresqlQueryExecutor(new ProjectConfig(), dataSource, metastore, new CustomDataSourceService(dataSource), false); eventStore = new PostgresqlEventStore(dataSource, new PostgresqlModule.PostgresqlVersion(dataSource), build); FastGenericFunnelQueryExecutor exec = new FastGenericFunnelQueryExecutor(new QueryExecutorService(queryExecutor, metastore, null, '"'), new ProjectConfig(), metastore); funnelQueryExecutor = new PostgresqlFunnelQueryExecutor(exec, new ProjectConfig(), metastore, queryExecutor); funnelQueryExecutor.setup(); super.setup(); }