private Map<String, Integer> generateColumnMap(List<String> variables, List<SchemaField> metadata) { HashMap<String, Integer> colMap = new HashMap<>(variables.size()); for (String var : variables) { for (int i = 0; i < metadata.size(); i++) { if (metadata.get(i).getName().equals(var)) { colMap.put(variables.get(i), i); break; } } } return colMap; }
private Map<String, Integer> generateColumnMap(List<String> variables, List<SchemaField> metadata) { HashMap<String, Integer> colMap = new HashMap<>(variables.size()); for (String var : variables) { for (int i = 0; i < metadata.size(); i++) { if (metadata.get(i).getName().equals(var)) { colMap.put(variables.get(i), i); break; } } } return colMap; }
protected boolean testDeviceIdExists(Optional<RetentionAction> firstAction, Map<String, List<SchemaField>> collections) { if (firstAction.isPresent()) { List<SchemaField> schemaFields = collections.get(firstAction.get().collection()); if (schemaFields == null) { throw new RakamException("The collection in first action does not exist.", HttpResponseStatus.BAD_REQUEST); } return schemaFields.stream().anyMatch(e -> e.getName().equals("_device_id")); } else { return collections.entrySet().stream() .allMatch(e -> e.getValue().stream().anyMatch(field -> field.getName().equals("_device_id"))); } } }
private void addConditionalModuleField(Set<SchemaField> fields, String field, List<SchemaField> newFields) { if (fields.stream().anyMatch(f -> f.getName().equals(field))) { newFields.forEach(newField -> addModuleField(fields, newField)); } }
@Subscribe public void onCreateFields(SystemEvents.CollectionFieldCreatedEvent event) { if (event.fields.stream().anyMatch(f -> f.getName().equals("_user"))) { createInternal(event.project, event.collection); } }
@Subscribe public void onCreateCollection(SystemEvents.CollectionCreatedEvent event) { if (event.fields.stream().anyMatch(f -> f.getName().equals("_user"))) { createInternal(event.project, event.collection); } }
@Override public synchronized List<SchemaField> getOrCreateCollectionFields(String project, String collection, Set<SchemaField> fields) throws NotExistsException { ValidationUtil.checkCollectionValid(collection); Map<String, List<SchemaField>> list = collections.get(project); if (list == null) { throw new NotExistsException("Project"); } List<SchemaField> schemaFields = list.computeIfAbsent(collection, (key) -> new ArrayList<>()); fields.stream() .filter(field -> !schemaFields.stream().anyMatch(f -> f.getName().equals(field.getName()))) .forEach(schemaFields::add); return schemaFields; }
private boolean getHasDeviceId(String project, String collection) { return metastore.getCollection(project, collection).stream() .anyMatch(f -> f.getName().equals("_device_id")); }
private void checkFields(List<SchemaField> fields) { SchemaField[] collisions = fields.stream() .filter(newField -> constantFields.stream() .anyMatch(f -> f.getName().equals(newField.getName()) && !f.getType().equals(newField.getType()))) .toArray(SchemaField[]::new); checkState(collisions.length == 0, "Module field collides with existing field that has another type exists: ", Arrays.toString(collisions)); collisions = dependentFields.values().stream() .flatMap(col -> col.stream()) .filter(field -> fields.stream().anyMatch(f -> f.getName().equals(field.getName()) && !f.getType().equals(field.getType()))) .toArray(SchemaField[]::new); checkState(collisions.length == 0, "Fields already exist in dependency table: ", Arrays.toString(collisions)); }
protected boolean testDeviceIdExists(FunnelStep firstAction, Map<String, List<SchemaField>> collections) { List<SchemaField> schemaFields = collections.get(firstAction.getCollection()); if (schemaFields == null) { throw new RakamException("The collection in first action does not exist.", HttpResponseStatus.BAD_REQUEST); } return schemaFields.stream().anyMatch(e -> e.getName().equals("_device_id")); } }
public Map<String, FieldType> loadColumns(String project) { Map<String, FieldType> columns = getMetadata(project).stream() .collect(Collectors.toMap(col -> col.getName(), col -> col.getType())); return columns; }
private boolean check(String project, String collection, SchemaField existing, SchemaField moduleField) { if (existing.getName().equals(moduleField.getName())) { if (!existing.getType().equals(moduleField.getType())) { throw new IllegalStateException(format("Module field '%s' type does not match existing field in event of project %s.%s. Existing type: %s, Module field type: %s. \n" + "Please change the schema manually of disable the module.", existing.getName(), project, collection, existing.getType(), moduleField.getType())); } return true; } return false; } }
public KafkaEventSupplier(String project, List<CollectionStreamQuery> collections, List<String> columns, StreamResponse response) { this.collectionNames = collections.stream().map(c -> c.getCollection()).collect(Collectors.toSet()); this.lastOffsets = offsetManager.getOffset(project, collectionNames); this.collections = collections; this.project = project; List<SchemaField> sample = getMetadata().get(collections.get(0)); if (columns != null) { this.columns = columns.stream().map(colName -> sample.stream() .filter(field -> field.getName().equals(colName)).findFirst().get()) .collect(Collectors.toList()); } else { this.columns = null; } this.response = response; }
private void addModuleField(Set<SchemaField> fields, SchemaField newField) { Iterator<SchemaField> iterator = fields.iterator(); while (iterator.hasNext()) { SchemaField field = iterator.next(); if (field.getName().equals(newField.getName())) { if (field.getType().equals(newField.getType())) { return; } else { iterator.remove(); break; } } } fields.add(newField); }
@Inject public JsonEventDeserializer(Metastore metastore, ApiKeyService apiKeyService, ConfigManager configManager, SchemaChecker schemaChecker, ProjectConfig projectConfig, FieldDependency fieldDependency) { this.metastore = metastore; this.conditionalMagicFields = fieldDependency.dependentFields; this.apiKeyService = apiKeyService; this.schemaChecker = schemaChecker; this.projectConfig = projectConfig; this.configManager = configManager; this.constantFields = fieldDependency.constantFields; conditionalFieldMapping = conditionalMagicFields.values().stream() .flatMap(e -> e.stream()).collect(toImmutableMap(e -> e.getName(), e -> e.getType())); }
public Recipe export(String project) { final Map<String, Recipe.CollectionDefinition> collections = metastore.getCollections(project).entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> { List<Map<String, Recipe.SchemaFieldInfo>> map = e.getValue().stream() .map(a -> ImmutableMap.of(a.getName(), new Recipe.SchemaFieldInfo(a.getCategory(), a.getType()))) .collect(Collectors.toList()); return new Recipe.CollectionDefinition(map); })); final List<MaterializedView> materializedViews = materializedViewService.list(project).stream() .map(m -> new MaterializedView(m.tableName, m.name, m.query, m.updateInterval, m.incremental, m.realTime, m.options)) .collect(Collectors.toList()); return new Recipe(Recipe.Strategy.SPECIFIC, collections, materializedViews); }
public void onCreateCollectionFields(String project, String collection, List<SchemaField> fields) { for (SchemaField field : fields) { try { // We cant't use CONCURRENTLY because it causes dead-lock with ALTER TABLE and it's slow. projectConfig.getTimeColumn(); executor.executeRawStatement(String.format("CREATE INDEX %s %s ON %s.%s USING %s(%s)", postgresql9_5 ? "IF NOT EXISTS" : "", checkCollection(String.format("%s_%s_%s_auto_index", project, collection, field.getName())), project, checkCollection(collection), (postgresql9_5 && field.getName().equals(projectConfig.getTimeColumn())) ? "BRIN" : "BTREE", checkTableColumn(field.getName()))); } catch (Exception e) { if (postgresql9_5) { throw e; } } } } }
public static Schema.Field generateAvroField(SchemaField field) { return new Schema.Field(field.getName(), generateAvroSchema(field.getType()), null, NullNode.getInstance()); }
@Test() public void testUserAgentNotExists() throws Exception { UserAgentEventMapper mapper = new UserAgentEventMapper(new WebsiteMapperConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_user_agent"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_user_agent", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_user_agent", true); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertNull(resp); for (SchemaField field : fields) { assertNull(event.getAttribute(field.getName())); } }
@Test() public void testDisableUserAgent() throws Exception { UserAgentEventMapper mapper = new UserAgentEventMapper(new WebsiteMapperConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_user_agent"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_user_agent", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_user_agent", false); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, () -> new DefaultHttpHeaders().set("User-Agent", USER_AGENT), InetAddress.getLocalHost(), null); assertNull(resp); for (SchemaField field : fields) { assertNull(event.getAttribute(field.getName())); } } }