@Override public void addFieldDependency(FieldDependencyBuilder builder) { builder.addFields("_user_agent", ImmutableList.of( new SchemaField("_user_agent_family", FieldType.STRING), new SchemaField("_user_agent_version", FieldType.STRING), new SchemaField("_os", FieldType.STRING), new SchemaField("_os_version", FieldType.STRING), new SchemaField("_device_family", FieldType.STRING) )); }
@Override public void addFieldDependency(FieldDependencyBuilder builder) { builder.addFields("_referrer", ImmutableList.of( new SchemaField("_referrer_medium", STRING), new SchemaField("_referrer_source", STRING), new SchemaField("_referrer_term", STRING), new SchemaField("_referrer_domain", STRING), new SchemaField("_referrer_path", STRING) )); } }
@Override public List<SchemaField> getMetadata(RequestContext context) { return ImmutableList.of(new SchemaField("id", STRING)); }
@JsonIgnore public List<SchemaField> build() { return columns.stream() .map(column -> { Map.Entry<String, SchemaFieldInfo> next = column.entrySet().iterator().next(); return new SchemaField(next.getKey(), next.getValue().type); }).collect(Collectors.toList()); } }
public List<SchemaField> convertToSchema(ResultSet dbColumns) throws SQLException { List<SchemaField> schemaFields = Lists.newArrayList(); while (dbColumns.next()) { String columnName = dbColumns.getString("COLUMN_NAME"); FieldType fieldType = fromSql(dbColumns.getInt("DATA_TYPE"), dbColumns.getString("TYPE_NAME"), s -> fromPrestoType(s)); schemaFields.add(new SchemaField(columnName, fieldType)); } return schemaFields.isEmpty() ? null : schemaFields; }
@Override public void addFieldDependency(FieldDependencyBuilder builder) { List<SchemaField> fields = CITY_DATABASE_ATTRIBUTES.stream() .map(attr -> new SchemaField("_" + attr, getType(attr))) .collect(Collectors.toList()); builder.addFields("_ip", fields); }
@Override public void addFieldDependency(FieldDependencyBuilder builder) { builder.addFields(of(new SchemaField(projectConfig.getTimeColumn(), FieldType.TIMESTAMP))); }
@Override public List<SchemaField> getCollection(String project, String collection) { return dao.listTableColumns(project, collection).stream() // this field should be removed since the server sets it .filter(a -> !a.getColumnName().equals(prestoConfig.getCheckpointColumn())) .map(column -> { TypeSignature typeSignature = column.getDataType().getTypeSignature(); return new SchemaField(column.getColumnName(), PrestoQueryExecution.fromPrestoType(typeSignature.getBase(), typeSignature.getParameters().stream() .filter(param -> param.getKind() == TYPE) .map(param -> param.getTypeSignature().getBase()).iterator())); }).collect(Collectors.toList()); }
private List<SchemaField> generateSchema(Map<String, Object> properties) { return properties.entrySet().stream() .map(entry -> new SchemaField(entry.getKey(), getType(entry.getValue()))) .collect(Collectors.toList()); } }
@Test public void testGetAttributesWithFilter() { getMetastore().createProject(PROJECT_NAME); getMetastore().getOrCreateCollectionFields(PROJECT_NAME, "test", ImmutableSet.of(new SchemaField("_time", TIMESTAMP), new SchemaField("test", STRING))); ImmutableSet<SchemaField> schema = ImmutableSet.of(new SchemaField("test", STRING)); getMetastore().getOrCreateCollectionFields(PROJECT_NAME, "test", schema); assertEquals(getMetastore().getAttributes(PROJECT_NAME, "test", "test", Optional.empty(), Optional.empty(), Optional.of("T")).join().size(), 0); } }
@Test public void testGetAttributesWithTime() { getMetastore().createProject(PROJECT_NAME); getMetastore().getOrCreateCollectionFields(PROJECT_NAME, "test", ImmutableSet.of(new SchemaField("_time", TIMESTAMP), new SchemaField("test", STRING))); ImmutableSet<SchemaField> schema = ImmutableSet.of(new SchemaField("test", STRING)); getMetastore().getOrCreateCollectionFields(PROJECT_NAME, "test", schema); assertEquals(getMetastore().getAttributes(PROJECT_NAME, "test", "test", Optional.of(LocalDate.parse("2017-01-01")), Optional.of(LocalDate.parse("2017-02-01")), Optional.empty()).join().size(), 0); }
@Test public void testGetAttributesWithTimeAndFilter() { getMetastore().createProject(PROJECT_NAME); getMetastore().getOrCreateCollectionFields(PROJECT_NAME, "test", ImmutableSet.of(new SchemaField("_time", TIMESTAMP), new SchemaField("test", STRING))); ImmutableSet<SchemaField> schema = ImmutableSet.of(new SchemaField("test", STRING)); getMetastore().getOrCreateCollectionFields(PROJECT_NAME, "test", schema); assertEquals(getMetastore().getAttributes(PROJECT_NAME, "test", "test", Optional.of(LocalDate.parse("2017-01-01")), Optional.of(LocalDate.parse("2017-02-01")), Optional.of("T")).join().size(), 0); }
@Test(expectedExceptions = IllegalStateException.class) public void testInvalidDuplicateFieldNames() throws Exception { getMetastore().createProject(PROJECT_NAME); getMetastore().getOrCreateCollectionFields(PROJECT_NAME, "testcollection", ImmutableSet.of(new SchemaField("test", STRING), new SchemaField("test", LONG))); }
@Test public void testGetAttributes() { getMetastore().createProject(PROJECT_NAME); getMetastore().getOrCreateCollectionFields(PROJECT_NAME, "test", ImmutableSet.of(new SchemaField("_time", TIMESTAMP), new SchemaField("test", STRING))); ImmutableSet<SchemaField> schema = ImmutableSet.of(new SchemaField("test", STRING)); getMetastore().getOrCreateCollectionFields(PROJECT_NAME, "test", schema); assertEquals(getMetastore().getAttributes(PROJECT_NAME, "test", "test", Optional.empty(), Optional.empty(), Optional.empty()).join().size(), 0); }
@Test() public void testObjectSentToInvalidScalarValue() throws Exception { metastore.getOrCreateCollectionFields("test", "test", ImmutableSet.of(new SchemaField("test", FieldType.INTEGER))); Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> props = ImmutableMap.of( "test", ImmutableList.of("test")); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "api", api, "collection", "test", "properties", props)); Event event = mapper.readValue(bytes, Event.class); assertNull(event.getAttribute("test")); }
@Test public void testCollectionMethods() throws Exception { getMetastore().createProject(PROJECT_NAME); ImmutableSet<SchemaField> schema = ImmutableSet.of(new SchemaField("test1", STRING), new SchemaField("test2", STRING)); getMetastore().getOrCreateCollectionFields(PROJECT_NAME, "testcollection1", schema); getMetastore().getOrCreateCollectionFields(PROJECT_NAME, "testcollection2", schema); assertEquals(ImmutableSet.of("testcollection1", "testcollection2"), ImmutableSet.copyOf(getMetastore().getCollectionNames(PROJECT_NAME))); Map<String, List<SchemaField>> testing = getMetastore().getCollections(PROJECT_NAME); assertEquals(testing.size(), 2); assertTrue(ImmutableSet.copyOf(testing.get("testcollection1")).containsAll(schema)); assertTrue(ImmutableSet.copyOf(testing.get("testcollection2")).containsAll(schema)); }
@Test public void testCollectionFieldsOrdering() throws Exception { getMetastore().createProject(PROJECT_NAME); ImmutableSet.Builder<SchemaField> builder = ImmutableSet.builder(); for (FieldType fieldType : FieldType.values()) { builder.add(new SchemaField(fieldType.name(), fieldType)); } getMetastore().getOrCreateCollectionFields(PROJECT_NAME, "testcollection", builder.build()); for (int i = 0; i < 100; i++) { assertTrue(getMetastore().getCollection(PROJECT_NAME, "testcollection").containsAll(builder.build())); } }
@Test public void testAllSchemaTypes() throws Exception { getMetastore().createProject(PROJECT_NAME); ImmutableSet.Builder<SchemaField> builder = ImmutableSet.builder(); for (FieldType fieldType : FieldType.values()) { builder.add(new SchemaField(fieldType.name(), fieldType)); } getMetastore().getOrCreateCollectionFields(PROJECT_NAME, "testcollection", builder.build()); assertTrue(getMetastore().getCollection(PROJECT_NAME, "testcollection").containsAll(builder.build())); }
@Test public void testCreateCollection() throws Exception { getMetastore().createProject(PROJECT_NAME); ImmutableSet<SchemaField> schema = ImmutableSet.of(new SchemaField("test", STRING)); getMetastore().getOrCreateCollectionFields(PROJECT_NAME, "test", schema); assertTrue(getMetastore().getCollection(PROJECT_NAME, "test").containsAll(schema)); }
@Test public void testCreateFields() throws Exception { getMetastore().createProject(PROJECT_NAME); getMetastore().getOrCreateCollectionFields(PROJECT_NAME, "test", ImmutableSet.of()); ImmutableSet<SchemaField> schema = ImmutableSet.of(new SchemaField("test", STRING)); getMetastore().getOrCreateCollectionFields(PROJECT_NAME, "test", schema); assertTrue(getMetastore().getCollection(PROJECT_NAME, "test").containsAll(schema)); }