@Override public void addFieldDependency(FieldDependencyBuilder builder) { builder.addFields("_user_agent", ImmutableList.of( new SchemaField("_user_agent_family", FieldType.STRING), new SchemaField("_user_agent_version", FieldType.STRING), new SchemaField("_os", FieldType.STRING), new SchemaField("_os_version", FieldType.STRING), new SchemaField("_device_family", FieldType.STRING) )); }
private boolean check(String project, String collection, SchemaField existing, SchemaField moduleField) { if (existing.getName().equals(moduleField.getName())) { if (!existing.getType().equals(moduleField.getType())) { throw new IllegalStateException(format("Module field '%s' type does not match existing field in event of project %s.%s. Existing type: %s, Module field type: %s. \n" + "Please change the schema manually of disable the module.", existing.getName(), project, collection, existing.getType(), moduleField.getType())); } return true; } return false; } }
public Recipe export(String project) { final Map<String, Recipe.CollectionDefinition> collections = metastore.getCollections(project).entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> { List<Map<String, Recipe.SchemaFieldInfo>> map = e.getValue().stream() .map(a -> ImmutableMap.of(a.getName(), new Recipe.SchemaFieldInfo(a.getCategory(), a.getType()))) .collect(Collectors.toList()); return new Recipe.CollectionDefinition(map); })); final List<MaterializedView> materializedViews = materializedViewService.list(project).stream() .map(m -> new MaterializedView(m.tableName, m.name, m.query, m.updateInterval, m.incremental, m.realTime, m.options)) .collect(Collectors.toList()); return new Recipe(Recipe.Strategy.SPECIFIC, collections, materializedViews); }
.filter(f -> f.getName().equals(name)).findAny(); .filter(e -> e.getName().equals(name)) .findAny().orElse(new SchemaField(name, type)); newFields.add(field); .filter(i -> finalFields.get(i).getName().equals(colName)).findAny().getAsInt()) .toArray();
String colName = columns.getString("COLUMN_NAME"); strings.add(colName); currentFields.add(new SchemaField(colName, fromSql(columns.getInt("DATA_TYPE"), columns.getString("TYPE_NAME"), this::fromPrestoType))); List<SchemaField> schemaFields = fields.stream().filter(f -> !strings.contains(f.getName())).collect(Collectors.toList()); Runnable task; if (currentFields.isEmpty()) { return f; }) .map(f -> format("\"%s\" %s", f.getName(), toSql(f.getType()))) .collect(Collectors.joining(", ")); if (queryEnd.isEmpty()) { .map(f -> format("ALTER TABLE %s.\"%s\".\"%s\" ADD COLUMN \"%s\" %s", config.getColdStorageConnector(), project, collection, f.getName(), toSql(f.getType()))) .forEach(q -> { try {
private Map<String, Integer> generateColumnMap(List<String> variables, List<SchemaField> metadata) { HashMap<String, Integer> colMap = new HashMap<>(variables.size()); for (String var : variables) { for (int i = 0; i < metadata.size(); i++) { if (metadata.get(i).getName().equals(var)) { colMap.put(variables.get(i), i); break; } } } return colMap; }
.map(e -> { FieldType type; if (e.getName().equals("_user")) { type = configManager.setConfigOnce(project, USER_TYPE.name(), STRING); } else { type = e.getType(); SchemaField schemaField = new SchemaField(e.getName(), type, e.getDescriptiveName(), e.getDescription(), e.getCategory()); return schemaField; }) .filter(f -> fields.stream().anyMatch(field -> field.getName().equals(f.getName()) && !f.getType().equals(field.getType()))) .collect(Collectors.toList()); SchemaField existingField = fields.stream().filter(field -> field.getName().equals(f.getName())).findAny().get(); return format("Recipe: [%s : %s], CollectionDefinition: [%s, %s]", f.getName(), f.getType(), existingField.getName(), existingField.getType()); }).collect(Collectors.joining(", ")); String message = overrideExisting ? "Overriding collection fields is not possible." : "Collision in collection fields.";
columns.add(new SchemaField(metaData.getColumnName(i), type)); continue; FieldType type = schemaField.getType(); int columnIndex = i + 1; switch (type) { columns.set(i, new SchemaField(metaData.getColumnName(i + 1), STRING));
public static byte[] exportAsAvro(QueryResult result) { Schema avroSchema = AvroUtil.convertAvroSchema(result.getMetadata()); ByteArrayOutputStream out = new ByteArrayOutputStream(); DatumWriter writer = new FilteredRecordWriter(avroSchema, GenericData.get()); BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); GenericData.Record record = new GenericData.Record(avroSchema); for (List<Object> row : result.getResult()) { List<SchemaField> metadata = result.getMetadata(); for (int i = 0; i < row.size(); i++) { record.put(i, getAvroValue(row.get(i), metadata.get(i).getType())); } try { writer.write(record, encoder); } catch (Exception e) { throw new RuntimeException("Couldn't serialize event", e); } } return out.toByteArray(); }
.filter(d -> d.getName().equals(dimension.get())).findAny().get().getType().getPrettyName().equals("TIMESTAMP")) { if (!segment.isPresent() || !timeStampMapping.containsKey(FunnelTimestampSegments.valueOf(segment.get().toUpperCase()))) { throw new RakamException("When dimension is of type TIMESTAMP, segmenting should be done on timestamp field.", BAD_REQUEST); new SchemaField("step", STRING), new SchemaField("dimension", STRING), new SchemaField("count", LONG)); } else { newResult = IntStream.range(0, steps.size()) new SchemaField("step", STRING), new SchemaField("count", LONG));
private Map<String, Integer> generateColumnMap(List<String> variables, List<SchemaField> metadata) { HashMap<String, Integer> colMap = new HashMap<>(variables.size()); for (String var : variables) { for (int i = 0; i < metadata.size(); i++) { if (metadata.get(i).getName().equals(var)) { colMap.put(variables.get(i), i); break; } } } return colMap; }
ImmutableSet.of(new SchemaField(attr, attrValue.fieldType))); for (SchemaField field : fields) { for (Schema.Field oldField : oldFields) { if (oldField.name().equals(field.getName())) { continue outer;
new SchemaField("dimension", result.getMetadata().get(0).getType()), new SchemaField("lead", INTEGER), new SchemaField("value", INTEGER)), rows, result.getProperties()); });
private String buildValues(GenericRecord properties, List<SchemaField> schema) { StringBuilder builder = new StringBuilder("select "); int size = properties.getSchema().getFields().size(); appendValue(builder, Instant.now().toEpochMilli(), FieldType.TIMESTAMP); for (int i = 0; i < size; i++) { builder.append(", "); appendValue(builder, properties.get(i), schema.get(i).getType()); } return builder.toString(); }
private void checkFields(List<SchemaField> fields) { SchemaField[] collisions = fields.stream() .filter(newField -> constantFields.stream() .anyMatch(f -> f.getName().equals(newField.getName()) && !f.getType().equals(newField.getType()))) .toArray(SchemaField[]::new); checkState(collisions.length == 0, "Module field collides with existing field that has another type exists: ", Arrays.toString(collisions)); collisions = dependentFields.values().stream() .flatMap(col -> col.stream()) .filter(field -> fields.stream().anyMatch(f -> f.getName().equals(field.getName()) && !f.getType().equals(field.getType()))) .toArray(SchemaField[]::new); checkState(collisions.length == 0, "Fields already exist in dependency table: ", Arrays.toString(collisions)); }
if (dimension.isPresent()) { SchemaField column = metastore.getCollection(context.project, steps.get(0).getCollection()).stream() .filter(c -> c.getName().equals(dimension.get())) .findAny().orElseThrow(() -> new RakamException("Dimension is not exist.", BAD_REQUEST)); if (column.getType() != TIMESTAMP) { throw new RakamException("Segment is supported only for TIMESTAMP columns.", BAD_REQUEST); new SchemaField("step", STRING), new SchemaField("dimension", STRING), new SchemaField("count", LONG)); new SchemaField("step", STRING), new SchemaField("count", LONG));
@Override public void addFieldDependency(FieldDependencyBuilder builder) { builder.addFields("_referrer", ImmutableList.of( new SchemaField("_referrer_medium", STRING), new SchemaField("_referrer_source", STRING), new SchemaField("_referrer_term", STRING), new SchemaField("_referrer_domain", STRING), new SchemaField("_referrer_path", STRING) )); } }
protected boolean testDeviceIdExists(Optional<RetentionAction> firstAction, Map<String, List<SchemaField>> collections) { if (firstAction.isPresent()) { List<SchemaField> schemaFields = collections.get(firstAction.get().collection()); if (schemaFields == null) { throw new RakamException("The collection in first action does not exist.", HttpResponseStatus.BAD_REQUEST); } return schemaFields.stream().anyMatch(e -> e.getName().equals("_device_id")); } else { return collections.entrySet().stream() .allMatch(e -> e.getValue().stream().anyMatch(field -> field.getName().equals("_device_id"))); } } }
int[] indexes = header.getValue(); List<FieldType> types = Arrays.stream(indexes) .mapToObj(i -> header.getKey().get(i).getType()).collect(Collectors.toList());