public static Optional<Schema> getKeySchema(Field field) { switch (field.schema().getType()) { case RECORD: return getKeySchemaFromRecord(field.schema()); default: if (field.doc() != null && field.doc().toLowerCase().endsWith(SCHEMA_DEDUP_FIELD_ANNOTATOR)) { return Optional.of(field.schema()); } else { return Optional.absent(); } } }
public static List<String> generateColumnComments(Schema schema) { List<Schema.Field> fields = schema.getFields(); List<String> fieldComments = new ArrayList<String>(fields.size()); for (Schema.Field field : fields) { String fieldComment = field.doc() == null ? "" : field.doc(); fieldComments.add(fieldComment); } return fieldComments; } }
private List<Schema.Field> getFields(Schema.Field schemaField) { List<Schema.Field> fields = new ArrayList<Schema.Field>(); JsonNode nullDefault = JsonNodeFactory.instance.nullNode(); if (schemaField.schema().getType() == Schema.Type.RECORD) { for (Schema.Field field : schemaField.schema().getFields()) { fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), nullDefault)); } } else { fields.add(new Schema.Field(schemaField.name(), schemaField.schema(), schemaField.doc(), nullDefault)); } return fields; }
for (Schema.Field field : fields) { Schema.Field newField = new Schema.Field(field.name(), replace.get(field.schema()), field.doc(), field.defaultVal(), field.order()); copyAllProperties(field, newField); newFields.add(newField);
for (Schema.Field field : fields) { Schema.Field newField = new Schema.Field(field.name(), replace.get(field.schema()), copyDocs ? field.doc() : null, field.defaultVal(), field.order()); copyProperties.copy(field, newField); newFields.add(newField);
void fieldsToJson(Names names, JsonGenerator gen) throws IOException { gen.writeStartArray(); for (Field f : fields) { gen.writeStartObject(); gen.writeStringField("name", f.name()); gen.writeFieldName("type"); f.schema().toJson(names, gen); if (f.doc() != null) gen.writeStringField("doc", f.doc()); if (f.defaultValue() != null) { gen.writeFieldName("default"); gen.writeTree(f.defaultValue()); } if (f.order() != Field.Order.ASCENDING) gen.writeStringField("order", f.order().name); if (f.aliases != null && f.aliases.size() != 0) { gen.writeFieldName("aliases"); gen.writeStartArray(); for (String alias : f.aliases) gen.writeString(alias); gen.writeEndArray(); } f.writeProps(gen); gen.writeEndObject(); } gen.writeEndArray(); } }
void fieldsToJson(Names names, JsonGenerator gen) throws IOException { gen.writeStartArray(); for (Field f : fields) { gen.writeStartObject(); gen.writeStringField("name", f.name()); gen.writeFieldName("type"); f.schema().toJson(names, gen); if (f.doc() != null) gen.writeStringField("doc", f.doc()); if (f.defaultValue() != null) { gen.writeFieldName("default"); gen.writeTree(f.defaultValue()); } if (f.order() != Field.Order.ASCENDING) gen.writeStringField("order", f.order().name); if (f.aliases != null && f.aliases.size() != 0) { gen.writeFieldName("aliases"); gen.writeStartArray(); for (String alias : f.aliases) gen.writeString(alias); gen.writeEndArray(); } f.writeProps(gen); gen.writeEndObject(); } gen.writeEndArray(); } }
private Schema removeFieldsFromRecords(Schema schema, Map<String, Schema> schemaMap) { Schema newRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError()); // Put an incomplete schema into schemaMap to avoid re-processing a recursive field. // The fields in the incomplete schema will be populated once the current schema is completely processed. schemaMap.put(schema.getFullName(), newRecord); List<Field> newFields = Lists.newArrayList(); for (Field field : schema.getFields()) { if (!this.shouldRemove(field)) { Field newField; if (this.children.containsKey(field.name())) { newField = new Field(field.name(), this.children.get(field.name()).removeFields(field.schema(), schemaMap), field.doc(), field.defaultValue()); } else { newField = new Field(field.name(), DO_NOTHING_INSTANCE.removeFields(field.schema(), schemaMap), field.doc(), field.defaultValue()); } newFields.add(newField); } } newRecord.setFields(newFields); return newRecord; }
public static Optional<Schema> getKeySchemaFromRecord(Schema record) { Preconditions.checkArgument(record.getType() == Schema.Type.RECORD); List<Field> fields = Lists.newArrayList(); for (Field field : record.getFields()) { Optional<Schema> newFieldSchema = getKeySchema(field); if (newFieldSchema.isPresent()) { fields.add(new Field(field.name(), newFieldSchema.get(), field.doc(), field.defaultValue())); } } if (!fields.isEmpty()) { Schema newSchema = Schema.createRecord(record.getName(), record.getDoc(), record.getName(), false); newSchema.setFields(fields); return Optional.of(newSchema); } else { return Optional.absent(); } }
private static Optional<Schema> removeUncomparableFieldsFromRecord(Schema record, Set<Schema> processed) { Preconditions.checkArgument(record.getType() == Schema.Type.RECORD); if (processed.contains(record)) { return Optional.absent(); } processed.add(record); List<Field> fields = Lists.newArrayList(); for (Field field : record.getFields()) { Optional<Schema> newFieldSchema = removeUncomparableFields(field.schema(), processed); if (newFieldSchema.isPresent()) { fields.add(new Field(field.name(), newFieldSchema.get(), field.doc(), field.defaultValue())); } } Schema newSchema = Schema.createRecord(record.getName(), record.getDoc(), record.getNamespace(), false); newSchema.setFields(fields); return Optional.of(newSchema); }
/** * Makes sure that "doc" tags are transcribed in the schemas. * Note that there are docs both for fields and for the records * themselves. */ @Test public void testDocs() { Schema schema = Schema.parse(SCHEMA_WITH_DOC_TAGS); assertEquals("This is not a world record.", schema.getDoc()); assertEquals("Inner Fixed", schema.getField("inner_fixed").doc()); assertEquals("Very Inner Fixed", schema.getField("inner_fixed").schema().getDoc()); assertEquals("Inner String", schema.getField("inner_string").doc()); assertEquals("Inner Enum", schema.getField("inner_enum").doc()); assertEquals("Very Inner Enum", schema.getField("inner_enum").schema().getDoc()); assertEquals("Inner Union", schema.getField("inner_union").doc()); }
private static List<Schema.Field> _cloneFieldsAndResetPosition( List<Schema.Field> fields) { List<Schema.Field> copyFieldList = new ArrayList<>(); for (Schema.Field schemaEntry : fields) { Schema.Field field = new Schema.Field( schemaEntry.name(), schemaEntry.schema(), schemaEntry.doc(), schemaEntry.defaultVal(), schemaEntry.order()); Map<String, Object> objectProperties = schemaEntry.getObjectProps(); for (Map.Entry<String, Object> entry : objectProperties.entrySet()) { field.addProp(entry.getKey(), entry.getValue()); } copyFieldList.add(field); } return copyFieldList; }
@Override public Schema convertSchemaImpl(Schema inputSchema, WorkUnitState workUnitState) throws SchemaConversionException { if (inputSchema == null) { throw new SchemaConversionException("input schema is empty"); } List<Schema.Field> fields = Lists.newArrayList(); for (Schema.Field field : inputSchema.getFields()) { Schema.Field newField = new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue(), field.order()); fields.add(newField); } Schema.Field requestResponseField = new Schema.Field(HTTP_REQUEST_RESPONSE_FIELD, HttpRequestResponseRecord.getClassSchema(), "http output schema contains request url and return result", null); fields.add(requestResponseField); Schema combinedSchema = Schema.createRecord(inputSchema.getName(), inputSchema.getDoc() + " (Http request and response are contained)", inputSchema.getNamespace(), false); combinedSchema.setFields(fields); return combinedSchema; }
@Test public void testFields() { Schema rec = SchemaBuilder.record("Rec").fields() .name("documented").doc("documented").type().nullType().noDefault() .name("ascending").orderAscending().type().booleanType().noDefault() .name("descending").orderDescending().type().floatType().noDefault() .name("ignored").orderIgnore().type().doubleType().noDefault() .name("aliased").aliases("anAlias").type().stringType().noDefault() .endRecord(); Assert.assertEquals("documented", rec.getField("documented").doc()); Assert.assertEquals(Order.ASCENDING, rec.getField("ascending").order()); Assert.assertEquals(Order.DESCENDING, rec.getField("descending").order()); Assert.assertEquals(Order.IGNORE, rec.getField("ignored").order()); Assert.assertTrue(rec.getField("aliased").aliases().contains("anAlias")); }
@Test public void testFieldDocs() { String schemaStr = "{\"name\": \"Rec\",\"type\": \"record\",\"fields\" : ["+ "{\"name\": \"f\", \"type\": \"int\", \"doc\": \"test\"}]}"; // check field doc is parsed correctly Schema schema = Schema.parse(schemaStr); assertEquals("test", schema.getField("f").doc()); // check print/read cycle preserves field doc schema = Schema.parse(schema.toString()); assertEquals("test", schema.getField("f").doc()); }
@Override public Schema.Field apply(Field input) { //this should never happen but the API has marked input as Nullable if (null == input) { return null; } Field field = new Field(input.name(), input.schema(), input.doc(), input.defaultValue(), input.order()); return field; } });
private static void _copyFields( List<Schema.Field> fields, List<Schema.Field> copiedFieldList) { for (Schema.Field field : fields) { Schema.Field newField = new Schema.Field( field.name(), field.schema(), field.doc(), field.defaultVal()); _copyFieldProperties(field.getObjectProps(), newField); copiedFieldList.add(newField); } }
/** * Convert to the output schema of a field */ protected Field convertFieldSchema(Schema inputSchema, Field field, WorkUnitState workUnit) throws SchemaConversionException { if (field.name().equals(payloadField)) { // Create a payload field with latest schema return createLatestPayloadField(field); } // Make a copy of the field to the output schema return new Field(field.name(), field.schema(), field.doc(), field.defaultValue(), field.order()); }