/** Utility for template use. Returns true if the encode/decode * logic in record.vm can handle the schema being presented. */ public boolean isCustomCodable(Schema schema) { if (schema.isError()) return false; return isCustomCodable(schema, new HashSet<Schema>()); }
schema.isError() ? ERROR_RESERVED_WORDS : ACCESSOR_MUTATOR_RESERVED_WORDS, true);
@Override public Schema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException { Preconditions.checkArgument(inputSchema.getFields().equals(gobblinTrackingEventSchema.getFields())); Schema outputSchema = Schema .createRecord(ConfigUtils.getString(config, NEW_SCHEMA_NAME, inputSchema.getName()), inputSchema.getDoc(), inputSchema.getNamespace(), inputSchema.isError()); outputSchema.setFields(newFields); return outputSchema; }
@Override public Schema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException { List<Field> outputSchemaFields = new ArrayList<>(); for (Field field : inputSchema.getFields()) { outputSchemaFields.add(convertFieldSchema(inputSchema, field, workUnit)); } Schema outputSchema = Schema .createRecord(inputSchema.getName(), inputSchema.getDoc(), inputSchema.getNamespace(), inputSchema.isError()); outputSchema.setFields(outputSchemaFields); return outputSchema; }
/*** * Flatten Record schema * @param schema Record Schema to flatten * @param shouldPopulateLineage If lineage information should be tagged in the field, this is true when we are * un-nesting fields * @param flattenComplexTypes Flatten complex types recursively other than Record and Option * @return Flattened Record Schema */ private Schema flattenRecord(Schema schema, boolean shouldPopulateLineage, boolean flattenComplexTypes) { Preconditions.checkNotNull(schema); Preconditions.checkArgument(Schema.Type.RECORD.equals(schema.getType())); Schema flattenedSchema; List<Schema.Field> flattenedFields = new ArrayList<>(); if (schema.getFields().size() > 0) { for (Schema.Field oldField : schema.getFields()) { List<Schema.Field> newFields = flattenField(oldField, ImmutableList.<String>of(), shouldPopulateLineage, flattenComplexTypes, Optional.<Schema>absent()); if (null != newFields && newFields.size() > 0) { flattenedFields.addAll(newFields); } } } flattenedSchema = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError()); flattenedSchema.setFields(flattenedFields); return flattenedSchema; }
@Override public SchemaVisitorAction visitNonTerminal(final Schema nt) { Schema.Type type = nt.getType(); if (type == RECORD) { Schema newSchema = Schema.createRecord(nt.getName(), copyDocs ? nt.getDoc() : null, nt.getNamespace(), nt.isError()); copyProperties.copy(nt, newSchema); replace.put(nt, newSchema); } return SchemaVisitorAction.CONTINUE; }
public static Schema newSchema( Schema metadataSchema, String newSchemaName, List<Schema.Field> moreFields, int insertPoint) { Schema newSchema = Schema.createRecord( newSchemaName, metadataSchema.getDoc(), metadataSchema.getNamespace(), metadataSchema.isError()); List<Schema.Field> fields = metadataSchema.getFields(); List<Schema.Field> copyFieldList = _cloneFieldsAndResetPosition(fields); copyFieldList.addAll(insertPoint, moreFields); newSchema.setFields(copyFieldList); Map<String, Object> objectProperties = metadataSchema.getObjectProps(); for (Map.Entry<String, Object> entry : objectProperties.entrySet()) { newSchema.addProp(entry.getKey(), entry.getValue()); } return newSchema; }
/** * Copies the input {@link org.apache.avro.Schema} but changes the schema name. * @param schema {@link org.apache.avro.Schema} to copy. * @param newName name for the copied {@link org.apache.avro.Schema}. * @return A {@link org.apache.avro.Schema} that is a copy of schema, but has the name newName. */ public static Schema switchName(Schema schema, String newName) { if (schema.getName().equals(newName)) { return schema; } Schema newSchema = Schema.createRecord(newName, schema.getDoc(), schema.getNamespace(), schema.isError()); List<Field> fields = schema.getFields(); Iterable<Field> fieldsNew = Iterables.transform(fields, new Function<Field, Field>() { @Override public Schema.Field apply(Field input) { //this should never happen but the API has marked input as Nullable if (null == input) { return null; } Field field = new Field(input.name(), input.schema(), input.doc(), input.defaultValue(), input.order()); return field; } }); newSchema.setFields(Lists.newArrayList(fieldsNew)); return newSchema; }
private Schema removeFieldsFromRecords(Schema schema, Map<String, Schema> schemaMap) { Schema newRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError()); // Put an incomplete schema into schemaMap to avoid re-processing a recursive field. // The fields in the incomplete schema will be populated once the current schema is completely processed. schemaMap.put(schema.getFullName(), newRecord); List<Field> newFields = Lists.newArrayList(); for (Field field : schema.getFields()) { if (!this.shouldRemove(field)) { Field newField; if (this.children.containsKey(field.name())) { newField = new Field(field.name(), this.children.get(field.name()).removeFields(field.schema(), schemaMap), field.doc(), field.defaultValue()); } else { newField = new Field(field.name(), DO_NOTHING_INSTANCE.removeFields(field.schema(), schemaMap), field.doc(), field.defaultValue()); } newFields.add(newField); } } newRecord.setFields(newFields); return newRecord; }
@Override public SchemaVisitorAction visitNonTerminal(final Schema nt) { Schema.Type type = nt.getType(); if (type == Schema.Type.RECORD) { if (SchemaResolver.isUnresolvedSchema(nt)) { // unresolved schema will get a replacement that we already encountered, // or we will attempt to resolve. final String unresolvedSchemaName = SchemaResolver.getUnresolvedSchemaName(nt); Schema resSchema = symbolTable.apply(unresolvedSchemaName); if (resSchema == null) { throw new AvroTypeException("Unable to resolve " + unresolvedSchemaName); } Schema replacement = replace.get(resSchema); if (replacement == null) { replace.put(nt, Schemas.visit(resSchema, new ResolvingVisitor(resSchema, new IdentityHashMap<>(), symbolTable))); } else { replace.put(nt, replacement); } } else { // create a fieldless clone. Fields will be added in afterVisitNonTerminal. Schema newSchema = Schema.createRecord(nt.getName(), nt.getDoc(), nt.getNamespace(), nt.isError()); copyAllProperties(nt, newSchema); replace.put(nt, newSchema); } } return SchemaVisitorAction.CONTINUE; }
.createRecord(inputSchema.getName(), inputSchema.getDoc(), inputSchema.getNamespace(), inputSchema.isError()); outputSchema.setFields(fields); return outputSchema;
Schema.createRecord(newSchema.getName(), newSchema.getDoc(), newSchema.getNamespace(), newSchema.isError()); mergedSchema.setFields(combinedFields); return mergedSchema;
public static Schema appendFields( Schema schema, List<Schema.Field> fields) { if (schema.getType() != Schema.Type.RECORD) { TalendRuntimeExceptionBuilder talendRuntimeExceptionBuilder = TalendRuntimeException.build( CommonErrorCodes.UNEXPECTED_EXCEPTION); talendRuntimeExceptionBuilder.setAndThrow( "Schema type is not supported: " + schema.getType()); } Schema newSchema = Schema.createRecord( schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError()); List<Schema.Field> copiedFieldList = new ArrayList<>(); _copyFields(schema.getFields(), copiedFieldList); _copyFields(fields, copiedFieldList); newSchema.setFields(copiedFieldList); _copyFieldProperties(schema.getObjectProps(), newSchema); return newSchema; }
if (aliases.containsKey(name)) name = aliases.get(name); result = Schema.createRecord(name.full, s.getDoc(), null, s.isError()); seen.put(s, result); List<Field> newFields = new ArrayList<Field>();
case RECORD: result = Schema.createRecord(s.getFullName(), s.getDoc(), null, s.isError()); for (String alias : s.getAliases())
if (aliases.containsKey(name)) name = aliases.get(name); result = Schema.createRecord(name.full, s.getDoc(), null, s.isError()); seen.put(s, result); List<Field> newFields = new ArrayList<>();
schema.isError()); newSchema.setFields(newFields); break;
@Test public void testP2() throws Exception { Schema e1 = ReflectData.get().getSchema(E1.class); assertEquals(Schema.Type.RECORD, e1.getType()); assertTrue(e1.isError()); Field message = e1.getField("detailMessage"); assertNotNull("field 'detailMessage' should not be null", message); Schema messageSchema = message.schema(); assertEquals(Schema.Type.UNION, messageSchema.getType()); assertEquals(Schema.Type.NULL, messageSchema.getTypes().get(0).getType()); assertEquals(Schema.Type.STRING, messageSchema.getTypes().get(1).getType()); Protocol p2 = ReflectData.get().getProtocol(P2.class); Protocol.Message m = p2.getMessages().get("error"); // check error schema is union Schema response = m.getErrors(); assertEquals(Schema.Type.UNION, response.getType()); assertEquals(Schema.Type.STRING, response.getTypes().get(0).getType()); assertEquals(e1, response.getTypes().get(1)); }
@Test public void testRecord() { Schema schema = SchemaBuilder .record("myrecord").namespace("org.example").aliases("oldrecord").fields() .name("f0").aliases("f0alias").type().stringType().noDefault() .name("f1").doc("This is f1").type().longType().noDefault() .name("f2").type().nullable().booleanType().booleanDefault(true) .endRecord(); Assert.assertEquals("myrecord", schema.getName()); Assert.assertEquals("org.example", schema.getNamespace()); Assert.assertEquals("org.example.oldrecord", schema.getAliases().iterator().next()); Assert.assertFalse(schema.isError()); List<Schema.Field> fields = schema.getFields(); Assert.assertEquals(3, fields.size()); Assert.assertEquals( new Schema.Field("f0", Schema.create(Schema.Type.STRING), null, null), fields.get(0)); Assert.assertTrue(fields.get(0).aliases().contains("f0alias")); Assert.assertEquals( new Schema.Field("f1", Schema.create(Schema.Type.LONG), "This is f1", null), fields.get(1)); List<Schema> types = new ArrayList<>(); types.add(Schema.create(Schema.Type.BOOLEAN)); types.add(Schema.create(Schema.Type.NULL)); Schema optional = Schema.createUnion(types); Assert.assertEquals(new Schema.Field("f2", optional, null, true), fields.get(2)); }