public static boolean isOptionalPrimitive(Schema schema) { return schema.getType().equals(Schema.Type.UNION) && schema.getTypes().size() == 2 && ( (schema.getTypes().get(0).getType().equals(Schema.Type.NULL) && (isPrimitive(schema.getTypes().get(1)) || isPrimitiveArray(schema.getTypes().get(1)))) || (schema.getTypes().get(1).getType().equals(Schema.Type.NULL) && (isPrimitive(schema.getTypes().get(0)) || isPrimitiveArray(schema.getTypes().get(0)))) ); }
/** * Check is record need deal locator. * * @param schema the input schema * @return boolean 'true' if record need deal locator */ public static boolean isRecordNeedDeallocator(Schema schema) { if (schema.getType() == Type.RECORD) { for (Field f : schema.getFields()) { Type type = f.schema().getType(); if (type == Type.ARRAY || type == Type.BYTES || type == Type.STRING || type == Type.FIXED || type == Type.RECORD || type == Type.UNION) { return true; } } } return false; }
/** * Helper method to extract the supported Avro schema from the given Avro field schema. * <p>Currently we support INT/LONG/FLOAT/DOUBLE/BOOLEAN/STRING/ENUM */ private static org.apache.avro.Schema extractSupportedSchema(org.apache.avro.Schema fieldSchema) { org.apache.avro.Schema.Type fieldType = fieldSchema.getType(); if (fieldType == org.apache.avro.Schema.Type.UNION) { org.apache.avro.Schema nonNullSchema = null; for (org.apache.avro.Schema childFieldSchema : fieldSchema.getTypes()) { if (childFieldSchema.getType() != org.apache.avro.Schema.Type.NULL) { if (nonNullSchema == null) { nonNullSchema = childFieldSchema; } else { throw new IllegalStateException("More than one non-null schema in UNION schema"); } } } if (nonNullSchema != null) { return extractSupportedSchema(nonNullSchema); } else { throw new IllegalStateException("Cannot find non-null schema in UNION schema"); } } else if (fieldType == org.apache.avro.Schema.Type.RECORD) { List<Field> recordFields = fieldSchema.getFields(); Preconditions.checkState(recordFields.size() == 1, "Not one field in the RECORD schema"); return extractSupportedSchema(recordFields.get(0).schema()); } else { return fieldSchema; } }
private static TypeInfo generateRecordTypeInfo(Schema schema, Set<Schema> seenSchemas) throws AvroSerdeException { assert schema.getType().equals(Schema.Type.RECORD); if (seenSchemas == null) { seenSchemas = Collections.newSetFromMap(new IdentityHashMap<Schema, Boolean>()); } else if (seenSchemas.contains(schema)) { throw new AvroSerdeException( "Recursive schemas are not supported. Recursive schema was " + schema .getFullName()); } seenSchemas.add(schema); List<Schema.Field> fields = schema.getFields(); List<String> fieldNames = new ArrayList<String>(fields.size()); List<TypeInfo> typeInfos = new ArrayList<TypeInfo>(fields.size()); for(int i = 0; i < fields.size(); i++) { fieldNames.add(i, fields.get(i).name()); typeInfos.add(i, generateTypeInfo(fields.get(i).schema(), seenSchemas)); } return TypeInfoFactory.getStructTypeInfo(fieldNames, typeInfos); }
/* package private */void compareArrayTypes(Schema oldSchema, Schema newSchema, List<Message> messages, String name) { if(oldSchema == null || newSchema == null || oldSchema.getType() != Schema.Type.ARRAY) { throw new IllegalArgumentException("Old schema must be ARRAY type. Name=" + name + ". Type=" + oldSchema); } if(newSchema.getType() != Schema.Type.ARRAY) { messages.add(new Message(Level.ERROR, "Illegal type change from " + oldSchema.getType() + " to " + newSchema.getType() + " for field " + name)); return; } // Compare the array element types compareTypes(oldSchema.getElementType(), newSchema.getElementType(), messages, name + ".<array element>"); }
/** * Gets the schema by full name. * * @param arraySchema the array schema * @param fullName the full name * @return the schema by full name */ private static Schema getSchemaByFullName(Schema arraySchema, String fullName) { if (arraySchema.getElementType().getType() == Schema.Type.UNION) { List<Schema> itemTypes = arraySchema.getElementType().getTypes(); return getSchemaByFullName(itemTypes, fullName); } else { return arraySchema.getElementType().getFullName().equals( fullName) ? arraySchema.getElementType() : null; } }
/** * Return whether the Avro field is a single-value field. */ public static boolean isSingleValueField(Field field) { try { org.apache.avro.Schema fieldSchema = extractSupportedSchema(field.schema()); return fieldSchema.getType() != org.apache.avro.Schema.Type.ARRAY; } catch (Exception e) { throw new RuntimeException("Caught exception while extracting non-null schema from field: " + field.name(), e); } }
/** * Check if schemas are equals to each other. * * @param s1 the schema one * @param s2 the schema two * @return boolean 'true' if the input schemas are equals */ public static boolean isEqualSchemas(Schema s1, Schema s2) { if (!(s1.getType().equals(s2.getType()) && s1.getFullName().equals(s2.getFullName()))) { return false; } switch (s1.getType()) { case RECORD: return isEqualRecords(s1, s2); case UNION: return isEqualUnions(s1, s2); case ARRAY: return isEqualSchemas(s1.getElementType(), s2.getElementType()); case MAP: return isEqualSchemas(s1.getValueType(), s2.getValueType()); case ENUM: return isEqualEnums(s1, s2); case FIXED: return s1.getFixedSize() == s2.getFixedSize(); default: return true; } }
private static TypeInformation<?> convertToTypeInfo(Schema schema) { switch (schema.getType()) { case RECORD: final List<Schema.Field> fields = schema.getFields(); final Schema.Field field = fields.get(i); types[i] = convertToTypeInfo(field.schema()); names[i] = field.name(); case ARRAY: return Types.OBJECT_ARRAY(convertToTypeInfo(schema.getElementType())); case MAP: return Types.MAP(Types.STRING, convertToTypeInfo(schema.getValueType())); case UNION: final Schema actualSchema; if (schema.getTypes().size() == 2 && schema.getTypes().get(0).getType() == Schema.Type.NULL) { actualSchema = schema.getTypes().get(1); } else if (schema.getTypes().size() == 2 && schema.getTypes().get(1).getType() == Schema.Type.NULL) { actualSchema = schema.getTypes().get(0); } else if (schema.getTypes().size() == 1) { return Types.VOID; throw new IllegalArgumentException("Unsupported Avro type '" + schema.getType() + "'.");
/* package private */Schema stripOptionalTypeUnion(Schema schema) { if(schema.getType() == Schema.Type.UNION && schema.getTypes().size() == 2 && schema.getTypes().contains(NULL_TYPE_SCHEMA)) { return schema.getTypes().get(0).equals(NULL_TYPE_SCHEMA) ? schema.getTypes().get(1) : schema.getTypes().get(0); } return schema; }
/** * This is a test to validate support of maps in {@link org.apache.gobblin.util.AvroUtils#getFieldValue(GenericRecord, String)} * and {@link org.apache.gobblin.util.AvroUtils#getFieldSchema(Schema, String)} * @throws IOException */ @Test public void testGetObjectFromMap() throws IOException { final String TEST_FIELD_LOCATION = "Map.stringKey.Field"; String avroFilePath = this.AVRO_DIR + "avroDir/avroUtilsTestFile.avro"; GenericRecord record = getRecordFromFile(avroFilePath).get(0); Assert.assertEquals(AvroUtils.getFieldValue(record, TEST_FIELD_LOCATION).get().toString(), "stringValue2"); Assert.assertEquals(AvroUtils.getFieldSchema(record.getSchema(), TEST_FIELD_LOCATION).get().getType(), Schema.Type.STRING); }
/** * Returns the number of parameter units required by fields for the * AllArgsConstructor. * * @param record a Record schema */ protected int calcAllArgConstructorParameterUnits(Schema record) { if (record.getType() != Schema.Type.RECORD) throw new RuntimeException("This method must only be called for record schemas."); return record.getFields().size(); }
/** * Returns true if this field is optional. Optional fields are represented * as a type union containing the null type. * * @param field * @return */ /* package private */boolean isOptional(Field field) { if(field.schema().getType() == Type.UNION) { for(Schema nestedType: field.schema().getTypes()) { if(nestedType.getType() == Type.NULL) { return true; } } } return false; }
public static boolean isPrimitiveArray(Schema schema) { return schema.getType().equals(Schema.Type.ARRAY) && isPrimitive(schema.getElementType()); }
private void handleSessionSiblingField( final GenericRecord randomAvroMessage, final List<Object> genericRowValues, final String sessionisationValue, final Schema.Field field ) { try { final Schema.Type type = field.schema().getType(); if (type == Schema.Type.INT) { genericRowValues.add(mapSessionValueToSibling(sessionisationValue, field)); } else { genericRowValues.add(randomAvroMessage.get(field.name())); } } catch (final Exception err) { genericRowValues.add(randomAvroMessage.get(field.name())); } }
private static JsonNode injectUuidsFromJsonNodes(JsonNode json, Schema schema) { if (json == null) { return json; } switch (schema.getType()) { case RECORD: schema.getFields().stream() .filter(f -> !f.name().equals(UUID_FIELD)) .forEach(f -> injectUuidsFromJsonNodes(json.get(f.name()), f.schema())); boolean addressable = schema.getFields().stream().filter(f -> f.name().equals( UUID_FIELD)).findFirst().isPresent(); if (addressable) { ((ObjectNode) json).put(UUID_FIELD, (Integer) null); } break; case UNION: schema.getTypes() .forEach(s -> injectUuidsFromJsonNodes(json.get(s.getName()), s)); break; case ARRAY: json.getElements().forEachRemaining((elem) -> injectUuids(elem, schema.getElementType())); break; default: return json; } return json; }
/** Utility for template use. For a two-branch union type with * one null branch, returns the index of the null branch. It's an * error to use on anything other than a two-branch union with on * null branch. */ public int getNonNullIndex(Schema s) { if (s.getType() != Schema.Type.UNION || s.getTypes().size() != 2 || ! s.getTypes().contains(NULL_SCHEMA)) throw new IllegalArgumentException("Can only be used on 2-branch union with a null branch: " + s); return (s.getTypes().get(0).equals(NULL_SCHEMA) ? 1 : 0); }