private static String extractAvroTypeFromUnion(Schema.Field field) { if (field.schema().getTypes().size() >= 3) { LOG.warn("Avro schema field " + field.name() + " has 3 or more types: using the first non-null type"); } for (Schema schema : field.schema().getTypes()) { if (!schema.getType().toString().equalsIgnoreCase("NULL")) { return schema.getType().toString(); } } String message = "Avro schema field " + field.name() + " is a union, but it does not contain a non-null field type."; LOG.error(message); throw new RuntimeException(message); }
private static HiveAttribute convertAvroSchemaFieldToHiveAttribute(Schema.Field field) { String avroFieldType = field.schema().getType().toString(); if (avroFieldType.equalsIgnoreCase("UNION")) { avroFieldType = extractAvroTypeFromUnion(field); } if (HiveAttribute.fromAvroType(avroFieldType) == null) { throw new RuntimeException("Hive does not support attribute type '" + avroFieldType + "'"); } return new HiveAttribute(field.name(), HiveAttribute.fromAvroType(avroFieldType)); }
"Currently only supports element unions of a type and null (" + s.toString() +")"); default: throw new ExecException("Unknown type: " + s.getType().toString());
default: throw new UnsupportedOperationException( "carbon not support " + type.toString() + " avro type yet");
throw new IncorrectTypeException( "Incorrect type - can't insert a " + value.getClass().getCanonicalName() + " into an Avro record of type " + changedField.schema().getType().toString());
break; default: key = memberAvroSchema.getType().toString().toLowerCase();
if (!SUPPORTED_TYPES.contains(st)) { throw new IllegalArgumentException(String.format( "Unsupported type '%s' for field '%s'", st.toString(), f.name()));
protected Map.Entry<String, Schema> findUnionMember(DataSchema dataSchema, Schema avroSchema) { AvroOverride avroOverride = getAvroOverride(dataSchema); String key = (avroOverride == null ? dataSchema.getUnionMemberKey() : avroOverride.getAvroSchemaFullName()); List<Schema> members = avroSchema.getTypes(); for (Schema member : members) { String name; switch (member.getType()) { case ENUM: case FIXED: case RECORD: name = member.getFullName(); break; default: name = member.getType().toString().toLowerCase(); } if (name.equals(key)) return new AbstractMap.SimpleEntry<String, Schema>(name, member); } appendMessage("cannot find %1$s in union %2$s", key, avroSchema); return null; }
/** * Provide the schema for the generated rows. * * @param schema the Schema to use in the generated rows. * @return the {@link Read} with the specified schema */ public Read withSchema(Schema schema) { checkArgument(schema != null, "RowGeneratorIO.read().withSchema(schema) called with null schema"); checkArgument(schema.getType() == Schema.Type.RECORD, "RowGeneratorIO.read().withSchema(schema) called with a non-record schema: %s", schema.getType().toString()); this.schema = schema; return builder().setSchema(schema).build(); }
private static String extractAvroTypeFromUnion(Schema.Field field) { if (field.schema().getTypes().size() >= 3) { LOG.warn("Avro schema field " + field.name() + " has 3 or more types: using the first non-null type"); } for (Schema schema : field.schema().getTypes()) { if (!schema.getType().toString().equalsIgnoreCase("NULL")) { return schema.getType().toString(); } } String message = "Avro schema field " + field.name() + " is a union, but it does not contain a non-null field type."; LOG.error(message); throw new RuntimeException(message); }
private static Object getDefaultValue(@NonNull final Schema.Type type) { if (type == Schema.Type.STRING) { return STRING_VALUE_DEFAULT; } else if (type == Schema.Type.BOOLEAN) { return BOOLEAN_VALUE_DEFAULT; } else if (type == Schema.Type.LONG) { return System.currentTimeMillis(); } else if (type == Schema.Type.UNION) { return null; } else { log.warn("Found unknown type {}, returning null", type.toString()); return null; } }
/** * Provide the schema for the generated rows. * * @param schema the Schema to use in the generated rows. * @return the {@link Read} with the specified schema */ public Read withSchema(Schema schema) { checkArgument(schema != null, "RowGeneratorIO.read().withSchema(schema) called with null schema"); checkArgument(schema.getType() == Schema.Type.RECORD, "RowGeneratorIO.read().withSchema(schema) called with a non-record schema: %s", schema.getType().toString()); this.schema = schema; return builder().setSchema(schema).build(); }
private static String extractAvroTypeFromUnion(Schema.Field field) { if (field.schema().getTypes().size() >= 3) { LOG.warn("Avro schema field " + field.name() + " has 3 or more types: using the first non-null type"); } for (Schema schema : field.schema().getTypes()) { if (!schema.getType().toString().equalsIgnoreCase("NULL")) { return schema.getType().toString(); } } String message = "Avro schema field " + field.name() + " is a union, but it does not contain a non-null field type."; LOG.error(message); throw new RuntimeException(message); }
@Override public Object get(String name, Document document) { if (document.get(name).equals(Schema.Type.NULL.toString())) return null; return document.get(name); }
@Override public String objectToString(Object object) { if (object==null) return Schema.Type.NULL.toString(); return object.toString(); }
private static HiveAttribute convertAvroSchemaFieldToHiveAttribute(Schema.Field field) { String avroFieldType = field.schema().getType().toString(); if (avroFieldType.equalsIgnoreCase("UNION")) { avroFieldType = extractAvroTypeFromUnion(field); } if (HiveAttribute.fromAvroType(avroFieldType) == null) { throw new RuntimeException("Hive does not support attribute type '" + avroFieldType + "'"); } return new HiveAttribute(field.name(), HiveAttribute.fromAvroType(avroFieldType)); }
private static HiveAttribute convertAvroSchemaFieldToHiveAttribute(Schema.Field field) { String avroFieldType = field.schema().getType().toString(); if (avroFieldType.equalsIgnoreCase("UNION")) { avroFieldType = extractAvroTypeFromUnion(field); } if (HiveAttribute.fromAvroType(avroFieldType) == null) { throw new RuntimeException("Hive does not support attribute type '" + avroFieldType + "'"); } return new HiveAttribute(field.name(), HiveAttribute.fromAvroType(avroFieldType)); }
@ProcessElement public void processElement(ProcessContext ctx) { GenericRecord genericRecord = ctx.element(); Schema schema = new Schema.Parser().parse(schemaJson); StringBuilder row = new StringBuilder(); for (Schema.Field field : schema.getFields()) { String fieldType = field.schema().getType().toString().toLowerCase(); if (!acceptedTypes.contains(fieldType)) { LOG.error("Data transformation doesn't support: " + fieldType); throw new IllegalArgumentException("Field type " + fieldType + " is not supported."); } if (row.length() > 0) { row.append(delimiter); } row.append(genericRecord.get(field.name())); } ctx.output(row.toString()); } }
throw new IncorrectTypeException( "Incorrect type - can't insert a " + value.getClass().getCanonicalName() + " into an Avro record of type " + changedField.schema().getType().toString());