public static String generateHiveDDL(Schema avroSchema, String tableName) { Schema.Type schemaType = avroSchema.getType(); StringBuilder sb = new StringBuilder("CREATE EXTERNAL TABLE IF NOT EXISTS "); sb.append(tableName); sb.append(" ("); if (Schema.Type.RECORD.equals(schemaType)) { List<String> hiveColumns = new ArrayList<>(); List<Schema.Field> fields = avroSchema.getFields(); if (fields != null) { hiveColumns.addAll( fields.stream().map(field -> field.name() + " " + getHiveTypeFromAvroType(field.schema())).collect(Collectors.toList())); } sb.append(StringUtils.join(hiveColumns, ", ")); sb.append(") STORED AS ORC"); return sb.toString(); } else { throw new IllegalArgumentException("Avro schema is of type " + schemaType.getName() + ", not RECORD"); } }
protected String getIndexedRecordId(IndexedRecord indexedRecord) throws IOException { Schema indexRecordSchema = indexedRecord.getSchema(); List<Schema.Field> indexRecordFields = indexRecordSchema.getFields(); Stream<Schema.Field> stream = indexRecordFields.stream(); Schema.Field idField = stream.filter( field -> AvroConstants.ID.equals(field.name()) ).findFirst( ).orElseThrow( () -> new IOException( String.format( "Unable to find '%s' field in the incoming indexed record", AvroConstants.ID)) ); Schema fieldSchema = idField.schema(); Schema unwrappedSchema = AvroUtils.unwrapIfNullable(fieldSchema); Schema.Type fieldType = unwrappedSchema.getType(); if (fieldType == Schema.Type.STRING) { return (String)indexedRecord.get(idField.pos()); } throw new IOException( i18nMessages.getMessage( "error.unsupported.field.schema", idField.name(), fieldType.getName())); }
public EnumConverter(String fieldName, boolean nullable, String sourceType, Schema schemaNode) { super(fieldName, nullable, sourceType); this.enumSet.addAll(schemaNode.getEnumSymbols()); this.enumName = schemaNode.getType().getName(); this.schema = schemaNode; }
switch (st) { default: // boolean, bytes, double, float, int, long, null, string return o.append('"').append(st.getName()).append('"'); o.append("{\"type\":\"").append(st.getName()).append("\""); if (st == Schema.Type.ARRAY) build(env, s.getElementType(), o.append(",\"items\":")); env.put(name, qname); o.append("{\"name\":").append(qname); o.append(",\"type\":\"").append(st.getName()).append("\""); if (st == Schema.Type.ENUM) { o.append(",\"symbols\":[");
converter = JsonElementConversionWithAvroSchemaFactory.getConvertor(field.name(), type.getName(), schema, workUnit, nullable, ignoreFields); avroRecord.put(field.name(), converter.convert(inputRecord.get(field.name())));
throw new IllegalArgumentException("Error converting Avro type " + avroType.getName() + " to Hive type");
return Type.NULL.getName(); if (isRecord(datum)) return getRecordSchema(datum).getFullName(); return getEnumSchema(datum).getFullName(); if (isArray(datum)) return Type.ARRAY.getName(); if (isMap(datum)) return Type.MAP.getName(); if (isFixed(datum)) return getFixedSchema(datum).getFullName(); if (isString(datum)) return Type.STRING.getName(); if (isBytes(datum)) return Type.BYTES.getName(); if (isInteger(datum)) return Type.INT.getName(); if (isLong(datum)) return Type.LONG.getName(); if (isFloat(datum)) return Type.FLOAT.getName(); if (isDouble(datum)) return Type.DOUBLE.getName(); if (isBoolean(datum)) return Type.BOOLEAN.getName(); throw new AvroRuntimeException (String.format("Unknown datum type %s: %s",
return Type.NULL.getName(); if (isRecord(datum)) return getRecordSchema(datum).getFullName(); return getEnumSchema(datum).getFullName(); if (isArray(datum)) return Type.ARRAY.getName(); if (isMap(datum)) return Type.MAP.getName(); if (isFixed(datum)) return getFixedSchema(datum).getFullName(); if (isString(datum)) return Type.STRING.getName(); if (isBytes(datum)) return Type.BYTES.getName(); if (isInteger(datum)) return Type.INT.getName(); if (isLong(datum)) return Type.LONG.getName(); if (isFloat(datum)) return Type.FLOAT.getName(); if (isDouble(datum)) return Type.DOUBLE.getName(); if (isBoolean(datum)) return Type.BOOLEAN.getName(); throw new AvroRuntimeException (String.format("Unknown datum type %s: %s",
final byte[] rawFingerprint = SchemaNormalization.parsingFingerprint(fingerprintAlgorithm, schema); avroMetadata.put(SCHEMA_FINGERPRINT_ATTR, Hex.encodeHexString(rawFingerprint)); avroMetadata.put(SCHEMA_TYPE_ATTR, schema.getType().getName()); avroMetadata.put(SCHEMA_NAME_ATTR, schema.getName()); } catch (NoSuchAlgorithmException e) {
private ObjectNode _createApioExpectedForm( IndexedRecord indexedRecord, boolean excludeId) throws IOException { Schema indexRecordSchema = indexedRecord.getSchema(); List<Schema.Field> indexRecordFields = indexRecordSchema.getFields(); ObjectNode objectNode = _mapper.createObjectNode(); for (Schema.Field field : indexRecordFields) { String fieldName = field.name(); if (excludeId && fieldName.equals(AvroConstants.ID)) { continue; } Schema fieldSchema = field.schema(); Schema unwrappedSchema = AvroUtils.unwrapIfNullable(fieldSchema); Schema.Type fieldType = unwrappedSchema.getType(); if (fieldType == Schema.Type.STRING) { objectNode.put( fieldName, (String)indexedRecord.get(field.pos())); } else if (fieldType == Schema.Type.NULL) { objectNode.put(fieldName, ""); } else { throw new IOException( i18nMessages.getMessage( "error.unsupported.field.schema", field.name(), fieldType.getName())); } } return objectNode; }
/** * Helper method to build H2 field name and type. * * @param fieldName Field name * @param avroFieldType Avro field type * @param nullable Whether the column is nullable * @return H2 field name and type */ @Nonnull private static String buildH2FieldNameAndType(@Nonnull String fieldName, @Nonnull Schema.Type avroFieldType, boolean nullable) { String avroFieldTypeName = avroFieldType.getName(); String h2FieldType; switch (avroFieldTypeName) { case "int": h2FieldType = "bigint"; break; case "string": h2FieldType = "varchar(128)"; break; default: h2FieldType = avroFieldTypeName; break; } if (nullable) { return fieldName + " " + h2FieldType; } else { return fieldName + " " + h2FieldType + " not null"; } }
public static TypeInfo getPrimitiveOrcTypeFromPrimitiveAvroType(Schema.Type avroType) throws IllegalArgumentException { if (avroType == null) { throw new IllegalArgumentException("Avro type is null"); } switch (avroType) { case INT: return TypeInfoFactory.getPrimitiveTypeInfo("int"); case LONG: return TypeInfoFactory.getPrimitiveTypeInfo("bigint"); case BOOLEAN: return TypeInfoFactory.getPrimitiveTypeInfo("boolean"); case BYTES: return TypeInfoFactory.getPrimitiveTypeInfo("binary"); case DOUBLE: return TypeInfoFactory.getPrimitiveTypeInfo("double"); case FLOAT: return TypeInfoFactory.getPrimitiveTypeInfo("float"); case STRING: return TypeInfoFactory.getPrimitiveTypeInfo("string"); default: throw new IllegalArgumentException("Avro type " + avroType.getName() + " is not a primitive type"); } }
@Override protected String getSchemaName(Object datum) { if (datum != null) { Class c = datum.getClass(); if (isStringable(c)) return Schema.Type.STRING.getName(); } return super.getSchemaName(datum); }
protected PrimitiveBuilder(Completion<R> context, NameContext names, Schema.Type type) { this.context = context; this.immutable = names.getFullname(type.getName()); }
protected PrimitiveBuilder(Completion<R> context, NameContext names, Schema.Type type) { this.context = context; this.immutable = names.getFullname(type.getName()); }
@Override protected String getSchemaName(Object datum) { // support implicit conversion from thrift's i16 // to avro INT for thrift's optional fields if (datum instanceof Short) return Schema.Type.INT.getName(); // support implicit conversion from thrift's byte // to avro INT for thrift's optional fields if (datum instanceof Byte) return Schema.Type.INT.getName(); return super.getSchemaName(datum); }
@Override protected String getSchemaName(Object datum) { if (datum != null) { Class c = datum.getClass(); if (isStringable(c)) return Schema.Type.STRING.getName(); } return super.getSchemaName(datum); }
private Schema primitive(Schema.Type type, Schema bare) { // test creation of bare schema by name Schema bareByName = SchemaBuilder.builder().type(type.getName()); Assert.assertEquals(Schema.create(type), bareByName); Assert.assertEquals(bareByName, bare); // return a schema with custom prop set Schema p = Schema.create(type); p.addProp("p", "v"); return p; }
public MapConverter(String fieldName, boolean nullable, String sourceType, Schema schemaNode, WorkUnitState state, List<String> ignoreFields) throws UnsupportedDateTypeException { super(fieldName, nullable, sourceType); super.setElementConverter( getConvertor(fieldName, schemaNode.getValueType().getType().getName(), schemaNode.getValueType(), state, isNullable(), ignoreFields)); }
public ArrayConverter(String fieldName, boolean nullable, String sourceType, Schema schemaNode, WorkUnitState state, List<String> ignoreFields) throws UnsupportedDateTypeException { super(fieldName, nullable, sourceType); super.setElementConverter( getConvertor(fieldName, schemaNode.getElementType().getType().getName(), schemaNode.getElementType(), state, isNullable(), ignoreFields)); }