/** * Converts Hive schema to avro schema * * @param columnNames Names of the hive columns * @param columnTypes Hive Column types * @param namespace Namespace of Avro schema * @param name Avro schema name * @param doc Avro schema doc * @return Avro Schema */ public Schema convert(List<String> columnNames, List<TypeInfo> columnTypes, List<String> columnComments, String namespace, String name, String doc) { List<Schema.Field> fields = new ArrayList<Schema.Field>(); for (int i = 0; i < columnNames.size(); ++i) { final String comment = columnComments.size() > i ? columnComments.get(i) : null; final Schema.Field avroField = createAvroField(columnNames.get(i), columnTypes.get(i), comment); fields.addAll(getFields(avroField)); } if (name == null || name.isEmpty()) { name = "baseRecord"; } Schema avroSchema = Schema.createRecord(name, doc, namespace, false); avroSchema.setFields(fields); return avroSchema; }
private Schema createAvroRecord(TypeInfo typeInfo) { List<Schema.Field> childFields = new ArrayList<Schema.Field>(); final List<String> allStructFieldNames = ((StructTypeInfo) typeInfo).getAllStructFieldNames(); final List<TypeInfo> allStructFieldTypeInfos = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos(); if (allStructFieldNames.size() != allStructFieldTypeInfos.size()) { throw new IllegalArgumentException("Failed to generate avro schema from hive schema. " + "name and column type differs. names = " + allStructFieldNames + ", types = " + allStructFieldTypeInfos); } for (int i = 0; i < allStructFieldNames.size(); ++i) { final TypeInfo childTypeInfo = allStructFieldTypeInfos.get(i); final Schema.Field grandChildSchemaField = createAvroField(allStructFieldNames.get(i), childTypeInfo, childTypeInfo.toString()); final List<Schema.Field> grandChildFields = getFields(grandChildSchemaField); childFields.addAll(grandChildFields); } Schema recordSchema = Schema.createRecord("record_" + recordCounter, typeInfo.toString(), null, false); ++recordCounter; recordSchema.setFields(childFields); return recordSchema; }
/** * Converts Hive schema to avro schema * * @param columnNames Names of the hive columns * @param columnTypes Hive Column types * @param namespace Namespace of Avro schema * @param name Avro schema name * @param doc Avro schema doc * @return Avro Schema */ public Schema convert(List<String> columnNames, List<TypeInfo> columnTypes, List<String> columnComments, String namespace, String name, String doc) { List<Schema.Field> fields = new ArrayList<Schema.Field>(); for (int i = 0; i < columnNames.size(); ++i) { final String comment = columnComments.size() > i ? columnComments.get(i) : null; final Schema.Field avroField = createAvroField(columnNames.get(i), columnTypes.get(i), comment); fields.addAll(getFields(avroField)); } if (name == null || name.isEmpty()) { name = "baseRecord"; } Schema avroSchema = Schema.createRecord(name, doc, namespace, false); avroSchema.setFields(fields); return avroSchema; }
/** * Converts Hive schema to avro schema * * @param columnNames Names of the hive columns * @param columnTypes Hive Column types * @param namespace Namespace of Avro schema * @param name Avro schema name * @param doc Avro schema doc * @return Avro Schema */ public Schema convert(List<String> columnNames, List<TypeInfo> columnTypes, List<String> columnComments, String namespace, String name, String doc) { List<Schema.Field> fields = new ArrayList<Schema.Field>(); for (int i = 0; i < columnNames.size(); ++i) { final String comment = columnComments.size() > i ? columnComments.get(i) : null; final Schema.Field avroField = createAvroField(columnNames.get(i), columnTypes.get(i), comment); fields.addAll(getFields(avroField)); } if (name == null || name.isEmpty()) { name = "baseRecord"; } Schema avroSchema = Schema.createRecord(name, doc, namespace, false); avroSchema.setFields(fields); return avroSchema; }
/** * Converts Hive schema to avro schema * * @param columnNames Names of the hive columns * @param columnTypes Hive Column types * @param namespace Namespace of Avro schema * @param name Avro schema name * @param doc Avro schema doc * @return Avro Schema */ public Schema convert(List<String> columnNames, List<TypeInfo> columnTypes, List<String> columnComments, String namespace, String name, String doc) { List<Schema.Field> fields = new ArrayList<Schema.Field>(); for (int i = 0; i < columnNames.size(); ++i) { final String comment = columnComments.size() > i ? columnComments.get(i) : null; final Schema.Field avroField = createAvroField(columnNames.get(i), columnTypes.get(i), comment); fields.addAll(getFields(avroField)); } if (name == null || name.isEmpty()) { name = "baseRecord"; } Schema avroSchema = Schema.createRecord(name, doc, namespace, false); avroSchema.setFields(fields); return avroSchema; }
/** * Converts Hive schema to avro schema * * @param columnNames Names of the hive columns * @param columnTypes Hive Column types * @param namespace Namespace of Avro schema * @param name Avro schema name * @param doc Avro schema doc * @return Avro Schema */ public Schema convert(List<String> columnNames, List<TypeInfo> columnTypes, List<String> columnComments, String namespace, String name, String doc) { List<Schema.Field> fields = new ArrayList<Schema.Field>(); for (int i = 0; i < columnNames.size(); ++i) { final String comment = columnComments.size() > i ? columnComments.get(i) : null; final Schema.Field avroField = createAvroField(columnNames.get(i), columnTypes.get(i), comment); fields.addAll(getFields(avroField)); } if (name == null || name.isEmpty()) { name = "baseRecord"; } Schema avroSchema = Schema.createRecord(name, doc, namespace, false); avroSchema.setFields(fields); return avroSchema; }
private Schema createAvroRecord(TypeInfo typeInfo) { List<Schema.Field> childFields = new ArrayList<Schema.Field>(); final List<String> allStructFieldNames = ((StructTypeInfo) typeInfo).getAllStructFieldNames(); final List<TypeInfo> allStructFieldTypeInfos = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos(); if (allStructFieldNames.size() != allStructFieldTypeInfos.size()) { throw new IllegalArgumentException("Failed to generate avro schema from hive schema. " + "name and column type differs. names = " + allStructFieldNames + ", types = " + allStructFieldTypeInfos); } for (int i = 0; i < allStructFieldNames.size(); ++i) { final TypeInfo childTypeInfo = allStructFieldTypeInfos.get(i); final Schema.Field grandChildSchemaField = createAvroField(allStructFieldNames.get(i), childTypeInfo, childTypeInfo.toString()); final List<Schema.Field> grandChildFields = getFields(grandChildSchemaField); childFields.addAll(grandChildFields); } Schema recordSchema = Schema.createRecord("record_" + recordCounter, typeInfo.toString(), null, false); ++recordCounter; recordSchema.setFields(childFields); return recordSchema; }
private Schema createAvroRecord(TypeInfo typeInfo) { List<Schema.Field> childFields = new ArrayList<Schema.Field>(); final List<String> allStructFieldNames = ((StructTypeInfo) typeInfo).getAllStructFieldNames(); final List<TypeInfo> allStructFieldTypeInfos = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos(); if (allStructFieldNames.size() != allStructFieldTypeInfos.size()) { throw new IllegalArgumentException("Failed to generate avro schema from hive schema. " + "name and column type differs. names = " + allStructFieldNames + ", types = " + allStructFieldTypeInfos); } for (int i = 0; i < allStructFieldNames.size(); ++i) { final TypeInfo childTypeInfo = allStructFieldTypeInfos.get(i); final Schema.Field grandChildSchemaField = createAvroField(allStructFieldNames.get(i), childTypeInfo, childTypeInfo.toString()); final List<Schema.Field> grandChildFields = getFields(grandChildSchemaField); childFields.addAll(grandChildFields); } Schema recordSchema = Schema.createRecord("record_" + recordCounter, typeInfo.toString(), null, false); ++recordCounter; recordSchema.setFields(childFields); return recordSchema; }
private Schema createAvroRecord(TypeInfo typeInfo) { List<Schema.Field> childFields = new ArrayList<Schema.Field>(); final List<String> allStructFieldNames = ((StructTypeInfo) typeInfo).getAllStructFieldNames(); final List<TypeInfo> allStructFieldTypeInfos = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos(); if (allStructFieldNames.size() != allStructFieldTypeInfos.size()) { throw new IllegalArgumentException("Failed to generate avro schema from hive schema. " + "name and column type differs. names = " + allStructFieldNames + ", types = " + allStructFieldTypeInfos); } for (int i = 0; i < allStructFieldNames.size(); ++i) { final TypeInfo childTypeInfo = allStructFieldTypeInfos.get(i); final Schema.Field grandChildSchemaField = createAvroField(allStructFieldNames.get(i), childTypeInfo, childTypeInfo.toString()); final List<Schema.Field> grandChildFields = getFields(grandChildSchemaField); childFields.addAll(grandChildFields); } Schema recordSchema = Schema.createRecord("record_" + recordCounter, typeInfo.toString(), null, false); ++recordCounter; recordSchema.setFields(childFields); return recordSchema; }
private Schema createAvroRecord(TypeInfo typeInfo) { List<Schema.Field> childFields = new ArrayList<Schema.Field>(); final List<String> allStructFieldNames = ((StructTypeInfo) typeInfo).getAllStructFieldNames(); final List<TypeInfo> allStructFieldTypeInfos = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos(); if (allStructFieldNames.size() != allStructFieldTypeInfos.size()) { throw new IllegalArgumentException("Failed to generate avro schema from hive schema. " + "name and column type differs. names = " + allStructFieldNames + ", types = " + allStructFieldTypeInfos); } for (int i = 0; i < allStructFieldNames.size(); ++i) { final TypeInfo childTypeInfo = allStructFieldTypeInfos.get(i); final Schema.Field grandChildSchemaField = createAvroField(allStructFieldNames.get(i), childTypeInfo, childTypeInfo.toString()); final List<Schema.Field> grandChildFields = getFields(grandChildSchemaField); childFields.addAll(grandChildFields); } Schema recordSchema = Schema.createRecord("record_" + recordCounter, typeInfo.toString(), null, false); ++recordCounter; recordSchema.setFields(childFields); return recordSchema; }