StructTypeInfo rootType = new StructTypeInfo(); rootType.setAllStructFieldNames(columnNames); rootType.setAllStructFieldTypeInfos(fieldTypes); inspector = OrcStruct.createObjectInspector(rootType);
StructTypeInfo rootType = new StructTypeInfo(); rootType.setAllStructFieldNames(columnNames); rootType.setAllStructFieldTypeInfos(fieldTypes); inspector = OrcStruct.createObjectInspector(rootType);
static ListTypeInfo toStructListTypeInfo(MapTypeInfo mapTypeInfo) { final StructTypeInfo structTypeInfo = new StructTypeInfo(); structTypeInfo.setAllStructFieldNames(Lists.newArrayList("keys", "values")); structTypeInfo.setAllStructFieldTypeInfos(Lists.newArrayList( mapTypeInfo.getMapKeyTypeInfo(), mapTypeInfo.getMapValueTypeInfo())); final ListTypeInfo structListTypeInfo = new ListTypeInfo(); structListTypeInfo.setListElementTypeInfo(structTypeInfo); return structListTypeInfo; }
sti.setAllStructFieldNames(structTypeInfo.getAllStructFieldNames()); return new ExprNodeConstantDesc(sti, newValues);
final TypeInfo valueTypeInfo = mapTypeInfo.getMapValueTypeInfo(); final StructTypeInfo mapStructTypeInfo = new StructTypeInfo(); mapStructTypeInfo.setAllStructFieldNames(Lists.newArrayList("keys", "values")); mapStructTypeInfo.setAllStructFieldTypeInfos( Lists.newArrayList(keyTypeInfo, valueTypeInfo));
TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty); StructTypeInfo rootType = new StructTypeInfo(); rootType.setAllStructFieldNames(columnNames); rootType.setAllStructFieldTypeInfos(fieldTypes); inspector = OrcStruct.createObjectInspector(rootType);
rowType.setAllStructFieldNames(columnNames); rowType.setAllStructFieldTypeInfos(fieldTypes); return OrcRecordUpdater.createEventSchema
private static List<OrcProto.Type> getSchemaOnRead(int numCols, Configuration conf) { String columnTypeProperty = conf.get(serdeConstants.LIST_COLUMN_TYPES); final String columnNameProperty = conf.get(serdeConstants.LIST_COLUMNS); if (columnTypeProperty == null || columnNameProperty == null) { return null; } ArrayList<String> columnNames = Lists.newArrayList(columnNameProperty.split(",")); ArrayList<TypeInfo> fieldTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty); StructTypeInfo structTypeInfo = new StructTypeInfo(); // Column types from conf includes virtual and partition columns at the end. We consider only // the actual columns in the file. structTypeInfo.setAllStructFieldNames(Lists.newArrayList(columnNames.subList(0, numCols))); structTypeInfo.setAllStructFieldTypeInfos(Lists.newArrayList(fieldTypes.subList(0, numCols))); ObjectInspector oi = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(structTypeInfo); return getOrcTypes(oi); }
public Writable serialize(Object o, ObjectInspector objectInspector) { //overwrite field names (as they get lost by Hive) StructTypeInfo structTypeInfo = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(objectInspector); structTypeInfo.setAllStructFieldNames(columnNames); List<TypeInfo> info = structTypeInfo.getAllStructFieldTypeInfos(); List<String> names = structTypeInfo.getAllStructFieldNames(); Map<String, Object> recordMap = new HashMap<>(); List<Object> recordObjects = ((StructObjectInspector) objectInspector).getStructFieldsDataAsList(o); List<Schema.Field> fields = schema.getFields(); for (int structIndex = 0; structIndex < info.size(); structIndex++) { Object obj = recordObjects.get(structIndex); TypeInfo objType = info.get(structIndex); Schema.Field field = fields.get(structIndex); Schema fieldSchema = field.getSchema().isNullable() ? field.getSchema().getNonNullable() : field.getSchema(); if (obj instanceof LazyNonPrimitive || obj instanceof LazyPrimitive) { // In case the SerDe that deserialized the object is the one of a native table recordMap.put(names.get(structIndex), fromLazyObject(objType, obj, fieldSchema)); } else if (obj instanceof Writable) { // Native tables sometimes introduce primitive Writable objects at this point recordMap.put(names.get(structIndex), fromWritable((Writable) obj, fieldSchema)); } else { // In case the deserializer is the DatasetSerDe recordMap.put(names.get(structIndex), serialize(obj, objType, fieldSchema)); } } // TODO Improve serialization logic - CDAP-11 return new Text(GSON.toJson(recordMap)); }
public Writable serialize(Object o, ObjectInspector objectInspector) { //overwrite field names (as they get lost by Hive) StructTypeInfo structTypeInfo = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(objectInspector); structTypeInfo.setAllStructFieldNames(columnNames); List<TypeInfo> info = structTypeInfo.getAllStructFieldTypeInfos(); List<String> names = structTypeInfo.getAllStructFieldNames(); Map<String, Object> recordMap = new HashMap<>(); List<Object> recordObjects = ((StructObjectInspector) objectInspector).getStructFieldsDataAsList(o); List<Schema.Field> fields = schema.getFields(); for (int structIndex = 0; structIndex < info.size(); structIndex++) { Object obj = recordObjects.get(structIndex); TypeInfo objType = info.get(structIndex); Schema.Field field = fields.get(structIndex); Schema fieldSchema = field.getSchema().isNullable() ? field.getSchema().getNonNullable() : field.getSchema(); if (obj instanceof LazyNonPrimitive || obj instanceof LazyPrimitive) { // In case the SerDe that deserialized the object is the one of a native table recordMap.put(names.get(structIndex), fromLazyObject(objType, obj, fieldSchema)); } else if (obj instanceof Writable) { // Native tables sometimes introduce primitive Writable objects at this point recordMap.put(names.get(structIndex), fromWritable((Writable) obj, fieldSchema)); } else { // In case the deserializer is the DatasetSerDe recordMap.put(names.get(structIndex), serialize(obj, objType, fieldSchema)); } } // TODO Improve serialization logic - CDAP-11 return new Text(GSON.toJson(recordMap)); }
public static Schema convertTable(String table, Collection<FieldSchema> columns, @Nullable PartitionStrategy strategy) { ArrayList<String> fieldNames = Lists.newArrayList(); ArrayList<TypeInfo> fieldTypes = Lists.newArrayList(); LinkedList<String> start = Lists.newLinkedList(); Collection<String[]> requiredFields = requiredFields(strategy); List<Schema.Field> fields = Lists.newArrayList(); for (FieldSchema column : columns) { // pass null for the initial path to exclude the table name TypeInfo type = parseTypeInfo(column.getType()); fieldNames.add(column.getName()); fieldTypes.add(type); fields.add(convertField(start, column.getName(), type, requiredFields)); } StructTypeInfo struct = new StructTypeInfo(); struct.setAllStructFieldNames(fieldNames); struct.setAllStructFieldTypeInfos(fieldTypes); Schema recordSchema = Schema.createRecord(table, doc(struct), null, false); recordSchema.setFields(fields); return recordSchema; }
public static Schema convertTable(String table, Collection<FieldSchema> columns, @Nullable PartitionStrategy strategy) { ArrayList<String> fieldNames = Lists.newArrayList(); ArrayList<TypeInfo> fieldTypes = Lists.newArrayList(); LinkedList<String> start = Lists.newLinkedList(); Collection<String[]> requiredFields = requiredFields(strategy); List<Schema.Field> fields = Lists.newArrayList(); for (FieldSchema column : columns) { // pass null for the initial path to exclude the table name TypeInfo type = parseTypeInfo(column.getType()); fieldNames.add(column.getName()); fieldTypes.add(type); fields.add(convertField(start, column.getName(), type, requiredFields)); } StructTypeInfo struct = new StructTypeInfo(); struct.setAllStructFieldNames(fieldNames); struct.setAllStructFieldTypeInfos(fieldTypes); Schema recordSchema = Schema.createRecord(table, doc(struct), null, false); recordSchema.setFields(fields); return recordSchema; }
typeInfos.add(info); ((StructTypeInfo)ti).setAllStructFieldNames(names); ((StructTypeInfo)ti).setAllStructFieldTypeInfos(typeInfos); break;