private static GroupType buildProjectedGroupType( GroupType originalType, List<Type> types) { if (types == null || types.isEmpty()) { return null; } return new GroupType(originalType.getRepetition(), originalType.getName(), types); }
public ListDataWriter(ListObjectInspector inspector, GroupType groupType) { this.inspector = inspector; // Get the internal array structure GroupType repeatedType = groupType.getType(0).asGroupType(); this.repeatedGroupName = repeatedType.getName(); Type elementType = repeatedType.getType(0); this.elementName = elementType.getName(); ObjectInspector elementInspector = this.inspector.getListElementObjectInspector(); this.elementWriter = createWriter(elementInspector, elementType); }
public MapDataWriter(MapObjectInspector inspector, GroupType groupType) { this.inspector = inspector; // Get the internal map structure (MAP_KEY_VALUE) GroupType repeatedType = groupType.getType(0).asGroupType(); this.repeatedGroupName = repeatedType.getName(); // Get key element information Type keyType = repeatedType.getType(0); ObjectInspector keyInspector = this.inspector.getMapKeyObjectInspector(); this.keyName = keyType.getName(); this.keyWriter = createWriter(keyInspector, keyType); // Get value element information Type valuetype = repeatedType.getType(1); ObjectInspector valueInspector = this.inspector.getMapValueObjectInspector(); this.valueName = valuetype.getName(); this.valueWriter = createWriter(valueInspector, valuetype); }
private HiveCollectionConverter(GroupType collectionType, ConverterParent parent, int index, boolean isMap, TypeInfo hiveTypeInfo) { setMetadata(parent.getMetadata()); this.collectionType = collectionType; this.parent = parent; this.index = index; Type repeatedType = collectionType.getType(0); if (isMap) { this.innerConverter = new KeyValueConverter( repeatedType.asGroupType(), this, hiveTypeInfo); } else if (isElementType(repeatedType, collectionType.getName())) { this.innerConverter = getConverterFromDescription(repeatedType, 0, this, extractListCompatibleType(hiveTypeInfo)); } else { this.innerConverter = new ElementConverter( repeatedType.asGroupType(), this, extractListCompatibleType(hiveTypeInfo)); } }
Schema convert(GroupType parquetSchema) { return convertFields(parquetSchema.getName(), parquetSchema.getFields()); }
public Group addGroup(String field) { if (LOG.isDebugEnabled()) { LOG.debug("add group {} to {}", field, getType().getName()); } return addGroup(getType().getFieldIndex(field)); }
/** * {@inheritDoc} */ @Override public void writeToStringBuilder(StringBuilder sb, String indent) { sb.append(indent) .append(getRepetition().name().toLowerCase(Locale.ENGLISH)) .append(" group ") .append(getName()) .append(getOriginalType() == null ? "" : " (" + getOriginalType() +")") .append(getId() == null ? "" : " = " + getId()) .append(" {\n"); membersDisplayString(sb, indent + " "); sb.append(indent) .append("}"); }
/** * {@inheritDoc} */ @Override public void writeToStringBuilder(StringBuilder sb, String indent) { sb.append(indent) .append(getRepetition().name().toLowerCase(Locale.ENGLISH)) .append(" group ") .append(getName()) .append(getLogicalTypeAnnotation() == null ? "" : " (" + getLogicalTypeAnnotation().toString() +")") .append(getId() == null ? "" : " = " + getId()) .append(" {\n"); membersDisplayString(sb, indent + " "); sb.append(indent) .append("}"); }
public Group addGroup(String field) { if (LOG.isDebugEnabled()) { LOG.debug("add group {} to {}", field, getType().getName()); } return addGroup(getType().getFieldIndex(field)); }
/** * @param newFields a list of types to use as fields in a copy of this group * @return a group with the same attributes and new fields. */ public GroupType withNewFields(List<Type> newFields) { return new GroupType(getRepetition(), getName(), getOriginalType(), newFields, getId()); }
/** * @param newFields a list of types to use as fields in a copy of this group * @return a group with the same attributes and new fields. */ public GroupType withNewFields(List<Type> newFields) { return new GroupType(getRepetition(), getName(), getLogicalTypeAnnotation(), newFields, getId()); }
private Converter groupConverter(OutputMutator mutator, List<Field> arrowSchema, GroupType groupType, PathSegment colNextChild, final String nameForChild) { Collection<SchemaPath> c = new ArrayList<>(); while (colNextChild != null) { if (colNextChild.isNamed()) { break; } colNextChild = colNextChild.getChild(); } if (colNextChild != null) { SchemaPath s = new SchemaPath(colNextChild.getNameSegment()); c.add(s); } if (arrowSchema != null) { return groupConverterFromArrowSchema(nameForChild, groupType.getName(), groupType, c); } return defaultGroupConverter(mutator, groupType, nameForChild, c, null); }
/** * @param id the field id * @return a new GroupType with the same fields and a new id */ @Override public GroupType withId(int id) { return new GroupType(getRepetition(), getName(), getLogicalTypeAnnotation(), fields, new ID(id)); }
/** * @param id the field id * @return a new GroupType with the same fields and a new id */ @Override public GroupType withId(int id) { return new GroupType(getRepetition(), getName(), getOriginalType(), fields, new ID(id)); }
@Override protected Type union(Type toMerge, boolean strict) { if (toMerge.isPrimitive()) { throw new IncompatibleSchemaModificationException("can not merge primitive type " + toMerge + " into group type " + this); } return new GroupType(toMerge.getRepetition(), getName(), toMerge.getLogicalTypeAnnotation(), mergeFields(toMerge.asGroupType()), getId()); }
@Override protected Type union(Type toMerge, boolean strict) { if (toMerge.isPrimitive()) { throw new IncompatibleSchemaModificationException("can not merge primitive type " + toMerge + " into group type " + this); } return new GroupType(toMerge.getRepetition(), getName(), toMerge.getOriginalType(), mergeFields(toMerge.asGroupType()), getId()); }
@Override public Type map(GroupType map, Type key, Type value) { GroupType repeated = map.getType(0).asGroupType(); Type originalKey = repeated.getType(0); Type originalValue = repeated.getType(1); int keyId = getId(originalKey); int valueId = getId(originalValue); if (selectedIds.contains(keyId) || selectedIds.contains(valueId)) { return map; } else if (value != null) { if (value != originalValue) { return Types.map(map.getRepetition()) .key(originalKey) .value(value) .id(getId(map)) .named(map.getName()); } return map; } return null; }
@Override public Type list(GroupType list, Type element) { GroupType repeated = list.getType(0).asGroupType(); Type originalElement = repeated.getType(0); int elementId = getId(originalElement); if (selectedIds.contains(elementId)) { return list; } else if (element != null) { if (element != originalElement) { // the element type was projected return Types.list(list.getRepetition()) .element(element) .id(getId(list)) .named(list.getName()); } return list; } return null; }
@Override public void visit(GroupType groupType) { SchemaElement element = new SchemaElement(groupType.getName()); element.setRepetition_type(toParquetRepetition(groupType.getRepetition())); if (groupType.getOriginalType() != null) { element.setConverted_type(getConvertedType(groupType.getOriginalType())); } if (groupType.getId() != null) { element.setField_id(groupType.getId().intValue()); } visitChildren(result, groupType, element); }
@Override public void visit(GroupType groupType) { SchemaElement element = new SchemaElement(groupType.getName()); element.setRepetition_type(toParquetRepetition(groupType.getRepetition())); if (groupType.getLogicalTypeAnnotation() != null) { element.setConverted_type(convertToConvertedType(groupType.getLogicalTypeAnnotation())); element.setLogicalType(convertToLogicalType(groupType.getLogicalTypeAnnotation())); } if (groupType.getId() != null) { element.setField_id(groupType.getId().intValue()); } visitChildren(result, groupType, element); }