private static void showDetails(PrettyPrintWriter out, GroupType type, int depth, MessageType container, List<String> cpath) { String name = Strings.repeat(".", depth) + type.getName(); Repetition rep = type.getRepetition(); int fcount = type.getFieldCount(); out.format("%s: %s F:%d%n", name, rep, fcount); cpath.add(type.getName()); for (Type ftype : type.getFields()) { showDetails(out, ftype, depth + 1, container, cpath); } cpath.remove(cpath.size() - 1); }
/** * {@inheritDoc} */ @Override protected int typeHashCode() { int c = 17; c += 31 * getRepetition().hashCode(); c += 31 * getName().hashCode(); c += 31 * getFields().hashCode(); return c; }
/** * {@inheritDoc} */ @Override public void writeToStringBuilder(StringBuilder sb, String indent) { sb.append(indent) .append(getRepetition().name().toLowerCase()) .append(" group ") .append(getName()) .append(getOriginalType() == null ? "" : " (" + getOriginalType() +")") .append(" {\n"); membersDisplayString(sb, indent + " "); sb.append(indent) .append("}"); }
/** * {@inheritDoc} */ @Override public void writeToStringBuilder(StringBuilder sb, String indent) { sb.append(indent) .append(getRepetition().name().toLowerCase()) .append(" group ") .append(getName()) .append(getOriginalType() == null ? "" : " (" + getOriginalType() +")") .append(getId() == null ? "" : " = " + getId()) .append(" {\n"); membersDisplayString(sb, indent + " "); sb.append(indent) .append("}"); }
/** * {@inheritDoc} */ @Override protected boolean typeEquals(Type other) { Type otherType = (Type) other; if (otherType.isPrimitive()) { return false; } else { GroupType groupType = otherType.asGroupType(); return getRepetition() == groupType.getRepetition() && getName().equals(groupType.getName()) && getFields().equals(groupType.getFields()); } }
public ParquetMapConverter(Type type, String columnName, GroupType mapType, int fieldIndex) { checkArgument( mapType.getFieldCount() == 1, "Expected MAP column '%s' to only have one field, but has %s fields", mapType.getName(), mapType.getFieldCount()); this.mapType = type; this.fieldIndex = fieldIndex; parquet.schema.Type entryType = mapType.getFields().get(0); entryConverter = new ParquetMapEntryConverter(type, columnName + ".entry", entryType.asGroupType()); }
private String path(List<GroupType> path, Type type) { String pathString = ""; if (path.size() > 0) { for (int i = 1; i < path.size(); i++) { pathString += path.get(i).getName() + "."; } } pathString += type.getName() + "."; return pathString; }
/** * @param newFields * @return a group with the same attributes and new fields. */ public GroupType withNewFields(List<Type> newFields) { return new GroupType(getRepetition(), getName(), getOriginalType(), newFields, getId()); }
/** * @param id the field id * @return a new GroupType with the same fields and a new id */ @Override public GroupType withId(int id) { return new GroupType(getRepetition(), getName(), getOriginalType(), fields, new ID(id)); }
public Group addGroup(String field) { if (DEBUG) logger.debug("add group "+field+" to "+getType().getName()); return addGroup(getType().getFieldIndex(field)); }
public Group addGroup(String field) { if (DEBUG) logger.debug("add group "+field+" to "+getType().getName()); return addGroup(getType().getFieldIndex(field)); }
@Override protected Type union(Type toMerge, boolean strict) { if (toMerge.isPrimitive()) { throw new IncompatibleSchemaModificationException("can not merge primitive type " + toMerge + " into group type " + this); } return new GroupType(toMerge.getRepetition(), getName(), mergeFields(toMerge.asGroupType())); }
public ParquetListConverter(Type prestoType, String columnName, GroupType listType, int fieldIndex) { checkArgument( listType.getFieldCount() == 1, "Expected LIST column '%s' to only have one field, but has %s fields", columnName, listType.getFieldCount()); checkArgument(ARRAY.equals(prestoType.getTypeSignature().getBase())); this.arrayType = prestoType; this.fieldIndex = fieldIndex; // The Parquet specification requires that the element value of a // LIST type be wrapped in an inner repeated group, like so: // // optional group listField (LIST) { // repeated group list { // optional int element // } // } // // However, some parquet libraries don't follow this spec. The // compatibility rules used here are specified in the Parquet // documentation at http://git.io/vOpNz. parquet.schema.Type elementType = listType.getType(0); if (isElementType(elementType, listType.getName())) { elementConverter = createConverter(prestoType.getTypeParameters().get(0), columnName + ".element", elementType, 0); } else { elementConverter = new ParquetListEntryConverter(prestoType.getTypeParameters().get(0), columnName, elementType.asGroupType()); } }
public static GroupType listType(Repetition repetition, String alias, Type nestedType) { GroupType repeatedField; if (nestedType.isPrimitive()) { repeatedField = new GroupType( Repetition.REPEATED, "bag", nestedType); } else { final GroupType nestedGroupType = nestedType.asGroupType(); repeatedField = new GroupType( Repetition.REPEATED, nestedGroupType.getName(), nestedGroupType.getFields()); } return listWrapper( repetition, alias, LIST, repeatedField ); } }
@Override public void visit(GroupType groupType) { SchemaElement element = new SchemaElement(groupType.getName()); element.setRepetition_type(toParquetRepetition(groupType.getRepetition())); if (groupType.getOriginalType() != null) { element.setConverted_type(getConvertedType(groupType.getOriginalType())); } visitChildren(result, groupType, element); }
private HiveCollectionConverter(GroupType collectionType, ConverterParent parent, int index, boolean isMap) { setMetadata(parent.getMetadata()); this.collectionType = collectionType; this.parent = parent; this.index = index; Type repeatedType = collectionType.getType(0); if (isMap) { this.innerConverter = new KeyValueConverter( repeatedType.asGroupType(), this); } else if (isElementType(repeatedType, collectionType.getName())) { this.innerConverter = getConverterFromDescription(repeatedType, 0, this); } else { this.innerConverter = new ElementConverter( repeatedType.asGroupType(), this); } }