Refine search
GroupType repeatedType = type.getType(0).asGroupType(); recordConsumer.startField(repeatedType.getName(), 0); Type keyType = repeatedType.getType(0); String keyName = keyType.getName(); ObjectInspector keyInspector = inspector.getMapKeyObjectInspector(); Type valuetype = repeatedType.getType(1); String valueName = valuetype.getName(); ObjectInspector valueInspector = inspector.getMapValueObjectInspector();
public void add(int fieldIndex, Binary value) { switch (this.getType().getType(fieldIndex).asPrimitiveType().getPrimitiveTypeName()) { case BINARY: this.add(fieldIndex, new BinaryValue(value)); break; case INT96: this.add(fieldIndex, new Int96Value(value)); break; default: throw new UnsupportedOperationException( this.getType().asPrimitiveType().getName() + " not supported for Binary"); } }
if (type.isPrimitive()) { checkInspectorCategory(inspector, ObjectInspector.Category.PRIMITIVE); writePrimitive(value, (PrimitiveObjectInspector) inspector); GroupType groupType = type.asGroupType(); OriginalType originalType = type.getOriginalType();
public void add(int fieldIndex, Primitive value) { Type type = this.schema.getType(fieldIndex); List<Object> list = this.data[fieldIndex]; if (!type.isRepetition(REPEATED) && !list.isEmpty()) { throw new IllegalStateException( "field " + fieldIndex + " (" + type.getName() + ") can not have more than one value: " + list); } else { list.add(value); } }
public static ColumnIO getArrayElementColumn(ColumnIO columnIO) { while (columnIO instanceof GroupColumnIO && !columnIO.getType().isRepetition(REPEATED)) { columnIO = ((GroupColumnIO) columnIO).getChild(0); } /* If array has a standard 3-level structure with middle level repeated group with a single field: * optional group my_list (LIST) { * repeated group element { * required binary str (UTF8); * }; * } */ if (columnIO instanceof GroupColumnIO && columnIO.getType().getOriginalType() == null && ((GroupColumnIO) columnIO).getChildrenCount() == 1 && !columnIO.getName().equals("array") && !columnIO.getName().equals(columnIO.getParent().getName() + "_tuple")) { return ((GroupColumnIO) columnIO).getChild(0); } /* Backward-compatibility support for 2-level arrays where a repeated field is not a group: * optional group my_list (LIST) { * repeated int32 element; * } */ return columnIO; }
return Optional.empty(); boolean required = columnIO.getType().getRepetition() != OPTIONAL; int repetitionLevel = columnIO.getRepetitionLevel(); int definitionLevel = columnIO.getDefinitionLevel(); RichColumnDescriptor column = new RichColumnDescriptor(primitiveColumnIO.getColumnDescriptor(), columnIO.getType().asPrimitiveType()); return Optional.of(new PrimitiveField(type, repetitionLevel, definitionLevel, required, column, primitiveColumnIO.getId()));
private void validate(PrimitiveTypeName p) { Type currentType = types.peek().asGroupType().getType(fields.peek()); int c = fieldValueCount.pop() + 1; fieldValueCount.push(c); if (DEBUG) LOG.debug("validate " + p + " for " + currentType.getName()); switch (currentType.getRepetition()) { case OPTIONAL: case REQUIRED: if (c > 1) { throw new InvalidRecordException("repeated value when the type is not repeated in " + currentType); } break; case REPEATED: break; default: throw new InvalidRecordException("unknown repetition " + currentType.getRepetition() + " in " + currentType); } if (!currentType.isPrimitive() || currentType.asPrimitiveType().getPrimitiveTypeName() != p) { throw new InvalidRecordException("expected type " + currentType + " but got "+ p); } }
if (parquetType.isPrimitive()) { final PrimitiveType.PrimitiveTypeName parquetPrimitiveTypeName = parquetType.asPrimitiveType() .getPrimitiveTypeName(); final OriginalType originalType = parquetType.getOriginalType(); if (originalType == OriginalType.DECIMAL) { final DecimalMetadata decimalMetadata = parquetType.asPrimitiveType().getDecimalMetadata(); return field.append("DECIMAL(").append(decimalMetadata.getPrecision()).append(" , ") .append(decimalMetadata.getScale()).append(")").toString(); GroupType parquetGroupType = parquetType.asGroupType(); OriginalType originalType = parquetGroupType.getOriginalType(); if (originalType != null) { switch (originalType) { case LIST: if (parquetGroupType.getFieldCount() != 1) { throw new UnsupportedOperationException("Invalid list type " + parquetGroupType); Type elementType = parquetGroupType.getType(0); if (!elementType.isRepetition(Type.Repetition.REPEATED)) { throw new UnsupportedOperationException("Invalid list type " + parquetGroupType); .isPrimitive()) { throw new UnsupportedOperationException("Invalid map type " + parquetGroupType); GroupType mapKeyValType = parquetGroupType.getType(0).asGroupType(); if (!keyType.isPrimitive() || !keyType.asPrimitiveType().getPrimitiveTypeName() .equals(PrimitiveType.PrimitiveTypeName.BINARY) || !keyType.getOriginalType().equals(OriginalType.UTF8)) {
public ColumnDescriptor getColumnDescription(String[] path) { int maxRep = getMaxRepetitionLevel(path); int maxDef = getMaxDefinitionLevel(path); PrimitiveType type = getType(path).asPrimitiveType(); return new ColumnDescriptor(path, type.getPrimitiveTypeName(), type.getTypeLength(), maxRep, maxDef); }
void checkGroupContains(Type subType) { if (subType.isPrimitive()) { throw new InvalidRecordException(subType + " found: expected " + this); } List<Type> fields = subType.asGroupType().getFields(); for (Type otherType : fields) { Type thisType = this.getType(otherType.getName()); thisType.checkContains(otherType); } }
private boolean hasMissingRequiredFieldInGroupType(GroupType requested, GroupType fullSchema) { for (Type field : fullSchema.getFields()) { if (requested.containsField(field.getName())) { Type requestedType = requested.getType(field.getName()); // if a field is in requested schema and the type of it is a group type, then do recursive check if (!field.isPrimitive()) { if (hasMissingRequiredFieldInGroupType(requestedType.asGroupType(), field.asGroupType())) { return true; } else { continue;// check next field } } } else { if (field.getRepetition() == Type.Repetition.REQUIRED) { return true; // if a field is missing in requested schema and it's required } else { continue; // the missing field is not required, then continue checking next field } } } return false; }
private Type filterBag(GroupType bagType, FieldSchema bagFieldSchema) throws FrontendException { if (DEBUG) LOG.debug("filtering BAG schema:\n" + bagType + "\nwith:\n " + bagFieldSchema); if (bagType.getFieldCount() != 1) { throw new RuntimeException("not unwrapping the right type, this should be a Bag: " + bagType); } Type nested = bagType.getType(0); FieldSchema innerField = bagFieldSchema.schema.getField(0); if (nested.isPrimitive() || nested.getOriginalType() == OriginalType.MAP || nested.getOriginalType() == OriginalType.LIST) { // Bags always contain tuples => we skip the extra tuple that was inserted in that case. innerField = innerField.schema.getField(0); } return bagType.withNewFields(filter(nested, innerField)); } }
public Group addGroup(int fieldIndex) { ParquetGroup g = new ParquetGroup(this.schema.getType(fieldIndex).asGroupType()); this.data[fieldIndex].add(g); return g; }
private void visitChildren(GroupColumnIO newIO, GroupType groupType, GroupType requestedGroupType) { GroupColumnIO oldIO = current; current = newIO; for (Type type : groupType.getFields()) { // if the file schema does not contain the field it will just stay null if (requestedGroupType.containsField(type.getName())) { currentRequestedIndex = requestedGroupType.getFieldIndex(type.getName()); currentRequestedType = requestedGroupType.getType(currentRequestedIndex); if (currentRequestedType.getRepetition().isMoreRestrictiveThan(type.getRepetition())) { incompatibleSchema(type, currentRequestedType); } type.accept(this); } } current = oldIO; }
private Column convertPrimitiveField(final Type fieldType) { final String fieldName = fieldType.getName(); final PrimitiveTypeName parquetPrimitiveTypeName = fieldType.asPrimitiveType().getPrimitiveTypeName(); final OriginalType originalType = fieldType.getOriginalType(); return parquetPrimitiveTypeName.convert( new PrimitiveType.PrimitiveTypeNameConverter<Column, RuntimeException>() {
@Override public void write(TupleEntry record) { recordConsumer.startMessage(); final List<Type> fields = rootSchema.getFields(); for (int i = 0; i < fields.size(); i++) { Type field = fields.get(i); if (record == null || record.getObject(field.getName()) == null) { continue; } recordConsumer.startField(field.getName(), i); if (field.isPrimitive()) { writePrimitive(record, field.asPrimitiveType()); } else { throw new UnsupportedOperationException("Complex type not implemented"); } recordConsumer.endField(field.getName(), i); } recordConsumer.endMessage(); }
public SchemaIntersection(MessageType fileSchema, Fields requestedFields) { if(requestedFields == Fields.UNKNOWN) requestedFields = Fields.ALL; Fields newFields = Fields.NONE; List<Type> newSchemaFields = new ArrayList<Type>(); int schemaSize = fileSchema.getFieldCount(); for (int i = 0; i < schemaSize; i++) { Type type = fileSchema.getType(i); Fields name = new Fields(type.getName()); if(requestedFields.contains(name)) { newFields = newFields.append(name); newSchemaFields.add(type); } } this.sourceFields = newFields; this.requestedSchema = new MessageType(fileSchema.getName(), newSchemaFields); }
public static parquet.schema.Type getParquetTypeByName(String columnName, MessageType messageType) { if (messageType.containsField(columnName)) { return messageType.getType(columnName); } // parquet is case-sensitive, but hive is not. all hive columns get converted to lowercase // check for direct match above but if no match found, try case-insensitive match for (parquet.schema.Type type : messageType.getFields()) { if (type.getName().equalsIgnoreCase(columnName)) { return type; } } return null; }
.toArray(String[]::new); ColumnPath columnPath = ColumnPath.get(path); PrimitiveTypeName primitiveTypeName = messageType.getType(columnPath.toArray()).asPrimitiveType().getPrimitiveTypeName(); ColumnChunkMetaData column = ColumnChunkMetaData.get( columnPath,
/** * {@inheritDoc} */ @Override protected boolean typeEquals(Type other) { if (other.isPrimitive()) { PrimitiveType primitiveType = other.asPrimitiveType(); return getRepetition() == primitiveType.getRepetition() && getPrimitiveTypeName().equals(primitiveType.getPrimitiveTypeName()) && getName().equals(primitiveType.getName()); } else { return false; } }