return Optional.empty(); boolean required = columnIO.getType().getRepetition() != OPTIONAL; int repetitionLevel = columnIO.getRepetitionLevel(); int definitionLevel = columnIO.getDefinitionLevel();
private void add(int fieldIndex, Primitive value) { Type type = schema.getType(fieldIndex); List<Object> list = data[fieldIndex]; if (type.getRepetition() != Type.Repetition.REPEATED && !list.isEmpty()) { throw new IllegalStateException("field "+fieldIndex+" (" + type.getName() + ") can not have more than one value: " + list); } list.add(value); }
private void validateMissingFields(int index) { for (int i = previousField.peek() + 1; i < index; i++) { Type type = types.peek().asGroupType().getType(i); if (type.getRepetition() == Repetition.REQUIRED) { throw new InvalidRecordException("required field is missing " + type); } } }
private void validate(PrimitiveTypeName p) { Type currentType = types.peek().asGroupType().getType(fields.peek()); int c = fieldValueCount.pop() + 1; fieldValueCount.push(c); if (DEBUG) LOG.debug("validate " + p + " for " + currentType.getName()); switch (currentType.getRepetition()) { case OPTIONAL: case REQUIRED: if (c > 1) { throw new InvalidRecordException("repeated value when the type is not repeated in " + currentType); } break; case REPEATED: break; default: throw new InvalidRecordException("unknown repetition " + currentType.getRepetition() + " in " + currentType); } if (!currentType.isPrimitive() || currentType.asPrimitiveType().getPrimitiveTypeName() != p) { throw new InvalidRecordException("expected type " + currentType + " but got "+ p); } }
private void validate(PrimitiveTypeName p) { Type currentType = types.peek().asGroupType().getType(fields.peek()); int c = fieldValueCount.pop() + 1; fieldValueCount.push(c); if (DEBUG) LOG.debug("validate " + p + " for " + currentType.getName()); switch (currentType.getRepetition()) { case OPTIONAL: case REQUIRED: if (c > 1) { throw new InvalidRecordException("repeated value when the type is not repeated in " + currentType); } break; case REPEATED: break; default: throw new InvalidRecordException("unknown repetition " + currentType.getRepetition() + " in " + currentType); } if (!currentType.isPrimitive() || currentType.asPrimitiveType().getPrimitiveTypeName() != p) { throw new InvalidRecordException("expected type " + p + " but got "+ currentType); } }
@Override void setLevels(int r, int d, String[] fieldPath, int[] indexFieldPath, List<ColumnIO> repetition, List<ColumnIO> path) { super.setLevels(r, d, fieldPath, indexFieldPath, repetition, path); for (ColumnIO child : this.children) { String[] newFieldPath = Arrays.copyOf(fieldPath, fieldPath.length + 1); int[] newIndexFieldPath = Arrays.copyOf(indexFieldPath, indexFieldPath.length + 1); newFieldPath[fieldPath.length] = child.getType().getName(); newIndexFieldPath[indexFieldPath.length] = this.getType().asGroupType().getFieldIndex(child.getType().getName()); List<ColumnIO> newRepetition; if (child.getType().getRepetition() == REPEATED) { newRepetition = new ArrayList<ColumnIO>(repetition); newRepetition.add(child); } else { newRepetition = repetition; } List<ColumnIO> newPath = new ArrayList<ColumnIO>(path); newPath.add(child); child.setLevels( // the type repetition level increases whenever there's a possible repetition child.getType().getRepetition() == REPEATED ? r + 1 : r, // the type definition level increases whenever a field can be missing (not required) child.getType().getRepetition() != REQUIRED ? d + 1 : d, newFieldPath, newIndexFieldPath, newRepetition, newPath ); } }
private void validate(PrimitiveTypeName... ptypes) { Type currentType = types.peek().asGroupType().getType(fields.peek()); int c = fieldValueCount.pop() + 1; fieldValueCount.push(c); if (DEBUG) LOG.debug("validate " + Arrays.toString(ptypes) + " for " + currentType.getName()); switch (currentType.getRepetition()) { case OPTIONAL: case REQUIRED: if (c > 1) { throw new InvalidRecordException("repeated value when the type is not repeated in " + currentType); } break; case REPEATED: break; default: throw new InvalidRecordException("unknown repetition " + currentType.getRepetition() + " in " + currentType); } if (!currentType.isPrimitive()) { throw new InvalidRecordException( "expected type in " + Arrays.toString(ptypes) + " but got " + currentType); } for (PrimitiveTypeName p : ptypes) { if (currentType.asPrimitiveType().getPrimitiveTypeName() == p) { return; // type is valid } } throw new InvalidRecordException( "expected type in " + Arrays.toString(ptypes) + " but got " + currentType); }
private void visitChildren(GroupColumnIO newIO, GroupType groupType, GroupType requestedGroupType) { GroupColumnIO oldIO = current; current = newIO; for (Type type : groupType.getFields()) { // if the file schema does not contain the field it will just stay null if (requestedGroupType.containsField(type.getName())) { currentRequestedIndex = requestedGroupType.getFieldIndex(type.getName()); currentRequestedType = requestedGroupType.getType(currentRequestedIndex); if (currentRequestedType.getRepetition().isMoreRestrictiveThan(type.getRepetition())) { incompatibleSchema(type, currentRequestedType); } type.accept(this); } } current = oldIO; }
if (toMerge.containsField(type.getName())) { Type fieldToMerge = toMerge.getType(type.getName()); if (fieldToMerge.getRepetition().isMoreRestrictiveThan(type.getRepetition())) { throw new IncompatibleSchemaModificationException("repetition constraint is more restrictive: can not merge type " + fieldToMerge + " into " + type);
@Override protected Type union(Type toMerge, boolean strict) { if (toMerge.isPrimitive()) { throw new IncompatibleSchemaModificationException("can not merge primitive type " + toMerge + " into group type " + this); } return new GroupType(toMerge.getRepetition(), getName(), mergeFields(toMerge.asGroupType())); }
ColumnIO getParent(int r) { if (getRepetitionLevel() == r && getType().getRepetition() == Repetition.REPEATED) { return this; } else if (getParent()!=null && getParent().getDefinitionLevel()>=r) { return getParent().getParent(r); } else { throw new InvalidRecordException("no parent("+r+") for "+Arrays.toString(this.getFieldPath())); } }
@Override protected Type union(Type toMerge, boolean strict) { if (!toMerge.isPrimitive() || (strict && !primitive.equals(toMerge.asPrimitiveType().getPrimitiveTypeName()))) { throw new IncompatibleSchemaModificationException("can not merge type " + toMerge + " into " + this); } Types.PrimitiveBuilder<PrimitiveType> builder = Types.primitive( primitive, toMerge.getRepetition()); if (PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY == primitive) { builder.length(length); } return builder.named(getName()); } }
private boolean hasMissingRequiredFieldInGroupType(GroupType requested, GroupType fullSchema) { for (Type field : fullSchema.getFields()) { if (requested.containsField(field.getName())) { Type requestedType = requested.getType(field.getName()); // if a field is in requested schema and the type of it is a group type, then do recursive check if (!field.isPrimitive()) { if (hasMissingRequiredFieldInGroupType(requestedType.asGroupType(), field.asGroupType())) { return true; } else { continue;// check next field } } } else { if (field.getRepetition() == Type.Repetition.REQUIRED) { return true; // if a field is missing in requested schema and it's required } else { continue; // the missing field is not required, then continue checking next field } } } return false; }