public void add(int fieldIndex, Primitive value) { Type type = this.schema.getType(fieldIndex); List<Object> list = this.data[fieldIndex]; if (!type.isRepetition(REPEATED) && !list.isEmpty()) { throw new IllegalStateException( "field " + fieldIndex + " (" + type.getName() + ") can not have more than one value: " + list); } else { list.add(value); } }
public Group addGroup(int fieldIndex) { ParquetGroup g = new ParquetGroup(this.schema.getType(fieldIndex).asGroupType()); this.data[fieldIndex].add(g); return g; }
/** * Add a {@link Group} given a String key. * @param key * @param object */ private void addGroup(String key, Group object) { int fieldIndex = getIndex(key); this.schema.getType(fieldIndex).asGroupType(); this.data[fieldIndex].add(object); } }
public void add(int fieldIndex, Binary value) { switch (this.getType().getType(fieldIndex).asPrimitiveType().getPrimitiveTypeName()) { case BINARY: this.add(fieldIndex, new BinaryValue(value)); break; case INT96: this.add(fieldIndex, new Int96Value(value)); break; default: throw new UnsupportedOperationException( this.getType().asPrimitiveType().getName() + " not supported for Binary"); } }
/** * It writes all the fields contained inside a group to the RecordConsumer. * * @param value The list of values contained in the group. * @param inspector The object inspector used to get the correct value type. * @param type Type that contains information about the group schema. */ private void writeGroupFields(final Object value, final StructObjectInspector inspector, final GroupType type) { if (value != null) { List<? extends StructField> fields = inspector.getAllStructFieldRefs(); List<Object> fieldValuesList = inspector.getStructFieldsDataAsList(value); for (int i = 0; i < type.getFieldCount(); i++) { Type fieldType = type.getType(i); String fieldName = fieldType.getName(); Object fieldValue = fieldValuesList.get(i); if (fieldValue != null) { ObjectInspector fieldInspector = fields.get(i).getFieldObjectInspector(); recordConsumer.startField(fieldName, i); writeValue(fieldValue, fieldInspector, fieldType); recordConsumer.endField(fieldName, i); } } } }
private void writeSingleLevelArray(final Object value, final ListObjectInspector inspector, final GroupType type) { // Get the internal array structure Type elementType = type.getType(0); recordConsumer.startGroup(); List<?> arrayValues = inspector.getList(value); if (!arrayValues.isEmpty()) { recordConsumer.startField(elementType.getName(), 0); ObjectInspector elementInspector = inspector.getListElementObjectInspector(); for (Object element : arrayValues) { if (element == null) { throw new IllegalArgumentException("Array elements are requires in given schema definition"); } writeValue(element, elementInspector, elementType); } recordConsumer.endField(elementType.getName(), 0); } recordConsumer.endGroup(); }
GroupType repeatedType = type.getType(0).asGroupType(); recordConsumer.startField(repeatedType.getName(), 0); Type keyType = repeatedType.getType(0); String keyName = keyType.getName(); ObjectInspector keyInspector = inspector.getMapKeyObjectInspector(); Type valuetype = repeatedType.getType(1); String valueName = valuetype.getName(); ObjectInspector valueInspector = inspector.getMapValueObjectInspector();
@Override protected Type getType(String[] path, int depth) { if (depth == path.length) { return this; } return getType(path[depth]).getType(path, depth + 1); }
/** * {@inheritDoc} */ public void startGroup() { previousField.push(-1); types.push(types.peek().asGroupType().getType(fields.peek())); delegate.startGroup(); }
public void add(int fieldIndex, Primitive value) { Type type = this.schema.getType(fieldIndex); List<Object> list = this.data[fieldIndex]; if (!type.isRepetition(REPEATED) && !list.isEmpty()) { throw new IllegalStateException( "field " + fieldIndex + " (" + type.getName() + ") can not have more than one value: " + list); } else { list.add(value); } }
/** * Add a {@link Group} given a String key. * @param key * @param object */ private void addGroup(String key, Group object) { int fieldIndex = getIndex(key); this.schema.getType(fieldIndex).asGroupType(); this.data[fieldIndex].add(object); } }
public Group addGroup(int fieldIndex) { ParquetGroup g = new ParquetGroup(this.schema.getType(fieldIndex).asGroupType()); this.data[fieldIndex].add(g); return g; }
@Override protected boolean containsPath(String[] path, int depth) { if (depth == path.length) { return false; } return containsField(path[depth]) && getType(path[depth]).containsPath(path, depth + 1); }
public TupleConverter(GroupType parquetSchema) { int schemaSize = parquetSchema.getFieldCount(); this.converters = new Converter[schemaSize]; for (int i = 0; i < schemaSize; i++) { Type type = parquetSchema.getType(i); converters[i] = newConverter(type, i); } }
@Override protected int getMaxRepetitionLevel(String[] path, int depth) { int myVal = isRepetition(Repetition.REPEATED) ? 1 : 0; if (depth == path.length) { return myVal; } return myVal + getType(path[depth]).getMaxRepetitionLevel(path, depth + 1); }
private Type filterMap(GroupType mapType, FieldSchema mapFieldSchema) throws FrontendException { if (DEBUG) LOG.debug("filtering MAP schema:\n" + mapType + "\nwith:\n " + mapFieldSchema); if (mapType.getFieldCount() != 1) { throw new RuntimeException("not unwrapping the right type, this should be a Map: " + mapType); } GroupType nested = mapType.getType(0).asGroupType(); if (nested.getFieldCount() != 2) { throw new RuntimeException("this should be a Map Key/Value: " + mapType); } FieldSchema innerField = mapFieldSchema.schema.getField(0); return mapType.withNewFields(nested.withNewFields(nested.getType(0), filter(nested.getType(1), innerField))); }
void checkGroupContains(Type subType) { if (subType.isPrimitive()) { throw new InvalidRecordException(subType + " found: expected " + this); } List<Type> fields = subType.asGroupType().getFields(); for (Type otherType : fields) { Type thisType = this.getType(otherType.getName()); thisType.checkContains(otherType); } }
private PrimitiveConverter getPrimitiveConverter(ColumnDescriptor path) { Type currentType = schema; Converter currentConverter = recordConverter; for (String fieldName : path.getPath()) { final GroupType groupType = currentType.asGroupType(); int fieldIndex = groupType.getFieldIndex(fieldName); currentType = groupType.getType(fieldName); currentConverter = currentConverter.asGroupConverter().getConverter(fieldIndex); } PrimitiveConverter converter = currentConverter.asPrimitiveConverter(); return converter; }