public static int getFieldIndex(MessageType fileSchema, String name) { try { return fileSchema.getFieldIndex(name.toLowerCase(Locale.ENGLISH)); } catch (InvalidRecordException e) { for (parquet.schema.Type type : fileSchema.getFields()) { if (type.getName().equalsIgnoreCase(name)) { return fileSchema.getFieldIndex(type.getName()); } } return -1; } }
public void add(int fieldIndex, Primitive value) { Type type = this.schema.getType(fieldIndex); List<Object> list = this.data[fieldIndex]; if (!type.isRepetition(REPEATED) && !list.isEmpty()) { throw new IllegalStateException( "field " + fieldIndex + " (" + type.getName() + ") can not have more than one value: " + list); } else { list.add(value); } }
public String toString(String indent) { StringBuilder result = new StringBuilder(); int i = 0; for (Type field : this.schema.getFields()) { String name = field.getName(); List<Object> values = this.data[i]; for (Object value : values) { result.append(indent).append(name); if (value == null) { result.append(": NULL\n"); } else if (value instanceof Group) { result.append("\n").append(((ParquetGroup) value).toString(indent + " ")); } else { result.append(": ").append(value.toString()).append("\n"); } } i++; } return result.toString(); }
public static parquet.schema.Type getParquetTypeByName(String columnName, MessageType messageType) { if (messageType.containsField(columnName)) { return messageType.getType(columnName); } // parquet is case-sensitive, but hive is not. all hive columns get converted to lowercase // check for direct match above but if no match found, try case-insensitive match for (parquet.schema.Type type : messageType.getFields()) { if (type.getName().equalsIgnoreCase(columnName)) { return type; } } return null; }
String columnName = useParquetColumnNames ? name : fileSchema.getFields().get(column.getHiveColumnIndex()).getName(); fieldsBuilder.add(constructField(type, lookupColumnByName(messageColumnIO, columnName)));
public static GroupType mapType(Repetition repetition, String alias, String mapAlias, Type keyType, Type valueType) { //support projection only on key of a map if (valueType == null) { return listWrapper( repetition, alias, MAP_KEY_VALUE, new GroupType( Repetition.REPEATED, mapAlias, keyType)); } else { if (!valueType.getName().equals("value")) { throw new RuntimeException(valueType.getName() + " should be value"); } return listWrapper( repetition, alias, MAP_KEY_VALUE, new GroupType( Repetition.REPEATED, mapAlias, keyType, valueType)); } }
public static GroupType mapType(Repetition repetition, String alias, String mapAlias, Type keyType, Type valueType) { //support projection only on key of a map if (valueType == null) { return listWrapper( repetition, alias, MAP_KEY_VALUE, new GroupType( Repetition.REPEATED, mapAlias, keyType)); } else { if (!valueType.getName().equals("value")) { throw new RuntimeException(valueType.getName() + " should be value"); } return listWrapper( repetition, alias, MAP_KEY_VALUE, new GroupType( Repetition.REPEATED, mapAlias, keyType, valueType)); } }
private void writeSingleLevelArray(final Object value, final ListObjectInspector inspector, final GroupType type) { // Get the internal array structure Type elementType = type.getType(0); recordConsumer.startGroup(); List<?> arrayValues = inspector.getList(value); if (!arrayValues.isEmpty()) { recordConsumer.startField(elementType.getName(), 0); ObjectInspector elementInspector = inspector.getListElementObjectInspector(); for (Object element : arrayValues) { if (element == null) { throw new IllegalArgumentException("Array elements are requires in given schema definition"); } writeValue(element, elementInspector, elementType); } recordConsumer.endField(elementType.getName(), 0); } recordConsumer.endGroup(); }
/** * It writes all the fields contained inside a group to the RecordConsumer. * * @param value The list of values contained in the group. * @param inspector The object inspector used to get the correct value type. * @param type Type that contains information about the group schema. */ private void writeGroupFields(final Object value, final StructObjectInspector inspector, final GroupType type) { if (value != null) { List<? extends StructField> fields = inspector.getAllStructFieldRefs(); List<Object> fieldValuesList = inspector.getStructFieldsDataAsList(value); for (int i = 0; i < type.getFieldCount(); i++) { Type fieldType = type.getType(i); String fieldName = fieldType.getName(); Object fieldValue = fieldValuesList.get(i); if (fieldValue != null) { ObjectInspector fieldInspector = fields.get(i).getFieldObjectInspector(); recordConsumer.startField(fieldName, i); writeValue(fieldValue, fieldInspector, fieldType); recordConsumer.endField(fieldName, i); } } } }
String elementName = elementType.getName();
/** * returns the name of the corresponding field * @param index the index of the desired field in this type * @return the name of the field at this index */ public String getFieldName(int index) { return fields.get(index).getName(); }
@Override public String toString() { return this.getClass().getSimpleName()+" "+type.getName() +" r:"+repetitionLevel +" d:"+definitionLevel +" "+Arrays.toString(fieldPath); }
public GroupType asGroupType() { if (isPrimitive()) { throw new ClassCastException(this.getName() + " is not a group"); } return (GroupType)this; }
@Override protected List<String[]> getPaths(int depth) { List<String[]> result = new ArrayList<String[]>(); for (Type field : fields) { List<String[]> paths = field.getPaths(depth + 1); for (String[] path : paths) { path[depth] = field.getName(); result.add(path); } } return result; }
private void add(int fieldIndex, Primitive value) { Type type = schema.getType(fieldIndex); List<Object> list = data[fieldIndex]; if (!type.isRepetition(Type.Repetition.REPEATED) && !list.isEmpty()) { throw new IllegalStateException("field "+fieldIndex+" (" + type.getName() + ") can not have more than one value: " + list); } list.add(value); }
private FieldSchema getFieldSchema(Type parquetType) throws FrontendException { final String fieldName = parquetType.getName(); if (parquetType.isPrimitive()) { return getSimpleFieldSchema(fieldName, parquetType); } else { return getComplexFieldSchema(fieldName, parquetType); } }
private void endListWrapper() { if (size > 0) { recordConsumer.endField(listContent.getType().getName(), 0); } recordConsumer.endGroup(); end(); }
private void startListWrapper() { start(); recordConsumer.startGroup(); if (size > 0) { recordConsumer.startField(listContent.getType().getName(), 0); currentProtocol = contentProtocol; } }
@Override public void writeMapBegin(TMap map) throws TException { start(); recordConsumer.startGroup(); countToConsume = map.size; if (countToConsume > 0) { recordConsumer.startField(mapContent.getType().getName(), 0); currentProtocol = keyProtocol; } }