public ParquetGroup(GroupType schema) { this.schema = schema; this.data = new List[schema.getFields().size()]; for (int i = 0; i < schema.getFieldCount(); ++i) { this.data[i] = new ArrayList(); } }
/** * It writes all the fields contained inside a group to the RecordConsumer. * * @param value The list of values contained in the group. * @param inspector The object inspector used to get the correct value type. * @param type Type that contains information about the group schema. */ private void writeGroupFields(final Object value, final StructObjectInspector inspector, final GroupType type) { if (value != null) { List<? extends StructField> fields = inspector.getAllStructFieldRefs(); List<Object> fieldValuesList = inspector.getStructFieldsDataAsList(value); for (int i = 0; i < type.getFieldCount(); i++) { Type fieldType = type.getType(i); String fieldName = fieldType.getName(); Object fieldValue = fieldValuesList.get(i); if (fieldValue != null) { ObjectInspector fieldInspector = fields.get(i).getFieldObjectInspector(); recordConsumer.startField(fieldName, i); writeValue(fieldValue, fieldInspector, fieldType); recordConsumer.endField(fieldName, i); } } } }
@SuppressWarnings("unchecked") public SimpleGroup(GroupType schema) { this.schema = schema; this.data = new List[schema.getFields().size()]; for (int i = 0; i < schema.getFieldCount(); i++) { this.data[i] = new ArrayList<Object>(); } }
@SuppressWarnings("unchecked") public SimpleGroup(GroupType schema) { this.schema = schema; this.data = new List[schema.getFields().size()]; for (int i = 0; i < schema.getFieldCount(); i++) { this.data[i] = new ArrayList<Object>(); } }
public HiveStructConverter(final GroupType requestedSchema, final GroupType tableSchema, Map<String, String> metadata) { this(requestedSchema, null, 0, tableSchema); setMetadata(metadata); this.reuseWritableArray = true; this.writables = new Writable[tableSchema.getFieldCount()]; }
public ParquetGroup(GroupType schema) { this.schema = schema; this.data = new List[schema.getFields().size()]; for (int i = 0; i < schema.getFieldCount(); ++i) { this.data[i] = new ArrayList(); } }
public ParquetListEntryConverter(Type prestoType, String columnName, GroupType elementType) { checkArgument( elementType.getOriginalType() == null, "Expected LIST column '%s' field to be type STRUCT, but is %s", columnName, elementType); checkArgument( elementType.getFieldCount() == 1, "Expected LIST column '%s' element to have one field, but has %s fields", columnName, elementType.getFieldCount()); elementConverter = createConverter(prestoType, columnName + ".element", elementType.getType(0), 0); }
public ParquetMapConverter(Type type, String columnName, GroupType mapType, int fieldIndex) { checkArgument( mapType.getFieldCount() == 1, "Expected MAP column '%s' to only have one field, but has %s fields", mapType.getName(), mapType.getFieldCount()); this.mapType = type; this.fieldIndex = fieldIndex; parquet.schema.Type entryType = mapType.getFields().get(0); entryConverter = new ParquetMapEntryConverter(type, columnName + ".entry", entryType.asGroupType()); }
/** * {@inheritDoc} */ public void endGroup() { delegate.endGroup(); validateMissingFields(types.peek().asGroupType().getFieldCount()); types.pop(); previousField.pop(); }
private void visitChildren(final List<SchemaElement> result, GroupType groupType, SchemaElement element) { element.setNum_children(groupType.getFieldCount()); result.add(element); for (parquet.schema.Type field : groupType.getFields()) { addToList(result, field); } } });
public TupleConverter(GroupType parquetSchema) { int schemaSize = parquetSchema.getFieldCount(); this.converters = new Converter[schemaSize]; for (int i = 0; i < schemaSize; i++) { Type type = parquetSchema.getType(i); converters[i] = newConverter(type, i); } }
/** * {@inheritDoc} */ public void endGroup() { delegate.endGroup(); validateMissingFields(types.peek().asGroupType().getFieldCount()); types.pop(); previousField.pop(); }
/** * {@inheritDoc} */ public void endMessage() { delegate.endMessage(); validateMissingFields(types.peek().asGroupType().getFieldCount()); previousField.pop(); }
public SimpleRecordConverter(GroupType schema, String name, SimpleRecordConverter parent) { this.converters = new Converter[schema.getFieldCount()]; this.parent = parent; this.name = name; int i = 0; for (Type field: schema.getFields()) { converters[i++] = createConverter(field); } }
/** * {@inheritDoc} */ public void endMessage() { delegate.endMessage(); validateMissingFields(types.peek().asGroupType().getFieldCount()); previousField.pop(); }
private static void showDetails(PrettyPrintWriter out, GroupType type, int depth, MessageType container, List<String> cpath) { String name = Strings.repeat(".", depth) + type.getName(); Repetition rep = type.getRepetition(); int fcount = type.getFieldCount(); out.format("%s: %s F:%d%n", name, rep, fcount); cpath.add(type.getName()); for (Type ftype : type.getFields()) { showDetails(out, ftype, depth + 1, container, cpath); } cpath.remove(cpath.size() - 1); }
private Type getType(boolean columnIndexAccess, String alias, int index) { if(columnIndexAccess) { if(index < parquetSchema.getFieldCount()) { return parquetSchema.getType(index); } } else { return parquetSchema.getType(parquetSchema.getFieldIndex(alias)); } return null; }
MapConverter(GroupType parquetSchema, FieldSchema pigSchema, ParentValueContainer parent, boolean numbersDefaultToZero, boolean columnIndexAccess) throws FrontendException { if (parquetSchema.getFieldCount() != 1) { throw new IllegalArgumentException("maps have only one field. " + parquetSchema); } this.parent = parent; keyValue = new MapKeyValueConverter(parquetSchema.getType(0).asGroupType(), pigSchema.schema.getField(0), numbersDefaultToZero, columnIndexAccess); }
private Type filterMap(GroupType mapType, FieldSchema mapFieldSchema) throws FrontendException { if (DEBUG) LOG.debug("filtering MAP schema:\n" + mapType + "\nwith:\n " + mapFieldSchema); if (mapType.getFieldCount() != 1) { throw new RuntimeException("not unwrapping the right type, this should be a Map: " + mapType); } GroupType nested = mapType.getType(0).asGroupType(); if (nested.getFieldCount() != 2) { throw new RuntimeException("this should be a Map Key/Value: " + mapType); } FieldSchema innerField = mapFieldSchema.schema.getField(0); return mapType.withNewFields(nested.withNewFields(nested.getType(0), filter(nested.getType(1), innerField))); }
private Type filterBag(GroupType bagType, FieldSchema bagFieldSchema) throws FrontendException { if (DEBUG) LOG.debug("filtering BAG schema:\n" + bagType + "\nwith:\n " + bagFieldSchema); if (bagType.getFieldCount() != 1) { throw new RuntimeException("not unwrapping the right type, this should be a Bag: " + bagType); } Type nested = bagType.getType(0); FieldSchema innerField = bagFieldSchema.schema.getField(0); if (nested.isPrimitive() || nested.getOriginalType() == OriginalType.MAP || nested.getOriginalType() == OriginalType.LIST) { // Bags always contain tuples => we skip the extra tuple that was inserted in that case. innerField = innerField.schema.getField(0); } return bagType.withNewFields(filter(nested, innerField)); } }