public static GroupColumnIO getMapKeyValueColumn(GroupColumnIO groupColumnIO) { while (groupColumnIO.getChildrenCount() == 1) { groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0); } return groupColumnIO; }
@Override public MessageType getType() { return (MessageType) super.getType(); } }
public List<String[]> getColumnNames() { return super.getColumnNames(); }
public static ColumnIO getArrayElementColumn(ColumnIO columnIO) { while (columnIO instanceof GroupColumnIO && !columnIO.getType().isRepetition(REPEATED)) { columnIO = ((GroupColumnIO) columnIO).getChild(0); } /* If array has a standard 3-level structure with middle level repeated group with a single field: * optional group my_list (LIST) { * repeated group element { * required binary str (UTF8); * }; * } */ if (columnIO instanceof GroupColumnIO && columnIO.getType().getOriginalType() == null && ((GroupColumnIO) columnIO).getChildrenCount() == 1 && !columnIO.getName().equals("array") && !columnIO.getName().equals(columnIO.getParent().getName() + "_tuple")) { return ((GroupColumnIO) columnIO).getChild(0); } /* Backward-compatibility support for 2-level arrays where a repeated field is not a group: * optional group my_list (LIST) { * repeated int32 element; * } */ return columnIO; }
public ListWriteProtocol(GroupColumnIO columnIO, ThriftField values, Events returnClause) { super(returnClause); this.listContent = columnIO.getChild(0); this.contentProtocol = getProtocol(values, listContent, new Events() { int consumedRecords = 0; @Override public void start() { } @Override public void end() { ++ consumedRecords; if (consumedRecords == size) { currentProtocol = ListWriteProtocol.this; consumedRecords = 0; } } }); }
private void writeNullToLeaves(GroupColumnIO group) { IntArrayList nullCache = groupNullCache.get(group); if (nullCache == null || nullCache.isEmpty()) return; int parentDefinitionLevel = group.getParent().getDefinitionLevel(); for (ColumnWriter leafWriter : groupToLeafWriter.get(group)) { for (IntIterator iter = nullCache.iterator(); iter.hasNext();) { int repetitionLevel = iter.nextInt(); leafWriter.writeNull(repetitionLevel, parentDefinitionLevel); } } nullCache.clear(); }
@Override public void startMessage() { if (DEBUG) log("< MESSAGE START >"); currentColumnIO = MessageColumnIO.this; r[0] = 0; int numberOfFieldsToVisit = ((GroupColumnIO) currentColumnIO).getChildrenCount(); fieldsWritten[0].reset(numberOfFieldsToVisit); if (DEBUG) printState(); }
@Override public void visit(GroupType groupType) { if (currentRequestedType.isPrimitive()) { incompatibleSchema(groupType, currentRequestedType); } GroupColumnIO newIO = new GroupColumnIO(groupType, current, currentRequestedIndex); current.add(newIO); visitChildren(newIO, groupType, currentRequestedType.asGroupType()); }
@Override public void visit(PrimitiveType primitiveType) { if (!currentRequestedType.isPrimitive() || (this.strictTypeChecking && currentRequestedType.asPrimitiveType().getPrimitiveTypeName() != primitiveType.getPrimitiveTypeName())) { incompatibleSchema(primitiveType, currentRequestedType); } PrimitiveColumnIO newIO = new PrimitiveColumnIO(primitiveType, current, currentRequestedIndex, leaves.size()); current.add(newIO); leaves.add(newIO); }
private void buildGroupToLeafWriterMap(PrimitiveColumnIO primitive, ColumnWriter writer) { GroupColumnIO parent = primitive.getParent(); do { getLeafWriters(parent).add(writer); parent = parent.getParent(); } while (parent != null); }
public static ColumnIO getArrayElementColumn(ColumnIO columnIO) { while (columnIO instanceof GroupColumnIO && !columnIO.getType().isRepetition(REPEATED)) { columnIO = ((GroupColumnIO) columnIO).getChild(0); } /* If array has a standard 3-level structure with middle level repeated group with a single field: * optional group my_list (LIST) { * repeated group element { * required binary str (UTF8); * }; * } */ if (columnIO instanceof GroupColumnIO && columnIO.getType().getOriginalType() == null && ((GroupColumnIO) columnIO).getChildrenCount() == 1 && !columnIO.getName().equals("array") && !columnIO.getName().equals(columnIO.getParent().getName() + "_tuple")) { return ((GroupColumnIO) columnIO).getChild(0); } /* Backward-compatibility support for 2-level arrays where a repeated field is not a group: * optional group my_list (LIST) { * repeated int32 element; * } */ return columnIO; }
public MapWriteProtocol(GroupColumnIO columnIO, MapType type, Events returnClause) { super(returnClause); this.mapContent = (GroupColumnIO)columnIO.getChild(0); this.key = mapContent.getChild(0); this.value = mapContent.getChild(1); this.keyProtocol = getProtocol(type.getKey(), this.key, new Events() { @Override
private void writeNullToLeaves(GroupColumnIO group) { IntArrayList nullCache = groupNullCache.get(group); if (nullCache == null || nullCache.isEmpty()) return; int parentDefinitionLevel = group.getParent().getDefinitionLevel(); for (ColumnWriter leafWriter : groupToLeafWriter.get(group)) { for (IntIterator iter = nullCache.iterator(); iter.hasNext();) { int repetitionLevel = iter.nextInt(); leafWriter.writeNull(repetitionLevel, parentDefinitionLevel); } } nullCache.clear(); }
@Override public void startMessage() { if (DEBUG) log("< MESSAGE START >"); currentColumnIO = MessageColumnIO.this; r[0] = 0; int numberOfFieldsToVisit = ((GroupColumnIO) currentColumnIO).getChildrenCount(); fieldsWritten[0].reset(numberOfFieldsToVisit); if (DEBUG) printState(); }
@Override public void visit(GroupType groupType) { if (currentRequestedType.isPrimitive()) { incompatibleSchema(groupType, currentRequestedType); } GroupColumnIO newIO = new GroupColumnIO(groupType, current, currentRequestedIndex); current.add(newIO); visitChildren(newIO, groupType, currentRequestedType.asGroupType()); }
@Override public void visit(PrimitiveType primitiveType) { if (!currentRequestedType.isPrimitive() || (this.strictTypeChecking && currentRequestedType.asPrimitiveType().getPrimitiveTypeName() != primitiveType.getPrimitiveTypeName())) { incompatibleSchema(primitiveType, currentRequestedType); } PrimitiveColumnIO newIO = new PrimitiveColumnIO(primitiveType, current, currentRequestedIndex, leaves.size()); current.add(newIO); leaves.add(newIO); }
private void buildGroupToLeafWriterMap(PrimitiveColumnIO primitive, ColumnWriter writer) { GroupColumnIO parent = primitive.getParent(); do { getLeafWriters(parent).add(writer); parent = parent.getParent(); } while (parent != null); }
public static GroupColumnIO getMapKeyValueColumn(GroupColumnIO groupColumnIO) { while (groupColumnIO.getChildrenCount() == 1) { groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0); } return groupColumnIO; }
@Override public void startField(String field, int index) { try { if (DEBUG) log("startField({}, {})", field, index); currentColumnIO = ((GroupColumnIO) currentColumnIO).getChild(index); emptyField = true; if (DEBUG) printState(); } catch (RuntimeException e) { throw new ParquetEncodingException("error starting field " + field + " at " + index, e); } }
ColumnIO getParent(int r) { if (getRepetitionLevel() == r && getType().isRepetition(Repetition.REPEATED)) { return this; } else if (getParent()!=null && getParent().getDefinitionLevel()>=r) { return getParent().getParent(r); } else { throw new InvalidRecordException("no parent("+r+") for "+Arrays.toString(this.getFieldPath())); } }