void add(ColumnIO child) { children.add(child); childrenByName.put(child.getType().getName(), child); ++ childrenSize; }
void add(ColumnIO child) { children.add(child); childrenByName.put(child.getType().getName(), child); ++ childrenSize; }
@Override void setLevels(int r, int d, String[] fieldPath, int[] indexFieldPath, List<ColumnIO> repetition, List<ColumnIO> path) { super.setLevels(r, d, fieldPath, indexFieldPath, repetition, path); for (ColumnIO child : this.children) { String[] newFieldPath = Arrays.copyOf(fieldPath, fieldPath.length + 1); int[] newIndexFieldPath = Arrays.copyOf(indexFieldPath, indexFieldPath.length + 1); newFieldPath[fieldPath.length] = child.getType().getName(); newIndexFieldPath[indexFieldPath.length] = child.getIndex(); List<ColumnIO> newRepetition; if (child.getType().isRepetition(REPEATED)) { newRepetition = new ArrayList<ColumnIO>(repetition); newRepetition.add(child); } else { newRepetition = repetition; } List<ColumnIO> newPath = new ArrayList<ColumnIO>(path); newPath.add(child); child.setLevels( // the type repetition level increases whenever there's a possible repetition child.getType().isRepetition(REPEATED) ? r + 1 : r, // the type definition level increases whenever a field can be missing (not required) !child.getType().isRepetition(REQUIRED) ? d + 1 : d, newFieldPath, newIndexFieldPath, newRepetition, newPath ); } }
@Override void setLevels(int r, int d, String[] fieldPath, int[] indexFieldPath, List<ColumnIO> repetition, List<ColumnIO> path) { super.setLevels(r, d, fieldPath, indexFieldPath, repetition, path); for (ColumnIO child : this.children) { String[] newFieldPath = Arrays.copyOf(fieldPath, fieldPath.length + 1); int[] newIndexFieldPath = Arrays.copyOf(indexFieldPath, indexFieldPath.length + 1); newFieldPath[fieldPath.length] = child.getType().getName(); newIndexFieldPath[indexFieldPath.length] = child.getIndex(); List<ColumnIO> newRepetition; if (child.getType().isRepetition(REPEATED)) { newRepetition = new ArrayList<ColumnIO>(repetition); newRepetition.add(child); } else { newRepetition = repetition; } List<ColumnIO> newPath = new ArrayList<ColumnIO>(path); newPath.add(child); child.setLevels( // the type repetition level increases whenever there's a possible repetition child.getType().isRepetition(REPEATED) ? r + 1 : r, // the type definition level increases whenever a field can be missing (not required) !child.getType().isRepetition(REQUIRED) ? d + 1 : d, newFieldPath, newIndexFieldPath, newRepetition, newPath ); } }
private void writeNull(ColumnIO undefinedField, int r, int d) { if (undefinedField.getType().isPrimitive()) { columnWriter[((PrimitiveColumnIO) undefinedField).getId()].writeNull(r, d); } else { GroupColumnIO groupColumnIO = (GroupColumnIO) undefinedField; // only cache the repetition level, the definition level should always be the definition level of the parent node cacheNullForGroup(groupColumnIO, r); } }
private void endListWrapper() { if (size > 0) { recordConsumer.endField(listContent.getType().getName(), 0); } recordConsumer.endGroup(); end(); }
private void startListWrapper() { start(); recordConsumer.startGroup(); if (size > 0) { recordConsumer.startField(listContent.getType().getName(), 0); currentProtocol = contentProtocol; } }
private void writeNull(ColumnIO undefinedField, int r, int d) { if (undefinedField.getType().isPrimitive()) { columnWriter[((PrimitiveColumnIO) undefinedField).getId()].writeNull(r, d); } else { GroupColumnIO groupColumnIO = (GroupColumnIO) undefinedField; // only cache the repetition level, the definition level should always be the definition level of the parent node cacheNullForGroup(groupColumnIO, r); } }
public static ColumnIO getArrayElementColumn(ColumnIO columnIO) { while (columnIO instanceof GroupColumnIO && !columnIO.getType().isRepetition(REPEATED)) { columnIO = ((GroupColumnIO) columnIO).getChild(0); } /* If array has a standard 3-level structure with middle level repeated group with a single field: * optional group my_list (LIST) { * repeated group element { * required binary str (UTF8); * }; * } */ if (columnIO instanceof GroupColumnIO && columnIO.getType().getOriginalType() == null && ((GroupColumnIO) columnIO).getChildrenCount() == 1 && !columnIO.getName().equals("array") && !columnIO.getName().equals(columnIO.getParent().getName() + "_tuple")) { return ((GroupColumnIO) columnIO).getChild(0); } /* Backward-compatibility support for 2-level arrays where a repeated field is not a group: * optional group my_list (LIST) { * repeated int32 element; * } */ return columnIO; }
public static ColumnIO getArrayElementColumn(ColumnIO columnIO) { while (columnIO instanceof GroupColumnIO && !columnIO.getType().isRepetition(REPEATED)) { columnIO = ((GroupColumnIO) columnIO).getChild(0); } /* If array has a standard 3-level structure with middle level repeated group with a single field: * optional group my_list (LIST) { * repeated group element { * required binary str (UTF8); * }; * } */ if (columnIO instanceof GroupColumnIO && columnIO.getType().getOriginalType() == null && ((GroupColumnIO) columnIO).getChildrenCount() == 1 && !columnIO.getName().equals("array") && !columnIO.getName().equals(columnIO.getParent().getName() + "_tuple")) { return ((GroupColumnIO) columnIO).getChild(0); } /* Backward-compatibility support for 2-level arrays where a repeated field is not a group: * optional group my_list (LIST) { * repeated int32 element; * } */ return columnIO; }
ColumnIO getParent(int r) { if (getRepetitionLevel() == r && getType().isRepetition(Repetition.REPEATED)) { return this; } else if (getParent()!=null && getParent().getDefinitionLevel()>=r) { return getParent().getParent(r); } else { throw new InvalidRecordException("no parent("+r+") for "+Arrays.toString(this.getFieldPath())); } }
ColumnIO getParent(int r) { if (getRepetitionLevel() == r && getType().isRepetition(Repetition.REPEATED)) { return this; } else if (getParent()!=null && getParent().getDefinitionLevel()>=r) { return getParent().getParent(r); } else { throw new InvalidRecordException("no parent("+r+") for "+Arrays.toString(this.getFieldPath())); } }
return Optional.empty(); boolean required = columnIO.getType().getRepetition() != OPTIONAL; int repetitionLevel = columnRepetitionLevel(columnIO); int definitionLevel = columnDefinitionLevel(columnIO); RichColumnDescriptor column = new RichColumnDescriptor(primitiveColumnIO.getColumnDescriptor(), columnIO.getType().asPrimitiveType()); return Optional.of(new PrimitiveField(type, repetitionLevel, definitionLevel, required, column, primitiveColumnIO.getId()));