@Override List<String[]> getColumnNames() { return Arrays.asList(new String[][] { getFieldPath() }); }
@Override List<String[]> getColumnNames() { return Arrays.asList(new String[][] { getFieldPath() }); }
public MessageColumnIORecordConsumer(ColumnWriteStore columns) { int maxDepth = 0; this.columnWriter = new ColumnWriter[MessageColumnIO.this.getLeaves().size()]; for (PrimitiveColumnIO primitiveColumnIO : MessageColumnIO.this.getLeaves()) { maxDepth = Math.max(maxDepth, primitiveColumnIO.getFieldPath().length); columnWriter[primitiveColumnIO.getId()] = columns.getColumnWriter(primitiveColumnIO.getColumnDescriptor()); } currentIndex = new int[maxDepth]; r = new int[maxDepth]; }
public MessageColumnIORecordConsumer(ColumnWriteStore columns) { this.columns = columns; int maxDepth = 0; this.columnWriter = new ColumnWriter[MessageColumnIO.this.getLeaves().size()]; for (PrimitiveColumnIO primitiveColumnIO : MessageColumnIO.this.getLeaves()) { maxDepth = Math.max(maxDepth, primitiveColumnIO.getFieldPath().length); columnWriter[primitiveColumnIO.getId()] = columns.getColumnWriter(primitiveColumnIO.getColumnDescriptor()); } fieldsWritten = new FieldsMarker[maxDepth]; for (int i = 0; i < maxDepth; i++) { fieldsWritten[i] = new FieldsMarker(); } r = new int[maxDepth]; }
private void validateFSA(int[][] expectedFSA, MessageColumnIO columnIO, RecordReaderImplementation<?> recordReader) { log("FSA: ----"); List<PrimitiveColumnIO> leaves = columnIO.getLeaves(); for (int i = 0; i < leaves.size(); ++i) { PrimitiveColumnIO primitiveColumnIO = leaves.get(i); log(Arrays.toString(primitiveColumnIO.getFieldPath())); for (int r = 0; r < expectedFSA[i].length; r++) { int next = expectedFSA[i][r]; log(" "+r+" -> "+ (next==leaves.size() ? "end" : Arrays.toString(leaves.get(next).getFieldPath()))+": "+recordReader.getNextLevel(i, r)); assertEquals(Arrays.toString(primitiveColumnIO.getFieldPath())+": "+r+" -> ", next, recordReader.getNextReader(i, r)); } } log("----"); }
} else { // otherwise close until the next common parent levelToClose[i][nextRepLevel] = getCommonParentLevel( leafColumnIO.getFieldPath(), leaves[nextColIdx].getFieldPath()); if (levelToClose[i][nextRepLevel] > leaves[i].getFieldPath().length-1) { throw new ParquetEncodingException(Arrays.toString(leaves[i].getFieldPath())+" -("+nextRepLevel+")-> "+levelToClose[i][nextRepLevel]);
private State(int id, PrimitiveColumnIO primitiveColumnIO, ColumnReader column, int[] nextLevel, GroupConverter[] groupConverterPath, PrimitiveConverter primitiveConverter) { this.id = id; this.primitiveColumnIO = primitiveColumnIO; this.maxDefinitionLevel = primitiveColumnIO.getDefinitionLevel(); this.maxRepetitionLevel = primitiveColumnIO.getRepetitionLevel(); this.column = column; this.nextLevel = nextLevel; this.groupConverterPath = groupConverterPath; this.primitiveConverter = primitiveConverter; this.primitive = primitiveColumnIO.getType().asPrimitiveType().getPrimitiveTypeName(); this.fieldPath = primitiveColumnIO.getFieldPath(); this.primitiveField = fieldPath[fieldPath.length - 1]; this.indexFieldPath = primitiveColumnIO.getIndexFieldPath(); this.primitiveFieldIndex = indexFieldPath[indexFieldPath.length - 1]; }
} else { // otherwise close until the next common parent nextLevel[i][r] = getCommonParentLevel( primitiveColumnIO.getFieldPath(), leaves[next].getFieldPath()); if (nextLevel[i][r] > leaves[i].getFieldPath().length-1) { throw new ParquetEncodingException(Arrays.toString(leaves[i].getFieldPath())+" -("+r+")-> "+nextLevel[i][r]);
private State(int id, PrimitiveColumnIO primitiveColumnIO, ColumnReader column, int[] nextLevel, GroupConverter[] groupConverterPath, PrimitiveConverter primitiveConverter) { this.id = id; this.primitiveColumnIO = primitiveColumnIO; this.maxDefinitionLevel = primitiveColumnIO.getDefinitionLevel(); this.maxRepetitionLevel = primitiveColumnIO.getRepetitionLevel(); this.column = column; this.nextLevel = nextLevel; this.groupConverterPath = groupConverterPath; this.primitiveConverter = primitiveConverter; this.primitive = primitiveColumnIO.getType().asPrimitiveType().getPrimitiveTypeName(); this.fieldPath = primitiveColumnIO.getFieldPath(); this.primitiveField = fieldPath[fieldPath.length - 1]; this.indexFieldPath = primitiveColumnIO.getIndexFieldPath(); this.primitiveFieldIndex = indexFieldPath[indexFieldPath.length - 1]; }