public VectorSchemaRoot(Iterable<FieldVector> vectors) { this( StreamSupport.stream(vectors.spliterator(), false).map(t -> t.getField()).collect(Collectors.toList()), StreamSupport.stream(vectors.spliterator(), false).collect(Collectors.toList()), 0 ); }
private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) { nodes.add(new ArrowFieldNode(vector.getValueCount(), includeNullCount ? vector.getNullCount() : -1)); List<ArrowBuf> fieldBuffers = vector.getFieldBuffers(); List<BufferType> expectedBuffers = TypeLayout.getTypeLayout(vector.getField().getType()).getBufferTypes(); if (fieldBuffers.size() != expectedBuffers.size()) { throw new IllegalArgumentException(String.format( "wrong number of buffers for field %s in vector %s. found: %s", vector.getField(), vector.getClass().getSimpleName(), fieldBuffers)); } buffers.addAll(fieldBuffers); for (FieldVector child : vector.getChildrenFromFields()) { appendNodes(child, nodes, buffers); } }
public ArrowType getVectorType() { return dictionary.getField().getType(); }
protected List<String> getChildFieldNames() { return getChildren().stream() .map(child -> child.getField().getName()) .collect(Collectors.toList()); }
@Override public Field getField() { List<Field> children = Collections.singletonList(getDataVector().getField()); return new Field(name, fieldType, children); }
/** * Creates the vector schema from incoming container and referenced fields. * @param input * @param referencedFields * @return the vector schema root. */ public static VectorSchemaRoot getSchemaRoot(VectorAccessible input, Set referencedFields) { List<FieldVector> fv = ImmutableList.copyOf(input) .stream() .map(vw -> ((FieldVector)vw.getValueVector())) .filter(fVec -> referencedFields.contains(fVec.getField())) .collect(Collectors.toList()); List<Field> fields = fv.stream() .map(fieldVec -> fieldVec.getField()) .collect(Collectors.toList()); Schema schemaWithOnlyReferencedFields = new Schema(fields); VectorSchemaRoot root = new VectorSchemaRoot( schemaWithOnlyReferencedFields, fv, 0 ); return root; } }
@Override public Field getField() { return new Field(name, fieldType, Collections.singletonList(getDataVector().getField())); }
@Override public String toString() { return "VectorPivotDef [type=" + type + ", nullByteOffset=" + nullByteOffset + ", nullBitOffset=" + nullBitOffset + ", offset=" + offset + ", vector=" + Describer.describe(vector.getIncoming().getField()) + "]"; }
private TreeNode visitValueVectorReadExpression(ValueVectorReadExpression readExpr, Void value) { FieldVector vector = incoming.getValueAccessorById(FieldVector.class, readExpr.getTypedFieldId().getFieldIds()) .getValueVector(); referencedFields.add(vector.getField()); return TreeBuilder.makeField(vector.getField()); }
public static Reallocator getReallocator(FieldVector vect){ if(vect instanceof VarCharVector){ return new VarCharReallocator(((VarCharVector) vect)); }else if(vect instanceof VarBinaryVector){ return new VarBinaryReallocator(((VarBinaryVector) vect)); }else{ throw new IllegalStateException("Invalid vector: " + Describer.describe(vect.getField())); } } }
private void writeDictionaryBatches(JsonGenerator generator, Set<Long> dictionaryIdsUsed, DictionaryProvider provider) throws IOException { generator.writeArrayFieldStart("dictionaries"); for (Long id : dictionaryIdsUsed) { generator.writeStartObject(); generator.writeObjectField("id", id); generator.writeFieldName("data"); Dictionary dictionary = provider.lookup(id); FieldVector vector = dictionary.getVector(); List<Field> fields = Collections.singletonList(vector.getField()); List<FieldVector> vectors = Collections.singletonList(vector); VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getValueCount()); writeBatch(root); generator.writeEndObject(); } generator.writeEndArray(); }
/** * Take an expression tree and convert it into a Gandiva Expression. */ public static ExpressionTree serializeExpr(VectorAccessible incoming, LogicalExpression ex, FieldVector out, Set<Field> referencedFields) { GandivaExpressionBuilder serializer = new GandivaExpressionBuilder(incoming, referencedFields); TreeNode expr = ex.accept(serializer, null); return TreeBuilder.makeExpression(expr, out.getField()); }
public TransferImpl(FixedSizeListVector to) { this.to = to; to.addOrGetVector(vector.getField().getFieldType()); dataPair = vector.makeTransferPair(to.vector); }
/** * Loads the record batch in the vectors. * will not close the record batch * * @param recordBatch the batch to load */ public void load(ArrowRecordBatch recordBatch) { Iterator<ArrowBuf> buffers = recordBatch.getBuffers().iterator(); Iterator<ArrowFieldNode> nodes = recordBatch.getNodes().iterator(); for (FieldVector fieldVector : root.getFieldVectors()) { loadBuffers(fieldVector, fieldVector.getField(), buffers, nodes); } root.setRowCount(recordBatch.getLength()); if (nodes.hasNext() || buffers.hasNext()) { throw new IllegalArgumentException("not all nodes and buffers were consumed. nodes: " + Collections2.toList(nodes).toString() + " buffers: " + Collections2.toList(buffers).toString()); } }
public VectorSchemaRoot(FieldVector parent) { this(parent.getField().getChildren(), parent.getChildrenFromFields(), parent.getValueCount()); }
@Override @SuppressWarnings("unchecked") public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType type) { boolean created = false; if (vector == ZeroVector.INSTANCE) { vector = type.createNewSingleVector(DATA_VECTOR_NAME, allocator, null); this.reader = new UnionFixedSizeListReader(this); created = true; } // returned vector must have the same field if (!Objects.equals(vector.getField().getType(), type.getType())) { final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]", type.getType(), vector.getField().getType()); throw new SchemaChangeRuntimeException(msg); } return new AddOrGetResult<>((T) vector, created); }
/** * Load an ArrowDictionaryBatch to the readers dictionary vectors. * * @param dictionaryBatch dictionary batch to load */ protected void loadDictionary(ArrowDictionaryBatch dictionaryBatch) { long id = dictionaryBatch.getDictionaryId(); Dictionary dictionary = dictionaries.get(id); if (dictionary == null) { throw new IllegalArgumentException("Dictionary ID " + id + " not defined in schema"); } FieldVector vector = dictionary.getVector(); VectorSchemaRoot root = new VectorSchemaRoot( Collections.singletonList(vector.getField()), Collections.singletonList(vector), 0); VectorLoader loader = new VectorLoader(root); try { loader.load(dictionaryBatch.getDictionary()); } finally { dictionaryBatch.close(); } } }
public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldType) { boolean created = false; if (vector instanceof ZeroVector) { vector = fieldType.createNewSingleVector(DATA_VECTOR_NAME, allocator, callBack); // returned vector must have the same field created = true; if (callBack != null && // not a schema change if changing from ZeroVector to ZeroVector (fieldType.getType().getTypeID() != ArrowTypeID.Null)) { callBack.doWork(); } } if (vector.getField().getType().getTypeID() != fieldType.getType().getTypeID()) { final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]", fieldType.getType().getTypeID(), vector.getField().getType().getTypeID()); throw new SchemaChangeRuntimeException(msg); } return new AddOrGetResult<>((T) vector, created); }
public static void pivot(PivotDef pivot, int count, FixedBlockVector fixedBlock, VariableBlockVector variable) { fixedBlock.ensureAvailableBlocks(count); for(VectorPivotDef def : pivot.getFixedPivots()){ switch(def.getType()){ case BIT: pivotBit(def, fixedBlock, count); break; case FOUR_BYTE: pivot4Bytes(def, fixedBlock, count); break; case EIGHT_BYTE: pivot8Bytes(def, fixedBlock, count); break; case SIXTEEN_BYTE: pivot16Bytes(def, fixedBlock, count); break; case VARIABLE: default: throw new UnsupportedOperationException("Pivot: unknown type: " + Describer.describe(def.getIncomingVector().getField())); } } if(pivot.getVariableCount() > 0){ pivotVariableLengths(pivot.getVariablePivots(), fixedBlock, variable, count); } }