public ArrowWrapperWritable serializeBatch(VectorizedRowBatch vectorizedRowBatch, boolean isNative) { rootVector.setValueCount(0); if(rootVector.getChild(fieldName) != null) { fieldExists = true; final FieldVector arrowVector = rootVector.addOrGet(fieldName, fieldType, FieldVector.class); if(fieldExists) { arrowVector.setValueCount(isNative ? vectorizedRowBatch.size : batchSize); rootVector.setValueCount(batchSize); } else { rootVector.setValueCount(vectorizedRowBatch.size);
public ArrowWrapperWritable emptyBatch() { rootVector.setValueCount(0); for (int fieldIndex = 0; fieldIndex < fieldTypeInfos.size(); fieldIndex++) { final TypeInfo fieldTypeInfo = fieldTypeInfos.get(fieldIndex); final String fieldName = fieldNames.get(fieldIndex); final FieldType fieldType = toFieldType(fieldTypeInfo); final FieldVector arrowVector = rootVector.addOrGet(fieldName, fieldType, FieldVector.class); arrowVector.setInitialCapacity(0); arrowVector.allocateNew(); } VectorSchemaRoot vectorSchemaRoot = new VectorSchemaRoot(rootVector); return new ArrowWrapperWritable(vectorSchemaRoot, allocator, rootVector); }
@Override public UInt1Writer uInt1(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); UInt1Vector v = container.addOrGet(name, FieldType.nullable( MinorType.UINT1.getType() ), UInt1Vector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.UINT1); } } return writer; }
private VarBinaryVector getOrCreateColumnVector(StructVector sv, String qualifier) { int oldSize = sv.size(); VarBinaryVector v = sv.addOrGet(qualifier, FieldType.nullable(getArrowMinorType(COLUMN_TYPE.getMinorType()).getType()), VarBinaryVector.class); if (oldSize != sv.size()) { v.allocateNew(); } return v; }
final StructVector mapVector = new StructVector("map", allocator, mapType, null); final ArrowBuf tempBuf = allocator.buffer(2048)) { AllocationHelper.allocateNew(intVector, records); NullableStructWriter mapWriter = mapVector.getWriter(); for (int i = 0; i < records; i++) { intVector.set(i, intBaseValue + i); mapVector.setIndexDefined(i); mapVector.setValueCount(records); "int", i, "bits", listOf(true, false) ), mapVector.getObject(i) );
accessor = new StructAccessor(structVector); childColumns = new ArrowColumnVector[structVector.size()]; for (int i = 0; i < childColumns.length; ++i) { childColumns[i] = new ArrowColumnVector(structVector.getVectorById(i));
public void load(SerializedField metadata, ArrowBuf buf) { structVector.clear();; for (Field childField : structVector.getField().getChildren()) { children.add(childField.getName().toLowerCase()); FieldVector vector = structVector.getChild(fieldDef.getName()); if (vector == null) { structVector.putChild(fieldDef.getName(), vector); FieldVector childVector = structVector.getChild(remaingChild); childVector.allocateNew(); childVector.setValueCount(metadata.getValueCount());
@Override public FieldVector getNewVector( String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new StructVector(name, allocator, fieldType, schemaChangeCallback); }
public NullableStructWriter(StructVector container) { this.container = container; this.initialCapacity = 0; for (Field child : container.getField().getChildren()) { MinorType minorType = Types.getMinorTypeForArrowType(child.getType()); switch (minorType) { break; case UNION: UnionWriter writer = new UnionWriter(container.addOrGet(child.getName(), FieldType.nullable(MinorType.UNION.getType()), UnionVector.class), getNullableStructWriterFactory()); fields.put(handleCase(child.getName()), writer); break;
setReaderAndWriterIndex(); final ArrowBuf[] buffers; if (getBufferSize() == 0) { buffers = new ArrowBuf[0]; } else { buffer.retain(); clear();
public SerializedField getMetadata() { int bufferSize = structVector.getBufferSize(); SerializedField.Builder b = SerializedField.newBuilder() .setNamePart(NamePart.newBuilder().setName(structVector.getField().getName())) .setMajorType(Types.optional(MinorType.STRUCT)) .setBufferLength(bufferSize) .setValueCount(structVector.valueCount); b.addChild(buildValidityMetadata()); int expectedBufferSize = BaseValueVectorHelper.getValidityBufferSizeFromCount(structVector.valueCount); for(ValueVector v : structVector.getChildren()) { SerializedField metadata = TypeHelper.getMetadata(v); expectedBufferSize += metadata.getBufferLength(); b.addChild(metadata); } Preconditions.checkState(expectedBufferSize == bufferSize, "Invalid buffer count: %s != %s", expectedBufferSize, bufferSize); return b.build(); }
@Override public void start() { container.setIndexDefined(idx()); }
@Override public boolean isEmptyStruct() { return 0 == container.size(); }
StructVector structVector = new StructVector(field.getName(), allocator, callBack); if(!children.isEmpty()){ structVector.initializeChildrenFromFields(children);
addChild(addChild(addChild(l3StringsVector, LIST), LIST), VARCHAR); structVector = container.addOrGet(map); structVector.addOrGet("varchar", nullable(VARCHAR.getType()), VarCharVector.class); structVector.addOrGet("int", nullable(INT.getType()), IntVector.class); ListVector listVector = structVector.addOrGet("bits", nullable(LIST.getType()), ListVector.class); addChild(listVector, BIT);
@Override public void allocate() { container.allocateNew(); for(final FieldWriter w : fields.values()) { w.allocate(); } }
@Override public void clear() { container.clear(); for(final FieldWriter w : fields.values()) { w.clear(); } }
public void setValueCount(int count) { container.setValueCount(count); }
@Override public boolean allocateNewSafe() { /* Boolean to keep track if all the memory allocations were successful * Used in the case of composite vectors when we need to allocate multiple * buffers for multiple vectors. If one of the allocations failed we need to * clear all the memory that we allocated */ boolean success = false; try { clear(); allocateValidityBuffer(validityAllocationSizeInBytes); success = super.allocateNewSafe(); } finally { if (!success) { clear(); return false; } } return true; }
@Override public Field getField() { return container.getField(); }