ArrayAccessor(ListVector vector) { super(vector); this.accessor = vector; this.arrayData = new ArrowColumnVector(vector.getDataVector()); }
private void writeList(ListVector arrowVector, ListColumnVector hiveVector, ListTypeInfo typeInfo, int size, VectorizedRowBatch vectorizedRowBatch, boolean isNative) { final int OFFSET_WIDTH = 4; final TypeInfo elementTypeInfo = typeInfo.getListElementTypeInfo(); final ColumnVector hiveElementVector = hiveVector.child; final FieldVector arrowElementVector = (FieldVector) arrowVector.addOrGetVector(toFieldType(elementTypeInfo)).getVector(); arrowElementVector.setInitialCapacity(hiveVector.childCount); arrowElementVector.allocateNew(); write(arrowElementVector, hiveElementVector, elementTypeInfo, hiveVector.childCount, vectorizedRowBatch, isNative); final ArrowBuf offsetBuffer = arrowVector.getOffsetBuffer(); int nextOffset = 0; for (int rowIndex = 0; rowIndex < size; rowIndex++) { if (hiveVector.isNull[rowIndex]) { offsetBuffer.setInt(rowIndex * OFFSET_WIDTH, nextOffset); } else { offsetBuffer.setInt(rowIndex * OFFSET_WIDTH, nextOffset); nextOffset += (int) hiveVector.lengths[rowIndex]; arrowVector.setNotNull(rowIndex); } } offsetBuffer.setInt(size * OFFSET_WIDTH, nextOffset); }
@Override final boolean isNullAt(int rowId) { // TODO: Workaround if vector has all non-null values, see ARROW-1948 if (accessor.getValueCount() > 0 && accessor.getValidityBuffer().capacity() == 0) { return false; } else { return super.isNullAt(rowId); } }
public void load(SerializedField metadata, ArrowBuf buffer) { /* release the current buffers (if any) */ listVector.clear(); /* load inner offset buffer */ final SerializedField offsetMetadata = metadata.getChild(0); final int offsetLength = offsetMetadata.getBufferLength(); loadOffsetBuffer(offsetMetadata, buffer); /* load inner validity buffer */ final SerializedField bitMetadata = metadata.getChild(1); final int bitLength = bitMetadata.getBufferLength(); loadValidityBuffer(bitMetadata, buffer.slice(offsetLength, bitLength)); /* load inner data vector */ final SerializedField vectorMetadata = metadata.getChild(2); if (listVector.getDataVector() == BaseRepeatedValueVector.DEFAULT_DATA_VECTOR) { listVector.addOrGetVector(FieldType.nullable(getArrowMinorType(vectorMetadata.getMajorType().getMinorType()).getType())); } final int vectorLength = vectorMetadata.getBufferLength(); TypeHelper.load(listVector.vector, vectorMetadata, buffer.slice(offsetLength + bitLength, vectorLength)); listVector.setLastSet(metadata.getValueCount()); listVector.valueCount = metadata.getValueCount(); }
/** Helper method which creates a empty list vector */ private static ListVector testEmptyListVector() { final ListVector vector = new ListVector("emptyListVector", ALLOCATOR, FieldType.nullable(ArrowType.Null.INSTANCE), null); vector.allocateNew(); return vector; }
public TransferImpl(ListVector to) { this.to = to; to.addOrGetVector(vector.getField().getFieldType()); if (to.getDataVector() instanceof ZeroVector) { to.addOrGetVector(vector.getField().getFieldType()); } dataTransferPair = getDataVector().makeTransferPair(to.getDataVector()); }
@Override public void initializeChildrenFromFields(List<Field> children) { if (children.size() != 1) { throw new IllegalArgumentException("Lists have only one child. Found: " + children); } Field field = children.get(0); AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(field.getFieldType()); if (!addOrGetVector.isCreated()) { throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector()); } addOrGetVector.getVector().initializeChildrenFromFields(field.getChildren()); }
@Override public FieldVector getNewVector( String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new ListVector(name, allocator, fieldType, schemaChangeCallback); }
@Override final ColumnarArray getArray(int rowId) { ArrowBuf offsets = accessor.getOffsetBuffer(); int index = rowId * ListVector.OFFSET_WIDTH; int start = offsets.getInt(index); int end = offsets.getInt(index + ListVector.OFFSET_WIDTH); return new ColumnarArray(arrayData, start, end - start); } }
@Override public void set(ValueVector v, int index) { this.dataVector = ((ListVector)v).getDataVector(); if(obj != null){ UnionListWriter listWriter = ((ListVector)v).getWriter(); listWriter.setPosition(index); List<Integer> list = obj; listWriter.startList(); for (int i = 0; i < list.size(); i++) { listWriter.bigInt().writeBigInt(list.get(i)); } listWriter.endList(); } }
/** * Get the element in the list vector at a particular index. * @param index position of the element * @return Object at given position */ @Override public Object getObject(int index) { if (isSet(index) == 0) { return null; } final List<Object> vals = new JsonStringArrayList<>(); final int start = offsetBuffer.getInt(index * OFFSET_WIDTH); final int end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); final ValueVector vv = getDataVector(); for (int i = start; i < end; i++) { vals.add(vv.getObject(i)); } return vals; }
public SerializedField.Builder getMetadataBuilder() { return SerializedField.newBuilder() .setMajorType(MajorType.newBuilder().setMinorType(MinorType.LIST).setMode(DataMode.OPTIONAL).build()) .setNamePart(NamePart.newBuilder().setName(listVector.getField().getName())) .setValueCount(listVector.getValueCount()) .setBufferLength(listVector.getBufferSize()) .addChild(buildOffsetMetadata()) .addChild(buildValidityMetadata()) .addChild(TypeHelper.getMetadata(listVector.vector)); }
@Test public void list() throws InterruptedException { try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); ListVector inVector = new ListVector("input", allocator, null)) { UnionListWriter writer = inVector.getWriter(); writer.allocate(); writer.setPosition(0);
ListVector listVector = new ListVector(field.getName(), allocator, callBack); if(!children.isEmpty()){ listVector.initializeChildrenFromFields(children);
setReaderAndWriterIndex(); final ArrowBuf[] buffers; if (getBufferSize() == 0) { buffers = new ArrowBuf[0]; } else { buffer.retain(); clear();
@Override public void startList() { vector.startNewValue(idx()); writer.setPosition(vector.getOffsetBuffer().getInt((idx() + 1) * OFFSET_WIDTH)); }
@Override public void allocate() { vector.allocateNew(); }
private void writeMap(ListVector arrowVector, MapColumnVector hiveVector, MapTypeInfo typeInfo, int size, VectorizedRowBatch vectorizedRowBatch, boolean isNative) { final ListTypeInfo structListTypeInfo = toStructListTypeInfo(typeInfo); final ListColumnVector structListVector = toStructListVector(hiveVector); write(arrowVector, structListVector, structListTypeInfo, size, vectorizedRowBatch, isNative); final ArrowBuf validityBuffer = arrowVector.getValidityBuffer(); for (int rowIndex = 0; rowIndex < size; rowIndex++) { if (hiveVector.isNull[rowIndex]) { BitVectorHelper.setValidityBit(validityBuffer, rowIndex, 0); } else { BitVectorHelper.setValidityBitToOne(validityBuffer, rowIndex); } } }
/** * Slice this vector at desired index and length and transfer the * corresponding data to the target vector. * @param startIndex start position of the split in source vector. * @param length length of the split. */ @Override public void splitAndTransfer(int startIndex, int length) { final int startPoint = offsetBuffer.getInt(startIndex * OFFSET_WIDTH); final int sliceLength = offsetBuffer.getInt((startIndex + length) * OFFSET_WIDTH) - startPoint; to.clear(); to.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH); /* splitAndTransfer offset buffer */ for (int i = 0; i < length + 1; i++) { final int relativeOffset = offsetBuffer.getInt((startIndex + i) * OFFSET_WIDTH) - startPoint; to.offsetBuffer.setInt(i * OFFSET_WIDTH, relativeOffset); } /* splitAndTransfer validity buffer */ splitAndTransferValidityBuffer(startIndex, length, to); /* splitAndTransfer data buffer */ dataTransferPair.splitAndTransfer(startPoint, sliceLength); to.lastSet = length; to.setValueCount(length); }
/** * Allocate memory for the vector. We internally use a default value count * of 4096 to allocate memory for at least these many elements in the * vector. * * @return false if memory allocation fails, true otherwise. */ public boolean allocateNewSafe() { boolean success = false; try { /* we are doing a new allocation -- release the current buffers */ clear(); /* allocate validity buffer */ allocateValidityBuffer(validityAllocationSizeInBytes); /* allocate offset and data buffer */ success = super.allocateNewSafe(); } finally { if (!success) { clear(); return false; } } return true; }