/** * Convert a field vector to a column vector * @param fieldVector the field vector to convert * @param type the type of the column vector * @return the converted ndarray */ public static INDArray convertArrowVector(FieldVector fieldVector,ColumnType type) { DataBuffer buffer = null; int cols = fieldVector.getValueCount(); ByteBuffer direct = ByteBuffer.allocateDirect(fieldVector.getDataBuffer().capacity()); direct.order(ByteOrder.nativeOrder()); fieldVector.getDataBuffer().getBytes(0,direct); direct.rewind(); switch(type) { case Integer: buffer = Nd4j.createBuffer(direct, DataBuffer.Type.INT,cols,0); break; case Float: buffer = Nd4j.createBuffer(direct, DataBuffer.Type.FLOAT,cols); break; case Double: buffer = Nd4j.createBuffer(direct, DataBuffer.Type.DOUBLE,cols); break; case Long: buffer = Nd4j.createBuffer(direct, DataBuffer.Type.LONG,cols); break; } return Nd4j.create(buffer,new int[] {cols,1}); }
private void resetFirstAccumulatorVector() { Preconditions.checkArgument(accumulators.length == 1, "Error: incorrect number of batches in accumulator"); final FieldVector vector = accumulators[0]; Preconditions.checkArgument(vector != null, "Error: expecting a valid accumulator"); final ArrowBuf validityBuffer = vector.getValidityBuffer(); final ArrowBuf dataBuffer = vector.getDataBuffer(); validityBuffer.readerIndex(0); validityBuffer.writerIndex(0); dataBuffer.readerIndex(0); dataBuffer.writerIndex(0); initialize(vector); vector.setValueCount(0); }
Preconditions.checkArgument(rawDataLength == bitsLength + dataLength, "Error, read incorrect accumulator vector buffer length"); final ArrowBuf validityBuffer = vector.getValidityBuffer(); final ArrowBuf dataBuffer = vector.getDataBuffer(); readIntoArrowBuf(validityBuffer, bitsLength, input); readIntoArrowBuf(dataBuffer, dataLength, input);
for (FieldVector vector : postSpillAccumulatorVectors) { final ArrowBuf validityBuffer = vector.getValidityBuffer(); final ArrowBuf dataBuffer = vector.getDataBuffer(); validityBuffer.readerIndex(0); validityBuffer.writerIndex(0);