@Override public void allocate() { vector.allocateNew(); }
public void allocate(){ vector.allocateNew(); }
/** * Returns a vector representing a tensor view * of each ndarray. * Each ndarray will be a "row" represented as a tensor object * with in the return {@link VarBinaryVector} * @param bufferAllocator the buffer allocator to use * @param name the name of the column * @param data the input arrays * @return */ public static VarBinaryVector vectorFor(BufferAllocator bufferAllocator,String name,INDArray[] data) { VarBinaryVector ret = new VarBinaryVector(name,bufferAllocator); ret.allocateNew(); for(int i = 0; i < data.length; i++) { //slice the databuffer to use only the needed portion of the buffer //for proper offset ByteBuffer byteBuffer = BinarySerde.toByteBuffer(data[i]); ret.set(i,byteBuffer,0,byteBuffer.capacity()); } return ret; }
@Override public int next() { final Stopwatch watch = Stopwatch.createStarted(); if (keyVector != null) { keyVector.clear(); keyVector.allocateNew(); } if (valueVector != null) { valueVector.clear(); valueVector.allocateNew(); } int recordCount = 0; int batchSize = 0; try { while (recordCount < numRowsPerBatch && batchSize < numBytesPerBatch && reader.next(key, value)) { keyVector.setSafe(recordCount, key.getBytes(), 0, key.getLength()); valueVector.setSafe(recordCount, value.getBytes(), 0, value.getLength()); batchSize += (key.getLength() + value.getLength()); ++recordCount; } keyVector.setValueCount(recordCount); valueVector.setValueCount(recordCount); logger.debug("Read {} records in {} ms", recordCount, watch.elapsed(TimeUnit.MILLISECONDS)); return recordCount; } catch (IOException ioe) { close(); throw UserException.dataReadError(ioe).addContext("File Path", split.getPath().toString()).build(logger); } }
static void populate(VarBinaryVector vector, byte[][] values){ vector.allocateNew(); Random r = new Random(); for(int i =0; i < values.length; i++){ byte[] val = values[i]; if(val != null){ vector.setSafe(i, val, 0, val.length); } else { // add noise. this confirms that after pivot, noise is gone. byte[] bytes = new byte[r.nextInt(15)]; r.nextBytes(bytes); vector.setSafe(i, bytes, 0, bytes.length); vector.setNull(i); } } vector.setValueCount(values.length); }
private VarBinaryVector getOrCreateColumnVector(StructVector sv, String qualifier) { int oldSize = sv.size(); VarBinaryVector v = sv.addOrGet(qualifier, FieldType.nullable(getArrowMinorType(COLUMN_TYPE.getMinorType()).getType()), VarBinaryVector.class); if (oldSize != sv.size()) { v.allocateNew(); } return v; }
private static VectorContainer buildBinaryGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) { final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.Binary(), null); final VectorContainer input = new VectorContainer(bufferAllocator); final VarBinaryVector binaryVector = input.addOrGet(field); binaryVector.allocateNew(); final SortedSet<Binary> values = new TreeSet<>(); for (Dictionary dictionary : dictionaries) { for (int i = 0; i <= dictionary.getMaxId(); ++i) { values.add(dictionary.decodeToBinary(i)); } } if (existingDict != null) { final VarBinaryVector existingDictValues = existingDict.getValueAccessorById(VarBinaryVector.class, 0).getValueVector(); for (int i = 0; i < existingDict.getRecordCount(); ++i) { values.add(Binary.fromConstantByteArray(existingDictValues.get(i))); } } final Iterator<Binary> iter = values.iterator(); int recordCount = 0; while (iter.hasNext()) { final byte[] data = iter.next().getBytes(); binaryVector.setSafe(recordCount++, data, 0, data.length); } binaryVector.setValueCount(recordCount); input.setRecordCount(recordCount); input.buildSchema(BatchSchema.SelectionVectorMode.NONE); return input; }
private static Pair<VarBinaryVector, ResultVerifier> testVarBinaryVector(final int startIndexInCurrentOutput, final int startIndexInJob) { VarBinaryVector colVarBinaryV = new VarBinaryVector("colVarBinary", allocator); colVarBinaryV.allocateNew(500, 5); colVarBinaryV.set(0, "value1".getBytes()); colVarBinaryV.set(1,
if (rowKeyVector != null) { rowKeyVector.clear(); rowKeyVector.allocateNew();
public void loadData(SerializedField metadata, ArrowBuf buffer) { /* clear the current buffers (if any) */ vector.clear(); /* get the metadata children */ final SerializedField bitsField = metadata.getChild(0); final SerializedField valuesField = metadata.getChild(1); final int valuesLength = buffer.capacity(); vector.allocateNew(valuesLength, metadata.getValueCount()); /* set inner validity buffer */ setValidityBuffer(bitsField); /* load inner offset and value buffers */ vector.offsetBuffer.close(); vector.valueBuffer.close(); loadOffsetAndDataBuffer(valuesField, buffer.slice(0, valuesLength)); vector.setLastSet(metadata.getValueCount() - 1); }
binaryVector.allocateNew(); binaryVector.setSafe(0, "abc".getBytes(UTF8), 0, 3); binaryVector.setSafe(1, "bcd".getBytes(UTF8), 0, 3);