@Override @SuppressWarnings("unchecked") public VectorWrapper<T> cloneAndTransfer(BufferAllocator allocator, CallBack callback) { if(vectors.size() == 0){ return new HyperVectorWrapper<T>(field, (T[])vectors.toArray(), false); }else { T[] newVectors = (T[]) Array.newInstance(vectors.get(0).getClass(), vectors.size()); for(int i =0; i < vectors.size(); i++){ newVectors[i] = (T) vectors.get(i).getTransferPair(vectors.get(i).getField().getName(), allocator, callback).getTo(); } return new HyperVectorWrapper<T>(field, (T[])vectors.toArray(), false); } }
@Override public VectorAccessible setup(VectorAccessible accessible) throws Exception { state.is(State.NEEDS_SETUP); List<TransferPair> pairs = new ArrayList<>(); for(VectorWrapper<?> w : accessible){ TransferPair pair = w.getValueVector().getTransferPair(context.getAllocator()); pairs.add(pair); outgoing.add(pair.getTo()); } transfers = ImmutableList.copyOf(pairs); outgoing.buildSchema(SelectionVectorMode.NONE); outgoing.setInitialCapacity(context.getTargetBatchSize()); state = State.CAN_CONSUME; return outgoing; }
@Override public VectorAccessible setup(VectorAccessible left, VectorAccessible right) throws Exception { state.is(State.NEEDS_SETUP); List<Field> leftFields = left.getSchema().getFields(); List<Field> rightFields = right.getSchema().getFields(); Preconditions.checkArgument(leftFields.size() == rightFields.size(), "Field counts don't match. Left: \n%s\nRight:\n%s", left.getSchema(), right.getSchema()); for(int i =0; i < leftFields.size(); i++){ Field leftField = leftFields.get(i); Field rightField = rightFields.get(i); Preconditions.checkArgument(leftField.getType().equals(rightField.getType()), "Field types don't match. Left: \n%s\nRight:\n%s", left.getSchema(), right.getSchema()); } List<TransferPair> leftTransfers = new ArrayList<>(); List<TransferPair> rightTransfers = new ArrayList<>(); final Iterator<VectorWrapper<?>> rightIter = right.iterator(); for(VectorWrapper<?> leftWrapper : left){ ValueVector vector = leftWrapper.getValueVector(); TransferPair pair = vector.getTransferPair(context.getAllocator()); leftTransfers.add(pair); outgoing.add(pair.getTo()); assert rightIter.hasNext(); ValueVector rightVector = rightIter.next().getValueVector(); rightTransfers.add(rightVector.makeTransferPair(pair.getTo())); } this.leftTransfers = ImmutableList.copyOf(leftTransfers); this.rightTransfers = ImmutableList.copyOf(rightTransfers); outgoing.buildSchema(SelectionVectorMode.NONE); state = State.CAN_CONSUME_L; return outgoing; }
FieldVector vector = (FieldVector) output.getTransferPair(output.getAllocator()).getTo(); TransferPair tp = vector.makeTransferPair(output); ((FixedWidthVector) vector).allocateNew(LBlockHashTableNoSpill.MAX_VALUES_PER_BATCH);
private VectorContainer transferBatch(VectorAccessible batch) { @SuppressWarnings("resource") // TODO better way to write this? VectorContainer container = new VectorContainer(); final List<ValueVector> vectors = Lists.newArrayList(); for (VectorWrapper<?> v : batch) { if (v.isHyper()) { throw new UnsupportedOperationException("Record batch data can't be created based on a hyper batch."); } TransferPair tp = v.getValueVector().getTransferPair(allocator); tp.transfer(); vectors.add(tp.getTo()); } container.addCollection(vectors); container.setRecordCount(batch.getRecordCount()); container.buildSchema(SelectionVectorMode.NONE); return container; }
@SuppressWarnings("unchecked") @Override public VectorWrapper<T> cloneAndTransfer(BufferAllocator allocator, CallBack callback) { try{ TransferPair tp = vector.getTransferPair(vector.getField().getName(), allocator, callback); tp.transfer(); return new SimpleVectorWrapper<T>((T) tp.getTo()); }catch(RuntimeException ex){ throw ex; } }
/** * Decodes a dictionary encoded array using the provided dictionary. * * @param indices dictionary encoded values, must be int type * @param dictionary dictionary used to decode the values * @return vector with values restored from dictionary */ public static ValueVector decode(ValueVector indices, Dictionary dictionary) { int count = indices.getValueCount(); ValueVector dictionaryVector = dictionary.getVector(); int dictionaryCount = dictionaryVector.getValueCount(); // copy the dictionary values into the decoded vector TransferPair transfer = dictionaryVector.getTransferPair(indices.getAllocator()); transfer.getTo().allocateNewSafe(); for (int i = 0; i < count; i++) { Object index = indices.getObject(i); if (index != null) { int indexAsInt = ((Number) index).intValue(); if (indexAsInt > dictionaryCount) { throw new IllegalArgumentException("Provided dictionary does not contain value for index " + indexAsInt); } transfer.copyValueSafe(indexAsInt, i); } } // TODO do we need to worry about the field? ValueVector decoded = transfer.getTo(); decoded.setValueCount(count); return decoded; }
TransferPair pair = incomingVector.getTransferPair(context.getAllocator()); transfers.add(pair); outgoing.add(pair.getTo());
@Override public int outputData() throws Exception { state.is(State.CAN_PRODUCE); final int records = incoming.getRecordCount(); for(JsonConverter<?> converter : converters){ converter.convert(records); } for(TransferPair transfer : transfers){ transfer.transfer(); transfer.getTo().setValueCount(records); } outgoing.setRecordCount(records); state = State.CAN_CONSUME; return records; }
public RecordBatchData(VectorAccessible batch, BufferAllocator allocator, boolean applySVMode) { this.recordCount = batch.getRecordCount(); final List<ValueVector> vectors = Lists.newArrayList(); if (applySVMode && batch.getSchema().getSelectionVectorMode() == SelectionVectorMode.TWO_BYTE) { this.sv2 = batch.getSelectionVector2().clone(); } else { this.sv2 = null; } for (VectorWrapper<?> v : batch) { if (v.isHyper()) { throw new UnsupportedOperationException("Record batch data can't be created based on a hyper batch."); } TransferPair tp = v.getValueVector().getTransferPair(allocator); tp.transfer(); vectors.add(tp.getTo()); } container.addCollection(vectors); container.setRecordCount(recordCount); container.buildSchema(applySVMode ? batch.getSchema().getSelectionVectorMode() : SelectionVectorMode.NONE); }
private void addBatchHelper(final ArrowBuf buffer) { resizeAttempted = true; /* the memory for accumulator vector comes from the operator's allocator that manages * memory for all data structures required for hash agg processing. * in other words, we should not use output.getAllocator() here as that allocator is * _only_ for for managing memory of output batch. */ FieldVector vector = (FieldVector) output.getTransferPair(computationVectorAllocator).getTo(); /* store the new vector and increment batches before allocating memory */ accumulators[batches] = vector; final int oldBatches = batches; batches++; /* if this step or memory allocation inside any child of NestedAccumulator fails, * we have captured enough info to rollback the operation. */ allocateAccumulatorForNewBatch(vector, buffer); /* need to clear the data since allocate new doesn't do so and we want to start with clean memory */ initialize(vector); bitAddresses[oldBatches] = vector.getValidityBufferAddress(); valueAddresses[oldBatches] = vector.getDataBufferAddress(); checkNotNull(); }
ValueVector vector = wrapper.getValueVector(); TransferPair pair = vector.getTransferPair(context.getAllocator()); outgoing.add(pair.getTo()); transfers.add(pair);
private void loadNextBatch(boolean first) throws IOException{ Preconditions.checkArgument(batchIndex + 1 < batchIndexMax, "You tried to go beyond end of available batches to read."); container.zeroVectors(); /* uncompress the data when de-serializing the spilled data into ArrowBufs */ final VectorAccessibleSerializable serializer = new VectorAccessibleSerializable(allocator, compressSpilledBatch, compressSpilledBatchAllocator); serializer.readFromStream(inputStream); final VectorContainer incoming = serializer.get(); Iterator<VectorWrapper<?>> wrapperIterator = incoming.iterator(); if(first){ for(VectorWrapper<?> wrapper : incoming){ final ValueVector sourceVector = wrapper.getValueVector(); TransferPair pair = sourceVector.getTransferPair(allocator); final ValueVector targetVector = pair.getTo(); pair.transfer(); container.add(targetVector); } container.buildSchema(); }else{ for (VectorWrapper<?> w : container) { final ValueVector sourceVector = wrapperIterator.next().getValueVector(); final TransferPair pair = sourceVector.makeTransferPair(w.getValueVector()); pair.transfer(); } } recordIndexMax = incoming.getRecordCount(); batchIndex++; recordIndex = -1; }
private FieldWriter promoteToUnion() { String name = vector.getField().getName(); TransferPair tp = vector.getTransferPair(vector.getMinorType().name().toLowerCase(), vector.getAllocator()); tp.transfer(); if (parentContainer != null) { // TODO allow dictionaries in complex types unionVector = parentContainer.addOrGetUnion(name); unionVector.allocateNew(); } else if (listVector != null) { unionVector = listVector.promoteToUnion(); } unionVector.addVector((FieldVector) tp.getTo()); writer = new UnionWriter(unionVector, nullableStructWriterFactory); writer.setPosition(idx()); for (int i = 0; i <= idx(); i++) { unionVector.setType(i, vector.getMinorType()); } vector = null; state = State.UNION; return writer; }
for(VectorWrapper<?> w : onDeckInput){ TransferPair pair = w.getValueVector().getTransferPair(context.getAllocator()); atBatInput.add(pair.getTo()); transfers.add(pair);
final VarCharVector newVarCharVector = (VarCharVector) tp.getTo(); final int[][] startLengths = {{0, 201}, {201, 200}, {401, 99}};