@Override final int getInt(int rowId) { return accessor.get(rowId); } }
@Override public FieldVector getNewVector( String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new IntVector(name, fieldType, allocator); }
@Override public void allocate() { vector.allocateNew(); }
public void populate(final int count){ for (int i = 0; i < count; i++) { if(value != null) { ((IntVector) vector).setSafe(i, value); } } vector.setValueCount(count); }
/** * Same as {@link #set(int, int)} except that it handles the * case when index is greater than or equal to existing * value capacity {@link #getValueCapacity()}. * * @param index position of element * @param value value of element */ public void setSafe(int index, int value) { handleSafe(index); set(index, value); }
/** * * @param allocator * @param name * @param data * @return */ public static IntVector vectorFor(BufferAllocator allocator,String name,int[] data) { IntVector float8Vector = new IntVector(name,FieldType.nullable(new ArrowType.Int(32,true)),allocator); float8Vector.allocateNew(data.length); for(int i = 0; i < data.length; i++) { float8Vector.setSafe(i,data[i]); } float8Vector.setValueCount(data.length); return float8Vector; }
static Integer[] populate4ByteValuesWithoutNull(IntVector vector, int size){ vector.allocateNew(); Integer values[] = new Integer[size]; for(int i =0; i < size; i++){ values[i] = RAND.nextInt(); vector.setSafe(i, values[i]); } vector.setValueCount(size); return values; }
private IntVector intVector(String name) { IntVector vec = new IntVector(name, allocator); vec.allocateNew(5); vec.set(0, 20); vec.set(1, 50); vec.set(2, -2000); vec.set(3, 327345); vec.setNull(4); vec.setValueCount(COUNT); return vec; }
private static VectorContainer buildIntegerGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) { final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.Int(32, true), null); final VectorContainer input = new VectorContainer(bufferAllocator); final IntVector intVector = input.addOrGet(field); intVector.allocateNew(); final SortedSet<Integer> values = Sets.newTreeSet(); for (Dictionary dictionary : dictionaries) { for (int i = 0; i <= dictionary.getMaxId(); ++i) { values.add(dictionary.decodeToInt(i)); } } if (existingDict != null) { final IntVector existingDictValues = existingDict.getValueAccessorById(IntVector.class, 0).getValueVector(); for (int i = 0; i < existingDict.getRecordCount(); ++i) { values.add(existingDictValues.get(i)); } } final Iterator<Integer> iter = values.iterator(); int recordCount = 0; while (iter.hasNext()) { intVector.setSafe(recordCount++, iter.next()); } intVector.setValueCount(recordCount); input.setRecordCount(recordCount); input.buildSchema(BatchSchema.SelectionVectorMode.NONE); return input; }
).toField("map").getFieldType(); try (final IntVector intVector = new IntVector("int", allocator); final Float8Vector float8Vector = new Float8Vector("float8", allocator); final StructVector mapVector = new StructVector("map", allocator, mapType, null); NullableStructWriter mapWriter = mapVector.getWriter(); for (int i = 0; i < records; i++) { intVector.set(i, intBaseValue + i); float8Vector.set(i, doubleBaseValue + i); intVector.setValueCount(records); float8Vector.setValueCount(records); mapVector.setValueCount(records); assertEquals(intBaseValue + i, intVector.get(i)); assertEquals(doubleBaseValue + i, float8Vector.get(i), 0); assertEquals(
@Override public void writeField(Object pojo, int outboundIndex) throws IllegalArgumentException, IllegalAccessException { Integer i = (Integer) field.get(pojo); if (i != null) { vector.setSafe(outboundIndex, i); } } }
private void decodeDate(IntVector input, DateMilliVector output, IntVector dictionary) { // dates are stored as int32 in parquet dictionaries if (hasSv2) { final SelectionVector2 sv2 = incoming.getSelectionVector2(); for (int i = 0; i < recordsConsumedCurrentBatch; ++i) { final int svIndex = sv2.getIndex(i); if (input.isNull(svIndex)) { output.setNull(svIndex); } else { final int id = input.get(svIndex); output.setSafe(svIndex, dictionary.get(id) * (long) DateTimeConstants.MILLIS_PER_DAY); } } } else { for (int i = 0; i < recordsConsumedCurrentBatch; ++i) { if (input.isNull(i)) { output.setNull(i); } else { int id = input.get(i); output.setSafe(i, dictionary.get(id) * (long) DateTimeConstants.MILLIS_PER_DAY); } } } }
/** * * @param allocator * @param name * @return */ public static IntVector intVectorOf(BufferAllocator allocator,String name,int length) { IntVector float8Vector = new IntVector(name,FieldType.nullable(new ArrowType.Int(32,true)),allocator); float8Vector.allocateNew(length); float8Vector.setValueCount(length); return float8Vector; }
final SabotContext context = bit.getContext(); try (final IntVector intVector = new IntVector("int", allocator); final VarBinaryVector binVector = new VarBinaryVector("binary", allocator)) { vectorList.add(binVector); intVector.setSafe(0, 0); binVector.setSafe(0, "ZERO".getBytes(), 0, "ZERO".getBytes().length); intVector.setSafe(1, 1); binVector.setSafe(1, "ONE".getBytes(), 0, "ONE".getBytes().length); intVector.setSafe(2, 2); binVector.setSafe(2, "TWO".getBytes(), 0, "TWO".getBytes().length); intVector.setSafe(3, 3); binVector.setSafe(3, "THREE".getBytes(), 0, "TWO".getBytes().length); intVector.setValueCount(4); binVector.setValueCount(4);
private void decodeInt(IntVector input, IntVector output, IntVector dictionary) { if (hasSv2) { final SelectionVector2 sv2 = incoming.getSelectionVector2(); for (int i = 0, svIndex = sv2.getIndex(i); i < recordsConsumedCurrentBatch; ++i) { final int id = input.get(svIndex); output.copyFromSafe(id, svIndex, dictionary); } } else { for (int i = 0; i < recordsConsumedCurrentBatch; ++i) { int id = input.get(i); output.copyFromSafe(id, i, dictionary); } } }
IntVector in1MaxOutputVector = new IntVector("int-max", allocator); final MaxAccumulators.IntMaxAccumulator in1MaxAccum = new MaxAccumulators.IntMaxAccumulator(in1, in1MaxOutputVector, in1MaxOutputVector, MAX_VALUES_PER_BATCH, IntVector in1MinOutputVector = new IntVector("int-min", allocator); final MinAccumulators.IntMinAccumulator in1MinAccum = new MinAccumulators.IntMinAccumulator(in1, in1MinOutputVector, in1MinOutputVector, MAX_VALUES_PER_BATCH, postSpillAccumulatorVectorFields.add(in1MaxOutputVector.getField()); postSpillAccumulatorVectorFields.add(in1MinOutputVector.getField()); postSpillAccumulatorVectorFields.add(in1Count.getField()); postSpillAccumulatorVectorFields.add(in1Count1.getField());
@Override public void setup(VectorAccessible incoming) throws Exception { state.is(State.NEEDS_SETUP); Preconditions.checkState(incoming.getSchema().getSelectionVectorMode() == BatchSchema.SelectionVectorMode.NONE, "Vectorized Partition Sender doesn't support SV " + incoming.getSchema().getSelectionVectorMode()); checkSchema(incoming.getSchema()); // how many records we can keep in memory before we are forced to flush the outgoing batch numRecordsBeforeFlush = calculateBucketSize(incoming); stats.setLongStat(Metric.BUCKET_SIZE, numRecordsBeforeFlush); // final BufferAllocator allocator = context.getAllocator(); // we need to synchronize this to ensure no receiver termination message is lost synchronized (batchCreationLock) { initBatchesAndLookup(incoming); // some receivers may have finished already, make sure to terminate corresponding outgoing batches for (int index = 0; index < terminations.size(); index++) { final int p = terminations.buffer[index]; batches[p].terminate(); batches[p + numReceivers].terminate(); } terminations.clear(); } copiers = MultiDestCopier.getCopiers(VectorContainer.getFieldVectors(incoming), batches, copyWatches); copyIndices = new IntVector("copy-compound-indices", allocator); copyIndices.allocateNew(numRecordsBeforeFlush); initHashVector(incoming); state = State.CAN_CONSUME; }
@Override public void copy(int inputIdx, int count, int outputIdx) { final long[] input = inputVector.vector; if (inputVector.isRepeating) { if (inputVector.isNull[0]) { return; // If all repeating values are null, then there is no need to write anything to vector } final int value = (int) input[0]; for (int i = 0; i < count; i++, outputIdx++) { outputVector.set(outputIdx, value); } } else if (inputVector.noNulls) { for (int i = 0; i < count; i++, inputIdx++, outputIdx++) { outputVector.set(outputIdx, (int) input[inputIdx]); } } else { final boolean isNull[] = inputVector.isNull; for (int i = 0; i < count; i++, inputIdx++, outputIdx++) { if (!isNull[inputIdx]) { outputVector.set(outputIdx, (int) input[inputIdx]); } } } } }