@Override public Field getField() { return vector.getField(); }
public ArrowColumnVector(ValueVector vector) { super(ArrowUtils.fromArrowField(vector.getField()));
@Override public MajorType getType() { return getMajorTypeForField(v.getField()); } }
/** * Find an estimated size of the vectors. Currently we assume constant size for variable length columns. */ public static int estimateRecordSize(Map<String, ValueVector> vectorMap, int listSizeEstimate, int varFieldSizeEstimate) { int estimatedRecordSize = 0; for (final ValueVector v : vectorMap.values()) { estimatedRecordSize += BatchSchema.estimateFieldSize(v.getField(), listSizeEstimate, varFieldSizeEstimate); } return estimatedRecordSize; }
public ArrowColumnVector(ValueVector vector) { super(ArrowUtils.fromArrowField(vector.getField()));
@SuppressWarnings("unchecked") public void addVector(ValueVector v) { Preconditions.checkArgument(v.getClass() == this.getVectorClass(), "Cannot add vector type %s to hypervector type %s for field %s", v.getClass(), this.getVectorClass(), v.getField()); vectors.add((T)v); cachedVectorsValid = false; }
public void add(ValueVector[] hyperVector, boolean releasable) { assert hyperVector.length != 0; clearSchema(); Class<?> clazz = hyperVector[0].getClass(); ValueVector[] c = (ValueVector[]) Array.newInstance(clazz, hyperVector.length); for (int i = 0; i < hyperVector.length; i++) { c[i] = hyperVector[i]; } // todo: work with a merged schema. wrappers.add(HyperVectorWrapper.create(hyperVector[0].getField(), c, releasable)); }
@Override public Field getField() { List<Field> children = new ArrayList<>(); for (ValueVector child : getChildren()) { children.add(child.getField()); } return new Field(name, fieldType, children); }
@Override @SuppressWarnings("unchecked") public VectorWrapper<T> cloneAndTransfer(BufferAllocator allocator, CallBack callback) { if(vectors.size() == 0){ return new HyperVectorWrapper<T>(field, (T[])vectors.toArray(), false); }else { T[] newVectors = (T[]) Array.newInstance(vectors.get(0).getClass(), vectors.size()); for(int i =0; i < vectors.size(); i++){ newVectors[i] = (T) vectors.get(i).getTransferPair(vectors.get(i).getField().getName(), allocator, callback).getTo(); } return new HyperVectorWrapper<T>(field, (T[])vectors.toArray(), false); } }
@Override public TypedFieldId getFieldIdIfMatches(int id, SchemaPath expectedPath) { return FieldIdUtil2.getFieldId(getValueVector().getField(), id, expectedPath, true); }
@Override public VectorWrapper<?> getChildWrapper(int[] ids) { if (ids.length == 1) { return this; } ValueVector[] vectors = new ValueVector[this.vectors.size()]; int index = 0; for (ValueVector v : this.vectors) { ValueVector vector = v; for (int i = 1; i < ids.length; i++) { final AbstractStructVector mapLike = AbstractStructVector.class.cast(vector); if (mapLike == null) { return null; } vector = mapLike.getChildByOrdinal(ids[i]); } vectors[index] = vector; index++; } return new HyperVectorWrapper<ValueVector>(vectors[0].getField(), vectors); }
public TypedFieldId add(ValueVector vv) { clearSchema(); int i = wrappers.size(); wrappers.add(SimpleVectorWrapper.create(vv)); return new TypedFieldId(CompleteType.fromField(vv.getField()), i); }
private static Set<String> extractSchema(RecordBatchData batch) { Set<String> schema = Sets.newHashSet(); for (ValueVector vv : batch.getVectors()) { schema.add(vv.getField().getName()); } return schema; }
public void addField(ValueVector vector) { container.add(vector); fieldVectorMap.put(vector.getField().getName().toLowerCase(), vector); }
private static void assertContainsFields(Set<String> fields, List<ValueVector> original) { assertEquals(original.size(), fields.size()); for (ValueVector vv : original) { final String fieldName = vv.getField().getName(); assertTrue("original field '" + fieldName + "' is not part of the new schema", fields.contains(fieldName)); } }
@SuppressWarnings("unchecked") @Override public VectorWrapper<T> cloneAndTransfer(BufferAllocator allocator, CallBack callback) { try{ TransferPair tp = vector.getTransferPair(vector.getField().getName(), allocator, callback); tp.transfer(); return new SimpleVectorWrapper<T>((T) tp.getTo()); }catch(RuntimeException ex){ throw ex; } }
private void showSingleBatch (VectorAccessibleSerializable vcSerializable, boolean showHeader) { final VectorContainer vectorContainer = vcSerializable.get(); /* show the header of the batch */ if (showHeader) { System.out.println(getBatchMetaInfo(vcSerializable).toString()); System.out.println("Schema Information"); for (final VectorWrapper w : vectorContainer) { final Field field = w.getValueVector().getField(); System.out.println (String.format("name : %s, minor_type : %s, data_mode : %s", field.getName(), getMinorTypeForArrowType(field.getType()).toString(), field.isNullable() ? "nullable":"non-nullable" )); } } /* show the contents in the batch */ VectorUtil.showVectorAccessibleContent(vectorContainer); }
private void replace(ValueVector newVector, SchemaPath schemaPath) { List<ValueVector> vectors = Lists.newArrayList(); for (VectorWrapper w : container) { ValueVector vector = w.getValueVector(); if (vector.getField().getName().equals(schemaPath.getLastSegment().getNameSegment().getNameSegment())) { vectors.add(newVector); } else { vectors.add(w.getValueVector()); } container.remove(vector); } container.addCollection(vectors); }
private FieldWriter promoteToUnion() { String name = vector.getField().getName(); TransferPair tp = vector.getTransferPair(vector.getMinorType().name().toLowerCase(), vector.getAllocator()); tp.transfer(); if (parentContainer != null) { // TODO allow dictionaries in complex types unionVector = parentContainer.addOrGetUnion(name); unionVector.allocateNew(); } else if (listVector != null) { unionVector = listVector.promoteToUnion(); } unionVector.addVector((FieldVector) tp.getTo()); writer = new UnionWriter(unionVector, nullableStructWriterFactory); writer.setPosition(idx()); for (int i = 0; i <= idx(); i++) { unionVector.setType(i, vector.getMinorType()); } vector = null; state = State.UNION; return writer; }
private static ORCCopier createCopier(ValueVector output, ColumnVector input) { if (output instanceof BaseVariableWidthVector) { return new VarWidthCopier((BytesColumnVector) input, (BaseVariableWidthVector) output); } else if (output instanceof IntVector) { return new IntCopier((LongColumnVector) input, (IntVector) output); } else if (output instanceof BigIntVector) { return new BigIntCopier((LongColumnVector) input, (BigIntVector) output); } else if (output instanceof Float4Vector) { return new Float4Copier((DoubleColumnVector) input, (Float4Vector) output); } else if (output instanceof Float8Vector) { return new Float8Copier((DoubleColumnVector) input, (Float8Vector) output); } else if (output instanceof DateMilliVector) { return new DateMilliCopier((LongColumnVector) input, (DateMilliVector) output); } else if (output instanceof TimeStampMilliVector) { return new TimeStampMilliCopier((TimestampColumnVector) input, (TimeStampMilliVector) output); } else if (output instanceof DecimalVector) { return new DecimalCopier((DecimalColumnVector) input, (DecimalVector) output); } else if (output instanceof BitVector) { return new BitCopier((LongColumnVector) input, (BitVector) output); } throw UserException.unsupportedError() .message("Received unsupported type '%s' in Hive vectorized ORC reader", output.getField()) .build(logger); }