private void writeStruct(NonNullableStructVector arrowVector, StructColumnVector hiveVector, StructTypeInfo typeInfo, int size, VectorizedRowBatch vectorizedRowBatch, boolean isNative) { final List<String> fieldNames = typeInfo.getAllStructFieldNames(); final List<TypeInfo> fieldTypeInfos = typeInfo.getAllStructFieldTypeInfos(); final ColumnVector[] hiveFieldVectors = hiveVector.fields; final int fieldSize = fieldTypeInfos.size(); for (int fieldIndex = 0; fieldIndex < fieldSize; fieldIndex++) { final TypeInfo fieldTypeInfo = fieldTypeInfos.get(fieldIndex); final ColumnVector hiveFieldVector = hiveFieldVectors[fieldIndex]; final String fieldName = fieldNames.get(fieldIndex); final FieldVector arrowFieldVector = arrowVector.addOrGet(fieldName, toFieldType(fieldTypeInfos.get(fieldIndex)), FieldVector.class); arrowFieldVector.setInitialCapacity(size); arrowFieldVector.allocateNew(); write(arrowFieldVector, hiveFieldVector, fieldTypeInfo, size, vectorizedRowBatch, isNative); } final ArrowBuf validityBuffer = arrowVector.getValidityBuffer(); for (int rowIndex = 0; rowIndex < size; rowIndex++) { if (hiveVector.isNull[rowIndex]) { BitVectorHelper.setValidityBit(validityBuffer, rowIndex, 0); } else { BitVectorHelper.setValidityBitToOne(validityBuffer, rowIndex); } } }
@Override public StructWriter struct(String name) { String finalName = handleCase(name); FieldWriter writer = fields.get(finalName); if(writer == null){ int vectorCount=container.size(); StructVector vector = container.addOrGet(name, FieldType.nullable(MinorType.STRUCT.getType()), StructVector.class); writer = new PromotableWriter(vector, container, getNullableStructWriterFactory()); if(vectorCount != container.size()) { writer.allocate(); } writer.setPosition(idx()); fields.put(finalName, writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.STRUCT); } } return writer; }
@Override public ListWriter list(String name) { String finalName = handleCase(name); FieldWriter writer = fields.get(finalName); int vectorCount = container.size(); if(writer == null) { writer = new PromotableWriter(container.addOrGet(name, FieldType.nullable(MinorType.LIST.getType()), ListVector.class), container, getNullableStructWriterFactory()); if (container.size() > vectorCount) { writer.allocate(); } writer.setPosition(idx()); fields.put(finalName, writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.LIST); } } return writer; }
@Override public DateDayWriter dateDay(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); DateDayVector v = container.addOrGet(name, FieldType.nullable( MinorType.DATEDAY.getType() ), DateDayVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.DATEDAY); } } return writer; }
@Override public TimeStampNanoTZWriter timeStampNanoTZ(String name, String timezone) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); TimeStampNanoTZVector v = container.addOrGet(name, FieldType.nullable( new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, timezone) ), TimeStampNanoTZVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.TIMESTAMPNANOTZ); } } return writer; }
@Override public DecimalWriter decimal(String name, int scale, int precision) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); DecimalVector v = container.addOrGet(name, FieldType.nullable( new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale) ), DecimalVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.DECIMAL); } } return writer; }
@Override public UInt2Writer uInt2(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); UInt2Vector v = container.addOrGet(name, FieldType.nullable( MinorType.UINT2.getType() ), UInt2Vector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.UINT2); } } return writer; }
@Override public TimeMilliWriter timeMilli(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); TimeMilliVector v = container.addOrGet(name, FieldType.nullable( MinorType.TIMEMILLI.getType() ), TimeMilliVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.TIMEMILLI); } } return writer; }
@Override public DateMilliWriter dateMilli(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); DateMilliVector v = container.addOrGet(name, FieldType.nullable( MinorType.DATEMILLI.getType() ), DateMilliVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.DATEMILLI); } } return writer; }
@Override public TimeStampSecWriter timeStampSec(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); TimeStampSecVector v = container.addOrGet(name, FieldType.nullable( MinorType.TIMESTAMPSEC.getType() ), TimeStampSecVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.TIMESTAMPSEC); } } return writer; }
@Override public TimeStampMilliWriter timeStampMilli(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); TimeStampMilliVector v = container.addOrGet(name, FieldType.nullable( MinorType.TIMESTAMPMILLI.getType() ), TimeStampMilliVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.TIMESTAMPMILLI); } } return writer; }
@Override public IntervalDayWriter intervalDay(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); IntervalDayVector v = container.addOrGet(name, FieldType.nullable( MinorType.INTERVALDAY.getType() ), IntervalDayVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.INTERVALDAY); } } return writer; }
@Override public UInt1Writer uInt1(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); UInt1Vector v = container.addOrGet(name, FieldType.nullable( MinorType.UINT1.getType() ), UInt1Vector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.UINT1); } } return writer; }
@Override public SmallIntWriter smallInt(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); SmallIntVector v = container.addOrGet(name, FieldType.nullable( MinorType.SMALLINT.getType() ), SmallIntVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.SMALLINT); } } return writer; }
@Override public BigIntWriter bigInt(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); BigIntVector v = container.addOrGet(name, FieldType.nullable( MinorType.BIGINT.getType() ), BigIntVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.BIGINT); } } return writer; }
@Override public BitWriter bit(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); BitVector v = container.addOrGet(name, FieldType.nullable( MinorType.BIT.getType() ), BitVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.BIT); } } return writer; }
protected StructTransferPair(NonNullableStructVector from, NonNullableStructVector to, boolean allocate) { this.from = from; this.to = to; this.pairs = new TransferPair[from.size()]; this.to.ephPair = null; int i = 0; FieldVector vector; for (String child : from.getChildFieldNames()) { int preSize = to.size(); vector = from.getChild(child); if (vector == null) { continue; } //DRILL-1872: we add the child fields for the vector, looking up the field by name. For a map vector, // the child fields may be nested fields of the top level child. For example if the structure // of a child field is oa.oab.oabc then we add oa, then add oab to oa then oabc to oab. // But the children member of a Materialized field is a HashSet. If the fields are added in the // children HashSet, and the hashCode of the Materialized field includes the hash code of the // children, the hashCode value of oa changes *after* the field has been added to the HashSet. // (This is similar to what happens in ScanBatch where the children cannot be added till they are // read). To take care of this, we ensure that the hashCode of the MaterializedField does not // include the hashCode of the children but is based only on MaterializedField$key. final FieldVector newVector = to.addOrGet(child, vector.getField().getFieldType(), vector.getClass()); if (allocate && to.size() != preSize) { newVector.allocateNew(); } pairs[i++] = vector.makeTransferPair(newVector); } }
@Override public TinyIntWriter tinyInt(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); TinyIntVector v = container.addOrGet(name, FieldType.nullable( MinorType.TINYINT.getType() ), TinyIntVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.TINYINT); } } return writer; }
@Override public TimeStampNanoWriter timeStampNano(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); TimeStampNanoVector v = container.addOrGet(name, FieldType.nullable( MinorType.TIMESTAMPNANO.getType() ), TimeStampNanoVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.TIMESTAMPNANO); } } return writer; }
@Override public VarBinaryWriter varBinary(String name) { FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); VarBinaryVector v = container.addOrGet(name, FieldType.nullable( MinorType.VARBINARY.getType() ), VarBinaryVector.class); writer = new PromotableWriter(v, container, getNullableStructWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { if(this.initialCapacity > 0) { vector.setInitialCapacity(this.initialCapacity); } vector.allocateNewSafe(); } writer.setPosition(idx()); fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized ((PromotableWriter)writer).getWriter(MinorType.VARBINARY); } } return writer; }