public static Field getFieldForSerializedField(SerializedField serializedField) { String name = serializedField.getNamePart().getName(); org.apache.arrow.vector.types.Types.MinorType arrowMinorType = getArrowMinorType(serializedField.getMajorType().getMinorType()); switch(serializedField.getMajorType().getMinorType()) { case LIST: return new Field(name, true, arrowMinorType.getType(), ImmutableList.of(getFieldForSerializedField(serializedField.getChild(2)))); case STRUCT: { ImmutableList.Builder<Field> builder = ImmutableList.builder(); List<SerializedField> childList = serializedField.getChildList(); Preconditions.checkState(childList.size() > 0, "children should start with validity vector buffer"); SerializedField bits = childList.get(0); Preconditions.checkState(bits.getNamePart().getName().equals("$bits$"), "children should start with validity vector buffer: %s", childList); for (int i = 1; i < childList.size(); i++) { SerializedField child = childList.get(i); final List<SerializedField> unionChilds = serializedField.getChild(1).getChildList(); final int typeIds[] = new int[unionChilds.size()]; for (int i=0; i < unionChilds.size(); i++) { return new Field(name, true, new Decimal(serializedField.getMajorType().getPrecision(), serializedField.getMajorType().getScale()), null); default: return new Field(name, true, arrowMinorType.getType(), null);
public Builder mergeFrom(com.dremio.exec.proto.UserBitShared.SerializedField other) { if (other == com.dremio.exec.proto.UserBitShared.SerializedField.getDefaultInstance()) return this; if (other.hasMajorType()) { mergeMajorType(other.getMajorType()); if (other.hasNamePart()) { mergeNamePart(other.getNamePart()); if (other.hasValueCount()) { setValueCount(other.getValueCount()); if (other.hasVarByteLength()) { setVarByteLength(other.getVarByteLength()); if (other.hasBufferLength()) { setBufferLength(other.getBufferLength()); this.mergeUnknownFields(other.getUnknownFields()); return this;
final String fieldName = field.getNamePart().getName();
final List<SerializedField> childList = metadata.getChildList(); int bufOffset = 0; int readableBytes = buf.readableBytes(); structVector.valueCount = metadata.getValueCount(); FieldVector childVector = structVector.getChild(remaingChild); childVector.allocateNew(); childVector.setValueCount(metadata.getValueCount());
metadata = UserBitShared.SerializedField.getDefaultInstance(); when(TypeHelper.getMetadata(valueVector)).thenReturn(metadata);
Assert.assertEquals(MinorType.VARCHAR, results.get(0).getHeader().getDef().getField(1).getChild(2).getMajorType().getMinorType()); } finally { for (QueryDataBatch batch : results) {
/** * computes the number of buffers for a given serialized field * @param field serialized field * @param buffers total buffers in the batch * @param buffersStart starting buffer for the passed field * * @return number of buffers for the field */ private static int fieldBuffersCount(SerializedField field, ByteBuf[] buffers, final int buffersStart) { int totalBufferWidth = 0; int lastIndex = buffersStart; while (totalBufferWidth < field.getBufferLength() && lastIndex < buffersStart + buffers.length) { ByteBuf buf = buffers[lastIndex]; totalBufferWidth += buf.readableBytes(); ++lastIndex; } if (totalBufferWidth != field.getBufferLength()) { throw new IllegalStateException("not enough buffers for field " + field.getNamePart().getName() + " of type " + field.getMajorType()); } return lastIndex - buffersStart; }
public void load(SerializedField metadata, ArrowBuf buf) { final List<SerializedField> fields = metadata.getChildList(); structVector.valueCount = metadata.getValueCount(); int bufOffset = 0; for (final SerializedField child : fields) { final Field fieldDef = SerializedFieldHelper.create(child); FieldVector vector = structVector.getChild(fieldDef.getName()); if (vector == null) { // if we arrive here, we didn't have a matching vector. vector = BasicTypeHelper.getNewVector(fieldDef, structVector.allocator); structVector.putChild(fieldDef.getName(), vector); } if (child.getValueCount() == 0) { vector.clear(); } else { TypeHelper.load(vector, child, buf.slice(bufOffset, child.getBufferLength())); } bufOffset += child.getBufferLength(); } Preconditions.checkState(bufOffset == buf.capacity()); }
public void load(SerializedField metadata, ArrowBuf buffer) { /* release the current buffers (if any) */ listVector.clear(); /* load inner offset buffer */ final SerializedField offsetMetadata = metadata.getChild(0); final int offsetLength = offsetMetadata.getBufferLength(); loadOffsetBuffer(offsetMetadata, buffer); /* load inner validity buffer */ final SerializedField bitMetadata = metadata.getChild(1); final int bitLength = bitMetadata.getBufferLength(); loadValidityBuffer(bitMetadata, buffer.slice(offsetLength, bitLength)); /* load inner data vector */ final SerializedField vectorMetadata = metadata.getChild(2); if (listVector.getDataVector() == BaseRepeatedValueVector.DEFAULT_DATA_VECTOR) { listVector.addOrGetVector(FieldType.nullable(getArrowMinorType(vectorMetadata.getMajorType().getMinorType()).getType())); } final int vectorLength = vectorMetadata.getBufferLength(); TypeHelper.load(listVector.vector, vectorMetadata, buffer.slice(offsetLength + bitLength, vectorLength)); listVector.setLastSet(metadata.getValueCount()); listVector.valueCount = metadata.getValueCount(); }
@Test public void testColumnOrderingWithUnionVector() throws Exception { List<QueryDataBatch> results = null; try { results = testRunAndReturn(QueryType.SQL, "SELECT * FROM cp.\"type_changes.json\""); final RecordBatchDef def = results.get(0).getHeader().getDef(); assertEquals(2, def.getFieldCount()); assertEquals("a", def.getField(0).getNamePart().getName()); assertEquals("b", def.getField(1).getNamePart().getName()); } finally { if (results != null) { for(QueryDataBatch r : results) { r.release(); } } } }
private static boolean typeEqual(SerializedField field1, SerializedField field2) { if (!field1.getMajorType().equals(field2.getMajorType())) { return false; } if (field1.getChildCount() != field2.getChildCount()) { return false; } for(int i=0; i<field1.getChildCount(); i++) { final SerializedField childField1 = field1.getChild(i); final SerializedField childField2 = field2.getChild(i); if (!typeEqual(childField1, childField2)) { return false; } } return true; }
public void load(SerializedField metadata, ArrowBuf buffer) { /* release the current buffers (if any) */ vector.clear(); /* get the metadata for all children */ final SerializedField offsetMetadata = metadata.getChild(0); final SerializedField vectorMetadata = metadata.getChild(1); final int offsetLength = offsetMetadata.getBufferLength(); final int vectorLength = vectorMetadata.getBufferLength(); /* load inner offset buffer */ loadOffsetBuffer(offsetMetadata, buffer); /* load inner data vector */ if (vector.getDataVector() == BaseRepeatedValueVector.DEFAULT_DATA_VECTOR) { vector.addOrGetVector(FieldType.nullable(getArrowMinorType(metadata.getMajorType().getMinorType()).getType())); } TypeHelper.load(vector.vector, vectorMetadata, buffer.slice(offsetLength, vectorLength)); }
public void load(SerializedField metadata, ArrowBuf buffer) { /* clear the current buffers (if any) */ vector.clear(); /* get the metadata children */ final SerializedField bitsField = metadata.getChild(0); final SerializedField valuesField = metadata.getChild(1); final int bitsLength = bitsField.getBufferLength(); final int capacity = buffer.capacity(); final int valuesLength = capacity - bitsLength; /* load inner validity buffer */ loadValidityBuffer(bitsField, buffer); /* load inner value buffer */ loadDataBuffer(valuesField, buffer.slice(bitsLength, valuesLength)); vector.valueCount = metadata.getValueCount(); }
public void loadData(SerializedField metadata, ArrowBuf buffer) { /* clear the current buffers (if any) */ vector.clear(); /* get the metadata children */ final SerializedField bitsField = metadata.getChild(0); final SerializedField valuesField = metadata.getChild(1); final int valuesLength = buffer.capacity(); vector.allocateNew(metadata.getValueCount()); /* set inner validity buffer */ setValidityBuffer(bitsField); /* load inner value buffer */ vector.valueBuffer.close(); loadDataBuffer(valuesField, buffer.slice(0, valuesLength)); }
public void loadData(SerializedField metadata, ArrowBuf buffer) { /* clear the current buffers (if any) */ vector.clear(); /* get the metadata children */ final SerializedField bitsField = metadata.getChild(0); final SerializedField valuesField = metadata.getChild(1); final int valuesLength = buffer.capacity(); vector.allocateNew(metadata.getValueCount()); /* set inner validity buffer */ setValidityBuffer(bitsField); /* load inner value buffer */ vector.valueBuffer.close(); loadDataBuffer(valuesField, buffer.slice(0, valuesLength)); }
public void loadData(SerializedField metadata, ArrowBuf buffer) { /* clear the current buffers (if any) */ vector.clear(); /* get the metadata children */ final SerializedField bitsField = metadata.getChild(0); final SerializedField valuesField = metadata.getChild(1); final int valuesLength = buffer.capacity(); vector.allocateNew(metadata.getValueCount()); /* set inner validity buffer */ setValidityBuffer(bitsField); /* load inner value buffer */ vector.valueBuffer.close(); loadDataBuffer(valuesField, buffer.slice(0, valuesLength)); }
public void loadData(SerializedField metadata, ArrowBuf buffer) { /* clear the current buffers (if any) */ vector.clear(); /* get the metadata children */ final SerializedField bitsField = metadata.getChild(0); final SerializedField valuesField = metadata.getChild(1); final int valuesLength = buffer.capacity(); vector.allocateNew(metadata.getValueCount()); /* set inner validity buffer */ setValidityBuffer(bitsField); /* load inner value buffer */ vector.valueBuffer.close(); loadDataBuffer(valuesField, buffer.slice(0, valuesLength)); }
public void loadData(SerializedField metadata, ArrowBuf buffer) { /* clear the current buffers (if any) */ vector.clear(); /* get the metadata children */ final SerializedField bitsField = metadata.getChild(0); final SerializedField valuesField = metadata.getChild(1); final int valuesLength = buffer.capacity(); vector.allocateNew(metadata.getValueCount()); /* set inner validity buffer */ setValidityBuffer(bitsField); /* load inner value buffer */ vector.valueBuffer.close(); loadDataBuffer(valuesField, buffer.slice(0, valuesLength)); }
public void loadData(SerializedField metadata, ArrowBuf buffer) { /* clear the current buffers (if any) */ vector.clear(); /* get the metadata children */ final SerializedField bitsField = metadata.getChild(0); final SerializedField valuesField = metadata.getChild(1); final int valuesLength = buffer.capacity(); vector.allocateNew(metadata.getValueCount()); /* set inner validity buffer */ setValidityBuffer(bitsField); /* load inner value buffer */ vector.valueBuffer.close(); loadDataBuffer(valuesField, buffer.slice(0, valuesLength)); }
public void loadData(SerializedField metadata, ArrowBuf buffer) { /* clear the current buffers (if any) */ vector.clear(); /* get the metadata children */ final SerializedField bitsField = metadata.getChild(0); final SerializedField valuesField = metadata.getChild(1); final int valuesLength = buffer.capacity(); vector.allocateNew(metadata.getValueCount()); /* set inner validity buffer */ setValidityBuffer(bitsField); /* load inner value buffer */ vector.valueBuffer.close(); loadDataBuffer(valuesField, buffer.slice(0, valuesLength)); }