public boolean isComparable() { switch(type.getTypeID()) { case Struct: case List: return false; default: return true; } }
public boolean isNumeric() { switch(type.getTypeID()){ case Decimal: case FloatingPoint: case Int: return true; default: return false; } }
public boolean isTemporal() { switch(type.getTypeID()){ case Date: case Time: case Timestamp: return true; default: return false; } }
public boolean isScalar() { switch(type.getTypeID()){ case List: case Struct: case Union: return false; default: return true; } }
public boolean isFixedWidthScalar() { switch(type.getTypeID()){ case List: case Struct: case Union: case Binary: case Utf8: return false; default: return true; } }
public boolean isVariableWidthScalar() { switch(type.getTypeID()){ case Utf8: case Binary: return true; default: return false; } }
public boolean isText() { return type.getTypeID() == ArrowTypeID.Utf8; }
public boolean isStruct() { return type.getTypeID() == ArrowTypeID.Struct; }
public boolean isList() { return type.getTypeID() == ArrowTypeID.List; }
public boolean isBoolean() { return type.getTypeID() == ArrowTypeID.Bool; }
public boolean isUnion() { return type.getTypeID() == ArrowTypeID.Union; }
public boolean isDecimal() { return type.getTypeID() == ArrowTypeID.Decimal; }
private void toJSONString(String name, ArrowTypeID typeID, List<Field> children, JsonGenerator jsonGenerator) throws IOException { jsonGenerator.writeStartObject(); jsonGenerator.writeFieldName("name"); jsonGenerator.writeString(name); if (typeID != null) { jsonGenerator.writeFieldName("type"); jsonGenerator.writeString(typeID.name()); } if (children != null && children.size() > 0) { jsonGenerator.writeFieldName("children"); jsonGenerator.writeStartArray(children.size()); for(Field child : children) { toJSONString(child.getName(), child.getType().getTypeID(), child.getChildren(), jsonGenerator); } jsonGenerator.writeEndArray(); } jsonGenerator.writeEndObject(); }
public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldType) { boolean created = false; if (vector instanceof ZeroVector) { vector = fieldType.createNewSingleVector(DATA_VECTOR_NAME, allocator, callBack); // returned vector must have the same field created = true; if (callBack != null && // not a schema change if changing from ZeroVector to ZeroVector (fieldType.getType().getTypeID() != ArrowTypeID.Null)) { callBack.doWork(); } } if (vector.getField().getType().getTypeID() != fieldType.getType().getTypeID()) { final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]", fieldType.getType().getTypeID(), vector.getField().getType().getTypeID()); throw new SchemaChangeRuntimeException(msg); } return new AddOrGetResult<>((T) vector, created); }
private void initHashVector(VectorAccessible incoming) { Preconditions.checkArgument(config.getExpr() instanceof SchemaPath, "hash expression expected to be a SchemaPath but was : " + config.getExpr().getClass().getName()); final SchemaPath expr = (SchemaPath) config.getExpr(); final TypedFieldId typedFieldId = incoming.getSchema().getFieldId(expr); final Field field = incoming.getSchema().getColumn(typedFieldId.getFieldIds()[0]); Preconditions.checkArgument(field.getType().getTypeID() == ArrowTypeID.Int); partitionIndices = incoming.getValueAccessorById(IntVector.class, typedFieldId.getFieldIds()[0]).getValueVector(); }
public static FieldConverter getConverter(final RowBasedRecordWriter recordWriter, final int fieldId, final String fieldName, final MinorType type, final FieldReader reader) { if (reader.getField().getFieldType().getType().getTypeID() == ArrowTypeID.Union) { return recordWriter.getNewUnionConverter(fieldId, fieldName, reader); } return getFieldConverter(recordWriter,fieldId, fieldName, type, reader); }
Converter groupConverterFromArrowSchema(String nameForChild, String fieldName, GroupType groupType, Collection<SchemaPath> c) { final Field arrowField = Schema.findField(arrowSchema, fieldName); final ArrowTypeID arrowTypeType = arrowField.getType().getTypeID(); final List<Field> arrowChildren = arrowField.getChildren(); if (arrowTypeType == ArrowTypeID.Union) { // if it's a union we will add the children directly to the parent return new UnionGroupConverter(mutator, getWriterProvider(), groupType, c, options, arrowChildren, nameForChild, schemaHelper); } else if (arrowTypeType == ArrowTypeID.List) { // make sure the parquet schema matches the arrow schema and delegate handling the logical list to defaultGroupConverter() Preconditions.checkState(groupType.getOriginalType() == OriginalType.LIST, "parquet schema doesn't match the arrow schema for LIST " + nameForChild); } return defaultGroupConverter(mutator, groupType, nameForChild, c, arrowChildren); }
private static List<Field> maskFields(List<Field> fields, FieldSelection selection) { ImmutableList.Builder<Field> fieldsListBuilder = ImmutableList.builder(); for (Field field : fields) { FieldSelection childSelection = selection.getChild(field.getName()); if (!childSelection.isNeverValid()) { if (field.getType().getTypeID() == ArrowTypeID.List) { Field innerField = field.getChildren().get(0); List<Field> childFields = maskFields(innerField.getChildren(), selection.getChild(field.getName())); Field newInnerField = new Field(innerField.getName(), innerField.isNullable(), innerField.getType(), childFields); fieldsListBuilder.add(new Field(field.getName(), field.isNullable(), field.getType(), Collections.singletonList(newInnerField))); } else { List<Field> childFields = maskFields(field.getChildren(), selection.getChild(field.getName())); fieldsListBuilder.add(new Field(field.getName(), field.isNullable(), field.getType(), childFields)); } } } return fieldsListBuilder.build(); }
public RelDataType toCalciteType(RelDataTypeFactory typeFactory) { final MinorType type = toMinorType(); if (isList()) { // RelDataType childType = convertFieldToRelDataType(field.getChildren().iterator().next(), typeFactory); // return typeFactory.createArrayType(childType, -1); return typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.ANY), true); } if (isStruct()) { // return convertFieldsToStruct(field.getChildren(), typeFactory); return typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.ANY), true); } final SqlTypeName sqlTypeName = getCalciteTypeFromMinorType(type); if(isVariableWidthScalar()){ return typeFactory.createTypeWithNullability(typeFactory.createSqlType(sqlTypeName, 1 << 16), true); } if(isDecimal()){ return typeFactory.createTypeWithNullability(typeFactory.createSqlType(sqlTypeName, getPrecision(), getScale()), true); } if (this.type.getTypeID() == ArrowTypeID.Timestamp || this.type.getTypeID() == ArrowTypeID.Time) { return typeFactory.createTypeWithNullability(typeFactory.createSqlType(sqlTypeName, getPrecision()), true); } return typeFactory.createTypeWithNullability(typeFactory.createSqlType(sqlTypeName), true); }