final int dataTypeLength; if (globalDictionaryColumns.containsKey(fieldName)) { mt = MajorType.newBuilder().setMinorType(MinorType.INT).setMode(getDataMode(column)).build(); dataTypeLength = getTypeLengthInBits(PrimitiveType.PrimitiveTypeName.INT32); } else {
@Test @Ignore public void testUnionDistinctBothEmptyJson() throws Exception { final String rootEmpty = FileUtils.getResourceAsFile("/project/pushdown/empty.json").toURI().toString(); final String query = String.format( "select key from dfs_test.\"%s\" " + "union " + "select key from dfs_test.\"%s\"", rootEmpty, rootEmpty); final List<Pair<SchemaPath, MajorType>> expectedSchema = Lists.newArrayList(); final MajorType majorType = TypeProtos.MajorType.newBuilder() .setMinorType(TypeProtos.MinorType.INT) .setMode(TypeProtos.DataMode.OPTIONAL) .build(); expectedSchema.add(Pair.of(SchemaPath.getSimplePath("key"), majorType)); testBuilder() .sqlQuery(query) .schemaBaseLine(expectedSchema) .build() .run(); }
@Ignore("DX-4180") @Test public void testUnionDistinctBothEmptyBatch() throws Exception { String rootSimple = FileUtils.getResourceAsFile("/store/json/booleanData.json").toURI().toString(); final String query = String.format( "select key from dfs_test.\"%s\" where 1 = 0 " + "union " + "select key from dfs_test.\"%s\" where 1 = 0", rootSimple, rootSimple); final List<Pair<SchemaPath, MajorType>> expectedSchema = Lists.newArrayList(); final MajorType majorType = TypeProtos.MajorType.newBuilder() .setMinorType(TypeProtos.MinorType.INT) .setMode(TypeProtos.DataMode.OPTIONAL) .build(); expectedSchema.add(Pair.of(SchemaPath.getSimplePath("key"), majorType)); testBuilder() .sqlQuery(query) .schemaBaseLine(expectedSchema) .build() .run(); }
case CHAR: castType = MajorType.newBuilder() .setMinorType(MinorType.VARCHAR) .setMode(DataMode.OPTIONAL) .setWidth(call.getType().getPrecision()) .setMinorType(MinorType.DECIMAL) .setMode(DataMode.REQUIRED) .setPrecision(precision) case BOOLEAN: castType = Types.required(MinorType.BIT); break; case BINARY: castType = MajorType.newBuilder() .setMinorType(MinorType.VARBINARY) .setMode(DataMode.OPTIONAL) .setPrecision(call.getType().getPrecision())
public static MajorType getMajorTypeForArrowType(ArrowType arrowType, List<Field> children) { MajorType.Builder builder = MajorType.newBuilder() .setMinorType(getMinorTypeFromArrowMinorType(getMinorTypeForArrowType(arrowType))) .setMode(DataMode.OPTIONAL); ArrowTypeID fieldType = arrowType.getTypeID();
.unOrdered() .csvBaselineFile("testframework/decimal_test.tsv") .baselineTypes(MajorType.newBuilder().setMinorType(MinorType.DECIMAL).setMode(DataMode.REQUIRED).setPrecision(32).setScale(2).build())
.setMinorType(MinorType.DECIMAL38SPARSE) .setMode(DataMode.OPTIONAL) .setPrecision(decimalField.getMajorType().getPrecision())
/** * Helper method that converts Hive type definition to Dremio type definition. * * @param typeInfo Hive type info * @param options * @return */ public static MajorType getMajorTypeFromHiveTypeInfo(final TypeInfo typeInfo, final OptionManager options) { switch (typeInfo.getCategory()) { case PRIMITIVE: { PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo; MinorType minorType = getMinorTypeFromHivePrimitiveTypeInfo(primitiveTypeInfo, options); MajorType.Builder typeBuilder = MajorType.newBuilder().setMinorType(getMinorTypeFromArrowMinorType(minorType)) .setMode(DataMode.OPTIONAL); // Hive columns (both regular and partition) could have null values if (primitiveTypeInfo.getPrimitiveCategory() == PrimitiveCategory.DECIMAL) { DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo; typeBuilder.setPrecision(decimalTypeInfo.precision()) .setScale(decimalTypeInfo.scale()).build(); } return typeBuilder.build(); } case LIST: case MAP: case STRUCT: case UNION: default: throwUnsupportedHiveDataTypeError(typeInfo.getCategory().toString()); } return null; // never reached }
return; case 1: builder.setMinorType(com.dremio.common.types.TypeProtos.MinorType.valueOf(input.readEnum())); break; case 2:
public Builder mergeFrom(com.dremio.common.types.TypeProtos.MajorType other) { if (other == com.dremio.common.types.TypeProtos.MajorType.getDefaultInstance()) return this; if (other.hasMinorType()) { setMinorType(other.getMinorType());
@JsonIgnore public MajorType getMajorType() { MajorType.Builder b = MajorType.newBuilder(); b.setMode(mode); b.setMinorType(minorType); if (precision != null) { b.setPrecision(precision); } if (width != null) { b.setWidth(width); } if (scale != null) { b.setScale(scale); } return b.build(); }
public static MajorType optional(final MinorType type) { return MajorType.newBuilder().setMode(DataMode.OPTIONAL).setMinorType(type).build(); }
public static MajorType required(final MinorType type) { return MajorType.newBuilder().setMode(DataMode.REQUIRED).setMinorType(type).build(); }
public static MajorType withMode(final MinorType type, final DataMode mode) { return MajorType.newBuilder().setMode(mode).setMinorType(type).build(); }
@JsonIgnore public MajorType getMajorType() { MajorType.Builder b = MajorType.newBuilder(); b.setMode(mode); b.setMinorType(minorType); if (precision != null) { b.setPrecision(precision); } if (width != null) { b.setWidth(width); } if (scale != null) { b.setScale(scale); } return b.build(); }
public static MajorType repeated(final MinorType type) { return MajorType.newBuilder().setMode(REPEATED).setMinorType(type).build(); }
public SerializedField.Builder getMetadataBuilder() { return SerializedField.newBuilder() .setMajorType(MajorType.newBuilder().setMinorType(MinorType.LIST).setMode(DataMode.OPTIONAL).build()) .setNamePart(NamePart.newBuilder().setName(listVector.getField().getName())) .setValueCount(listVector.getValueCount()) .setBufferLength(listVector.getBufferSize()) .addChild(buildOffsetMetadata()) .addChild(buildValidityMetadata()) .addChild(TypeHelper.getMetadata(listVector.vector)); }
public static MajorType withScaleAndPrecision(final MinorType type, final DataMode mode, final int scale, final int precision) { return MajorType.newBuilder().setMinorType(type).setMode(mode).setScale(scale).setPrecision(precision).build(); }
public static TypeProtos.MajorType toMajorType(PrimitiveType.PrimitiveTypeName primitiveTypeName, int length, TypeProtos.DataMode mode, SchemaElement schemaElement, OptionManager options, Field arrowField, final boolean readInt96AsTimeStamp) { MinorType minorType = getMinorType(primitiveTypeName, length, schemaElement, options, arrowField, readInt96AsTimeStamp); TypeProtos.MajorType.Builder typeBuilder = TypeProtos.MajorType.newBuilder().setMinorType(minorType).setMode(mode); if (CoreDecimalUtility.isDecimalType(minorType)) { typeBuilder.setPrecision(schemaElement.getPrecision()).setScale(schemaElement.getScale()); } return typeBuilder.build(); } }