@Test @Ignore public void testUnionDistinctBothEmptyJson() throws Exception { final String rootEmpty = FileUtils.getResourceAsFile("/project/pushdown/empty.json").toURI().toString(); final String query = String.format( "select key from dfs_test.\"%s\" " + "union " + "select key from dfs_test.\"%s\"", rootEmpty, rootEmpty); final List<Pair<SchemaPath, MajorType>> expectedSchema = Lists.newArrayList(); final MajorType majorType = TypeProtos.MajorType.newBuilder() .setMinorType(TypeProtos.MinorType.INT) .setMode(TypeProtos.DataMode.OPTIONAL) .build(); expectedSchema.add(Pair.of(SchemaPath.getSimplePath("key"), majorType)); testBuilder() .sqlQuery(query) .schemaBaseLine(expectedSchema) .build() .run(); }
/** * <code>optional .common.MajorType major_type = 1;</code> * * <pre> * the type associated with this field. * </pre> */ public Builder mergeMajorType(com.dremio.common.types.TypeProtos.MajorType value) { if (majorTypeBuilder_ == null) { if (((bitField0_ & 0x00000001) == 0x00000001) && majorType_ != com.dremio.common.types.TypeProtos.MajorType.getDefaultInstance()) { majorType_ = com.dremio.common.types.TypeProtos.MajorType.newBuilder(majorType_).mergeFrom(value).buildPartial(); } else { majorType_ = value; } onChanged(); } else { majorTypeBuilder_.mergeFrom(value); } bitField0_ |= 0x00000001; return this; } /**
@Ignore("DX-4180") @Test public void testUnionDistinctBothEmptyBatch() throws Exception { String rootSimple = FileUtils.getResourceAsFile("/store/json/booleanData.json").toURI().toString(); final String query = String.format( "select key from dfs_test.\"%s\" where 1 = 0 " + "union " + "select key from dfs_test.\"%s\" where 1 = 0", rootSimple, rootSimple); final List<Pair<SchemaPath, MajorType>> expectedSchema = Lists.newArrayList(); final MajorType majorType = TypeProtos.MajorType.newBuilder() .setMinorType(TypeProtos.MinorType.INT) .setMode(TypeProtos.DataMode.OPTIONAL) .build(); expectedSchema.add(Pair.of(SchemaPath.getSimplePath("key"), majorType)); testBuilder() .sqlQuery(query) .schemaBaseLine(expectedSchema) .build() .run(); }
public static MajorType getMajorTypeForArrowType(ArrowType arrowType, List<Field> children) { MajorType.Builder builder = MajorType.newBuilder() .setMinorType(getMinorTypeFromArrowMinorType(getMinorTypeForArrowType(arrowType))) .setMode(DataMode.OPTIONAL);
return; case 1: builder.setMajorType(input.mergeObject(com.dremio.common.types.TypeProtos.MajorType.newBuilder(), com.dremio.common.types.SchemaTypeProtos.MajorType.MERGE));
TypeProtos.MajorType.Builder majorTypeBuilder = TypeProtos.MajorType.newBuilder() .setMinorType(MinorType.DECIMAL38SPARSE) .setMode(DataMode.OPTIONAL)
/** * Helper method that converts Hive type definition to Dremio type definition. * * @param typeInfo Hive type info * @param options * @return */ public static MajorType getMajorTypeFromHiveTypeInfo(final TypeInfo typeInfo, final OptionManager options) { switch (typeInfo.getCategory()) { case PRIMITIVE: { PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo; MinorType minorType = getMinorTypeFromHivePrimitiveTypeInfo(primitiveTypeInfo, options); MajorType.Builder typeBuilder = MajorType.newBuilder().setMinorType(getMinorTypeFromArrowMinorType(minorType)) .setMode(DataMode.OPTIONAL); // Hive columns (both regular and partition) could have null values if (primitiveTypeInfo.getPrimitiveCategory() == PrimitiveCategory.DECIMAL) { DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo; typeBuilder.setPrecision(decimalTypeInfo.precision()) .setScale(decimalTypeInfo.scale()).build(); } return typeBuilder.build(); } case LIST: case MAP: case STRUCT: case UNION: default: throwUnsupportedHiveDataTypeError(typeInfo.getCategory().toString()); } return null; // never reached }
@JsonIgnore public MajorType getMajorType() { MajorType.Builder b = MajorType.newBuilder(); b.setMode(mode); b.setMinorType(minorType); if (precision != null) { b.setPrecision(precision); } if (width != null) { b.setWidth(width); } if (scale != null) { b.setScale(scale); } return b.build(); }
public static MajorType optional(final MinorType type) { return MajorType.newBuilder().setMode(DataMode.OPTIONAL).setMinorType(type).build(); }
public Builder toBuilder() { return newBuilder(this); }
@JsonIgnore public MajorType getMajorType() { MajorType.Builder b = MajorType.newBuilder(); b.setMode(mode); b.setMinorType(minorType); if (precision != null) { b.setPrecision(precision); } if (width != null) { b.setWidth(width); } if (scale != null) { b.setScale(scale); } return b.build(); }
public Builder newBuilderForType() { return newBuilder(); } public static Builder newBuilder(com.dremio.common.types.TypeProtos.MajorType prototype) {
public static MajorType withScaleAndPrecision(final MinorType type, final DataMode mode, final int scale, final int precision) { return MajorType.newBuilder().setMinorType(type).setMode(mode).setScale(scale).setPrecision(precision).build(); }
public static MajorType required(final MinorType type) { return MajorType.newBuilder().setMode(DataMode.REQUIRED).setMinorType(type).build(); }
public static MajorType withMode(final MinorType type, final DataMode mode) { return MajorType.newBuilder().setMode(mode).setMinorType(type).build(); }
public SerializedField.Builder getMetadataBuilder() { return SerializedField.newBuilder() .setMajorType(MajorType.newBuilder().setMinorType(MinorType.LIST).setMode(DataMode.OPTIONAL).build()) .setNamePart(NamePart.newBuilder().setName(listVector.getField().getName())) .setValueCount(listVector.getValueCount()) .setBufferLength(listVector.getBufferSize()) .addChild(buildOffsetMetadata()) .addChild(buildValidityMetadata()) .addChild(TypeHelper.getMetadata(listVector.vector)); }
public static MajorType repeated(final MinorType type) { return MajorType.newBuilder().setMode(REPEATED).setMinorType(type).build(); }
public static Builder newBuilder(com.dremio.common.types.TypeProtos.MajorType prototype) { return newBuilder().mergeFrom(prototype); } public Builder toBuilder() { return newBuilder(this); }
public static TypeProtos.MajorType toMajorType(PrimitiveType.PrimitiveTypeName primitiveTypeName, int length, TypeProtos.DataMode mode, SchemaElement schemaElement, OptionManager options, Field arrowField, final boolean readInt96AsTimeStamp) { MinorType minorType = getMinorType(primitiveTypeName, length, schemaElement, options, arrowField, readInt96AsTimeStamp); TypeProtos.MajorType.Builder typeBuilder = TypeProtos.MajorType.newBuilder().setMinorType(minorType).setMode(mode); if (CoreDecimalUtility.isDecimalType(minorType)) { typeBuilder.setPrecision(schemaElement.getPrecision()).setScale(schemaElement.getScale()); } return typeBuilder.build(); } }
public com.dremio.common.types.TypeProtos.MajorType.Builder newMessage() { return com.dremio.common.types.TypeProtos.MajorType.newBuilder(); } public java.lang.String getFieldName(int number)