int choice = random.nextInt(20); switch (primitive.typeId()) { case BOOLEAN: return choice < 10;
private static Object generatePrimitive(Type.PrimitiveType primitive, Random random) { int choice = random.nextInt(20); switch (primitive.typeId()) { case BOOLEAN: return choice < 10;
int choice = random.nextInt(20); switch (primitive.typeId()) { case BOOLEAN: return choice < 10;
case INT_16: case INT_32: if (expected.typeId() == TypeID.LONG) { return new IntAsLongReader(desc); } else { return new BytesReader(desc); case INT32: if (expected != null && expected.typeId() == TypeID.LONG) { return new IntAsLongReader(desc); } else { if (expected != null && expected.typeId() == TypeID.DOUBLE) { return new ParquetValueReaders.FloatAsDoubleReader(desc); } else {
public Type primitive(PrimitiveType primitive, Type.Repetition repetition, int id, String name) { switch (primitive.typeId()) { case BOOLEAN: return Types.primitive(BOOLEAN, repetition).id(id).named(name);
case INT_16: case INT_32: if (expected != null && expected.typeId() == Types.LongType.get().typeId()) { return new IntAsLongReader(desc); } else { return new BytesReader(desc); case INT32: if (expected != null && expected.typeId() == TypeID.LONG) { return new IntAsLongReader(desc); } else { if (expected != null && expected.typeId() == TypeID.DOUBLE) { return new FloatAsDoubleReader(desc); } else {
case INT_16: case INT_32: if (expected != null && expected.typeId() == Types.LongType.get().typeId()) { return new IntAsLongReader(desc); } else { return new BytesReader(desc); case INT32: if (expected != null && expected.typeId() == TypeID.LONG) { return new IntAsLongReader(desc); } else { if (expected != null && expected.typeId() == TypeID.DOUBLE) { return new FloatAsDoubleReader(desc); } else {
@Override public DataType primitive(Type.PrimitiveType primitive) { switch (primitive.typeId()) { case BOOLEAN: return BooleanType$.MODULE$;
@Override public Type primitive(Type.PrimitiveType primitive) { Class<? extends DataType> expectedType = TYPES.get(primitive.typeId()); Preconditions.checkArgument(expectedType != null && expectedType.isInstance(current), "Cannot project %s to incompatible type: %s", primitive, current); // additional checks based on type switch (primitive.typeId()) { case DECIMAL: Types.DecimalType decimal = (Types.DecimalType) primitive; DecimalType d = (DecimalType) current; Preconditions.checkArgument(d.scale() == decimal.scale(), "Cannot project decimal with incompatible scale: %s != %s", d.scale(), decimal.scale()); Preconditions.checkArgument(d.precision() >= decimal.precision(), "Cannot project decimal with incompatible precision: %s < %s", d.precision(), decimal.precision()); break; case TIMESTAMP: Types.TimestampType timestamp = (Types.TimestampType) primitive; Preconditions.checkArgument(timestamp.shouldAdjustToUTC(), "Cannot project timestamp (without time zone) as timestamptz (with time zone)"); break; default: } return primitive; }
@Override public Type primitive(Type.PrimitiveType primitive) { Class<? extends DataType> expectedType = TYPES.get(primitive.typeId()); Preconditions.checkArgument(expectedType != null && expectedType.isInstance(current), "Cannot project %s to incompatible type: %s", primitive, current); // additional checks based on type switch (primitive.typeId()) { case DECIMAL: Types.DecimalType decimal = (Types.DecimalType) primitive; DecimalType d = (DecimalType) current; Preconditions.checkArgument(d.scale() == decimal.scale(), "Cannot project decimal with incompatible scale: %s != %s", d.scale(), decimal.scale()); Preconditions.checkArgument(d.precision() >= decimal.precision(), "Cannot project decimal with incompatible precision: %s < %s", d.precision(), decimal.precision()); break; case TIMESTAMP: Types.TimestampType timestamp = (Types.TimestampType) primitive; Preconditions.checkArgument(timestamp.shouldAdjustToUTC(), "Cannot project timestamp (without time zone) as timestamptz (with time zone)"); break; default: } return primitive; }
public static boolean isPromotionAllowed(Type from, Type.PrimitiveType to) { // Warning! Before changing this function, make sure that the type change doesn't introduce // compatibility problems in partitioning. if (from.equals(to)) { return true; } switch (from.typeId()) { case INTEGER: return to == Types.LongType.get(); case FLOAT: return to == Types.DoubleType.get(); case DECIMAL: Types.DecimalType fromDecimal = (Types.DecimalType) from; if (to.typeId() != Type.TypeID.DECIMAL) { return false; } Types.DecimalType toDecimal = (Types.DecimalType) to; return (fromDecimal.scale() == toDecimal.scale() && fromDecimal.precision() <= toDecimal.precision()); } return false; }
@Override public Object primitive(Type.PrimitiveType primitive) { Object result = generatePrimitive(primitive, random); // For the primitives that Avro needs a different type than Spark, fix // them here. switch (primitive.typeId()) { case STRING: return ((UTF8String) result).toString(); case FIXED: return new GenericData.Fixed(typeToSchema.get(primitive), (byte[]) result); case BINARY: return ByteBuffer.wrap((byte[]) result); case UUID: return UUID.nameUUIDFromBytes((byte[]) result); case DECIMAL: return ((Decimal) result).toJavaBigDecimal(); default: return result; } } }
@Override public Type primitive(Type.PrimitiveType primitive) { if (sourceType.equals(primitive)) { return primitive; // already correct } switch (primitive.typeId()) { case STRING: if (sourceType.typeId() == Type.TypeID.UUID) { return sourceType; } break; case BINARY: if (sourceType.typeId() == Type.TypeID.FIXED) { return sourceType; } break; default: } // nothing to fix up, let validation catch promotion errors return primitive; } }
@Override public Object primitive(Type.PrimitiveType primitive) { Object result = generatePrimitive(primitive, random); // For the primitives that Avro needs a different type than Spark, fix // them here. switch (primitive.typeId()) { case FIXED: return new GenericData.Fixed(typeToSchema.get(primitive), (byte[]) result); case BINARY: return ByteBuffer.wrap((byte[]) result); case UUID: return UUID.nameUUIDFromBytes((byte[]) result); default: return result; } } }
@Override public Object primitive(Type.PrimitiveType primitive) { Object result = generatePrimitive(primitive, random); switch (primitive.typeId()) { case BINARY: return ByteBuffer.wrap((byte[]) result); case UUID: return UUID.nameUUIDFromBytes((byte[]) result); default: return result; } } }
@SuppressWarnings("unchecked") private Class<T> javaType() { return (Class<T>) type.asPrimitiveType().typeId().javaClass(); } }