public SparkQueryResult(Dataset<Row> dataset) { // Tuple2<String, String>[] colNameAndColType = dataset.dtypes(); List<Integer> nullable = new ArrayList<>(); List<String> columnClassName = new ArrayList<>(); for (StructField structField : dataset.schema().fields()) { if (structField.nullable()) { nullable.add(columnNullable); } else { nullable.add(columnNoNulls); } columnNames.add(structField.name()); int type = SparkDataTypeConverter.typeInt(structField.dataType()); columnTypes.add(type); columnClassName.add(SparkDataTypeConverter.typeClassName(type)); columnTypeNames.add(structField.dataType().typeName()); } dbmsQueryResultMetaData.isNullable = nullable; dbmsQueryResultMetaData.columnClassName = columnClassName; result = dataset.collectAsList(); }
@Override public Type struct(StructType struct, List<Type> types) { StructField[] fields = struct.fields(); List<Types.NestedField> newFields = Lists.newArrayListWithExpectedSize(fields.length); boolean isRoot = root == struct; for (int i = 0; i < fields.length; i += 1) { StructField field = fields[i]; Type type = types.get(i); int id; if (isRoot) { // for new conversions, use ordinals for ids in the root struct id = i; } else { id = getNextId(); } if (field.nullable()) { newFields.add(Types.NestedField.optional(id, field.name(), type)); } else { newFields.add(Types.NestedField.required(id, field.name(), type)); } } return Types.StructType.of(newFields); }
if (field.nullable()) { LOG.trace("Setting Field[{} => null] for DataType[{}]", fieldName, fieldDataType); values.add(null);
Object fieldValue = toRowValue(record.get(fieldName), fields[i].dataType(), fieldPath); if (fieldValue == null && !fields[i].nullable()) { throw new IllegalArgumentException("Null value is not allowed for row field at " + fieldPath);
Object fieldValue = toRowValue(record.get(fieldName), fields[i].dataType(), fieldPath); if (fieldValue == null && !fields[i].nullable()) { throw new IllegalArgumentException("Null value is not allowed for row field at " + fieldPath);
@Override public Type field(Types.NestedField field, Supplier<Type> fieldResult) { Preconditions.checkArgument(current instanceof StructType, "Not a struct: %s", current); StructType struct = (StructType) current; // fields are resolved by name because Spark only sees the current table schema. if (struct.getFieldIndex(field.name()).isEmpty()) { // make sure that filter fields are projected even if they aren't in the requested schema. if (filterRefs.contains(field.fieldId())) { return field.type(); } return null; } int fieldIndex = struct.fieldIndex(field.name()); StructField f = struct.fields()[fieldIndex]; Preconditions.checkArgument(f.nullable() || field.isRequired(), "Cannot project an optional field as non-null: %s", field.name()); this.current = f.dataType(); try { return fieldResult.get(); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( "Invalid projection for field " + field.name() + ": " + e.getMessage(), e); } finally { this.current = struct; } }
Object fieldValue = toRowValue(record.get(fieldName), fields[i].dataType(), fieldPath); if (fieldValue == null && !fields[i].nullable()) { throw new IllegalArgumentException("Null value is not allowed for row field at " + fieldPath);
@Override public Type field(Types.NestedField field, Supplier<Type> fieldResult) { Preconditions.checkArgument(current instanceof StructType, "Not a struct: %s", current); StructType struct = (StructType) current; // fields are resolved by name because Spark only sees the current table schema. if (struct.getFieldIndex(field.name()).isEmpty()) { // make sure that filter fields are projected even if they aren't in the requested schema. if (filterRefs.contains(field.fieldId())) { return field.type(); } return null; } int fieldIndex = struct.fieldIndex(field.name()); StructField f = struct.fields()[fieldIndex]; Preconditions.checkArgument(f.nullable() || field.isRequired(), "Cannot project an optional field as non-null: %s", field.name()); this.current = f.dataType(); try { return fieldResult.get(); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( "Invalid projection for field " + field.name() + ": " + e.getMessage(), e); } finally { this.current = struct; } }
type, f.dataType(), value)); } else if (!f.nullable()) { throw new RuntimeException(String.format("Type[%s] - Value cannot be 'null': %s[null]", type, f.dataType())); fields[i].dataType(), value)); } else if (!fields[i].nullable()) { throw new RuntimeException(String.format("Type[%s] - Element cannot be 'null': %s[null]", type, fields[i].dataType())); type, fields[i].dataType(), value)); } else if (!fields[i].nullable()) { throw new RuntimeException(String.format("Type[%s] - Value cannot be 'null': %s[null]", type, fields[i].dataType()));
structField.nullable() ? fieldBuilder.type().nullable() : fieldBuilder.type(); if (dt.equals(DataTypes.TimestampType)) { fieldType.stringType().noDefault();
Schema fieldSchema = dataTypeToSchema(structField.dataType(), recordCounter); fields.add(Schema.Field.of(structField.name(), structField.nullable() ? Schema.nullableOf(fieldSchema) : fieldSchema));
Schema fieldSchema = dataTypeToSchema(structField.dataType(), recordCounter); fields.add(Schema.Field.of(structField.name(), structField.nullable() ? Schema.nullableOf(fieldSchema) : fieldSchema));
Schema fieldSchema = dataTypeToSchema(structField.dataType(), recordCounter); fields.add(Schema.Field.of(structField.name(), structField.nullable() ? Schema.nullableOf(fieldSchema) : fieldSchema));
private static Schema schemaFor(StructType structType, String record, String namespace, int recordCount) { // Increment after using the recordCount record = (null == record) ? "record" + recordCount++ : record; LOG.debug("Converting {} to Avro Record schema [{}:{}]", structType, record, namespace); SchemaBuilder.RecordBuilder<Schema> schema = SchemaBuilder.record(record); if (null != namespace) { schema.namespace(namespace); } schema.doc("Auto-generated from Spark DataFrame"); SchemaBuilder.FieldAssembler<Schema> assembler = schema.fields(); StructField[] structFields = structType.fields(); for (StructField f : structFields) { assembler.name(f.name()).type(typeFor(f.dataType(), f.nullable(), recordCount)).noDefault(); } return assembler.endRecord(); }