if (theDataset.schema().getFieldIndex(resultsField).isDefined()) { throw new RuntimeException("The field [" + resultsField + "] already exists in the dataset schema. Use the " + RESULTS_FIELD_CONFIG + " configuration parameter to customize the data quality check field name");
@Override public Type field(Types.NestedField field, Supplier<Type> fieldResult) { Preconditions.checkArgument(current instanceof StructType, "Not a struct: %s", current); StructType struct = (StructType) current; // fields are resolved by name because Spark only sees the current table schema. if (struct.getFieldIndex(field.name()).isEmpty()) { // make sure that filter fields are projected even if they aren't in the requested schema. if (filterRefs.contains(field.fieldId())) { return field.type(); } return null; } int fieldIndex = struct.fieldIndex(field.name()); StructField f = struct.fields()[fieldIndex]; Preconditions.checkArgument(f.nullable() || field.isRequired(), "Cannot project an optional field as non-null: %s", field.name()); this.current = f.dataType(); try { return fieldResult.get(); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( "Invalid projection for field " + field.name() + ": " + e.getMessage(), e); } finally { this.current = struct; } }
@Override public Type field(Types.NestedField field, Supplier<Type> fieldResult) { Preconditions.checkArgument(current instanceof StructType, "Not a struct: %s", current); StructType struct = (StructType) current; // fields are resolved by name because Spark only sees the current table schema. if (struct.getFieldIndex(field.name()).isEmpty()) { // make sure that filter fields are projected even if they aren't in the requested schema. if (filterRefs.contains(field.fieldId())) { return field.type(); } return null; } int fieldIndex = struct.fieldIndex(field.name()); StructField f = struct.fields()[fieldIndex]; Preconditions.checkArgument(f.nullable() || field.isRequired(), "Cannot project an optional field as non-null: %s", field.name()); this.current = f.dataType(); try { return fieldResult.get(); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( "Invalid projection for field " + field.name() + ": " + e.getMessage(), e); } finally { this.current = struct; } }