public static RecordField merge(final RecordField thisField, final RecordField otherField) { final String fieldName = thisField.getFieldName(); final Set<String> aliases = new HashSet<>(); aliases.addAll(thisField.getAliases()); aliases.addAll(otherField.getAliases()); final Object defaultValue; if (thisField.getDefaultValue() == null && otherField.getDefaultValue() != null) { defaultValue = otherField.getDefaultValue(); } else { defaultValue = thisField.getDefaultValue(); } final DataType dataType; if (thisField.getDataType().equals(otherField.getDataType())) { dataType = thisField.getDataType(); } else { dataType = RecordFieldType.CHOICE.getChoiceDataType(thisField.getDataType(), otherField.getDataType()); } return new RecordField(fieldName, dataType, defaultValue, aliases, thisField.isNullable() || otherField.isNullable()); }
@Override public RelDataType getRowType(final RelDataTypeFactory typeFactory) { if (relDataType != null) { return relDataType; } RecordSchema schema; try (final InputStream in = session.read(flowFile)) { final RecordReader recordParser = recordParserFactory.createRecordReader(flowFile, in, logger); schema = recordParser.getSchema(); } catch (final Exception e) { throw new ProcessException("Failed to determine schema of data records for " + flowFile, e); } final List<String> names = new ArrayList<>(); final List<RelDataType> types = new ArrayList<>(); final JavaTypeFactory javaTypeFactory = (JavaTypeFactory) typeFactory; for (final RecordField field : schema.getFields()) { names.add(field.getFieldName()); final RelDataType relDataType = getRelDataType(field.getDataType(), javaTypeFactory); types.add(javaTypeFactory.createTypeWithNullability(relDataType, field.isNullable())); } logger.debug("Found Schema: {}", new Object[] {schema}); if (recordSchema == null) { recordSchema = schema; } relDataType = typeFactory.createStructType(Pair.zip(names, types)); return relDataType; }
private static Field buildAvroField(final RecordField recordField) { final Schema schema = buildAvroSchema(recordField.getDataType(), recordField.getFieldName(), recordField.isNullable()); final Field field = new Field(recordField.getFieldName(), schema, null, recordField.getDefaultValue()); for (final String alias : recordField.getAliases()) { field.addAlias(alias); } return field; }
private Map<String, Object> checkTypes(final Map<String, Object> values, final RecordSchema schema) { for (final RecordField field : schema.getFields()) { Object value = getExplicitValue(field, values); if (value == null) { if (field.isNullable() || field.getDefaultValue() != null) { continue; } throw new SchemaValidationException("Field " + field.getFieldName() + " cannot be null"); } if (!DataTypeUtils.isCompatibleDataType(value, field.getDataType())) { throw new SchemaValidationException("Field " + field.getFieldName() + " has a value of " + value + ", which cannot be coerced into the appropriate data type of " + field.getDataType()); } } return values; }
if (!field.isNullable() && field.getDefaultValue() == null) { result.addValidationError(new StandardValidationError(concat(fieldPrefix, field), ValidationErrorType.MISSING_FIELD, "Field is required"));
for (final RecordField childField : schema.getFields()) { final Object childValue = record.getValue(childField); if (childValue == null && !childField.isNullable()) { logger.debug("Value is not compatible with schema because field {} has a null value, which is not allowed in the schema", childField.getFieldName()); return false;
public static RecordField merge(final RecordField thisField, final RecordField otherField) { final String fieldName = thisField.getFieldName(); final Set<String> aliases = new HashSet<>(); aliases.addAll(thisField.getAliases()); aliases.addAll(otherField.getAliases()); final Object defaultValue; if (thisField.getDefaultValue() == null && otherField.getDefaultValue() != null) { defaultValue = otherField.getDefaultValue(); } else { defaultValue = thisField.getDefaultValue(); } final DataType dataType; if (thisField.getDataType().equals(otherField.getDataType())) { dataType = thisField.getDataType(); } else { dataType = RecordFieldType.CHOICE.getChoiceDataType(thisField.getDataType(), otherField.getDataType()); } return new RecordField(fieldName, dataType, defaultValue, aliases, thisField.isNullable() || otherField.isNullable()); }
private static Field buildAvroField(final RecordField recordField) { final Schema schema = buildAvroSchema(recordField.getDataType(), recordField.getFieldName(), recordField.isNullable()); final Field field = new Field(recordField.getFieldName(), schema, null, recordField.getDefaultValue()); for (final String alias : recordField.getAliases()) { field.addAlias(alias); } return field; }
private Map<String, Object> checkTypes(final Map<String, Object> values, final RecordSchema schema) { for (final RecordField field : schema.getFields()) { Object value = getExplicitValue(field, values); if (value == null) { if (field.isNullable() || field.getDefaultValue() != null) { continue; } throw new SchemaValidationException("Field " + field.getFieldName() + " cannot be null"); } if (!DataTypeUtils.isCompatibleDataType(value, field.getDataType())) { throw new SchemaValidationException("Field " + field.getFieldName() + " has a value of " + value + ", which cannot be coerced into the appropriate data type of " + field.getDataType()); } } return values; }
if (!field.isNullable() && field.getDefaultValue() == null) { result.addValidationError(new StandardValidationError(concat(fieldPrefix, field), ValidationErrorType.MISSING_FIELD, "Field is required"));
for (final RecordField childField : schema.getFields()) { final Object childValue = record.getValue(childField); if (childValue == null && !childField.isNullable()) { logger.debug("Value is not compatible with schema because field {} has a null value, which is not allowed in the schema", childField.getFieldName()); return false;